scripts/improve-fake-data-realism.js

const fs = require('fs');
const path = require('path');

const BENCHMARKS_DIR = path.join(__dirname, '..', 'public', 'benchmarks');

// Model capabilities - which models support what modalities
const MODEL_CAPABILITIES = {
  'openai-gpt-4o': { text: true, vision: true, audio: false },
  'anthropic-claude-3-5-sonnet': { text: true, vision: true, audio: false },
  'meta-llama-3-70b': { text: true, vision: false, audio: false },
  'google-gemma-2-27b': { text: true, vision: false, audio: false },
  'mistral-mistral-large': { text: true, vision: false, audio: false },
  'alibaba-qwen-2-72b': { text: true, vision: false, audio: false }
};

// Realistic sample questions/responses for different benchmark types
const SAMPLE_TEMPLATES = {
  text_qa: [
    {
      input: "What is the capital of France and what is its approximate population?",
      ground_truth: "Paris, with approximately 2.1 million people in the city proper",
      responses: [
        "Paris is the capital of France, with a population of around 2.2 million in the city proper.",
        "The capital is Paris. Its population is approximately 2.1 million people.",
        "Paris is France's capital city with about 2 million residents."
      ]
    },
    {
      input: "Explain the concept of photosynthesis in simple terms.",
      ground_truth: "Photosynthesis is the process by which plants use sunlight, water, and carbon dioxide to create oxygen and energy in the form of sugar.",
      responses: [
        "Photosynthesis is how plants convert sunlight, water, and CO2 into glucose and oxygen.",
        "It's the process where plants use light energy to make food from water and carbon dioxide, releasing oxygen as a byproduct.",
        "Plants take in light, water, and carbon dioxide to produce sugars for energy and release oxygen."
      ]
    },
    {
      input: "What are the three branches of the US government?",
      ground_truth: "The three branches are: Legislative (Congress), Executive (President), and Judicial (Supreme Court and federal courts).",
      responses: [
        "The US government has three branches: Legislative, Executive, and Judicial.",
        "There are three branches: Congress (makes laws), the President (enforces laws), and the courts (interpret laws).",
        "Legislative branch, Executive branch, and Judicial branch are the three branches of government."
      ]
    }
  ],
  reasoning: [
    {
      input: "If it takes 5 machines 5 minutes to make 5 widgets, how long would it take 100 machines to make 100 widgets?",
      ground_truth: "5 minutes",
      responses: [
        "5 minutes - each machine makes one widget in 5 minutes",
        "It would take 5 minutes since the rate per machine remains constant",
        "5 minutes. The time doesn't change with parallel production."
      ]
    },
    {
      input: "All roses are flowers. Some flowers fade quickly. Can we conclude that some roses fade quickly?",
      ground_truth: "No, we cannot conclude that with certainty from the given premises.",
      responses: [
        "No, this conclusion doesn't necessarily follow from the premises",
        "We cannot make that conclusion - the flowers that fade quickly might not include roses",
        "No, the premises don't provide enough information to conclude this about roses specifically"
      ]
    }
  ],
  coding: [
    {
      input: "Write a Python function to reverse a string",
      ground_truth: "def reverse_string(s):\n    return s[::-1]",
      responses: [
        "def reverse_string(s):\n    return s[::-1]",
        "def reverse_string(text):\n    return ''.join(reversed(text))",
        "def reverse_string(s):\n    return s[::-1]  # Using slice notation"
      ]
    },
    {
      input: "Implement a function to check if a number is prime",
      ground_truth: "def is_prime(n):\n    if n < 2: return False\n    for i in range(2, int(n**0.5) + 1):\n        if n % i == 0: return False\n    return True",
      responses: [
        "def is_prime(n):\n    if n < 2: return False\n    for i in range(2, int(n**0.5) + 1):\n        if n % i == 0: return False\n    return True",
        "def is_prime(num):\n    if num <= 1: return False\n    if num == 2: return True\n    if num % 2 == 0: return False\n    for i in range(3, int(num**0.5) + 1, 2):\n        if num % i == 0: return False\n    return True",
        "def is_prime(n):\n    return n > 1 and all(n % i for i in range(2, int(n**0.5) + 1))"
      ]
    }
  ],
  vision: [
    {
      input: "Describe what you see in this image [image of a beach scene]",
      ground_truth: "A sandy beach with clear blue water, white clouds in the sky, and people relaxing under umbrellas",
      responses: [
        "I see a beautiful beach scene with golden sand, turquoise water, and a clear blue sky with scattered clouds. Several beach umbrellas dot the shoreline with people underneath.",
        "The image shows a coastal beach with white sand, calm blue ocean waters, and beachgoers relaxing. There are colorful umbrellas providing shade.",
        "A picturesque beach scene featuring sandy shores, azure waters, fluffy clouds, and vacationers enjoying the beach under various umbrellas."
      ]
    }
  ]
};

function getRandomElement(array) {
  return array[Math.floor(Math.random() * array.length)];
}

function generateVariedScore(baseScore, variance = 0.15) {
  const min = Math.max(0, baseScore - variance);
  const max = Math.min(1, baseScore + variance);
  return min + Math.random() * (max - min);
}

function generateRealisticSubtaskScores(overallScore) {
  // Generate varied subtask scores that average to approximately the overall score
  const variance = 0.08;
  const subtask_a = generateVariedScore(overallScore, variance);
  const subtask_b = generateVariedScore(overallScore, variance);
  
  return { subtask_a, subtask_b };
}

function generateRealisticSamples(modelName, numSamples = 10) {
  const samples = [];
  
  for (let i = 0; i < numSamples; i++) {
    // Pick random template type
    const templateTypes = ['text_qa', 'reasoning', 'coding'];
    const type = getRandomElement(templateTypes);
    const template = getRandomElement(SAMPLE_TEMPLATES[type]);
    const response = getRandomElement(template.responses);
    
    // Generate realistic score based on string similarity
    const baseScore = 0.6 + Math.random() * 0.35; // Range 0.6-0.95
    const score = Math.round(baseScore * 100) / 100;
    
    samples.push({
      sample_id: `sample_${i}`,
      input: template.input,
      ground_truth: template.ground_truth,
      response: response,
      score: score
    });
  }
  
  return samples;
}

function shouldRemoveBenchmark(modelId, benchmark) {
  const capabilities = MODEL_CAPABILITIES[modelId];
  if (!capabilities) return false;
  
  // Remove vision benchmarks for text-only models
  const visionBenchmarks = [
    'ImageNet', 'COCO', 'CIFAR', 'Pascal VOC', 'Cityscapes',
    'ADE20K', 'Kinetics', 'KITTI', 'Places365', 'UCF-101',
    'NYU Depth', 'CelebA', 'Visual Genome', 'LVIS', 'Mapillary',
    'MPII Human Pose', 'Open Images', 'ScanNet', 'nuScenes',
    'IntPhys', 'DigiData', 'ActivityNet', 'DAVIS', 'VQA',
    'CLEVR', 'Waymo', 'Fashion-MNIST', 'MNIST', 'SVHN',
    'Argoverse', 'BDD100K', 'Moments in Time', 'HMDB',
    'Something-Something', 'Epic-Kitchens', 'Charades'
  ];
  
  // Also check for vision-related keywords in input/output modality
  const hasVisionModality = 
    benchmark.factsheet?.input_modality?.toLowerCase().includes('vision') ||
    benchmark.factsheet?.input_modality?.toLowerCase().includes('image') ||
    benchmark.factsheet?.input_modality?.toLowerCase().includes('video') ||
    benchmark.factsheet?.output_modality?.toLowerCase().includes('image');
  
  const isVisionBenchmark = visionBenchmarks.some(vb => 
    benchmark.evaluation_name.includes(vb)
  ) || hasVisionModality;
  
  return isVisionBenchmark && !capabilities.vision;
}

function adjustScoreForModel(benchmarkName, modelId, originalScore) {
  // Adjust scores to be more realistic based on known model capabilities
  const adjustments = {
    'openai-gpt-4o': 1.0, // Keep as-is, top tier
    'anthropic-claude-3-5-sonnet': 0.98, // Very close to GPT-4
    'meta-llama-3-70b': 0.85, // Good but not best
    'google-gemma-2-27b': 0.80, // Smaller model
    'mistral-mistral-large': 0.88, // Solid performance
    'alibaba-qwen-2-72b': 0.83 // Good but less known
  };
  
  const factor = adjustments[modelId] || 0.85;
  
  // Add some benchmark-specific adjustments
  if (benchmarkName.includes('Coding') || benchmarkName.includes('HumanEval')) {
    if (modelId === 'anthropic-claude-3-5-sonnet') return originalScore * 1.05; // Claude is strong at coding
  }
  
  if (benchmarkName.includes('BBQ') || benchmarkName.includes('Fairness')) {
    if (modelId === 'anthropic-claude-3-5-sonnet') return originalScore * 1.02; // Claude focuses on safety
  }
  
  return Math.min(0.95, originalScore * factor);
}

function addRealisticVariation(timestamp) {
  // Add some random hours/minutes to make timestamps more realistic
  const date = new Date(timestamp);
  date.setHours(Math.floor(Math.random() * 24));
  date.setMinutes(Math.floor(Math.random() * 60));
  date.setSeconds(Math.floor(Math.random() * 60));
  return date.toISOString();
}

function improveBenchmarkData() {
  const files = fs.readdirSync(BENCHMARKS_DIR).filter(f => f.endsWith('.json'));
  
  for (const file of files) {
    const filePath = path.join(BENCHMARKS_DIR, file);
    const content = JSON.parse(fs.readFileSync(filePath, 'utf8'));
    
    const modelId = file.replace('.json', '');
    console.log(`Improving ${modelId}...`);
    
    // Filter out vision benchmarks for text-only models
    content.evaluation_results = content.evaluation_results.filter(
      benchmark => !shouldRemoveBenchmark(modelId, benchmark)
    );
    
    // Improve each benchmark result
    content.evaluation_results.forEach(benchmark => {
      const originalScore = benchmark.score_details.score;
      
      // Adjust score for model capability
      const adjustedScore = adjustScoreForModel(
        benchmark.evaluation_name,
        modelId,
        originalScore
      );
      
      // Generate varied subtask scores
      const subtasks = generateRealisticSubtaskScores(adjustedScore);
      
      benchmark.score_details.score = adjustedScore;
      benchmark.score_details.details.subtask_a = subtasks.subtask_a;
      benchmark.score_details.details.subtask_b = subtasks.subtask_b;
      
      // Add realistic variation to timestamp
      if (benchmark.evaluation_timestamp) {
        benchmark.evaluation_timestamp = addRealisticVariation(
          benchmark.evaluation_timestamp
        );
      }
    });
    
    // Update source metadata to be more realistic
    const modelInfo = content.model_info;
    content.source_metadata.source_organization_name = `${modelInfo.developer} Research`;
    content.source_metadata.source_name = `${modelInfo.name} Evaluation Suite`;
    content.source_data.dataset_name = `Multi-Domain Benchmark Collection`;
    
    // Generate realistic sample data
    content.detailed_evaluation_results_per_samples = generateRealisticSamples(
      modelInfo.name,
      10
    );
    
    // Write back to file
    fs.writeFileSync(filePath, JSON.stringify(content, null, 2));
    console.log(`  ✓ Improved ${content.evaluation_results.length} benchmarks`);
    console.log(`  ✓ Generated ${content.detailed_evaluation_results_per_samples.length} realistic samples`);
  }
  
  console.log('\n✅ All benchmark data has been improved for realism!');
}

// Run the improvement
improveBenchmarkData();