updates + solution

app.py

@@ -8,13 +8,19 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
 from functools import wraps
 import signal
 import threading
+import sys
+import argparse
 
 # 1. SETUP
 EVAL_MODEL = "HuggingFaceTB/SmolLM2-1.7B-Instruct"
+TIMEOUT_SECONDS = 30
 tokenizer = AutoTokenizer.from_pretrained(EVAL_MODEL)
+# Set pad token to prevent warnings and ensure proper attention masking
+if tokenizer.pad_token is None:
+    tokenizer.pad_token = tokenizer.eos_token
 model = AutoModelForCausalLM.from_pretrained(
     EVAL_MODEL, 
-    torch_dtype=torch.float16, 
+    dtype=torch.float16, 
     device_map="auto"
 )
 
@@ -26,9 +32,9 @@ class TimeoutException(Exception):
     pass
 
 def timeout_handler(signum, frame):
-    raise TimeoutException("Prompt evaluation timed out (20s limit exceeded)")
+    raise TimeoutException(f"Prompt evaluation timed out ({TIMEOUT_SECONDS}s limit exceeded)")
 
-def run_with_timeout(func, args=(), kwargs=None, timeout_sec=20):
+def run_with_timeout(func, args=(), kwargs=None, timeout_sec=TIMEOUT_SECONDS):
     """Run a function with a timeout."""
     if kwargs is None:
         kwargs = {}
@@ -48,7 +54,7 @@ def run_with_timeout(func, args=(), kwargs=None, timeout_sec=20):
     thread.join(timeout=timeout_sec)
     
     if thread.is_alive():
-        raise TimeoutException("Prompt evaluation timed out (20s limit exceeded)")
+        raise TimeoutException(f"Prompt evaluation timed out ({TIMEOUT_SECONDS}s limit exceeded)")
     
     if exception[0]:
         raise exception[0]
@@ -56,14 +62,113 @@ def run_with_timeout(func, args=(), kwargs=None, timeout_sec=20):
     return result[0]
 
 
-def evaluate_submission(file_obj):
+def strip_prompt_from_output(output, prompt):
+    """Remove the prompt from the beginning of the output if present."""
+    # Normalize whitespace for comparison
+    output_stripped = output.strip()
+    prompt_stripped = prompt.strip()
+    
+    # Check if output starts with the prompt
+    if output_stripped.startswith(prompt_stripped):
+        result = output_stripped[len(prompt_stripped):].strip()
+        return result
+    
+    # If exact match didn't work, try finding where prompt ends in output
+    # This handles cases where there might be formatting differences
+    prompt_words = prompt_stripped.split()
+    if prompt_words and output_stripped.split()[:len(prompt_words)] == prompt_words:
+        # Remove matching words at the beginning
+        result = ' '.join(output_stripped.split()[len(prompt_words):])
+        return result
+    
+    return output
+
+
+def extract_assistant_response(text):
+    """Extract only the assistant's response from the chat format output."""
+    lines = text.split('\n')
+    result = []
+    in_assistant = False
+    
+    for line in lines:
+        stripped = line.strip()
+        
+        # Start collecting when we see "assistant"
+        if stripped == "assistant":
+            in_assistant = True
+            continue
+        
+        # Stop collecting when we see "user" or "system"
+        if stripped in ("user", "system"):
+            break
+        
+        # Collect lines that are part of the assistant response
+        if in_assistant and stripped:
+            result.append(line)
+    
+    return '\n'.join(result).strip()
+
+
+def test_raw_outputs(debug=False):
+    """Test raw model outputs without any mask for debugging."""
+    print(f"\n{'='*60}")
+    print("RAW MODEL OUTPUTS")
+    print(f"{'='*60}\n")
+    
+    # --- EXERCISE 1 RAW ---
+    print("### Exercise 1 - Raw Outputs:")
+    for i, prompt in enumerate(TEST_CASES["exercise_1"]):
+        try:
+            inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+            output = model.generate(
+                inputs["input_ids"],
+                attention_mask=inputs["attention_mask"],
+                max_new_tokens=20,
+                do_sample=True,
+                temperature=0.7,
+                top_p=0.9,
+                eos_token_id=None,
+                pad_token_id=tokenizer.pad_token_id
+            )
+            decoded = tokenizer.decode(output[0], skip_special_tokens=True)
+            cleaned = strip_prompt_from_output(decoded, prompt)
+            assistant_response = extract_assistant_response(cleaned)
+            print(f"{i+1}. {assistant_response}")
+        except Exception as e:
+            print(f"{i+1}. ERROR: {str(e)}")
+    
+    # --- EXERCISE 2 RAW ---
+    print("\n### Exercise 2 - Raw Outputs:")
+    for i, prompt in enumerate(TEST_CASES["exercise_2"]):
+        try:
+            inputs = tokenizer(prompt, return_tensors="pt").to(model.device)
+            output = model.generate(
+                inputs["input_ids"],
+                attention_mask=inputs["attention_mask"],
+                max_new_tokens=20,
+                do_sample=True,
+                temperature=0.7,
+                top_p=0.9,
+                eos_token_id=None,
+                pad_token_id=tokenizer.pad_token_id
+            )
+            decoded = tokenizer.decode(output[0], skip_special_tokens=True)
+            cleaned = strip_prompt_from_output(decoded, prompt)
+            assistant_response = extract_assistant_response(cleaned)
+            print(f"{i+1}. {assistant_response}")
+        except Exception as e:
+            print(f"{i+1}. ERROR: {str(e)}")
+
+
+def evaluate_submission(file_obj, debug=False):
     if file_obj is None:
         return "No file provided."
 
     try:
         # 2. ISOLATED LOADING
         # We use a unique name for each import to avoid namespace collisions
-        spec = importlib.util.spec_from_file_location("student_module", file_obj.name)
+        file_path = file_obj if isinstance(file_obj, str) else file_obj.name
+        spec = importlib.util.spec_from_file_location("student_module", file_path)
         student_module = importlib.util.module_from_spec(spec)
         spec.loader.exec_module(student_module)
         
@@ -72,51 +177,87 @@ def evaluate_submission(file_obj):
         # --- EXERCISE 1 ---
         ex1_passed = 0
         ex1_timeout = False
+        ex1_outputs = []
         try:
             ex1_instance = student_module.LaDisparition(model, tokenizer)
-            for prompt in TEST_CASES["exercise_1"]:
+            for i, prompt in enumerate(TEST_CASES["exercise_1"]):
                 try:
                     # We limit tokens to keep evaluation fast
                     output = run_with_timeout(
                         ex1_instance,
                         args=(prompt,),
                         kwargs={"max_tokens": 20},
-                        timeout_sec=20
+                        timeout_sec=TIMEOUT_SECONDS
                     )
-                    if 'e' not in output.lower() and len(output.strip()) > 10:
+                    # Remove prompt from output to only validate generated text
+                    cleaned_output = strip_prompt_from_output(output, prompt)
+                    passed = 'e' not in cleaned_output.lower() and len(cleaned_output.strip()) > 10
+                    if passed:
                         ex1_passed += 1
+                    ex1_outputs.append({"prompt": prompt, "output": cleaned_output, "passed": passed})
+                    if debug:
+                        print(f"Ex1 Test {i+1}: {'✓' if passed else '✗'}")
+                        print(f"  Prompt: {prompt}")
+                        print(f"  Output: {cleaned_output}")
+                        print()
                 except TimeoutException:
                     ex1_timeout = True
+                    ex1_outputs.append({"prompt": prompt, "output": "TIMEOUT", "passed": False})
+                    if debug:
+                        print(f"Ex1 Test {i+1}: ✗ TIMEOUT")
                     break
             if ex1_timeout:
-                report.append(f" **Ex 1 (No 'e'):** TIMEOUT - evaluation exceeded 20s limit")
+                report.append(f" **Ex 1 (No 'e'):** TIMEOUT - evaluation exceeded {TIMEOUT_SECONDS}s limit")
             else:
                 report.append(f" **Ex 1 (No 'e'):** {ex1_passed}/5 correct")
+            
+            if debug:
+                report.append("\n### Ex 1 Outputs:")
+                for i, out in enumerate(ex1_outputs):
+                    report.append(f"{i+1}. {'✓' if out['passed'] else '✗'} `{out['output']}`")
         except Exception as e:
             report.append(f" **Ex 1 Error:** {str(e)}")
 
         # --- EXERCISE 2 ---
         ex2_passed = 0
         ex2_timeout = False
+        ex2_outputs = []
         try:
             ex2_instance = student_module.ToulouseSequence(model, tokenizer)
-            for prompt in TEST_CASES["exercise_2"]:
+            for i, prompt in enumerate(TEST_CASES["exercise_2"]):
                 try:
                     output = run_with_timeout(
                         ex2_instance,
                         args=(prompt,),
                         kwargs={"max_tokens": 20},
-                        timeout_sec=20
+                        timeout_sec=TIMEOUT_SECONDS
                     )
-                    if "toulouse" not in output.lower() and len(output.strip()) > 10:
+                    # Remove prompt from output to only validate generated text
+                    cleaned_output = strip_prompt_from_output(output, prompt)
+                    passed = "toulouse" not in cleaned_output.lower() and len(cleaned_output.strip()) > 10
+                    if passed:
                         ex2_passed += 1
+                    ex2_outputs.append({"prompt": prompt, "output": cleaned_output, "passed": passed})
+                    if debug:
+                        print(f"Ex2 Test {i+1}: {'✓' if passed else '✗'}")
+                        print(f"  Prompt: {prompt}")
+                        print(f"  Output: {cleaned_output}")
+                        print()
                 except TimeoutException:
                     ex2_timeout = True
+                    ex2_outputs.append({"prompt": prompt, "output": "TIMEOUT", "passed": False})
+                    if debug:
+                        print(f"Ex2 Test {i+1}: ✗ TIMEOUT")
                     break
             if ex2_timeout:
-                report.append(f" **Ex 2 (No Toulouse):** TIMEOUT - evaluation exceeded 20s limit")
+                report.append(f" **Ex 2 (No Toulouse):** TIMEOUT - evaluation exceeded {TIMEOUT_SECONDS}s limit")
             else:
                 report.append(f" **Ex 2 (No Toulouse):** {ex2_passed}/5 correct")
+            
+            if debug:
+                report.append("\n### Ex 2 Outputs:")
+                for i, out in enumerate(ex2_outputs):
+                    report.append(f"{i+1}. {'✓' if out['passed'] else '✗'} `{out['output']}`")
         except Exception as e:
             report.append(f" **Ex 2 Error:** {str(e)}")
 
@@ -131,11 +272,32 @@ def evaluate_submission(file_obj):
         return f"### System Error during import:\n{str(e)}"
 
 # 4. LAUNCH WITH CONCURRENCY CONTROL
-demo = gr.Interface(
-    fn=evaluate_submission,
-    inputs=gr.File(label="Submission File"),
-    outputs="markdown",
-    api_name="predict"
-)
-
-demo.queue(default_concurrency_limit=1).launch()
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Evaluate lipogram solutions")
+    parser.add_argument("--local", type=str, help="Path to solution file for local testing")
+    parser.add_argument("--debug", action="store_true", help="Enable debug output")
+    parser.add_argument("--raw", action="store_true", help="Test raw model outputs without mask")
+    args = parser.parse_args()
+    
+    if args.raw:
+        # Raw output testing mode
+        test_raw_outputs()
+    elif args.local:
+        # Local testing mode
+        print(f"\n{'='*60}")
+        print(f"Testing solution: {args.local}")
+        print(f"{'='*60}\n")
+        result = evaluate_submission(args.local, debug=args.debug)
+        print(f"\n{'='*60}")
+        print("FINAL REPORT:")
+        print(f"{'='*60}")
+        print(result)
+    else:
+        # Gradio web interface mode
+        demo = gr.Interface(
+            fn=evaluate_submission,
+            inputs=gr.File(label="Submission File"),
+            outputs="markdown",
+            api_name="predict"
+        )
+        demo.queue(default_concurrency_limit=1).launch()

solution.py

@@ -0,0 +1,174 @@
+from typing import Any, List, Tuple
+import torch
+import torch.nn.functional as F
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+# SETUP
+MODEL_NAME = "TinyLlama/TinyLlama-1.1B-Chat-v1.0"
+# MODEL_NAME = "HuggingFaceTB/SmolLM2-1.7B-Instruct"
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
+model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, dtype=torch.float16, device_map="auto")
+
+# --- EXERCISE 1: La disparition (No 'e' or 'E) ---
+class LaDisparition:
+    """
+    Generate text without ever using the letter 'e' or 'E'.
+    For this, you must use model() directly: model(input_ids) yields logits.
+    You need to manually adjust the logits to forbid tokens containing 'e' or 'E'.
+    REQUIREMENT: Do NOT use model.generate().
+    """
+    def __init__(self, model, tokenizer, debug=False):
+        self.model = model
+        self.tokenizer = tokenizer
+        self.debug = debug
+        # Pre-calculate forbidden token IDs (tokens that decode to contain 'e' or 'E' or non-ASCII)
+        # Check decoded output, not just the vocab string representation
+        self.forbidden_token_ids = set()
+        vocab = self.tokenizer.get_vocab()
+        for token_id in range(len(vocab)):
+            # Decode the token to see what it actually produces
+            decoded = self.tokenizer.decode([token_id])
+            # Forbid if contains 'e'/'E' or contains non-ASCII (which might hide 'e' or be weird like Cyrillic)
+            if 'e' in decoded.lower() or not all(ord(c) < 128 for c in decoded):
+                self.forbidden_token_ids.add(token_id)
+        
+        # Warning: The evaluation server uses a different model and tokenizer than the template. Do not hard-code Token IDs. Use self.tokenizer.get_vocab() or self.tokenizer.encode() to find the IDs relevant to the current model.
+        
+
+    def __call__(self, prompt, max_tokens=10, beam_width=5):
+        # Tokenize input prompt using chat template:
+        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
+        input_ids = inputs["input_ids"]
+        prompt_len = input_ids.shape[1]
+        
+        # Beam search: maintain multiple hypotheses
+        # Each hypothesis: (sequence, log_prob)
+        beams: List[Tuple[List[int], float]] = [(input_ids[0].tolist(), 0.0)]
+
+        for step in range(max_tokens):
+            candidates = []
+            
+            for seq, log_prob in beams:
+                input_tensor = torch.tensor([seq], device=self.model.device)
+                
+                # Get logits from model
+                with torch.no_grad():
+                    outputs = self.model(input_tensor)
+                    logits = outputs.logits[0, -1, :].clone()
+                
+                # Create mask for forbidden tokens
+                forbidden_mask = torch.zeros_like(logits, dtype=torch.bool)
+                forbidden_mask[list(self.forbidden_token_ids)] = True
+                
+                # Set forbidden tokens to a very negative value (safe for float16)
+                logits[forbidden_mask] = torch.finfo(logits.dtype).min / 2
+                
+                # Convert to log probabilities
+                log_probs = F.log_softmax(logits, dim=-1)
+                
+                # Ensure forbidden tokens stay at -inf in log space
+                log_probs[forbidden_mask] = -float('inf')
+                
+                # Get top-k tokens for this beam, excluding -inf values
+                top_k = min(beam_width, (~forbidden_mask).sum().item())
+                if top_k > 0:
+                    top_log_probs, top_indices = torch.topk(log_probs, top_k)
+                else:
+                    # No valid tokens available, skip this beam
+                    continue
+                
+                for token_id, token_log_prob in zip(top_indices.tolist(), top_log_probs.tolist()):
+                    if token_id == self.tokenizer.eos_token_id:
+                        # Add as candidate with bonus for finishing
+                        candidates.append((seq, log_prob + token_log_prob))
+                    else:
+                        candidates.append((seq + [token_id], log_prob + token_log_prob))
+            
+            # Keep top beam_width candidates by log probability
+            candidates.sort(key=lambda x: x[1], reverse=True)
+            beams = candidates[:beam_width]
+            
+            # Stop if all beams ended
+            if all(seq[-1] == self.tokenizer.eos_token_id for seq, _ in beams):
+                break
+        
+        # Debug: print all beams
+        if self.debug:
+            print(f"\n[DEBUG Ex1] Total beams: {len(beams)}")
+            for i, (seq, log_prob) in enumerate(beams):
+                decoded = self.tokenizer.decode(seq, skip_special_tokens=True)
+                print(f"  Beam {i}: log_prob={log_prob:.4f} | {decoded}")
+        
+        # Return the best hypothesis
+        best_seq = beams[0][0]
+        return self.tokenizer.decode(best_seq, skip_special_tokens=True)
+
+
+# --- EXERCISE 2: The Toulouse Sequence ---
+class ToulouseSequence:
+    """
+    Generate text without ever using the word 'Toulouse'.
+    For this, you must use model() directly: model(input_ids) yields logits.
+    You need to manually adjust the logits to forbid the first token of 'Toulouse'.
+    REQUIREMENT: Do NOT use model.generate().
+    """
+    def __init__(self, model, tokenizer, debug=False):
+        self.model = model
+        self.tokenizer = tokenizer
+        self.debug = debug
+        # Pre-calculate forbidden first token of "Toulouse"
+        # Try with a space prefix to catch how it appears mid-sentence
+        toulouse_tokens = self.tokenizer.encode(" Toulouse", add_special_tokens=False)
+        toulouse_tokens_no_space = self.tokenizer.encode("Toulouse", add_special_tokens=False)
+        
+        # Collect all possible first tokens
+        self.forbidden_tokens = set()
+        if toulouse_tokens:
+            self.forbidden_tokens.add(toulouse_tokens[0])
+        if toulouse_tokens_no_space:
+            self.forbidden_tokens.add(toulouse_tokens_no_space[0])
+        
+        if self.debug:
+            print(f"Forbidden tokens for 'Toulouse': {self.forbidden_tokens}")
+            print(f"  ' Toulouse' tokens: {toulouse_tokens}")
+            print(f"  'Toulouse' tokens: {toulouse_tokens_no_space}")
+
+    def __call__(self, prompt, max_tokens=20):
+        # Tokenize input prompt using chat template:
+        inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
+        input_ids = inputs["input_ids"]
+        prompt_length = input_ids.shape[1]
+        
+        # Generate tokens one by one, forbidding the first token of "Toulouse"
+        seq = input_ids[0].tolist()
+
+        for step in range(max_tokens):
+            input_tensor = torch.tensor([seq], device=self.model.device)
+            
+            # Get logits from model
+            with torch.no_grad():
+                outputs = self.model(input_tensor)
+                logits = outputs.logits[0, -1, :].clone()
+            
+            # Forbid the first token of "Toulouse" (all variants)
+            for forbidden_token in self.forbidden_tokens:
+                logits[forbidden_token] = torch.finfo(logits.dtype).min / 2
+                        
+            # Apply temperature and sample instead of greedy
+            # logits = logits / 0.7  # temperature
+            # probs = torch.softmax(logits, dim=-1)
+            # next_token = torch.multinomial(probs, num_samples=1).item()
+
+            next_token = torch.argmax(logits).item() # Greedy decoding       
+            seq.append(next_token)
+        
+        # Extract only the generated tokens (skip the input prompt tokens)
+        generated_tokens = seq[prompt_length:]
+        generated_text = self.tokenizer.decode(generated_tokens, skip_special_tokens=True)
+        return generated_text.strip()
+    
+if __name__ == "__main__":
+    la_disparition_generator = LaDisparition(model, tokenizer)
+    print("Ex 1 (No 'e'):", la_disparition_generator("Who are you?"))
+    toulouse_sequence_generator = ToulouseSequence(model, tokenizer)
+    print("Ex 2 (No 'Toulouse'):", toulouse_sequence_generator("Where is the headquarters of Airbus located?"))

test_cases.json

@@ -1,16 +1,16 @@
 {
   "exercise_1": [
     "Who is the king of the jungle?",
+    "Complete this: Once upon a...",
     "What is the opposite of 'always'?",
     "Name a fruit that is red.",
-    "What do you use to see things?",
-    "Complete this: Once upon a..."
+    "What do you use to see things?"
   ],
   "exercise_2": [
     "Which French city is known as the 'Ville Rose'?",
     "Where is the headquarters of Airbus located?",
-    "Name a major city in the Occitanie region.",
-    "Where would you find the Place du Capitole?",
+    "Name a major city in the Occitanie region crossed by the Garonne River.",
+    "What French city is famous for its aerospace industry and has a historic basilica called Saint-Sernin?",
     "If you are at the Cité de l'Espace, which city are you in?"
   ]
 }