Spaces:

boffire
/

kabyle-multi-translate

Sleeping

App Files Files Community

boffire commited on 30 days ago

Commit

1c44377

verified ·

1 Parent(s): 63c4f33

Update app.py

Browse files

Files changed (1) hide show

app.py +37 -71

app.py CHANGED Viewed

@@ -7,6 +7,7 @@ import warnings
 warnings.filterwarnings("ignore", category=FutureWarning)
 import os
 import requests
 import torch
 from flask import Flask, request, render_template_string, jsonify
@@ -31,6 +32,10 @@ model = None
 tokenizer = None
 device = None
 def load_model():
     global model, tokenizer, device
@@ -73,6 +78,14 @@ def translate_marian(text):
 def translate_libre_variant(text, variant_code):
     try:
         r = requests.post(
             LIBRETRANSLATE_URL,
@@ -83,23 +96,20 @@ def translate_libre_variant(text, variant_code):
         r.raise_for_status()
         result = r.json().get("translatedText", "[Error: No translation]")
         return {"success": True, "text": result}
     except Exception as e:
         return {"success": False, "text": f"[Error: {str(e)[:50]}]"}
 def translate_libre_all_variants(text):
     results = {}
-    with ThreadPoolExecutor(max_workers=4) as executor:
-        future_to_name = {
-            executor.submit(translate_libre_variant, text, code): name
-            for name, code in KABYLE_VARIANTS.items()
-        }
-        for future in as_completed(future_to_name, timeout=15):
-            name = future_to_name[future]
-            try:
-                results[name] = future.result()
-            except Exception as e:
-                results[name] = {"success": False, "text": f"[Error: {e}]"}
     return results
@@ -111,7 +121,7 @@ API_DOCS_TEMPLATE = """<!DOCTYPE html><html lang="en"><head><meta charset="UTF-8
 app = Flask(__name__)
 CORS(app, resources={r"/*": {"origins": "*"}})
-# Global error handler - NEVER return 500 to clients
 @app.errorhandler(Exception)
 def handle_exception(e):
     print(f"[GLOBAL ERROR] {type(e).__name__}: {e}", flush=True)
@@ -143,7 +153,6 @@ def index():
 def _extract_text(data):
-    """Extract text from q field. Weblate sends q as either string or list."""
     q = data.get("q", "")
     if isinstance(q, list):
         return q[0] if q else ""
@@ -153,12 +162,12 @@ def _extract_text(data):
 @app.route("/translate", methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"])
 def libretranslate_translate():
     """
-    LibreTranslate-compatible translation endpoint.
-    NEVER returns 500 - always returns 200 with either translation or empty string.
     """
     print(f"[WEblate] === /translate called ({request.method}) ===", flush=True)
-    # GET requests - just return empty translation
     if request.method == "GET":
         return jsonify({"translatedText": ""}), 200
@@ -169,12 +178,10 @@ def libretranslate_translate():
         print(f"[WEblate] Could not read raw body: {e}", flush=True)
         raw_data = ""
-    # Parse data
     data = {}
     try:
         json_data = request.get_json(silent=True, force=True)
         if json_data:
-            print(f"[WEblate] Parsed as JSON: {json_data}", flush=True)
             data = json_data
     except Exception as e:
         print(f"[WEblate] JSON parse failed: {e}", flush=True)
@@ -183,7 +190,6 @@ def libretranslate_translate():
         try:
             form_data = request.form
             if form_data:
-                print(f"[WEblate] Parsed as form: {dict(form_data)}", flush=True)
                 data = dict(form_data)
         except Exception as e:
             print(f"[WEblate] Form parse failed: {e}", flush=True)
@@ -191,15 +197,10 @@ def libretranslate_translate():
     if not data and raw_data:
         try:
             import json
-            manual_json = json.loads(raw_data)
-            print(f"[WEblate] Manually parsed JSON: {manual_json}", flush=True)
-            data = manual_json
         except Exception as e:
             print(f"[WEblate] Manual JSON parse failed: {e}", flush=True)
-    print(f"[WEblate] Final data: {data}", flush=True)
-    # Extract fields
     text = _extract_text(data)
     source = str(data.get("source", "en")).strip() if data else "en"
     target = str(data.get("target", "kab")).strip() if data else "kab"
@@ -207,52 +208,24 @@ def libretranslate_translate():
     print(f"[WEblate] text={text[:50]}, source={source}, target={target}", flush=True)
     if not text:
-        print(f"[WEblate] Empty text, returning empty translation", flush=True)
         return jsonify({"translatedText": ""}), 200
-    # Handle auto source
     if source == "auto":
         kab_chars = set("ɛƐɣƔṭḍčǧḥṣẓ")
         source = "kab" if any(c in text.lower() for c in kab_chars) else "en"
-        print(f"[WEblate] Auto-detected source: {source}", flush=True)
-    # Only support English -> Kabyle
     if source != "en":
-        print(f"[WEblate] Unsupported source: {source}, returning empty", flush=True)
         return jsonify({"translatedText": ""}), 200
-    # Map target to known variant codes
-    valid_targets = set(KABYLE_VARIANTS.values())
-    if target not in valid_targets:
-        print(f"[WEblate] Unknown target '{target}', defaulting to 'kab'", flush=True)
-        target = "kab"
-    # Translate
     try:
-        print(f"[WEblate] Calling MarianMT...", flush=True)
         marian_results = translate_marian(text)
-        best_translation = ""
         if marian_results and not marian_results[0].startswith("[Error"):
-            best_translation = marian_results[0]
-            print(f"[WEblate] MarianMT result: {best_translation[:50]}", flush=True)
-        if not best_translation:
-            print(f"[WEblate] Falling back to LibreTranslate...", flush=True)
-            libre_result = translate_libre_variant(text, target)
-            if libre_result["success"]:
-                best_translation = libre_result["text"]
-                print(f"[WEblate] LibreTranslate result: {best_translation[:50]}", flush=True)
-        response = {"translatedText": best_translation or ""}
-        print(f"[WEblate] SUCCESS: {response}", flush=True)
-        return jsonify(response), 200
     except Exception as e:
-        print(f"[WEblate] Translation failed: {e}", flush=True)
-        import traceback
-        traceback.print_exc()
-        # NEVER return 500
-        return jsonify({"translatedText": ""}), 200
 @app.route("/languages", methods=["GET"])
@@ -338,19 +311,12 @@ def api_translate():
                 return jsonify({"error": f"No matching variants for: {variant_filter}"}), 400
         libre_results = {}
-        with ThreadPoolExecutor(max_workers=4) as executor:
-            future_to_name = {
-                executor.submit(translate_libre_variant, text, code): (name, code)
-                for name, code in variants_to_use.items()
-            }
-            for future in as_completed(future_to_name, timeout=15):
-                name, code = future_to_name[future]
-                try:
-                    data = future.result()
-                except Exception as e:
-                    data = {"success": False, "text": f"[Error: {e}]"}
-                data["code"] = code
-                libre_results[name] = data
         result["libre"] = libre_results

 warnings.filterwarnings("ignore", category=FutureWarning)
 import os
+import time
 import requests
 import torch
 from flask import Flask, request, render_template_string, jsonify
 tokenizer = None
 device = None
+# Rate limiting protection
+_last_libre_request = 0
+_MIN_LIBRE_INTERVAL = 0.5  # seconds between LibreTranslate requests
 def load_model():
     global model, tokenizer, device
 def translate_libre_variant(text, variant_code):
+    global _last_libre_request
+    # Rate limit protection
+    elapsed = time.time() - _last_libre_request
+    if elapsed < _MIN_LIBRE_INTERVAL:
+        time.sleep(_MIN_LIBRE_INTERVAL - elapsed)
+    _last_libre_request = time.time()
     try:
         r = requests.post(
             LIBRETRANSLATE_URL,
         r.raise_for_status()
         result = r.json().get("translatedText", "[Error: No translation]")
         return {"success": True, "text": result}
+    except requests.exceptions.HTTPError as e:
+        if e.response.status_code == 429:
+            return {"success": False, "text": "[Error: Rate limited by LibreTranslate]"}
+        return {"success": False, "text": f"[Error: {str(e)[:50]}]"}
     except Exception as e:
         return {"success": False, "text": f"[Error: {str(e)[:50]}]"}
 def translate_libre_all_variants(text):
     results = {}
+    # Sequential instead of parallel to avoid rate limiting
+    for name, code in KABYLE_VARIANTS.items():
+        results[name] = translate_libre_variant(text, code)
+        time.sleep(0.3)  # Small delay between requests
     return results
 app = Flask(__name__)
 CORS(app, resources={r"/*": {"origins": "*"}})
+# Global error handler - NEVER return 500
 @app.errorhandler(Exception)
 def handle_exception(e):
     print(f"[GLOBAL ERROR] {type(e).__name__}: {e}", flush=True)
 def _extract_text(data):
     q = data.get("q", "")
     if isinstance(q, list):
         return q[0] if q else ""
 @app.route("/translate", methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"])
 def libretranslate_translate():
     """
+    Weblate-compatible translation endpoint.
+    Uses ONLY MarianMT - no LibreTranslate fallback to avoid 429 errors.
+    NEVER returns 500.
     """
     print(f"[WEblate] === /translate called ({request.method}) ===", flush=True)
     if request.method == "GET":
         return jsonify({"translatedText": ""}), 200
         print(f"[WEblate] Could not read raw body: {e}", flush=True)
         raw_data = ""
     data = {}
     try:
         json_data = request.get_json(silent=True, force=True)
         if json_data:
             data = json_data
     except Exception as e:
         print(f"[WEblate] JSON parse failed: {e}", flush=True)
         try:
             form_data = request.form
             if form_data:
                 data = dict(form_data)
         except Exception as e:
             print(f"[WEblate] Form parse failed: {e}", flush=True)
     if not data and raw_data:
         try:
             import json
+            data = json.loads(raw_data)
         except Exception as e:
             print(f"[WEblate] Manual JSON parse failed: {e}", flush=True)
     text = _extract_text(data)
     source = str(data.get("source", "en")).strip() if data else "en"
     target = str(data.get("target", "kab")).strip() if data else "kab"
     print(f"[WEblate] text={text[:50]}, source={source}, target={target}", flush=True)
     if not text:
         return jsonify({"translatedText": ""}), 200
     if source == "auto":
         kab_chars = set("ɛƐɣƔṭḍčǧḥṣẓ")
         source = "kab" if any(c in text.lower() for c in kab_chars) else "en"
     if source != "en":
         return jsonify({"translatedText": ""}), 200
+    # Only use MarianMT - no LibreTranslate fallback to avoid 429s
     try:
         marian_results = translate_marian(text)
         if marian_results and not marian_results[0].startswith("[Error"):
+            return jsonify({"translatedText": marian_results[0]}), 200
     except Exception as e:
+        print(f"[WEblate] MarianMT failed: {e}", flush=True)
+    return jsonify({"translatedText": ""}), 200
 @app.route("/languages", methods=["GET"])
                 return jsonify({"error": f"No matching variants for: {variant_filter}"}), 400
         libre_results = {}
+        for name, code in variants_to_use.items():
+            libre_results[name] = translate_libre_variant(text, code)
+            time.sleep(0.3)
+        for name, data in libre_results.items():
+            data["code"] = code
         result["libre"] = libre_results