Upload folder using huggingface_hub

docs.py

@@ -0,0 +1,169 @@
+import zipfile
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+from converter.utils import are_keys_sequential, getMarkedFields, toSnakeCase
+
+W = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"
+
+headerReplacements = [
+  ["next_message_id", "next"],
+  ["option_next", "options_next"],
+  ["option_next_id", "options_next"],
+  ["options_next_id", "options_next"]
+]
+
+avoidTransformMarkers = [{
+  "identifier": "pixabowl/posts",
+  "fields": ["id"]
+}]
+
+splitMarkers = [{
+  "identifier": "sidetrails/connections/items",
+  "fields": ["bio"]
+}]
+
+
+# --- local .docx -> Google Docs API body.content bridge ---
+
+def _para_text(p: ET.Element) -> str:
+  parts = []
+  for node in p.iter():
+    if node.tag == f"{W}t":
+      parts.append(node.text or '')
+    elif node.tag == f"{W}br":
+      parts.append('\n')
+  return ''.join(parts)
+
+def _cell_content(tc: ET.Element) -> list:
+  return [
+    {'paragraph': {'elements': [{'textRun': {'content': _para_text(p)}}]}}
+    for p in tc.findall(f"{W}p")
+  ]
+
+def _docx_to_body_content(path: Path) -> list:
+  with zipfile.ZipFile(path) as z:
+    root = ET.fromstring(z.read("word/document.xml"))
+  body = root.find(f"{W}body")
+  content = []
+  for child in body:
+    if child.tag == f"{W}p":
+      text = _para_text(child)
+      content.append({'paragraph': {'elements': [{'textRun': {'content': text}}]}})
+    elif child.tag == f"{W}tbl":
+      rows = [
+        {'tableCells': [{'content': _cell_content(tc)} for tc in tr.findall(f"{W}tc")]}
+        for tr in child.findall(f"{W}tr")
+      ]
+      content.append({'table': {'tableRows': rows}})
+  return content
+
+
+# --- semantic data extraction (unchanged from owner's docs.py) ---
+
+def read_paragraph_element(element):
+  text_run = element.get('textRun')
+  if not text_run:
+    return ''
+  return text_run.get('content')
+
+def readCell(elements):
+  text = ''
+  for value in elements:
+    if 'paragraph' in value:
+      para_elements = value.get('paragraph').get('elements')
+      for elem in para_elements:
+        text += read_paragraph_element(elem)
+  return text
+
+def getArrayFields(elements):
+  arrayFields = []
+  for value in elements:
+    if 'table' in value:
+      header = []
+      table = value.get('table')
+      multi = False
+      for rowIndex, row in enumerate(table.get('tableRows')):
+        cells = row.get('tableCells')
+        for cellIndex, cell in enumerate(cells):
+          cellContent = readCell(cell.get('content')).rstrip()
+          if rowIndex == 0:
+            cellContent = toSnakeCase(cellContent)
+            for replacement in headerReplacements:
+              cellContent = cellContent.replace(replacement[0], replacement[1])
+            header.append(cellContent)
+          elif cellIndex == 0:
+            multi = toSnakeCase(cellContent) == ''
+          elif multi:
+            if cellContent != '' and header[cellIndex] not in arrayFields:
+              arrayFields.append(header[cellIndex])
+  return arrayFields
+
+def readDocElements(elements, splitFields, avoidTransformFields):
+  fileData = []
+  arrayFields = getArrayFields(elements)
+
+  for value in elements:
+    if 'table' in value:
+      tableData = {}
+      header = []
+      table = value.get('table')
+      rowData = {}
+      rowKey = ''
+      multi = False
+      for rowIndex, row in enumerate(table.get('tableRows')):
+        newRowKey = ''
+        cells = row.get('tableCells')
+        for cellIndex, cell in enumerate(cells):
+          cellContent = readCell(cell.get('content')).rstrip()
+          if rowIndex == 0:
+            cellContent = toSnakeCase(cellContent)
+            for replacement in headerReplacements:
+              cellContent = cellContent.replace(replacement[0], replacement[1])
+            header.append(cellContent)
+          elif cellIndex == 0:
+            if avoidTransformFields is not None and header[cellIndex] in avoidTransformFields:
+              newRowKey = cellContent
+            else:
+              newRowKey = toSnakeCase(cellContent)
+            if newRowKey != '':
+              multi = False
+              rowKey = newRowKey
+              rowData = {}
+            else:
+              multi = True
+          else:
+            if (multi and cellContent == '') or header[cellIndex] == '':
+              continue
+            cellContent = cellContent if cellContent != '-' else None
+            if header[cellIndex] in arrayFields:
+              rowData.setdefault(header[cellIndex], [])
+              if cellContent is not None:
+                rowData[header[cellIndex]].append(cellContent)
+            else:
+              if splitFields is not None and header[cellIndex] in splitFields:
+                if cellContent != '':
+                  rowData[header[cellIndex]] = cellContent.splitlines()
+              else:
+                rowData[header[cellIndex]] = cellContent
+
+        if rowIndex != 0 and rowKey != '':
+          non_empty_cols = len(list(h for h in header if h != ''))
+          tableData[rowKey] = rowData if non_empty_cols > 2 else (list(rowData.values())[0] if rowData else None)
+      fileData.append(tableData)
+
+  if len(fileData) == 1:
+    fileData = fileData[0]
+    if are_keys_sequential(fileData):
+      fileData = list(fileData.values())
+
+  return fileData
+
+
+def readLocalDoc(path: Path, file_path: str):
+  body_content = _docx_to_body_content(path)
+  return readDocElements(
+    body_content,
+    getMarkedFields(file_path, splitMarkers),
+    getMarkedFields(file_path, avoidTransformMarkers),
+  )

sheets.py

@@ -0,0 +1,147 @@
+import re
+import zipfile
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+from converter.utils import are_keys_sequential, toSnakeCase
+from converter.docs import headerReplacements
+
+S = "{http://schemas.openxmlformats.org/spreadsheetml/2006/main}"
+R = "{http://schemas.openxmlformats.org/officeDocument/2006/relationships}"
+
+
+def _col_num(letters: str) -> int:
+  n = 0
+  for ch in letters:
+    n = n * 26 + (ord(ch.upper()) - 64)
+  return n - 1
+
+def _as_number(text: str) -> int | float:
+  f = float(text)
+  return int(f) if f.is_integer() else f
+
+def _shared_strings(z: zipfile.ZipFile) -> list:
+  if "xl/sharedStrings.xml" not in z.namelist():
+    return []
+  root = ET.fromstring(z.read("xl/sharedStrings.xml"))
+  return ["".join(t.text or "" for t in si.iter(f"{S}t"))
+          for si in root.findall(f"{S}si")]
+
+def _worksheet_path(z: zipfile.ZipFile) -> str:
+  wb = ET.fromstring(z.read("xl/workbook.xml"))
+  sheet = next(iter(wb.iter(f"{S}sheet")))
+  rels = ET.fromstring(z.read("xl/_rels/workbook.xml.rels"))
+  relmap = {r.get("Id"): r.get("Target") for r in rels}
+  target = relmap.get(sheet.get(f"{R}id"), "worksheets/sheet1.xml")
+  path = target[1:] if target.startswith("/") else "xl/" + target
+  if path not in z.namelist():
+    path = sorted(n for n in z.namelist()
+                  if n.startswith("xl/worksheets/") and n.endswith(".xml"))[0]
+  return path
+
+def _cell_value(c: ET.Element, shared: list) -> str | int | float:
+  t = c.get("t")
+  if t == "s":
+    v = c.find(f"{S}v")
+    return shared[int(v.text)] if v is not None else ""
+  if t == "inlineStr":
+    return "".join(x.text or "" for x in c.iter(f"{S}t"))
+  if t in ("str", "b"):
+    v = c.find(f"{S}v")
+    return v.text if v is not None and v.text is not None else ""
+  v = c.find(f"{S}v")
+  if v is None or v.text is None:
+    return ""
+  return _as_number(v.text)
+
+
+def readLocalSheet(path: Path, file_path: str):
+  with zipfile.ZipFile(path) as z:
+    shared = _shared_strings(z)
+    ws_path = _worksheet_path(z)
+    ws = ET.fromstring(z.read(ws_path))
+
+  grid: dict = {}
+  max_row = 0
+  for c in ws.iter(f"{S}c"):
+    ref = c.get("r")
+    if not ref:
+      continue
+    m = re.match(r"([A-Za-z]+)(\d+)", ref)
+    col, row = _col_num(m.group(1)), int(m.group(2))
+    val = _cell_value(c, shared)
+    if val != "":
+      grid[(row, col)] = val
+      max_row = max(max_row, row)
+
+  header_width = max((col for (row, col) in grid if row == 1), default=-1) + 1
+  raw_header = [str(grid.get((1, k), "")) for k in range(header_width)]
+  while raw_header and raw_header[-1] == "":
+    raw_header.pop()
+  ncols = len(raw_header)
+
+  header = []
+  for h in raw_header:
+    h = toSnakeCase(h)
+    for replacement in headerReplacements:
+      h = h.replace(replacement[0], replacement[1])
+    header.append(h)
+
+  last_row = 1
+  for (row, col), val in grid.items():
+    if row >= 2 and col < ncols and val != "":
+      last_row = max(last_row, row)
+
+  rows = [
+    {header[k]: grid.get((r, k), "") for k in range(ncols)}
+    for r in range(2, last_row + 1)
+  ]
+
+  # detect array fields: columns that have values in continuation rows (empty first col)
+  array_fields = []
+  for row in rows:
+    first_val = str(row.get(header[0], ""))
+    if toSnakeCase(first_val) == "":
+      for field in header[1:]:
+        val = row.get(field, "")
+        if val != "" and field not in array_fields:
+          array_fields.append(field)
+
+  tableData = {}
+  rowKey = ""
+  rowData: dict = {}
+  for row in rows:
+    first_val = str(row.get(header[0], ""))
+    new_key = toSnakeCase(first_val)
+
+    if new_key != "":
+      rowKey = new_key
+      rowData = {}
+      for field in header[1:]:
+        if field == "":
+          continue
+        val = row.get(field, "")
+        cell = str(val) if isinstance(val, (int, float)) else val
+        cell = cell if cell != "-" else None
+        if field in array_fields:
+          rowData[field] = [cell] if cell is not None else []
+        else:
+          rowData[field] = cell
+    elif rowKey != "":
+      for field in header[1:]:
+        if field == "":
+          continue
+        val = row.get(field, "")
+        if val == "":
+          continue
+        cell = str(val) if isinstance(val, (int, float)) else val
+        cell = cell if cell != "-" else None
+        if field in array_fields and cell is not None:
+          rowData.setdefault(field, []).append(cell)
+
+    if rowKey != "":
+      tableData[rowKey] = rowData if len([h for h in header if h != ""]) > 2 else (list(rowData.values())[0] if rowData else None)
+
+  if are_keys_sequential(tableData):
+    return list(tableData.values())
+  return tableData

utils.py

@@ -0,0 +1,68 @@
+import json
+import os
+from re import sub
+
+entityNameReplacements = [
+  ['adam_s', 'adams'],
+  ['zoey_s', 'zoeys'],
+  ['adams_phone', 'myphone'],
+  ['zoeys_phone', 'herphone'],
+  ['adams\'phone', 'myphone'],
+  ['zoeys\'phone', 'herphone'],
+  ['adamsphone', 'myphone'],
+  ['zoeysphone', 'herphone'],
+  ['prologue', 'episode0'],
+  ['flashback', 'fb'],
+  ['cutscenesandsequences', 'special'],
+  ['sequences', 'events'],
+  ['dad', 'zoeysdad'],
+  ['mom', 'zoeysmom'],
+  ['png', ''],
+  ['jpg', ''],
+  ['jpeg', ''],
+]
+
+skipMarkers = [
+  '(Not to be translated)',
+  'Media',
+  'Voiceovers and Subtitles',
+  'Store'
+]
+
+def toSimpleCase(s, joiner=''):
+  return joiner.join(
+      sub('([A-Z][a-z]+)', r' \1',
+      sub('([A-Z]+)', r' \1',
+      s.replace('-', ' ').replace('\'', '').replace('.', ''))).split()).lower()
+
+def toSnakeCase(s):
+  return toSimpleCase(s, '_')
+
+def toSafeEntityName(s):
+  s = toSimpleCase(s)
+  for replacement in entityNameReplacements:
+    s = s.replace(replacement[0], replacement[1])
+  return s
+
+def are_keys_sequential(d):
+  keys = list(d.keys())
+  cursor = None
+  for index, key in enumerate(keys):
+    if index == 0:
+      if key != "0" and key != "1":
+        return False
+    if not key.isnumeric():
+      return False
+    if index > 0:
+      if int(key) - cursor != 1:
+        return False
+    cursor = int(key)
+  return True
+
+def getMarkedFields(filePath, markers):
+  fileCheck = [m for m in markers if m["identifier"] in filePath]
+  file = None if len(fileCheck) == 0 else fileCheck[0]
+  return None if file is None else file["fields"]
+
+def is_skippable(name):
+  return any(marker in name for marker in skipMarkers)