[ENG-8475] Fix unable to render spreadsheet files (.xls and .xlsx) (#393)

Author: sh-andriy

mfr/extensions/tabular/libs/xlrd_tools.py

@@ -1,95 +1,38 @@
 import xlrd
 import zipfile
-from collections import OrderedDict
-from ..exceptions import TableTooBigError, MissingRequirementsError
 
-from ..utilities import header_population
-from mfr.extensions.tabular.compat import range, basestring
+from io import BytesIO
+from openpyxl import load_workbook
+from collections import OrderedDict
+from ..utilities import (
+ to_bytes,
+ parse_xls,
+ parse_xlsx
+)
 
 
 def xlsx_xlrd(fp):
- """Read and convert a xlsx file to JSON format using the xlrd library
- :param fp: File pointer object
- :return: tuple of table headers and data
  """
- MAX_SIZE = 10000
-
- try:
- wb = xlrd.open_workbook(fp.name)
- using_xlrd = True
- except xlrd.biffh.XLRDError:
- using_xlrd = False
- try:
- from openpyxl import load_workbook
- except ImportError:
- raise MissingRequirementsError(
- 'openpyxl is required to read .xlsx files',
- function_preference='openpyxl'
- )
- try:
- wb = load_workbook(fp.name, data_only=True)
- except zipfile.BadZipFile:
- raise xlrd.biffh.XLRDError("Excel xlsx file; not supported")
+ • .xls → xlrd
+ • .xlsx → openpyxl (xlrd ≥2.0 dropped xlsx support)
 
+ `fp` is the stream returned by WaterButler/MFR. It may already have been
+ read, so we always rewind and copy to an in‑memory buffer that openpyxl (and
+ ZipFile) can seek inside safely.
+ """
  sheets = OrderedDict()
 
- if using_xlrd:
- for sheet in wb.sheets():
- if sheet.ncols > MAX_SIZE or sheet.nrows > MAX_SIZE:
- raise TableTooBigError('Table is too large to render.', '.xlsx',
- nbr_cols=sheet.ncols, nbr_rows=sheet.nrows)
-
- if sheet.ncols < 1 or sheet.nrows < 1:
- sheets[sheet.name] = ([], [])
- continue
-
- fields = sheet.row_values(0) if sheet.nrows else []
-
- fields = [
- str(value)
- if not isinstance(value, basestring) and value is not None
- else value or f'Unnamed: {index + 1}'
- for index, value in enumerate(fields)
- ]
-
- data = []
- for i in range(1, sheet.nrows):
- row = []
- for cell in sheet.row(i):
- if cell.ctype == xlrd.XL_CELL_DATE:
- value = xlrd.xldate.xldate_as_datetime(cell.value, wb.datemode).isoformat()
- else:
- value = cell.value
- row.append(value)
- data.append(dict(zip(fields, row)))
-
- header = header_population(fields)
- sheets[sheet.name] = (header, data)
-
- else:
- for name in wb.sheetnames:
- ws = wb[name]
- nrows = ws.max_row
- ncols = ws.max_column
- if ncols > MAX_SIZE or nrows > MAX_SIZE:
- raise TableTooBigError('Table is too large to render.', '.xlsx',
- nbr_cols=ncols, nbr_rows=nrows)
-
- if nrows < 1 or ncols < 1:
- sheets[name] = ([], [])
- continue
-
- header_row = next(ws.iter_rows(min_row=1, max_row=1, values_only=True))
- fields = [
- str(val) if val is not None else f'Unnamed: {i + 1}'
- for i, val in enumerate(header_row)
- ]
-
- data = []
- for row in ws.iter_rows(min_row=2, max_row=nrows, max_col=ncols, values_only=True):
- data.append(dict(zip(fields, row)))
+ try:
+ wb = xlrd.open_workbook(file_contents=to_bytes(fp))
+ return parse_xls(wb, sheets)
+ except xlrd.biffh.XLRDError:
+ pass
 
- header = header_population(fields)
- sheets[name] = (header, data)
+ try:
+ wb = load_workbook(BytesIO(to_bytes(fp)), data_only=True, read_only=True)
+ except zipfile.BadZipFile as exc:
+ raise xlrd.biffh.XLRDError(
+ "Invalid xlsx file or corrupted ZIP structure"
+ ) from exc
 
- return sheets
+ return parse_xlsx(wb, sheets)

mfr/extensions/tabular/utilities.py

@@ -1,16 +1,20 @@
 import re
+import xlrd
+
 from http import HTTPStatus
 from subprocess import (check_call,
  TimeoutExpired,
  CalledProcessError)
 from tempfile import NamedTemporaryFile
 
 from mfr.extensions.tabular import compat
-from mfr.core.exceptions import SubprocessError
+from mfr.core.exceptions import SubprocessError, TooBigToRenderError
 from mfr.extensions.tabular.settings import (PSPP_CONVERT_BIN,
  PSPP_CONVERT_TIMEOUT)
 
 
+MAX_SIZE = 10_000
+
 def header_population(headers):
  """make column headers from a list
  :param headers: list of column headers
@@ -83,3 +87,91 @@ def sav_to_csv(fp):
  exporter_class='tabular'
  )
  return csv_file
+
+
+def to_bytes(fp):
+ """
+ Return *exactly* the original bytes of the Excel file and rewind *fp*.
+ Handles both binary and text wrappers that WaterButler may give us.
+ """
+ try:
+ fp.seek(0)
+ except Exception:
+ pass
+
+ raw = fp.read()
+ if isinstance(raw, bytes):
+ try:
+ fp.seek(0)
+ except Exception:
+ pass
+ return raw
+
+ if hasattr(fp, "buffer"):
+ buf = fp.buffer
+ try:
+ buf.seek(0)
+ except Exception:
+ pass
+ data = buf.read()
+ try:
+ buf.seek(0)
+ except Exception:
+ pass
+ else:
+ data = raw.encode("utf-8", "surrogateescape")
+
+ try:
+ fp.seek(0)
+ except Exception:
+ pass
+ return data
+
+
+def parse_xls(wb, sheets):
+ for sheet in wb.sheets():
+ verify_size(sheet.nrows, sheet.ncols, '.xls')
+ fields = fix_headers(sheet.row_values(0))
+ rows = [
+ dict(zip(fields, row_vals(sheet.row(r), wb.datemode)))
+ for r in range(1, sheet.nrows)
+ ]
+ sheets[sheet.name] = (header_population(fields), rows)
+ return sheets
+
+
+def parse_xlsx(wb, sheets):
+ for name in wb.sheetnames:
+ ws = wb[name]
+ verify_size(ws.max_row, ws.max_column, '.xlsx')
+ header_row = next(ws.iter_rows(max_row=1, values_only=True))
+ fields = fix_headers(header_row)
+ rows = [
+ dict(zip(fields, row))
+ for row in ws.iter_rows(min_row=2,
+ max_row=ws.max_row,
+ max_col=ws.max_column,
+ values_only=True)
+ ]
+ sheets[name] = (header_population(fields), rows)
+ return sheets
+
+
+def verify_size(rows, cols, ext):
+ if rows > MAX_SIZE or cols > MAX_SIZE:
+ raise TooBigToRenderError('Table is too large to render.', ext,
+ nbr_cols=cols, nbr_rows=rows)
+
+
+def fix_headers(raw):
+ return [str(v) if v not in (None, '') else f'Unnamed: {i + 1}' for i, v in enumerate(raw)]
+
+
+def row_vals(row, datemode):
+ out = []
+ for c in row:
+ if c.ctype == xlrd.XL_CELL_DATE:
+ out.append(xlrd.xldate.xldate_as_datetime(c.value, datemode).isoformat())
+ else:
+ out.append(c.value)
+ return out

tests/extensions/tabular/test_xlsx_tools.py

@@ -8,7 +8,7 @@
 class TestTabularPandaTools:
 
  def test_xlsx_xlrd(self):
- with open(os.path.join(BASE, 'files', 'test.xlsx')) as fp:
+ with open(os.path.join(BASE, 'files', 'test.xlsx'), 'rb') as fp:
  sheets = xlrd_tools.xlsx_xlrd(fp)
 
  sheet = sheets.popitem()[1]