ENH: Allow poorly formatted stata files to be read (#25967)

* ENH: Allow poorly formatted stata files to be read Add a fall back decode path that allows improperly formatted Stata files written in 118 format but using latin-1 encoded strings to be read closes #25960 * MAINT: Refactor decode Refactor decode and null terminate to use file encoding

Author: bashtage

doc/source/whatsnew/v0.25.0.rst

@@ -355,6 +355,7 @@ I/O
 - Improved performance in :meth:`pandas.read_stata` and :class:`pandas.io.stata.StataReader` when converting columns that have missing values (:issue:`25772`)
 - Bug in :func:`read_hdf` not properly closing store after a ``KeyError`` is raised (:issue:`25766`)
 - Bug in ``read_csv`` which would not raise ``ValueError`` if a column index in ``usecols`` was out of bounds (:issue:`25623`)
+- Improved :meth:`pandas.read_stata` and :class:`pandas.io.stata.StataReader` to read incorrectly formatted 118 format files saved by Stata (:issue:`25960`)
 
 Plotting
 ^^^^^^^^

pandas/io/stata.py

@@ -1137,7 +1137,7 @@ def _get_varlist(self):
  elif self.format_version == 118:
  b = 129
 
- return [self._null_terminate(self.path_or_buf.read(b))
+ return [self._decode(self.path_or_buf.read(b))
  for i in range(self.nvar)]
 
  # Returns the format list
@@ -1151,7 +1151,7 @@ def _get_fmtlist(self):
  else:
  b = 7
 
- return [self._null_terminate(self.path_or_buf.read(b))
+ return [self._decode(self.path_or_buf.read(b))
  for i in range(self.nvar)]
 
  # Returns the label list
@@ -1162,18 +1162,18 @@ def _get_lbllist(self):
  b = 33
  else:
  b = 9
- return [self._null_terminate(self.path_or_buf.read(b))
+ return [self._decode(self.path_or_buf.read(b))
  for i in range(self.nvar)]
 
  def _get_variable_labels(self):
  if self.format_version == 118:
  vlblist = [self._decode(self.path_or_buf.read(321))
  for i in range(self.nvar)]
  elif self.format_version > 105:
- vlblist = [self._null_terminate(self.path_or_buf.read(81))
+ vlblist = [self._decode(self.path_or_buf.read(81))
  for i in range(self.nvar)]
  else:
- vlblist = [self._null_terminate(self.path_or_buf.read(32))
+ vlblist = [self._decode(self.path_or_buf.read(32))
  for i in range(self.nvar)]
  return vlblist
 
@@ -1192,21 +1192,21 @@ def _get_data_label(self):
  return self._decode(self.path_or_buf.read(strlen))
  elif self.format_version == 117:
  strlen = struct.unpack('b', self.path_or_buf.read(1))[0]
- return self._null_terminate(self.path_or_buf.read(strlen))
+ return self._decode(self.path_or_buf.read(strlen))
  elif self.format_version > 105:
- return self._null_terminate(self.path_or_buf.read(81))
+ return self._decode(self.path_or_buf.read(81))
  else:
- return self._null_terminate(self.path_or_buf.read(32))
+ return self._decode(self.path_or_buf.read(32))
 
  def _get_time_stamp(self):
  if self.format_version == 118:
  strlen = struct.unpack('b', self.path_or_buf.read(1))[0]
  return self.path_or_buf.read(strlen).decode("utf-8")
  elif self.format_version == 117:
  strlen = struct.unpack('b', self.path_or_buf.read(1))[0]
- return self._null_terminate(self.path_or_buf.read(strlen))
+ return self._decode(self.path_or_buf.read(strlen))
  elif self.format_version > 104:
- return self._null_terminate(self.path_or_buf.read(18))
+ return self._decode(self.path_or_buf.read(18))
  else:
  raise ValueError()
 
@@ -1267,10 +1267,10 @@ def _read_old_header(self, first_char):
  .format(','.join(str(x) for x in typlist)))
 
  if self.format_version > 108:
- self.varlist = [self._null_terminate(self.path_or_buf.read(33))
+ self.varlist = [self._decode(self.path_or_buf.read(33))
  for i in range(self.nvar)]
  else:
- self.varlist = [self._null_terminate(self.path_or_buf.read(9))
+ self.varlist = [self._decode(self.path_or_buf.read(9))
  for i in range(self.nvar)]
  self.srtlist = struct.unpack(
  self.byteorder + ('h' * (self.nvar + 1)),
@@ -1327,13 +1327,20 @@ def _calcsize(self, fmt):
  struct.calcsize(self.byteorder + fmt))
 
  def _decode(self, s):
- s = s.partition(b"\0")[0]
- return s.decode('utf-8')
-
- def _null_terminate(self, s):
  # have bytes not strings, so must decode
  s = s.partition(b"\0")[0]
- return s.decode(self._encoding)
+ try:
+ return s.decode(self._encoding)
+ except UnicodeDecodeError:
+ # GH 25960, fallback to handle incorrect format produced when 117
+ # files are converted to 118 files in Stata
+ msg = """
+One or more strings in the dta file could not be decoded using {encoding}, and
+so the fallback encoding of latin-1 is being used. This can happen when a file
+has been incorrectly encoded by Stata or some other software. You should verify
+the string values returned are correct."""
+ warnings.warn(msg.format(encoding=self._encoding), UnicodeWarning)
+ return s.decode('latin-1')
 
  def _read_value_labels(self):
  if self._value_labels_read:
@@ -1363,7 +1370,7 @@ def _read_value_labels(self):
  if not slength:
  break # end of value label table (format < 117)
  if self.format_version <= 117:
- labname = self._null_terminate(self.path_or_buf.read(33))
+ labname = self._decode(self.path_or_buf.read(33))
  else:
  labname = self._decode(self.path_or_buf.read(129))
  self.path_or_buf.read(3) # padding
@@ -1385,12 +1392,8 @@ def _read_value_labels(self):
  self.value_label_dict[labname] = dict()
  for i in range(n):
  end = off[i + 1] if i < n - 1 else txtlen
- if self.format_version <= 117:
- self.value_label_dict[labname][val[i]] = (
- self._null_terminate(txt[off[i]:end]))
- else:
- self.value_label_dict[labname][val[i]] = (
- self._decode(txt[off[i]:end]))
+ self.value_label_dict[labname][val[i]] = \
+ self._decode(txt[off[i]:end])
  if self.format_version >= 117:
  self.path_or_buf.read(6) # </lbl>
  self._value_labels_read = True
@@ -1545,7 +1548,7 @@ def read(self, nrows=None, convert_dates=None,
  for col, typ in zip(data, self.typlist):
  if type(typ) is int:
  data[col] = data[col].apply(
- self._null_terminate, convert_dtype=True)
+ self._decode, convert_dtype=True)
 
  data = self._insert_strls(data)
 

pandas/tests/io/data/stata1_encoding_118.dta

@@ -66,6 +66,8 @@ def setup_method(self, datapath):
  self.dta4_117 = os.path.join(self.dirpath, 'stata4_117.dta')
 
  self.dta_encoding = os.path.join(self.dirpath, 'stata1_encoding.dta')
+ self.dta_encoding_118 = os.path.join(self.dirpath,
+ 'stata1_encoding_118.dta')
 
  self.csv14 = os.path.join(self.dirpath, 'stata5.csv')
  self.dta14_113 = os.path.join(self.dirpath, 'stata5_113.dta')
@@ -1608,3 +1610,18 @@ def test_strl_latin1(self):
  val = gso.split(b'\x00')[-2]
  size = gso[gso.find(b'\x82') + 1]
  assert len(val) == size - 1
+
+ def test_encoding_latin1_118(self):
+ # GH 25960
+ msg = """
+One or more strings in the dta file could not be decoded using utf-8, and
+so the fallback encoding of latin-1 is being used. This can happen when a file
+has been incorrectly encoded by Stata or some other software. You should verify
+the string values returned are correct."""
+ with tm.assert_produces_warning(UnicodeWarning) as w:
+ encoded = read_stata(self.dta_encoding_118)
+ assert len(w) == 151
+ assert w[0].message.args[0] == msg
+
+ expected = pd.DataFrame([['Düsseldorf']] * 151, columns=['kreis1849'])
+ tm.assert_frame_equal(encoded, expected)