Do not nix rows of empty

... but _ignore_ empty rows when inferring columns. This changes the behavior of test_spam_header, which previously ignored an empty row when the user explicitly stated the row number to use as header.

pandas/io/html.py

@@ -496,11 +496,7 @@ def _expand_colspan_rowspan(self, rows):
  all_texts.append(texts)
  remainder = next_remainder
 
- # ignore all-empty-text rows
- no_empty = [row for row in all_texts
- if any(text for text in row)]
-
- return no_empty
+ return all_texts
 
  def _handle_hidden_tables(self, tbl_list, attr_name):
  """
@@ -785,10 +781,16 @@ def _data_to_frame(**kwargs):
  header = kwargs.pop('header')
  kwargs['skiprows'] = _get_skiprows(kwargs['skiprows'])
  if head:
- rows = lrange(len(head))
  body = head + body
- if header is None: # special case when a table has <th> elements
- header = 0 if rows == [0] else rows
+
+ # Infer header when there is a <thead> or top <th>-only rows
+ if header is None:
+ if len(head) == 1:
+ header = 0
+ else:
+ # ignore all-empty-text rows
+ header = [i for i, row in enumerate(head)
+ if any(text for text in row)]
 
  if foot:
  body += foot

pandas/tests/io/test_html.py

@@ -15,6 +15,7 @@
  date_range, Series)
 from pandas.compat import (map, zip, StringIO, BytesIO,
  is_platform_windows, PY3, reload)
+from pandas.errors import ParserError
 from pandas.io.common import URLError, file_path_to_url
 import pandas.io.html
 from pandas.io.html import read_html
@@ -147,7 +148,7 @@ def test_banklist_no_match(self):
  assert isinstance(df, DataFrame)
 
  def test_spam_header(self):
- df = self.read_html(self.spam_data, '.*Water.*', header=1)[0]
+ df = self.read_html(self.spam_data, '.*Water.*', header=2)[0]
  assert df.columns[0] == 'Proximates'
  assert not df.empty
 
@@ -424,7 +425,7 @@ def test_multiple_tbody(self):
  </tbody>
  </table>''')[0]
 
- expected = DataFrame({'A': [1, 3], 'B': [2, 4]})
+ expected = DataFrame(data=[[1, 2], [3, 4]], columns=['A', 'B'])
 
  tm.assert_frame_equal(result, expected)
 
@@ -471,11 +472,8 @@ def test_thead_without_tr(self):
  </tbody>
  </table>''')[0]
 
- expected = DataFrame(data={
- 'Country': ['Ukraine'],
- 'Municipality': ['Odessa'],
- 'Year': [1944],
- })
+ expected = DataFrame(data=[['Ukraine', 'Odessa', 1944]],
+ columns=['Country', 'Municipality', 'Year'])
 
  tm.assert_frame_equal(result, expected)
 
@@ -502,9 +500,10 @@ def test_tfoot_read(self):
  </tfoot>
  </table>'''
 
- expected1 = DataFrame({'A': ['bodyA'], 'B': ['bodyB']})
- expected2 = DataFrame({'A': ['bodyA', 'footA'],
- 'B': ['bodyB', 'footB']})
+ expected1 = DataFrame(data=[['bodyA', 'bodyB']], columns=['A', 'B'])
+
+ expected2 = DataFrame(data=[['bodyA', 'bodyB'], ['footA', 'footB']],
+ columns=['A', 'B'])
 
  data1 = data_template.format(footer="")
  data2 = data_template.format(
@@ -532,7 +531,7 @@ def test_parse_header_of_non_string_column(self):
  </table>
  ''', header=0)[0]
 
- expected = DataFrame(data={'S': ['text'], 'I': [1944]})
+ expected = DataFrame([['text', 1944]], columns=('S', 'I'))
 
  tm.assert_frame_equal(result, expected)
 
@@ -676,11 +675,7 @@ def test_colspan_rowspan_1(self):
  </table>
  """)[0]
 
- expected = DataFrame(data={
- 'A': ['a'],
- 'B': ['b'],
- 'C': ['c'],
- })
+ expected = DataFrame([['a', 'b', 'c']], columns=['A', 'B', 'C'])
 
  tm.assert_frame_equal(result, expected)
 
@@ -708,13 +703,8 @@ def test_colspan_rowspan_copy_values(self):
  </table>
  """, header=0)[0]
 
- expected = DataFrame(data={
- 'X': ['A'],
- 'X.1': ['B'],
- 'Y': ['B'],
- 'Z': ['Z'],
- 'W': ['C'],
- })
+ expected = DataFrame(data=[['A', 'B', 'B', 'Z', 'C']],
+ columns=['X', 'X.1', 'Y', 'Z', 'W'])
 
  tm.assert_frame_equal(result, expected)
 
@@ -739,13 +729,8 @@ def test_colspan_rowspan_both_not_1(self):
  </table>
  """, header=0)[0]
 
- expected = DataFrame(data={
- 'A': ['A'],
- 'B': ['B'],
- 'B.1': ['B'],
- 'B.2': ['B'],
- 'C': ['D'],
- })
+ expected = DataFrame(data=[['A', 'B', 'B', 'B', 'D']],
+ columns=['A', 'B', 'B.1', 'B.2', 'C'])
 
  tm.assert_frame_equal(result, expected)
 
@@ -769,10 +754,7 @@ def test_rowspan_at_end_of_row(self):
  </table>
  """, header=0)[0]
 
- expected = DataFrame(data={
- 'A': ['C'],
- 'B': ['B']
- })
+ expected = DataFrame(data=[['C', 'B']], columns=['A', 'B'])
 
  tm.assert_frame_equal(result, expected)
 
@@ -788,14 +770,12 @@ def test_rowspan_only_rows(self):
  </table>
  """, header=0)[0]
 
- expected = DataFrame(data={
- 'A': ['A', 'A'],
- 'B': ['B', 'B'],
- })
+ expected = DataFrame(data=[['A', 'B'], ['A', 'B']],
+ columns=['A', 'B'])
 
  tm.assert_frame_equal(result, expected)
 
- def test_header_inferred_from_th_elements(self):
+ def test_header_inferred_from_rows_with_only_th(self):
  # GH17054
  result = self.read_html("""
  <table>
@@ -814,10 +794,9 @@ def test_header_inferred_from_th_elements(self):
  </table>
  """)[0]
 
- expected = DataFrame(data={
- ('A', 'a'): [1],
- ('B', 'b'): [2],
- })
+ columns = MultiIndex(levels=[['A', 'B'], ['a', 'b']],
+ labels=[[0, 1], [0, 1]])
+ expected = DataFrame(data=[[1, 2]], columns=columns)
 
  tm.assert_frame_equal(result, expected)
 
@@ -856,6 +835,23 @@ def test_wikipedia_states_table(self, datapath):
  result = self.read_html(data, 'Arizona', header=1)[0]
  assert result['sq mi'].dtype == np.dtype('float64')
 
+ def test_parser_error_on_empty_header_row(self):
+ with tm.assert_raises_regex(ParserError,
+ r"Passed header=\[0,1\] are "
+ r"too many rows for this "
+ r"multi_index of columns"):
+ self.read_html("""
+ <table>
+ <thead>
+ <tr><th></th><th></tr>
+ <tr><th>A</th><th>B</th></tr>
+ </thead>
+ <tbody>
+ <tr><td>a</td><td>b</td></tr>
+ </tbody>
+ </table>
+ """, header=[0, 1])
+
  def test_decimal_rows(self):
  # GH 12907
  result = self.read_html('''<html>
@@ -960,6 +956,49 @@ def test_keep_default_na(self):
  html_df = self.read_html(html_data, keep_default_na=True)[0]
  tm.assert_frame_equal(expected_df, html_df)
 
+ def test_preserve_empty_rows(self):
+ result = self.read_html("""
+ <table>
+ <tr>
+ <th>A</th>
+ <th>B</th>
+ </tr>
+ <tr>
+ <td>a</td>
+ <td>b</td>
+ </tr>
+ <tr>
+ <td></td>
+ <td></td>
+ </tr>
+ </table>
+ """)[0]
+
+ expected = DataFrame(data=[['a', 'b'], [np.nan, np.nan]],
+ columns=['A', 'B'])
+
+ tm.assert_frame_equal(result, expected)
+
+ def test_ignore_empty_rows_when_inferring_header(self):
+ result = self.read_html("""
+ <table>
+ <thead>
+ <tr><th></th><th></tr>
+ <tr><th>A</th><th>B</th></tr>
+ <tr><th>a</th><th>b</th></tr>
+ </thead>
+ <tbody>
+ <tr><td>1</td><td>2</td></tr>
+ </tbody>
+ </table>
+ """)[0]
+
+ columns = MultiIndex(levels=[['A', 'B'], ['a', 'b']],
+ labels=[[0, 1], [0, 1]])
+ expected = DataFrame(data=[[1, 2]], columns=columns)
+
+ tm.assert_frame_equal(result, expected)
+
  def test_multiple_header_rows(self):
  # Issue #13434
  expected_df = DataFrame(data=[("Hillary", 68, "D"),