stop using rpy2; shell to pspp-convert

 * rpy2 was only being used to convert SPSS .sav files to a format pandas can handle. pspp-convert is a program that can convert .sav to .csv. Update the tabular renderer to use that instead. As a side effect, it turns out the standard csv library is better at detecting data types than pandas is, so use that to render the resulting csv file

Author: felliott

.travis.yml

@@ -16,7 +16,6 @@ install:
  - travis_retry pip install --upgrade pip
  - travis_retry pip install wheel==0.26.0
  - travis_retry pip install invoke==0.11.1
- - sed 's/^rpy2==/#rpy2==/' -i requirements.txt
  - travis_retry invoke wheelhouse --develop
  - travis_retry invoke install --develop
 

Dockerfile

@@ -22,6 +22,9 @@ RUN apt-get update \
  # unoconv dependencies
  && apt-get install -y \
  unoconv \
+ # pspp dependencies
+ && apt-get install -y \
+ pspp \
  && apt-get clean \
  && apt-get autoremove -y \
  && rm -rf /var/lib/apt/lists/*

README.md

@@ -14,13 +14,13 @@ For MacOSX users:
 
 ```bash
 brew install python3
-brew install r
+brew install pspp
 ```
 For Ubuntu users:
 
 ```bash
 apt-get install python3
-apt-get install r-base
+apt-get install pspp
 ```
 
 After installing python3.5, create the virtual environment with the following commands:

docs/install.rst

@@ -14,7 +14,7 @@ Or download one of the following:
 * tarball_
 * zipball_
 
-Make sure that you have installed R, are using python3.5, and have installed invoke for your current python3 version.
+Make sure that you have installed pspp, are using python3.5, and have installed invoke for your current python3 version.
 
 Install the version of invoke found in the requirements.txt file. Currently 0.11.1
 

mfr/extensions/tabular/libs/__init__.py

@@ -23,6 +23,10 @@ def dta_pandas():
  return dta_pandas
 
 
+def sav_stdlib():
+ from ..libs.stdlib_tools import sav_stdlib
+ return sav_stdlib
+
 def sav_pandas():
  from ..libs.panda_tools import sav_pandas
  return sav_pandas

mfr/extensions/tabular/libs/panda_tools.py

@@ -1,7 +1,9 @@
+from tempfile import NamedTemporaryFile
+
 import numpy
 import pandas
-from tempfile import NamedTemporaryFile
-from ..utilities import header_population, strip_comments
+
+from mfr.extensions.tabular.utilities import header_population, strip_comments, sav_to_csv
 
 
 def csv_pandas(fp):
@@ -36,11 +38,14 @@ def dta_pandas(fp):
 
 
 def sav_pandas(fp):
- """Read and convert a sav file to JSON format using the pandas library
+ """Read and convert a .sav file to a .csv file via pspp, then convert that to JSON format
+ using the pandas library
+
  :param fp: File pointer object
  :return: tuple of table headers and data
  """
- dataframe = robjectify(fp)
+ csv_file = sav_to_csv(fp)
+ dataframe = pandas.read_csv(csv_file.name, low_memory=False)
  return data_from_dataframe(dataframe)
 
 
@@ -64,15 +69,3 @@ def data_from_dataframe(dataframe):
  data_row[name] = value
  data.append(data_row)
  return {'Sheet 1': (header, data)}
-
-
-def robjectify(fp):
- """Create a dataframe object using R"""
-
- import pandas.rpy.common as common
- import rpy2.robjects as robjects
- r = robjects
- r.r("require(foreign)")
- r.r('x <- read.spss("{}",to.data.frame=T)'.format(fp.name))
- r.r('row.names(x) = 0:(nrow(x)-1)')
- return common.load_data('x')

mfr/extensions/tabular/libs/stdlib_tools.py

@@ -1,6 +1,8 @@
-import csv
 import re
+import csv
+
 from ..exceptions import EmptyTableException
+from mfr.extensions.tabular import utilities
 
 
 def csv_stdlib(fp):
@@ -42,6 +44,19 @@ def csv_stdlib(fp):
  return {'Sheet 1': (columns, rows)}
 
 
+def sav_stdlib(fp):
+ """Read and convert a .sav file to .csv with pspp, then convert that to JSON format using
+ the python standard library
+
+ :param fp: File pointer object to a .sav file
+ :return: tuple of table headers and data
+ """
+ csv_file = utilities.sav_to_csv(fp)
+ with open(csv_file.name, 'r') as file:
+ csv_file.close()
+ return csv_stdlib(file)
+
+
 def _set_dialect_quote_attrs(dialect, data):
  """Set quote-related dialect attributes based on up to 2kb of csv data.
 

mfr/extensions/tabular/settings.py

@@ -20,7 +20,7 @@
  '.xlsx': [libs.xlsx_xlrd],
  '.xls': [libs.xlsx_xlrd],
  '.dta': [libs.dta_pandas],
- '.sav': [libs.sav_pandas],
+ '.sav': [libs.sav_stdlib],
  # '.ods': [libs.ods_ezodf],
 })
 
@@ -38,3 +38,5 @@
  'syncColumnCellResize': True,
  'multiColumnSort': True,
 })
+
+PSPP_CONVERT_BIN = config.get('PSPP_CONVERT_BIN', '/usr/bin/pspp-convert')

mfr/extensions/tabular/utilities.py

@@ -1,6 +1,9 @@
 import re
+import subprocess
+from tempfile import NamedTemporaryFile
 
-from mfr.extensions.tabular import compat
+from mfr.core import exceptions
+from mfr.extensions.tabular import compat, settings
 
 
 def header_population(headers):
@@ -35,3 +38,22 @@ def strip_comments(src, dest):
  data = data.encode('utf-8', 'ignore')
  dest.write(data)
  dest.seek(0)
+
+
+def sav_to_csv(fp):
+ """Converts a SPSS .sav to a .csv file by calling out to ``pspp-convert``.
+
+ :param fp: file pointer object to .sav file
+ :return: file pointer to .csv file. You are responsible for closing this.
+ """
+ csv_file = NamedTemporaryFile(mode='w+b', suffix='.csv')
+ try:
+ subprocess.check_call([
+ settings.PSPP_CONVERT_BIN,
+ fp.name,
+ csv_file.name,
+ ])
+ except subprocess.CalledProcessError:
+ raise exceptions.ExporterError(
+ 'Unable to convert the SPSS file to CSV, please try again later.', code=400)
+ return csv_file

requirements.txt

@@ -39,9 +39,6 @@ docutils==0.12
 pandas==0.17.1
 git+https://github.com/icereval/xlrd.git
 
-# Rpy
-rpy2==2.7.8
-
 # Md
 markdown==2.6.2