Merge branch 'feature/jamovi-renderer' into develop

 [SVCS-549] Closes: #279

Author: felliott

mfr/extensions/jamovi/README.md

@@ -0,0 +1,5 @@
+# jamovi .omv file renderer
+
+`.omv` is the native file format for the free and open source [jamovi statistical spreadsheet](https://www.jamovi.org). An `.omv` file is a 'compound' file format, containing data, analyses, and results.
+
+`.omv` files created by recent versions of jamovi contain an `index.html` file which represents the results of the analyses performed. The jamovi `.omv` file renderer extracts the contents of `index.html` from the archive and replaces image paths from the archive with equivalent data URIs. This then serves as the rendered content.

mfr/extensions/jamovi/__init__.py

@@ -0,0 +1 @@
+from .render import JamoviRenderer # noqa

mfr/extensions/jamovi/exceptions.py

@@ -0,0 +1,45 @@
+from mfr.core.exceptions import RendererError
+
+
+class JamoviRendererError(RendererError):
+
+ def __init__(self, message, *args, **kwargs):
+ super().__init__(message, *args, renderer_class='jamovi', **kwargs)
+
+
+class JamoviVersionError(JamoviRendererError):
+ """The jamovi related errors raised from a :class:`mfr.extentions.jamovi` and relating to minimum
+ data archive version should throw or subclass JamoviVersionError.
+ """
+
+ __TYPE = 'jamovi_version'
+
+ def __init__(self, message, *args, code: int=400, created_by: str='',
+ actual_version: str='', required_version: str='', **kwargs):
+ super().__init__(message, *args, code=code, **kwargs)
+ self.created_by = created_by
+ self.actual_version = actual_version
+ self.required_version = required_version
+ self.attr_stack.append([self.__TYPE, {
+ 'created_by': self.created_by,
+ 'actual_version': self.actual_version,
+ 'required_version': self.required_version,
+ }])
+
+
+class JamoviFileCorruptError(JamoviRendererError):
+ """The jamovi related errors raised from a :class:`mfr.extentions.jamovi` and relating to failure
+ while consuming jamovi files should inherit from JamoviFileCorruptError
+ """
+
+ __TYPE = 'jamovi_file_corrupt'
+
+ def __init__(self, message, *args, code: int=400, corruption_type: str='',
+ reason: str='', **kwargs):
+ super().__init__(message, *args, code=code, **kwargs)
+ self.corruption_type = corruption_type
+ self.reason = reason
+ self.attr_stack.append([self.__TYPE, {
+ 'corruption_type': self.corruption_type,
+ 'reason': self.reason,
+ }])

mfr/extensions/jamovi/html_processor.py

@@ -0,0 +1,71 @@
+import base64
+from html.parser import HTMLParser
+from io import StringIO
+
+
+class HTMLProcessor(HTMLParser):
+
+ # The HTMLProcessor replaces the src attribute in <image> tags with the base64 equivalent.
+ # The image content comes from the zip_file (specified with set_src_source()).
+ # It also strips <script> and <object> tags and on$foo attributes from the HTML (potential
+ # attack vectors)
+
+ FILTERED_TAGS = ['script', 'object']
+
+ def __init__(self, zip_file):
+ HTMLParser.__init__(self)
+ self._html = StringIO() # buffer for the processed HTML
+ self._zip_file = zip_file
+
+ # used to exclude the contents of script and object tags
+ self._excl_nested_level = 0
+
+ def handle_starttag(self, tag, attrs):
+ if tag in self.FILTERED_TAGS: # filter scripts and objects (attack vectors)
+ self._excl_nested_level += 1
+ return
+
+ self._html.write('<')
+ self._html.write(tag)
+
+ for attr in attrs:
+ if attr[0].startswith('on'):
+ # skip onclick="", on...="" attributes (attack vectors)
+ continue
+ self._html.write(' ')
+ self._html.write(attr[0])
+ if attr[1] is not None:
+ self._html.write('="')
+ if attr[0] == 'src':
+ self._insert_data_uri(attr[1])
+ else:
+ self._html.write(attr[1])
+
+ self._html.write('"')
+
+ self._html.write('>')
+
+ def _insert_data_uri(self, src):
+ with self._zip_file.open(src) as src_file:
+ src_data = src_file.read()
+ src_b64 = base64.b64encode(src_data)
+
+ self._html.write('data:image/png;base64,')
+ self._html.write(src_b64.decode('utf-8'))
+
+ def handle_endtag(self, tag):
+ if tag in self.FILTERED_TAGS:
+ if self._excl_nested_level > 0:
+ self._excl_nested_level -= 1
+ return
+
+ self._html.write('</')
+ self._html.write(tag)
+ self._html.write('>')
+
+ def handle_data(self, data):
+ if self._excl_nested_level == 0:
+ self._html.write(data)
+
+ def final_html(self):
+ return self._html.getvalue()

mfr/extensions/jamovi/render.py

@@ -0,0 +1,114 @@
+from distutils.version import LooseVersion
+import os
+from zipfile import ZipFile, BadZipFile
+
+from mako.lookup import TemplateLookup
+
+from mfr.core import extension
+from mfr.extensions.jamovi import exceptions as jamovi_exceptions
+from mfr.extensions.jamovi.html_processor import HTMLProcessor
+
+
+class JamoviRenderer(extension.BaseRenderer):
+
+ # Minimum data archive version supported
+ MINIMUM_VERSION = LooseVersion('1.0.2')
+
+ TEMPLATE = TemplateLookup(
+ directories=[
+ os.path.join(os.path.dirname(__file__), 'templates')
+ ]).get_template('viewer.mako')
+
+ MESSAGE_FILE_CORRUPT = 'This jamovi file is corrupt and cannot be viewed.'
+ MESSAGE_NO_PREVIEW = 'This jamovi file does not support previews.'
+
+ def render(self):
+ try:
+ with ZipFile(self.file_path) as zip_file:
+ self._check_file(zip_file)
+ body = self._render_html(zip_file, self.metadata.ext)
+ return self.TEMPLATE.render(base=self.assets_url, body=body)
+ except BadZipFile as err:
+ raise jamovi_exceptions.JamoviRendererError(
+ '{} {}.'.format(self.MESSAGE_FILE_CORRUPT, str(err)),
+ extension=self.metadata.ext,
+ corruption_type='bad_zip',
+ reason=str(err),
+ )
+
+ @property
+ def file_required(self):
+ return True
+
+ @property
+ def cache_result(self):
+ return True
+
+ def _render_html(self, zip_file, ext, *args, **kwargs):
+ index = None
+ try:
+ with zip_file.open('index.html') as index_data:
+ index = index_data.read().decode('utf-8')
+ except KeyError:
+ raise jamovi_exceptions.JamoviRendererError(
+ self.MESSAGE_NO_PREVIEW,
+ )
+
+ processor = HTMLProcessor(zip_file)
+ processor.feed(index)
+
+ return processor.final_html()
+
+ def _check_file(self, zip_file):
+ """Check if the file is OK (not corrupt)
+ :param zip_file: an opened ZipFile representing the jamovi file
+ :return: True
+ """
+ # Extract manifest file content
+ try:
+ with zip_file.open('META-INF/MANIFEST.MF') as manifest_data:
+ manifest = manifest_data.read().decode('utf-8')
+ except KeyError:
+ raise jamovi_exceptions.JamoviFileCorruptError(
+ '{} Missing META-INF/MANIFEST.MF'.format(self.MESSAGE_FILE_CORRUPT),
+ extension=self.metadata.ext,
+ corruption_type='key_error',
+ reason='zip missing ./META-INF/MANIFEST.MF',
+ )
+
+ lines = manifest.split('\n')
+
+ # Search for Data-Archive-Version
+ version_str = None
+ for line in lines:
+ key_value = line.split(':')
+ if len(key_value) == 2 and key_value[0].strip() == 'Data-Archive-Version':
+ version_str = key_value[1].strip()
+ break
+ else:
+ raise jamovi_exceptions.JamoviFileCorruptError(
+ '{} Data-Archive-Version not found.'.format(self.MESSAGE_FILE_CORRUPT),
+ extension=self.metadata.ext,
+ corruption_type='manifest_parse_error',
+ reason='Data-Archive-Version not found.',
+ )
+
+ # Check that the file is new enough (contains preview content)
+ archive_version = LooseVersion(version_str)
+ try:
+ if archive_version < self.MINIMUM_VERSION:
+ raise jamovi_exceptions.JamoviFileCorruptError(
+ '{} Data-Archive-Version is too old.'.format(self.MESSAGE_FILE_CORRUPT),
+ extension=self.metadata.ext,
+ corruption_type='manifest_parse_error',
+ reason='Data-Archive-Version not found.',
+ )
+ except TypeError:
+ raise jamovi_exceptions.JamoviFileCorruptError(
+ '{} Data-Archive-Version not parsable.'.format(self.MESSAGE_FILE_CORRUPT),
+ extension=self.metadata.ext,
+ corruption_type='manifest_parse_error',
+ reason='Data-Archive-Version ({}) not parsable.'.format(version_str),
+ )
+
+ return True

mfr/extensions/jamovi/templates/viewer.mako

@@ -0,0 +1,6 @@
+<div style="word-wrap: break-word; overflow: auto;" class="mfrViewer">
+ ${body}
+</div>
+
+<script src="/static/js/mfr.js"></script>
+<script src="/static/js/mfr.child.js"></script>

mfr/extensions/jasp/templates/viewer.mako

@@ -1,4 +1,4 @@
-<div style="word-wrap: break-word; overflow: auto" class="mfrViewer">
+<div style="word-wrap: break-word; overflow: auto;" class="mfrViewer">
 ${body}
 </div>
 

setup.py

@@ -795,6 +795,9 @@ def parse_requirements(requirements):
  #'.wmv = mfr.extensions.video:VideoRenderer',
  '.webm = mfr.extensions.video:VideoRenderer',
 
+ # jamovi
+ '.omv = mfr.extensions.jamovi:JamoviRenderer',
+
  # JASP
  '.jasp = mfr.extensions.jasp:JASPRenderer',
 

supportedextensions.md

@@ -35,6 +35,9 @@ Some file types may not be in the correct list. Please search for the file type
 ## JASP
 * .jasp
 
+## Jamovi
+* .omv
+
 ## Google Documents
 * .gdoc
 * .gsheet