Merge pull request hardbyte#1221 from Tbruno25/1210_add-gzipreader

1210 Add compressed file methods

Author: zariiii9003

can/__init__.py

@@ -25,7 +25,7 @@
 )
 
 from .io import Logger, SizedRotatingLogger, Printer, LogReader, MessageSync
-from .io import ASCWriter, ASCReader, GzipASCWriter, GzipASCReader
+from .io import ASCWriter, ASCReader
 from .io import BLFReader, BLFWriter
 from .io import CanutilsLogReader, CanutilsLogWriter
 from .io import CSVWriter, CSVReader

can/io/__init__.py

@@ -8,7 +8,7 @@
 from .player import LogReader, MessageSync
 
 # Format specific
-from .asc import ASCWriter, ASCReader, GzipASCWriter, GzipASCReader
+from .asc import ASCWriter, ASCReader
 from .blf import BLFReader, BLFWriter
 from .canutils import CanutilsLogReader, CanutilsLogWriter
 from .csv import CSVWriter, CSVReader

can/io/asc.py

@@ -5,7 +5,6 @@
  - https://bitbucket.org/tobylorenz/vector_asc/src/47556e1a6d32c859224ca62d075e1efcc67fa690/src/Vector/ASC/tests/unittests/data/CAN_Log_Trigger_3_2.asc?at=master&fileviewer=file-view-default
  - under `test/data/logfile.asc`
 """
-import gzip
 import re
 from typing import Any, Generator, List, Optional, Dict, Union, TextIO
 
@@ -405,69 +404,3 @@ def on_message_received(self, msg: Message) -> None:
  data=" ".join(data),
  )
  self.log_event(serialized, msg.timestamp)
-
-
-class GzipASCReader(ASCReader):
- """Gzipped version of :class:`~can.ASCReader`"""
-
- def __init__(
- self,
- file: Union[typechecking.FileLike, typechecking.StringPathLike],
- base: str = "hex",
- relative_timestamp: bool = True,
- ):
- """
- :param file: a path-like object or as file-like object to read from
- If this is a file-like object, is has to opened in text
- read mode, not binary read mode.
- :param base: Select the base(hex or dec) of id and data.
- If the header of the asc file contains base information,
- this value will be overwritten. Default "hex".
- :param relative_timestamp: Select whether the timestamps are
- `relative` (starting at 0.0) or `absolute` (starting at
- the system time). Default `True = relative`.
- """
- self._fileobj = None
- if file is not None and (hasattr(file, "read") and hasattr(file, "write")):
- # file is None or some file-like object
- self._fileobj = file
- super(GzipASCReader, self).__init__(
- gzip.open(file, mode="rt"), base, relative_timestamp
- )
-
- def stop(self) -> None:
- super(GzipASCReader, self).stop()
- if self._fileobj is not None:
- self._fileobj.close()
-
-
-class GzipASCWriter(ASCWriter):
- """Gzipped version of :class:`~can.ASCWriter`"""
-
- def __init__(
- self,
- file: Union[typechecking.FileLike, typechecking.StringPathLike],
- channel: int = 1,
- compresslevel: int = 6,
- ):
- """
- :param file: a path-like object or as file-like object to write to
- If this is a file-like object, is has to opened in text
- write mode, not binary write mode.
- :param channel: a default channel to use when the message does not
- have a channel set
- :param compresslevel: Gzip compresslevel, see
- :class:`~gzip.GzipFile` for details. The default is 6.
- """
- self._fileobj = None
- if file is not None and (hasattr(file, "read") and hasattr(file, "write")):
- # file is None or some file-like object
- self._fileobj = file
- super(GzipASCWriter, self).__init__(
- gzip.open(file, mode="wt", compresslevel=compresslevel), channel
- )
-
- def stop(self) -> None:
- super(GzipASCWriter, self).stop()
- if self._fileobj is not None:
- self._fileobj.close()

can/io/logger.py

@@ -6,13 +6,9 @@
 import pathlib
 from abc import ABC, abstractmethod
 from datetime import datetime
-from typing import (
- Any,
- Optional,
- Callable,
- cast,
- Type,
-)
+import gzip
+from typing import Any, Optional, Callable, TextIO, Type, Tuple, Union, cast
+
 from types import TracebackType
 
 from typing_extensions import Literal
@@ -21,7 +17,7 @@
 from ..message import Message
 from ..listener import Listener
 from .generic import BaseIOHandler, FileIOMessageWriter
-from .asc import ASCWriter, GzipASCWriter
+from .asc import ASCWriter
 from .blf import BLFWriter
 from .canutils import CanutilsLogWriter
 from .csv import CSVWriter
@@ -34,15 +30,18 @@ class Logger(BaseIOHandler, Listener): # pylint: disable=abstract-method
  """
  Logs CAN messages to a file.
 
- The format is determined from the file format which can be one of:
+ The format is determined from the file suffix which can be one of:
  * .asc: :class:`can.ASCWriter`
- * .asc.gz: :class:`can.CompressedASCWriter`
  * .blf :class:`can.BLFWriter`
  * .csv: :class:`can.CSVWriter`
  * .db: :class:`can.SqliteWriter`
  * .log :class:`can.CanutilsLogWriter`
  * .txt :class:`can.Printer`
 
+ Any of these formats can be used with gzip compression by appending
+ the suffix .gz (e.g. filename.asc.gz). However, third-party tools might not
+ be able to read these files.
+
  The **filename** may also be *None*, to fall back to :class:`can.Printer`.
 
  The log files may be incomplete until `stop()` is called due to buffering.
@@ -55,7 +54,6 @@ class Logger(BaseIOHandler, Listener): # pylint: disable=abstract-method
  fetched_plugins = False
  message_writers = {
  ".asc": ASCWriter,
- ".asc.gz": GzipASCWriter,
  ".blf": BLFWriter,
  ".csv": CSVWriter,
  ".db": SqliteWriter,
@@ -85,7 +83,11 @@ def __new__( # type: ignore
  )
  Logger.fetched_plugins = True
 
- suffix = "".join(s.lower() for s in pathlib.PurePath(filename).suffixes)
+ suffix = pathlib.PurePath(filename).suffix.lower()
+
+ if suffix == ".gz":
+ suffix, filename = Logger.compress(filename)
+
  try:
  return cast(
  Listener, Logger.message_writers[suffix](filename, *args, **kwargs)
@@ -95,6 +97,17 @@ def __new__( # type: ignore
  f'No write support for this unknown log format "{suffix}"'
  ) from None
 
+ @staticmethod
+ def compress(filename: StringPathLike) -> Tuple[str, Union[str, Any]]:
+ """
+ Return the suffix and io object of the decompressed file.
+ File will automatically recompress upon close.
+ """
+ real_suffix = pathlib.Path(filename).suffixes[-2].lower()
+ mode = "ab" if real_suffix == ".blf" else "at"
+
+ return real_suffix, gzip.open(filename, mode)
+
  def on_message_received(self, msg: Message) -> None:
  pass
 

can/io/player.py

@@ -3,7 +3,7 @@
 well as :class:`MessageSync` which plays back messages
 in the recorded order an time intervals.
 """
-
+import gzip
 import pathlib
 from time import time, sleep
 import typing
@@ -14,7 +14,7 @@
  import can
 
 from .generic import BaseIOHandler, MessageReader
-from .asc import ASCReader, GzipASCReader
+from .asc import ASCReader
 from .blf import BLFReader
 from .canutils import CanutilsLogReader
 from .csv import CSVReader
@@ -25,14 +25,17 @@ class LogReader(BaseIOHandler):
  """
  Replay logged CAN messages from a file.
 
- The format is determined from the file format which can be one of:
+ The format is determined from the file suffix which can be one of:
  * .asc
- * .asc.gz
  * .blf
  * .csv
  * .db
  * .log
 
+ Gzip compressed files can be used as long as the original
+ files suffix is one of the above (e.g. filename.asc.gz).
+
+
  Exposes a simple iterator interface, to use simply:
 
  >>> for msg in LogReader("some/path/to/my_file.log"):
@@ -50,7 +53,6 @@ class LogReader(BaseIOHandler):
  fetched_plugins = False
  message_readers = {
  ".asc": ASCReader,
- ".asc.gz": GzipASCReader,
  ".blf": BLFReader,
  ".csv": CSVReader,
  ".db": SqliteReader,
@@ -77,7 +79,11 @@ def __new__( # type: ignore
  )
  LogReader.fetched_plugins = True
 
- suffix = "".join(s.lower() for s in pathlib.PurePath(filename).suffixes)
+ suffix = pathlib.PurePath(filename).suffix.lower()
+
+ if suffix == ".gz":
+ suffix, filename = LogReader.decompress(filename)
+
  try:
  return typing.cast(
  MessageReader,
@@ -88,6 +94,18 @@ def __new__( # type: ignore
  f'No read support for this unknown log format "{suffix}"'
  ) from None
 
+ @staticmethod
+ def decompress(
+ filename: "can.typechecking.StringPathLike",
+ ) -> typing.Tuple[str, typing.Union[str, typing.Any]]:
+ """
+ Return the suffix and io object of the decompressed file.
+ """
+ real_suffix = pathlib.Path(filename).suffixes[-2].lower()
+ mode = "rb" if real_suffix == ".blf" else "rt"
+
+ return real_suffix, gzip.open(filename, mode)
+
 
 class MessageSync: # pylint: disable=too-few-public-methods
  """

test/data/test_CanMessage.asc.gz

@@ -11,7 +11,6 @@
 
 TODO: correctly set preserves_channel and adds_default_channel
 """
-import gzip
 import logging
 import unittest
 import tempfile
@@ -48,6 +47,7 @@ def test_extension_matching(self):
  suffix_variants = [
  suffix.upper(),
  suffix.lower(),
+ f"can.msg.ext{suffix}",
  "".join([c.upper() if i % 2 else c for i, c in enumerate(suffix)]),
  ]
  for suffix_variant in suffix_variants:
@@ -558,34 +558,6 @@ def test_ignore_comments(self):
  _msg_list = self._read_log_file("logfile.asc")
 
 
-class TestGzipASCFileFormat(ReaderWriterTest):
- """Tests can.GzipASCWriter and can.GzipASCReader"""
-
- def _setup_instance(self):
- super()._setup_instance_helper(
- can.GzipASCWriter,
- can.GzipASCReader,
- binary_file=True,
- check_comments=True,
- preserves_channel=False,
- adds_default_channel=0,
- )
-
- def assertIncludesComments(self, filename):
- """
- Ensures that all comments are literally contained in the given file.
-
- :param filename: the path-like object to use
- """
- if self.original_comments:
- # read the entire outout file
- with gzip.open(filename, "rt" if self.binary_file else "r") as file:
- output_contents = file.read()
- # check each, if they can be found in there literally
- for comment in self.original_comments:
- self.assertIn(comment, output_contents)
-
-
 class TestBlfFileFormat(ReaderWriterTest):
  """Tests can.BLFWriter and can.BLFReader.
 

test/logformats_test.py

@@ -7,6 +7,8 @@
 import unittest
 from unittest import mock
 from unittest.mock import Mock
+import gzip
+import os
 import sys
 import can
 import can.logger
@@ -104,5 +106,39 @@ def test_log_virtual_sizedlogger(self):
  self.mock_logger_sized.assert_called_once()
 
 
+class TestLoggerCompressedFile(unittest.TestCase):
+ def setUp(self) -> None:
+ # Patch VirtualBus object
+ self.patcher_virtual_bus = mock.patch(
+ "can.interfaces.virtual.VirtualBus", spec=True
+ )
+ self.MockVirtualBus = self.patcher_virtual_bus.start()
+ self.mock_virtual_bus = self.MockVirtualBus.return_value
+
+ self.testmsg = can.Message(
+ arbitration_id=0xC0FFEE, data=[0, 25, 0, 1, 3, 1, 4, 1], is_extended_id=True
+ )
+ self.baseargs = [sys.argv[0], "-i", "virtual"]
+
+ self.testfile = open("coffee.log.gz", "w+")
+
+ def test_compressed_logfile(self):
+ """
+ Basic test to verify Logger is able to write gzip files.
+ """
+ self.mock_virtual_bus.recv = Mock(side_effect=[self.testmsg, KeyboardInterrupt])
+ sys.argv = self.baseargs + ["--file_name", self.testfile.name]
+ can.logger.main()
+ with gzip.open(self.testfile.name, "rt") as testlog:
+ last_line = testlog.readlines()[-1]
+
+ self.assertEqual(last_line, "(0.000000) vcan0 00C0FFEE#0019000103010401\n")
+
+ def tearDown(self) -> None:
+ self.testfile.close()
+ os.remove(self.testfile.name)
+ self.patcher_virtual_bus.stop()
+
+
 if __name__ == "__main__":
  unittest.main()

test/test_logger.py

@@ -15,6 +15,9 @@
 
 
 class TestPlayerScriptModule(unittest.TestCase):
+
+ logfile = os.path.join(os.path.dirname(__file__), "data", "test_CanMessage.asc")
+
  def setUp(self) -> None:
  # Patch VirtualBus object
  patcher_virtual_bus = mock.patch("can.interfaces.virtual.VirtualBus", spec=True)
@@ -29,9 +32,6 @@ def setUp(self) -> None:
  self.addCleanup(patcher_sleep.stop)
 
  self.baseargs = [sys.argv[0], "-i", "virtual"]
- self.logfile = os.path.join(
- os.path.dirname(__file__), "data", "test_CanMessage.asc"
- )
 
  def assertSuccessfulCleanup(self):
  self.MockVirtualBus.assert_called_once()
@@ -111,5 +111,13 @@ def test_play_error_frame(self):
  self.assertSuccessfulCleanup()
 
 
+class TestPlayerCompressedFile(TestPlayerScriptModule):
+ """
+ Re-run tests using a compressed file.
+ """
+
+ logfile = os.path.join(os.path.dirname(__file__), "data", "test_CanMessage.asc.gz")
+
+
 if __name__ == "__main__":
  unittest.main()