Default to UTF-8 Encoding and Add Optional Encoding Argument (#81)

Author: Cleptomania

CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
+## [2.2.8] - UNRELEASED
+
+Converts all file loading to use UTF-8 encoding by default. In most cases, all Tiled files will be exported from Tiled in UTF-8 encoding, however the python `open()` function uses the system default locale. The only case where Tiled would not have used UTF-8 is for JSON files when Tiled was compiled against Qt 5, which is only in some builds of Tiled from older systems. All XML files exported from Tiled will always be UTF-8. If someone happens to have a JSON file which was exported from Tiled on an encoding other than UTF-8, or for some other reason is in a different encoding. This can be switched using a new optional argument named `encoding` in the various public API `parse` functions such as `parse_map()`. This value is handed down through the pipeline of file loading in pytiled-parser, and will apply to every file loaded during the chain from this. This means that every file in a chain(for example, Map, Tileset, and Template File) must share the same encoding. This new argument is a string which is ultimately passed to the Python [open()](https://docs.python.org/3/library/functions.html#open) function. This change does introduce breaking changes in the underlying API which is not intended to be public facing, but if you are going deeper than the top level parse functions, you may need to adjust for this, as many of the underlying internal functions now have a mandatory encoding argument.
+
 ## [2.2.7] - 2024-10-03
 
 Fixes a bug when using the TMX format, where multi-line String properties would not be correctly parsed, as they are placed differently in the XML than single line strings. (#75)

pytiled_parser/parser.py

@@ -14,64 +14,67 @@
 from pytiled_parser.world import parse_world as _parse_world
 
 
-def parse_map(file: Path) -> TiledMap:
+def parse_map(file: Path, encoding: str = "utf-8") -> TiledMap:
  """Parse the raw Tiled map into a pytiled_parser type
 
  Args:
  file: Path to the map file
+ encoding: The character encoding set to use when opening the file
 
  Returns:
  TiledMap: A parsed and typed TiledMap
  """
- parser = check_format(file)
+ parser = check_format(file, encoding)
 
  # The type ignores are because mypy for some reason thinks those functions return Any
  if parser == "tmx":
- return tmx_map_parse(file) # type: ignore
+ return tmx_map_parse(file, encoding) # type: ignore
  else:
  try:
- return json_map_parse(file) # type: ignore
+ return json_map_parse(file, encoding) # type: ignore
  except ValueError:
  raise UnknownFormat(
  "Unknown Map Format, please use either the TMX or JSON format. "
  "This message could also mean your map file is invalid or corrupted."
  )
 
 
-def parse_tileset(file: Path) -> Tileset:
+def parse_tileset(file: Path, encoding: str = "utf-8") -> Tileset:
  """Parse the raw Tiled Tileset into a pytiled_parser type
 
  Args:
  file: Path to the map file
+ encoding: The character encoding set to use when opening the file
 
  Returns:
  Tileset: A parsed and typed Tileset
  """
- parser = check_format(file)
+ parser = check_format(file, encoding)
 
  if parser == "tmx":
- with open(file) as map_file:
+ with open(file, encoding=encoding) as map_file:
  raw_tileset = etree.parse(map_file).getroot()
- return tmx_tileset_parse(raw_tileset, 1)
+ return tmx_tileset_parse(raw_tileset, 1, encoding)
  else:
  try:
- with open(file) as my_file:
+ with open(file, encoding=encoding) as my_file:
  raw_tileset = json.load(my_file)
- return json_tileset_parse(raw_tileset, 1)
+ return json_tileset_parse(raw_tileset, 1, encoding)
  except ValueError:
  raise UnknownFormat(
  "Unknowm Tileset Format, please use either the TSX or JSON format. "
  "This message could also mean your tileset file is invalid or corrupted."
  )
 
 
-def parse_world(file: Path) -> World:
+def parse_world(file: Path, encoding: str = "utf-8") -> World:
  """Parse the raw world file into a pytiled_parser type
 
  Args:
  file: Path to the world file
+ encoding: The character encoding set to use when opening the file
 
  Returns:
  World: A parsed and typed World
  """
- return _parse_world(file)
+ return _parse_world(file, encoding)

pytiled_parser/parsers/json/layer.py

@@ -298,6 +298,7 @@ def _parse_tile_layer(raw_layer: RawLayer) -> TileLayer:
 
 def _parse_object_layer(
  raw_layer: RawLayer,
+ encoding: str,
  parent_dir: Optional[Path] = None,
 ) -> ObjectLayer:
  """Parse the raw_layer to an ObjectLayer.
@@ -310,7 +311,7 @@ def _parse_object_layer(
  """
  objects = []
  for object_ in raw_layer["objects"]:
- objects.append(parse_object(object_, parent_dir))
+ objects.append(parse_object(object_, encoding, parent_dir))
 
  return ObjectLayer(
  tiled_objects=objects,
@@ -339,7 +340,7 @@ def _parse_image_layer(raw_layer: RawLayer) -> ImageLayer:
 
 
 def _parse_group_layer(
- raw_layer: RawLayer, parent_dir: Optional[Path] = None
+ raw_layer: RawLayer, encoding, parent_dir: Optional[Path] = None
 ) -> LayerGroup:
  """Parse the raw_layer to a LayerGroup.
 
@@ -352,13 +353,14 @@ def _parse_group_layer(
  layers = []
 
  for layer in raw_layer["layers"]:
- layers.append(parse(layer, parent_dir=parent_dir))
+ layers.append(parse(layer, encoding, parent_dir=parent_dir))
 
  return LayerGroup(layers=layers, **_parse_common(raw_layer).__dict__)
 
 
 def parse(
  raw_layer: RawLayer,
+ encoding: str,
  parent_dir: Optional[Path] = None,
 ) -> Layer:
  """Parse a raw Layer into a pytiled_parser object.
@@ -378,9 +380,9 @@ def parse(
  type_ = raw_layer["type"]
 
  if type_ == "objectgroup":
- return _parse_object_layer(raw_layer, parent_dir)
+ return _parse_object_layer(raw_layer, encoding, parent_dir)
  elif type_ == "group":
- return _parse_group_layer(raw_layer, parent_dir)
+ return _parse_group_layer(raw_layer, encoding, parent_dir)
  elif type_ == "imagelayer":
  return _parse_image_layer(raw_layer)
  elif type_ == "tilelayer":

pytiled_parser/parsers/json/tiled_map.py

@@ -55,7 +55,7 @@
 """
 
 
-def parse(file: Path) -> TiledMap:
+def parse(file: Path, encoding: str) -> TiledMap:
  """Parse the raw Tiled map into a pytiled_parser type.
 
  Args:
@@ -64,7 +64,7 @@ def parse(file: Path) -> TiledMap:
  Returns:
  TiledMap: A parsed TiledMap.
  """
- with open(file) as map_file:
+ with open(file, encoding=encoding) as map_file:
  raw_tiled_map = json.load(map_file)
 
  parent_dir = file.parent
@@ -76,20 +76,22 @@ def parse(file: Path) -> TiledMap:
  if raw_tileset.get("source") is not None:
  # Is an external Tileset
  tileset_path = Path(parent_dir / raw_tileset["source"])
- parser = check_format(tileset_path)
- with open(tileset_path) as raw_tileset_file:
+ parser = check_format(tileset_path, encoding)
+ with open(tileset_path, encoding=encoding) as raw_tileset_file:
  if parser == "tmx":
  raw_tileset_external = etree.parse(raw_tileset_file).getroot()
  tilesets[raw_tileset["firstgid"]] = parse_tmx_tileset(
  raw_tileset_external,
  raw_tileset["firstgid"],
+ encoding,
  external_path=tileset_path.parent,
  )
  else:
  try:
  tilesets[raw_tileset["firstgid"]] = parse_json_tileset(
  json.load(raw_tileset_file),
  raw_tileset["firstgid"],
+ encoding,
  external_path=tileset_path.parent,
  )
  except ValueError:
@@ -101,7 +103,7 @@ def parse(file: Path) -> TiledMap:
  # Is an embedded Tileset
  raw_tileset = cast(RawTileSet, raw_tileset)
  tilesets[raw_tileset["firstgid"]] = parse_json_tileset(
- raw_tileset, raw_tileset["firstgid"]
+ raw_tileset, raw_tileset["firstgid"], encoding
  )
 
  if isinstance(raw_tiled_map["version"], float): # pragma: no cover
@@ -113,7 +115,10 @@ def parse(file: Path) -> TiledMap:
  map_ = TiledMap(
  map_file=file,
  infinite=raw_tiled_map.get("infinite", False),
- layers=[parse_layer(layer_, parent_dir) for layer_ in raw_tiled_map["layers"]],
+ layers=[
+ parse_layer(layer_, encoding, parent_dir)
+ for layer_ in raw_tiled_map["layers"]
+ ],
  map_size=Size(raw_tiled_map["width"], raw_tiled_map["height"]),
  next_layer_id=raw_tiled_map.get("nextlayerid"),
  next_object_id=raw_tiled_map["nextobjectid"],
@@ -157,6 +162,7 @@ def parse(file: Path) -> TiledMap:
  map_.tilesets[new_firstgid] = parse_json_tileset(
  tiled_object.new_tileset,
  new_firstgid,
+ encoding,
  tiled_object.new_tileset_path,
  )
  tiled_object.gid = tiled_object.gid + (new_firstgid - 1)

pytiled_parser/parsers/json/tiled_object.py

@@ -291,6 +291,7 @@ def _get_parser(raw_object: RawObject) -> Callable[[RawObject], TiledObject]:
 
 def parse(
  raw_object: RawObject,
+ encoding: str,
  parent_dir: Optional[Path] = None,
 ) -> TiledObject:
  """Parse the raw object into a pytiled_parser version
@@ -314,7 +315,9 @@ def parse(
  "A parent directory must be specified when using object templates."
  )
  template_path = Path(parent_dir / raw_object["template"])
- template, new_tileset, new_tileset_path = load_object_template(template_path)
+ template, new_tileset, new_tileset_path = load_object_template(
+ template_path, encoding
+ )
 
  if isinstance(template, dict):
  loaded_template = template["object"]

pytiled_parser/parsers/json/tileset.py

@@ -161,7 +161,9 @@ def _parse_grid(raw_grid: RawGrid) -> Grid:
  )
 
 
-def _parse_tile(raw_tile: RawTile, external_path: Optional[Path] = None) -> Tile:
+def _parse_tile(
+ raw_tile: RawTile, encoding: str, external_path: Optional[Path] = None
+) -> Tile:
  """Parse the raw_tile to a Tile object.
 
  Args:
@@ -180,7 +182,7 @@ def _parse_tile(raw_tile: RawTile, external_path: Optional[Path] = None) -> Tile
  tile.animation.append(_parse_frame(frame))
 
  if raw_tile.get("objectgroup") is not None:
- tile.objects = parse_layer(raw_tile["objectgroup"])
+ tile.objects = parse_layer(raw_tile["objectgroup"], encoding)
 
  if raw_tile.get("properties") is not None:
  tile.properties = parse_properties(raw_tile["properties"])
@@ -231,6 +233,7 @@ def _parse_tile(raw_tile: RawTile, external_path: Optional[Path] = None) -> Tile
 def parse(
  raw_tileset: RawTileSet,
  firstgid: int,
+ encoding: str,
  external_path: Optional[Path] = None,
 ) -> Tileset:
  """Parse the raw tileset into a pytiled_parser type
@@ -309,12 +312,12 @@ def parse(
  assert raw_tile.get("id") is None
  raw_tile["id"] = int(raw_tile_id)
  tiles[raw_tile["id"]] = _parse_tile(
- raw_tile, external_path=external_path
+ raw_tile, encoding, external_path=external_path
  )
  else:
  for raw_tile in raw_tileset["tiles"]:
  tiles[raw_tile["id"]] = _parse_tile(
- raw_tile, external_path=external_path
+ raw_tile, encoding, external_path=external_path
  )
  tileset.tiles = tiles
 

pytiled_parser/parsers/tmx/layer.py

@@ -262,7 +262,7 @@ def _parse_tile_layer(raw_layer: etree.Element) -> TileLayer:
 
 
 def _parse_object_layer(
- raw_layer: etree.Element, parent_dir: Optional[Path] = None
+ raw_layer: etree.Element, encoding: str, parent_dir: Optional[Path] = None
 ) -> ObjectLayer:
  """Parse the raw_layer to an ObjectLayer.
 
@@ -274,7 +274,7 @@ def _parse_object_layer(
  """
  objects = []
  for object_ in raw_layer.findall("./object"):
- objects.append(parse_object(object_, parent_dir))
+ objects.append(parse_object(object_, encoding, parent_dir))
 
  object_layer = ObjectLayer(
  tiled_objects=objects,
@@ -316,7 +316,7 @@ def _parse_image_layer(raw_layer: etree.Element) -> ImageLayer:
 
 
 def _parse_group_layer(
- raw_layer: etree.Element, parent_dir: Optional[Path] = None
+ raw_layer: etree.Element, encoding: str, parent_dir: Optional[Path] = None
 ) -> LayerGroup:
  """Parse the raw_layer to a LayerGroup.
 
@@ -331,7 +331,7 @@ def _parse_group_layer(
  layers.append(_parse_tile_layer(layer))
 
  for layer in raw_layer.findall("./objectgroup"):
- layers.append(_parse_object_layer(layer, parent_dir))
+ layers.append(_parse_object_layer(layer, encoding, parent_dir))
 
  for layer in raw_layer.findall("./imagelayer"):
  layers.append(_parse_image_layer(layer))
@@ -350,6 +350,7 @@ def _parse_group_layer(
 
 def parse(
  raw_layer: etree.Element,
+ encoding: str,
  parent_dir: Optional[Path] = None,
 ) -> Layer:
  """Parse a raw Layer into a pytiled_parser object.
@@ -369,9 +370,9 @@ def parse(
  type_ = raw_layer.tag
 
  if type_ == "objectgroup":
- return _parse_object_layer(raw_layer, parent_dir)
+ return _parse_object_layer(raw_layer, encoding, parent_dir)
  elif type_ == "group":
- return _parse_group_layer(raw_layer, parent_dir)
+ return _parse_group_layer(raw_layer, encoding, parent_dir)
  elif type_ == "imagelayer":
  return _parse_image_layer(raw_layer)
  elif type_ == "layer":