Simplify build process; Update display_name to remove region information and instead introduce a new full_name field; Remove description which doesn't contain any useful information

Author: avojak

Makefile

@@ -1,14 +1,14 @@
-SHELL := /bin/bash
+SHELL := /bin/bash
+PYTHON_EXE := python3
 
-PYTHON_EXE := python3
-LIBRETRO_SUPER_GIT_URL := https://github.com/libretro/libretro-super.git
+LIBRETRO_DATABASE_GIT_URL := https://github.com/libretro/libretro-database.git
+LIBRETRO_DATABASE_TAG := v1.10.3
 
-BUILD_DIR := build
-LIBRETRO_SUPER_DIR := $(BUILD_DIR)/libretro-super
-RETROARCH_DIR := $(LIBRETRO_SUPER_DIR)/retroarch
-RDB_DIR := $(RETROARCH_DIR)/media/libretrodb/rdb
-LIBRETRO_DB_DIR := $(RETROARCH_DIR)/libretro-db
-LIBRETRO_DB_TOOL_EXE := $(LIBRETRO_DB_DIR)/libretrodb_tool
+BUILD_DIR := build
+LIBRETRO_DATABASE_DIR := $(BUILD_DIR)/libretro-database
+LIBRETRO_RDB_DIR := $(LIBRETRO_DATABASE_DIR)/rdb
+LIBRETRO_SUPER_DIR := $(LIBRETRO_DATABASE_DIR)/libretro-super
+LIBRETRO_DB_TOOL_EXE := $(LIBRETRO_SUPER_DIR)/retroarch/libretro-db/libretrodb_tool
 
 SQLITE_DATABASE_FILE := $(BUILD_DIR)/libretrodb.sqlite
 SQLITE_DATABASE_ARCHIVE := $(SQLITE_DATABASE_FILE).tgz
@@ -17,24 +17,25 @@ SQLITE_DATABASE_ARCHIVE := $(SQLITE_DATABASE_FILE).tgz
 $(BUILD_DIR):
 mkdir -p $(BUILD_DIR)
 
-# Retrieve the libretro-super repository
-$(LIBRETRO_SUPER_DIR): | $(BUILD_DIR)
-git clone $(LIBRETRO_SUPER_GIT_URL) $(LIBRETRO_SUPER_DIR)
+$(LIBRETRO_RDB_DIR): $(LIBRETRO_DATABASE_DIR) $(LIBRETRO_SUPER_DIR)
 
-# Retrieve the Retroarch data and build the database
-$(RETROARCH_DIR): $(LIBRETRO_SUPER_DIR)
-cd $(LIBRETRO_SUPER_DIR) && \
-./libretro-fetch.sh retroarch && \
-./libretro-build-database.sh
+# Retrieve the libretro-database repository
+$(LIBRETRO_DATABASE_DIR): | $(BUILD_DIR)
+git clone $(LIBRETRO_DATABASE_GIT_URL) $(LIBRETRO_DATABASE_DIR)
+git -C $(LIBRETRO_DATABASE_DIR) checkout tags/$(LIBRETRO_DATABASE_TAG)
 
-# Build the libretrodb_tool
-$(LIBRETRO_DB_TOOL_EXE): $(RETROARCH_DIR)
-$(MAKE) -C $(LIBRETRO_DB_DIR) libretrodb_tool
+# Build the libretro-database repository
+$(LIBRETRO_SUPER_DIR): | $(LIBRETRO_DATABASE_DIR)
+$(MAKE) -C $(LIBRETRO_DATABASE_DIR) build
+
+# Ensure that the libretrodb_tool is built
+$(LIBRETRO_DB_TOOL_EXE): | $(LIBRETRO_SUPER_DIR)
+$(MAKE) -C $(LIBRETRO_DATABASE_DIR) build
 
 # Generates the .sqlite database file
-$(SQLITE_DATABASE_FILE): $(LIBRETRO_DB_TOOL_EXE)
+$(SQLITE_DATABASE_FILE): $(LIBRETRO_RDB_DIR) $(LIBRETRO_DB_TOOL_EXE)
 $(PYTHON_EXE) main.py \
---rdb-dir=$(RDB_DIR) \
+--rdb-dir=$(LIBRETRO_RDB_DIR) \
 --output=$(SQLITE_DATABASE_FILE) \
 --libretrodb-tool=$(LIBRETRO_DB_TOOL_EXE)
 
@@ -44,9 +45,11 @@ $(SQLITE_DATABASE_ARCHIVE): $(SQLITE_DATABASE_FILE)
 
 .PHONY: database
 database: $(SQLITE_DATABASE_FILE)
+$(info $(shell du -sh $(SQLITE_DATABASE_FILE)))
 
 .PHONY: archive
 archive: $(SQLITE_DATABASE_ARCHIVE)
+$(info $(shell du -sh $(SQLITE_DATABASE_ARCHIVE)))
 
 .PHONY: all
 all: database archive

README.md

@@ -1,5 +1,5 @@
-<img src="https://img.shields.io/badge/size-41.6%20MB-blue"></img>
-<img src="https://img.shields.io/badge/compressed%20size-13.4%20MB-blue"></img>
+<img src="https://img.shields.io/badge/size-37%20MB-blue"></img>
+<img src="https://img.shields.io/badge/compressed%20size-13%20MB-blue"></img>
 
 # SQLite Libretro DB
 
@@ -44,8 +44,8 @@ build/libretrodb.sqlite.tgz
 | release_month | INTEGER |
 | region_id | INTEGER |
 | genre_id | INTEGER |
-| description | TEXT |
 | display_name | TEXT |
+| full_name | TEXT |
 | boxart_url | TEXT |
 | platform_id | INTEGER |
 
@@ -131,7 +131,7 @@ Output:
 
 | serial_id | release_year | release_month | display_name | developer_name | franchise_name | region_name | genre_name | rom_name | rom_md5 | platform_name | manufacturer_name |
 | --------- | ------------ | ------------- | ------------ | -------------- | -------------- | ----------- | ---------- | -------- | ------- | ------------- | ----------------- |
-| 41575245 | 2001 | 9 | Advance Wars (USA) | Intelligent Systems | Advance Wars | USA | Strategy | Advance Wars (USA).gba | 27F322F5CD535297AB21BC4A41CBFC12 | Game Boy Advance | Nintendo |
+| 41575245 | 2001 | 9 | Advance Wars | Intelligent Systems | Advance Wars | USA | Strategy | Advance Wars (USA).gba | 27F322F5CD535297AB21BC4A41CBFC12 | Game Boy Advance | Nintendo |
 
 ## Artwork
 

main.py

@@ -9,6 +9,22 @@
 from pathlib import Path
 from subprocess import PIPE
 
+class Logger:
+
+ def __init__(self):
+ self.CRED = '\033[91m'
+ self.CGREEN = '\033[92m'
+ self.CEND = '\033[0m'
+
+ def error(self, msg):
+ print("{cred}{msg}{cend}".format(cred=self.CRED, msg=msg, cend=self.CEND))
+
+ def info(self, msg):
+ print(msg)
+
+ def success(self, msg):
+ print("{cgreen}{msg}{cend}".format(cgreen=self.CGREEN, msg=msg, cend=self.CEND))
+
 class Platform:
 
  def __init__(self, name, manufacturer_id):
@@ -25,9 +41,10 @@ def __init__(self, name, md5):
 
 class Game:
 
- def __init__(self, display_name, serial, rom, developer_id, franchise_id, release_year, release_month, region_id, genre_id, description, platform_id):
+ def __init__(self, display_name, full_name, serial, rom, developer_id, franchise_id, release_year, release_month, region_id, genre_id, platform_id):
  self.id = None
  self.display_name = display_name
+ self.full_name = full_name
  self.serial = serial
  self.rom = rom
  self.developer_id = developer_id
@@ -36,7 +53,6 @@ def __init__(self, display_name, serial, rom, developer_id, franchise_id, releas
  self.release_month = release_month
  self.region_id = region_id
  self.genre_id = genre_id
- self.description = description
  self.platform_id = platform_id
 
  """
@@ -46,6 +62,8 @@ def join(self, other):
  # Join the top-level fields
  if self.display_name is None and other.display_name is not None:
  self.display_name = other.display_name
+ if self.full_name is None and other.full_name is not None:
+ self.full_name = other.full_name
  if self.serial is None and other.serial is not None:
  self.serial = other.serial
  if self.developer_id is None and other.developer_id is not None:
@@ -60,8 +78,6 @@ def join(self, other):
  self.region_id = other.region_id
  if self.genre_id is None and other.genre_id is not None:
  self.genre_id = other.genre_id
- if self.description is None and other.description is not None:
- self.description = other.description
  if self.platform_id is None and other.platform_id is not None:
  self.platform_id = other.platform_id
  # Join the ROM
@@ -75,6 +91,7 @@ def join(self, other):
 class Converter:
 
  def __init__(self, rdb_dir, output_file, libretrodb_tool):
+ self.logger = Logger()
  self.rdb_dir = self._validate_rdb_dir(rdb_dir)
  self.output_file = self._validate_output_file(output_file)
  self.libretrodb_tool = self._validate_libretrodb_tool(libretrodb_tool)
@@ -92,7 +109,7 @@ def __init__(self, rdb_dir, output_file, libretrodb_tool):
  """
  def _validate_rdb_dir(self, rdb_dir):
  if not os.path.isdir(rdb_dir):
- print("{} is not a directory".format(rdb_dir))
+ self.logger.error("{} is not a directory".format(rdb_dir))
  exit(1)
  return rdb_dir
 
@@ -101,7 +118,7 @@ def _validate_rdb_dir(self, rdb_dir):
  """
  def _validate_output_file(self, output_file):
  if os.path.isfile(output_file):
- print("{} already exists".format(output_file))
+ self.logger.error("{} already exists".format(output_file))
  exit(1)
  return output_file
 
@@ -123,7 +140,7 @@ def _validate_output_file(self, output_file):
  """
  def _validate_libretrodb_tool(self, libretrodb_tool):
  if not os.path.isfile(libretrodb_tool):
- print("{} not found".format(libretrodb_tool))
+ self.logger.error("{} not found".format(libretrodb_tool))
  exit(1)
  return libretrodb_tool
 
@@ -143,7 +160,7 @@ def run(self):
  # Iterate over the .rdb files in the input directory and parse all data
  for file in [f for f in os.listdir(self.rdb_dir) if os.path.isfile(os.path.join(self.rdb_dir, f))]:
  if not file.endswith('.rdb'):
- print("Skipping non-RDB file: {}".format(file))
+ self.logger.info("Skipping non-RDB file: {}".format(file))
  continue
  self._parse_platform_file(os.path.join(self.rdb_dir, file))
  # break # TODO: Remove this
@@ -174,7 +191,7 @@ def run(self):
  Parses a single platform .rdb file.
  """
  def _parse_platform_file(self, rdb_file):
- print(" > Parsing: {}".format(rdb_file))
+ self.logger.info("Parsing {}".format(rdb_file))
 
  # Try to parse out the manufacturer and platfrom from the .rdb filename
  system_fullname = Path(rdb_file).stem
@@ -209,8 +226,8 @@ def _parse_line(self, json_str, platform_id):
  try:
  json_obj = json.loads(json_str)
  except json.decoder.JSONDecodeError as e:
- print("Error while parsing JSON: {}".format(e.msg))
- print("Original JSON string: {}".format(json_str))
+ self.logger.error("Error while parsing JSON: {}".format(str(e)))
+ self.logger.error("Original JSON string: {}".format(json_str))
  return
 
  # Extract the fields from the JSON
@@ -224,8 +241,14 @@ def _parse_line(self, json_str, platform_id):
  rom_name = self._get_json_value(json_obj, 'rom_name')
  region = self._get_json_value(json_obj, 'region')
  genre = self._get_json_value(json_obj, 'genre')
- description = self._get_json_value(json_obj, 'description')
- display_name = self._get_json_value(json_obj, 'name')
+ # description = self._get_json_value(json_obj, 'description') # This field in the dataset doesn't currently provide any added value
+ full_name = self._get_json_value(json_obj, 'name')
+ 
+ # Build the display name from the full name, but ignore all the trailing parenthesis-wrapped meta-tags
+ if full_name is None:
+ display_name = None
+ else:
+ display_name = full_name.split("(")[0].strip() if "(" in full_name else full_name
 
  # Save potentially common references to developers, franchises, regions and genres, and assign an ID
  if developer is not None and developer not in self.developers:
@@ -244,7 +267,7 @@ def _parse_line(self, json_str, platform_id):
 
  # Build the ROM and Game objects. Note that ROMs and games should be 1:1.
  rom = ROM(rom_name, md5)
- game = Game(display_name, serial, rom, developer_id, franchise_id, release_year, release_month, region_id, genre_id, description, platform_id)
+ game = Game(display_name, full_name, serial, rom, developer_id, franchise_id, release_year, release_month, region_id, genre_id, platform_id)
  if md5 in self.games:
  self.games[md5].join(game)
  else:
@@ -257,60 +280,59 @@ def _parse_line(self, json_str, platform_id):
  Insert the manufacturers into the database.
  """
  def _insert_manufacturers(self, cursor):
- print(" > Inserting {} manufacturers into database…".format(len(self.manufacturers)))
  for key,value in self.manufacturers.items():
  (id, name) = (value, key)
  cursor.execute(self._load_sql("./sql/insert_manufacturer.sql"), (id, name))
+ self.logger.success("Inserted {} manufacturers into database".format(len(self.manufacturers)))
 
  """
  Insert the platforms into the database.
  """
  def _insert_platforms(self, cursor):
- print(" > Inserting {} platforms into database…".format(len(self.platforms)))
  for key,value in self.platforms.items():
  cursor.execute(self._load_sql("./sql/insert_platform.sql"), (value.id, value.name, value.manufacturer_id))
+ self.logger.success("Inserted {} platforms into database".format(len(self.platforms)))
 
  """
  Insert the developers into the database.
  """
  def _insert_developers(self, cursor):
- print(" > Inserting {} developers into database…".format(len(self.developers)))
  for key,value in self.developers.items():
  (id, name) = (value, key)
  cursor.execute(self._load_sql("./sql/insert_developer.sql"), (id, name))
+ self.logger.success("Inserted {} developers into database".format(len(self.developers)))
 
  """
  Insert the franchises into the database.
  """
  def _insert_franchises(self, cursor):
- print(" > Inserting {} franchises into database…".format(len(self.franchises)))
  for key,value in self.franchises.items():
  (id, name) = (value, key)
  cursor.execute(self._load_sql("./sql/insert_franchise.sql"), (id, name))
+ self.logger.success("Inserted {} franchises into database".format(len(self.franchises)))
 
  """
  Insert the regions into the database.
  """
  def _insert_regions(self, cursor):
- print(" > Inserting {} regions into database…".format(len(self.regions)))
  for key,value in self.regions.items():
  (id, name) = (value, key)
  cursor.execute(self._load_sql("./sql/insert_region.sql"), (id, name))
+ self.logger.success("Inserted {} regions into database".format(len(self.regions)))
 
  """
  Insert the genres into the database.
  """
  def _insert_genres(self, cursor):
- print(" > Inserting {} genres into database…".format(len(self.genres)))
  for key,value in self.genres.items():
  (id, name) = (value, key)
  cursor.execute(self._load_sql("./sql/insert_genre.sql"), (id, name))
+ self.logger.success("Inserted {} genres into database".format(len(self.genres)))
 
  """
  Insert the ROMs and games into the database.
  """
  def _insert_games(self, cursor):
- print(" > Inserting {} games into database…".format(len(self.games)))
  for key,value in self.games.items():
  game = value
  cursor.execute(self._load_sql("./sql/insert_rom.sql"), (game.rom.id, game.rom.name, game.rom.md5))
@@ -324,9 +346,10 @@ def _insert_games(self, cursor):
  game.release_month,
  game.region_id,
  game.genre_id,
- game.description,
  game.display_name,
+ game.full_name,
  game.platform_id))
+ self.logger.success("Inserted {} games into database".format(len(self.games)))
 
  def _get_json_value(self, json_obj, key):
  return json_obj[key] if key in json_obj else None

sql/create_tables.sql

@@ -8,8 +8,8 @@ CREATE TABLE games (
  release_month INTEGER,
  region_id INTEGER,
  genre_id INTEGER,
- description TEXT,
  display_name TEXT,
+ full_name TEXT,
  -- boxart_url TEXT,
  platform_id INTEGER
 );

sql/insert_game.sql

@@ -8,8 +8,8 @@ INSERT INTO games(
  release_month,
  region_id,
  genre_id,
- description,
  display_name,
+ full_name,
  -- boxart_url,
  platform_id
 ) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)