Merge pull request #1 from avojak/deduplicate-games

Deduplicate games

Author: avojak

README.md

@@ -1,5 +1,5 @@
-<img src="https://img.shields.io/badge/size-37%20MB-blue"></img>
-<img src="https://img.shields.io/badge/compressed%20size-13%20MB-blue"></img>
+<img src="https://img.shields.io/badge/size-35%20MB-blue"></img>
+<img src="https://img.shields.io/badge/compressed%20size-14%20MB-blue"></img>
 
 # SQLite Libretro DB
 
@@ -10,7 +10,8 @@ the same content* from the Libretro RetroArch database in a single SQLite databa
 
 ***Important note:*** The conversion tool here also does some basic deconfliction when there are multiple records for the same ROM MD5 checksum.
 The underlying assumption is that if two ROMs have the same checksum, they're the same, and the metadata should be merged in favor of non-null
-values. The primary use-case is for client applications to be able to query the database by MD5 checksum of a ROM file, so keep in mind that this mindset informed the database schema and how the utility decides which data is duplicated.
+values. The primary use-case is for client applications to be able to query the database by MD5 checksum of a ROM file, so keep in mind that 
+this mindset informed the database schema and how the utility decides which data is duplicated.
 
 ## Usage
 
@@ -37,8 +38,10 @@ build/libretrodb.sqlite.tgz
 | ------ | --------- |
 | id | INTEGER PRIMARY KEY |
 | serial_id | TEXT |
-| rom_id | INTEGER |
 | developer_id | INTEGER |
+| publisher_id | INTEGER |
+| rating_id | INTEGER |
+| users | INTEGER |
 | franchise_id | INTEGER |
 | release_year | INTEGER |
 | release_month | INTEGER |
@@ -54,6 +57,7 @@ build/libretrodb.sqlite.tgz
 | Column | Data Type |
 | ------ | --------- |
 | id | INTEGER PRIMARY KEY |
+| serial_id | INTEGER |
 | name | TEXT |
 | md5 | TEXT |
 
@@ -64,6 +68,20 @@ build/libretrodb.sqlite.tgz
 | id | INTEGER PRIMARY KEY |
 | name | TEXT |
 
+### `publishers`
+
+| Column | Data Type |
+| ------ | --------- |
+| id | INTEGER PRIMARY KEY |
+| name | TEXT |
+
+### `ratings`
+
+| Column | Data Type |
+| ------ | --------- |
+| id | INTEGER PRIMARY KEY |
+| name | TEXT |
+
 ### `franchises`
 
 | Column | Data Type |
@@ -108,7 +126,10 @@ SELECT games.serial_id,
 games.release_year,
 games.release_month,
 games.display_name,
+games.users,
 developers.name as developer_name,
+publishers.name as publisher_name,
+ratings.name as rating_name,
 franchises.name as franchise_name,
 regions.name as region_name,
 genres.name as genre_name,
@@ -119,11 +140,13 @@ SELECT games.serial_id,
 FROM games
 LEFT JOIN developers ON games.developer_id = developers.id
 LEFT JOIN franchises ON games.franchise_id = franchises.id
+LEFT JOIN publishers ON games.publisher_id = publishers.id
+LEFT JOIN ratings ON games.rating_id = ratings.id
 LEFT JOIN genres ON games.genre_id = genres.id
 LEFT JOIN platforms ON games.platform_id = platforms.id
 LEFT JOIN manufacturers ON platforms.manufacturer_id = manufacturers.id
 LEFT JOIN regions ON games.region_id = regions.id
-INNER JOIN roms ON games.rom_id = roms.id
+INNER JOIN roms ON games.serial_id = roms.serial_id
 WHERE roms.md5 = "27F322F5CD535297AB21BC4A41CBFC12";
 ```
 

main.py

@@ -34,20 +34,23 @@ def __init__(self, name, manufacturer_id):
 
 class ROM:
 
- def __init__(self, name, md5):
+ def __init__(self, serial, name, md5):
  self.id = None
+ self.serial = serial
  self.name = name
  self.md5 = md5
 
 class Game:
 
- def __init__(self, display_name, full_name, serial, rom, developer_id, franchise_id, release_year, release_month, region_id, genre_id, platform_id):
+ def __init__(self, display_name, full_name, serial, developer_id, publisher_id, rating_id, users, franchise_id, release_year, release_month, region_id, genre_id, platform_id):
  self.id = None
  self.display_name = display_name
  self.full_name = full_name
  self.serial = serial
- self.rom = rom
  self.developer_id = developer_id
+ self.publisher_id = publisher_id
+ self.rating_id = rating_id
+ self.users = users
  self.franchise_id = franchise_id
  self.release_year = release_year
  self.release_month = release_month
@@ -59,7 +62,6 @@ def __init__(self, display_name, full_name, serial, rom, developer_id, franchise
  Merge other game into self by deferring to non-null fields.
  """
  def join(self, other):
- # Join the top-level fields
  if self.display_name is None and other.display_name is not None:
  self.display_name = other.display_name
  if self.full_name is None and other.full_name is not None:
@@ -68,6 +70,12 @@ def join(self, other):
  self.serial = other.serial
  if self.developer_id is None and other.developer_id is not None:
  self.developer_id = other.developer_id
+ if self.publisher_id is None and other.publisher_id is not None:
+ self.publisher_id = other.publisher_id
+ if self.rating_id is None and other.rating_id is not None:
+ self.rating_id = other.rating_id
+ if self.users is None and other.users is not None:
+ self.users = other.users
  if self.franchise_id is None and other.franchise_id is not None:
  self.franchise_id = other.franchise_id
  if self.release_year is None and other.release_year is not None:
@@ -80,13 +88,6 @@ def join(self, other):
  self.genre_id = other.genre_id
  if self.platform_id is None and other.platform_id is not None:
  self.platform_id = other.platform_id
- # Join the ROM
- if self.rom is None and other.rom is not None:
- self.rom = other.rom
- if self.rom.name is None and other.rom.name is not None:
- self.rom.name = other.rom.name
- if self.rom.md5 is None and other.rom.md5 is not None:
- self.rom.md5 = other.rom.md5
 
 class Converter:
 
@@ -97,12 +98,15 @@ def __init__(self, rdb_dir, output_file, libretrodb_tool):
  self.libretrodb_tool = self._validate_libretrodb_tool(libretrodb_tool)
  # Create storage for parsed data prior to insertion in the database
  self.developers = dict()
+ self.publishers = dict()
+ self.ratings = dict()
  self.franchises = dict()
  self.regions = dict()
  self.genres = dict()
  self.platforms = dict()
  self.manufacturers = dict()
  self.games = dict()
+ self.roms = dict()
 
  """
  Ensure that the provided directory of .rdb files exists.
@@ -174,12 +178,15 @@ def run(self):
 
  # Insert data into the database
  self._insert_developers(cursor)
+ self._insert_publishers(cursor)
+ self._insert_ratings(cursor)
  self._insert_franchises(cursor)
  self._insert_genres(cursor)
  self._insert_manufacturers(cursor)
  self._insert_platforms(cursor)
  self._insert_regions(cursor)
  self._insert_games(cursor)
+ self._insert_roms(cursor)
 
  # Commit changes to the database
  connection.commit()
@@ -234,6 +241,9 @@ def _parse_line(self, json_str, platform_id):
  serial = self._get_json_value(json_obj, 'serial')
  md5 = self._get_json_value(json_obj, 'md5')
  developer = self._get_json_value(json_obj, 'developer')
+ publisher = self._get_json_value(json_obj, 'publisher')
+ rating = self._get_json_value(json_obj, 'esrb_rating')
+ users = self._get_json_value(json_obj, 'users')
  franchise = self._get_json_value(json_obj, 'franchise')
  release_year = self._get_json_value(json_obj, 'releaseyear')
  release_month = self._get_json_value(json_obj, 'releasemonth')
@@ -253,6 +263,10 @@ def _parse_line(self, json_str, platform_id):
  # Save potentially common references to developers, franchises, regions and genres, and assign an ID
  if developer is not None and developer not in self.developers:
  self.developers[developer] = len(self.developers) + 1
+ if publisher is not None and publisher not in self.publishers:
+ self.publishers[publisher] = len(self.publishers) + 1
+ if rating is not None and rating not in self.ratings:
+ self.ratings[rating] = len(self.ratings) + 1
  if franchise is not None and franchise not in self.franchises:
  self.franchises[franchise] = len(self.franchises) + 1
  if region is not None and region not in self.regions:
@@ -261,20 +275,25 @@ def _parse_line(self, json_str, platform_id):
  self.genres[genre] = len(self.genres) + 1
 
  developer_id = self.developers[developer] if developer is not None else None
+ publisher_id = self.publishers[publisher] if publisher is not None else None
+ rating_id = self.ratings[rating] if rating is not None else None
  franchise_id = self.franchises[franchise] if franchise is not None else None
  region_id = self.regions[region] if region is not None else None
  genre_id = self.genres[genre] if genre is not None else None
 
  # Build the ROM and Game objects. Note that ROMs and games should be 1:1.
- rom = ROM(rom_name, md5)
- game = Game(display_name, full_name, serial, rom, developer_id, franchise_id, release_year, release_month, region_id, genre_id, platform_id)
- if md5 in self.games:
- self.games[md5].join(game)
+ rom = ROM(serial, rom_name, md5)
+ rom_id = len(self.roms) + 1
+ rom.id = rom_id
+ self.roms[rom_id] = rom
+
+ game = Game(display_name, full_name, serial, developer_id, publisher_id, rating_id, users, franchise_id, release_year, release_month, region_id, genre_id, platform_id)
+ if serial in self.games:
+ self.games[serial].join(game)
  else:
  id = len(self.games) + 1
- self.games[md5] = game
- self.games[md5].id = id
- self.games[md5].rom.id = id
+ game.id = id
+ self.games[serial] = game
 
  """
  Insert the manufacturers into the database.
@@ -302,6 +321,24 @@ def _insert_developers(self, cursor):
  cursor.execute(self._load_sql("./sql/insert_developer.sql"), (id, name))
  self.logger.success("Inserted {} developers into database".format(len(self.developers)))
 
+ """
+ Insert the publishers into the database.
+ """
+ def _insert_publishers(self, cursor):
+ for key,value in self.publishers.items():
+ (id, name) = (value, key)
+ cursor.execute(self._load_sql("./sql/insert_publisher.sql"), (id, name))
+ self.logger.success("Inserted {} publishers into database".format(len(self.publishers)))
+ 
+ """
+ Insert the ratings into the database.
+ """
+ def _insert_ratings(self, cursor):
+ for key,value in self.ratings.items():
+ (id, name) = (value, key)
+ cursor.execute(self._load_sql("./sql/insert_rating.sql"), (id, name))
+ self.logger.success("Inserted {} ratings into database".format(len(self.ratings)))
+
  """
  Insert the franchises into the database.
  """
@@ -330,17 +367,18 @@ def _insert_genres(self, cursor):
  self.logger.success("Inserted {} genres into database".format(len(self.genres)))
 
  """
- Insert the ROMs and games into the database.
+ Insert the games into the database.
  """
  def _insert_games(self, cursor):
  for key,value in self.games.items():
  game = value
- cursor.execute(self._load_sql("./sql/insert_rom.sql"), (game.rom.id, game.rom.name, game.rom.md5))
  cursor.execute(self._load_sql("./sql/insert_game.sql"), (
  game.id, 
  game.serial,
- game.rom.id, 
  game.developer_id,
+ game.publisher_id,
+ game.rating_id,
+ game.users,
  game.franchise_id,
  game.release_year,
  game.release_month,
@@ -351,6 +389,15 @@ def _insert_games(self, cursor):
  game.platform_id))
  self.logger.success("Inserted {} games into database".format(len(self.games)))
 
+ """
+ Insert the ROMs into the database.
+ """
+ def _insert_roms(self, cursor):
+ for key,value in self.roms.items():
+ rom = value
+ cursor.execute(self._load_sql("./sql/insert_rom.sql"), (rom.id, rom.serial, rom.name, rom.md5))
+ self.logger.success("Inserted {} ROMs into database".format(len(self.roms)))
+
  def _get_json_value(self, json_obj, key):
  return json_obj[key] if key in json_obj else None
 

sql/create_tables.sql

@@ -1,8 +1,10 @@
 CREATE TABLE games (
  id INTEGER PRIMARY KEY,
  serial_id TEXT,
- rom_id INTEGER,
  developer_id INTEGER,
+ publisher_id INTEGER,
+ rating_id INTEGER,
+ users INTEGER,
  franchise_id INTEGER,
  release_year INTEGER,
  release_month INTEGER,
@@ -15,13 +17,22 @@ CREATE TABLE games (
 );
 CREATE TABLE roms (
  id INTEGER PRIMARY KEY,
+ serial_id TEXT,
  name TEXT,
  md5 TEXT
 );
 CREATE TABLE developers (
  id INTEGER PRIMARY KEY,
  name TEXT
 );
+CREATE TABLE publishers (
+ id INTEGER PRIMARY KEY,
+ name TEXT
+);
+CREATE TABLE ratings (
+ id INTEGER PRIMARY KEY,
+ name TEXT
+);
 CREATE TABLE franchises (
  id INTEGER PRIMARY KEY,
  name TEXT

sql/insert_game.sql

@@ -1,8 +1,10 @@
 INSERT INTO games(
  id,
  serial_id,
- rom_id,
  developer_id,
+ publisher_id,
+ rating_id,
+ users,
  franchise_id,
  release_year,
  release_month,
@@ -12,4 +14,4 @@ INSERT INTO games(
  full_name,
  -- boxart_url,
  platform_id
-) VALUES(?,?,?,?,?,?,?,?,?,?,?,?)
+) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)

sql/insert_publisher.sql

@@ -0,0 +1 @@
+INSERT INTO publishers(id,name) VALUES(?,?)

sql/insert_rating.sql

@@ -0,0 +1 @@
+INSERT INTO ratings(id,name) VALUES(?,?)

sql/insert_rom.sql

@@ -1 +1 @@
-INSERT INTO roms(id,name,md5) VALUES(?,?,?)
+INSERT INTO roms(id,serial_id,name,md5) VALUES(?,?,?,?)