Merge pull request #5 from Wikia/football-metadata

Extract even more data from football.wikia.com

Author: macbre

.gitignore

@@ -109,3 +109,6 @@ redisgraph/data/
 # graphs
 *.png
 *.svg
+
+# cache
+grapher/grapher/scripts/.cache/*.json

Makefile

@@ -0,0 +1,2 @@
+redis:
+redis-cli -p 56379

docker-compose.yml

@@ -1,6 +1,6 @@
 version: '2'
 services:
- sphinx:
+ db:
  image: redislabs/redisgraph:1.0.8
  ports:
  - "127.0.0.1:56379:6379" # bind to local interface only!

grapher/.pylintrc

@@ -1,2 +1,2 @@
 [MESSAGES CONTROL]
-disable=fixme,too-few-public-methods,useless-super-delegation,useless-object-inheritance
+disable=fixme,pointless-string-statement,too-few-public-methods,useless-super-delegation,useless-object-inheritance,wrong-import-order

grapher/grapher/graph/__init__.py

@@ -0,0 +1,4 @@
+"""
+Graph storage abstraction layer
+"""
+from .redis import RedisGraph

grapher/grapher/graph/base.py

@@ -0,0 +1,25 @@
+"""
+Base class
+"""
+import logging
+
+
+class BaseGraph(object):
+ """
+ Represents a collection of nodes (model objects) and relations (edges) between them
+ """
+ def __init__(self):
+ self.models = list()
+ self.logger = logging.getLogger(self.__class__.__name__)
+
+ def add(self, model):
+ """
+ :type model grapher.models.BaseModel
+ """
+ self.models.append(model)
+
+ def store(self, graph_name):
+ """
+ Save a given graph
+ """
+ raise NotImplementedError('store methods needs to be implemented')

grapher/grapher/graph/redis.py

@@ -0,0 +1,160 @@
+"""
+Redis storage
+"""
+import redis
+from redisgraph import Node, Edge, Graph
+from .base import BaseGraph
+
+
+class RedisGraph(BaseGraph):
+ """
+ Handles storing a collection of models in RedisGraph
+ """
+ def __init__(self, host, port=6379):
+ """
+ :type host str
+ :type port int
+ """
+ super(RedisGraph, self).__init__()
+
+ self.host = host
+ self.port = port
+
+ self.logger.info('Using redis: %s:%d', host, port)
+
+ @staticmethod
+ def encode_properties(properties):
+ """
+ :type properties dict
+ :rtype: dict
+ """
+ ret = dict()
+
+ # redisgraph library does not encode quotes
+ # (John_Faxe_Jensen:Person{name:\"John \"Faxe\" Jensen\",birthDate:1965,height:1.78})
+ for key, value in properties.items():
+ ret[key] = value.replace('"', '\\"') if isinstance(value, str) else value
+
+ return ret
+
+ @classmethod
+ def model_to_node(cls, model):
+ """
+ :type model grapher.models.BaseModel
+ :rtype: Node
+ """
+ properties = dict(name=model.get_name())
+ properties.update(model.properties)
+
+ return Node(
+ alias=model.get_node_name(),
+ properties=cls.encode_properties(properties) if properties else None,
+ )
+
+ @classmethod
+ def model_to_edges(cls, model):
+ """
+ :type model grapher.models.BaseModel
+ :rtype: list[Edge]
+ """
+ for (relation, target, properties) in model.get_all_relations():
+ # Edge(john, 'visited', japan, properties={'purpose': 'pleasure'})
+ yield Edge(
+ src_node=Node(alias=model.get_node_name()),
+ relation=relation,
+ dest_node=Node(alias=target),
+ properties=cls.encode_properties(properties) if properties else None
+ )
+
+ def _get_graph(self, graph_name):
+ """
+ :type graph_name str
+ :rtype: Graph
+ """
+ # https://github.com/RedisLabs/redisgraph-py#example-using-the-python-client
+ redis_graph = Graph(
+ name=graph_name,
+ redis_con=redis.Redis(self.host, self.port)
+ )
+
+ # add all nodes
+ for model in self.models:
+ redis_graph.add_node(self.model_to_node(model))
+
+ # and now add edges
+ for model in self.models:
+ for edge in self.model_to_edges(model):
+ try:
+ # add target node if needed
+ # we may want to refer to a node that was not indexed above
+ # e.g. English player in a Spanish club
+ if edge.dest_node.alias not in redis_graph.nodes:
+ node = Node(
+ alias=edge.dest_node.alias,
+ properties={'name': str(edge.dest_node.alias).split(':')[0]}
+ )
+ redis_graph.add_node(node)
+ self.logger.info('Adding missing node: %s', edge.dest_node.alias)
+
+ redis_graph.add_edge(edge)
+ except KeyError:
+ print(model)
+ # graph can be not complete, some nodes can be missing despite the relation
+ self.logger.error('add_edge failed', exc_info=True)
+
+ # assert valid nodes
+ # for _, node in redis_graph.nodes.items():
+ # print(node.alias, node.properties)
+ # print(str(node))
+ return redis_graph
+
+ def dump(self, graph_name):
+ """
+ Return a redisgraph command that would create a graph
+
+ :type graph_name str
+ :rtype: str
+ """
+ redis_graph = self._get_graph(graph_name)
+
+ # https://oss.redislabs.com/redisgraph/#with-redis-cli
+ # copied from redisgraph/client.py (commit function)
+ query = ''
+
+ for _, node in redis_graph.nodes.items():
+ query += str(node) + ','
+
+ for edge in redis_graph.edges:
+ query += str(edge) + ','
+
+ # Discard leading comma.
+ if query[-1] == ',':
+ query = query[:-1]
+
+ # encode "
+ query = query.replace('"', '\\"')
+
+ return 'GRAPH.QUERY {name} "CREATE {graph}"'.format(name=graph_name, graph=query)
+
+ def store(self, graph_name):
+ """
+ Store the graph in Redis
+
+ :type graph_name str
+ """
+ redis_graph = self._get_graph(graph_name)
+
+ # and save it
+ self.logger.info('Committing graph with %d nodes and %s edges',
+ len(redis_graph.nodes), len(redis_graph.edges))
+
+ try:
+ redis_graph.delete()
+ except redis.exceptions.ResponseError as ex:
+ # Graph was not found in database.
+ self.logger.info(ex)
+
+ redis_graph.commit()
+
+ redis_graph.redis_con.execute_command('SAvE')
+ self.logger.info('Committed and saved')

grapher/grapher/models/__init__.py

@@ -2,4 +2,4 @@
 Schema.org representations
 """
 from .base import BaseModel
-from .person import PersonModel
+from .football import PersonModel, SportsTeamModel

grapher/grapher/models/base.py

@@ -2,6 +2,7 @@
 Base model
 """
 import json
+import re
 from collections import OrderedDict
 
 
@@ -10,9 +11,11 @@ class BaseModel(object):
  Base schema.org model for keeping metadata
  """
  def __init__(self, model_type, name):
+ assert name is not None, 'name of a model cannot be None'
+
  self.type = model_type
  self.name = name
- self.properties = OrderedDict()
+ self.properties = OrderedDict(name=name)
  self.relations = list()
 
  def get_type(self):
@@ -27,11 +30,36 @@ def get_name(self):
  """
  return self.name
 
- def get_node_name(self):
+ @staticmethod
+ def encode_name(name):
  """
+ :type name str
  :rtype: str
  """
- return '{}:{}'.format(self.get_type(), self.get_name())
+ # remove UTF characters
+ name = name.encode('ascii', 'ignore').decode('ascii')
+
+ # Must begin with an alphabetic letter
+ # Can contain numbers, but not as the first character
+ # Cannot contain symbols (an exception to this rule is using underscore)
+ #
+ # https://neo4j.com/docs/cypher-manual/current/syntax/naming/
+ name = re.sub(r'^\d+', '', name) # remove digits from the beginning of the string
+ return re.sub(r'[^a-z0-9]+', '_', name, flags=re.IGNORECASE).strip('_')
+
+ def get_node_name(self):
+ """
+ Return node name for using in Cypher queries, e.g. "Foo:Type"
+
+ :rtype: str|None
+ """
+ if self.get_name() is None:
+ return None
+
+ return '{}:{}'.format(
+ self.encode_name(self.get_name()),
+ self.encode_name(self.get_type())
+ )
 
  def add_property(self, key, value):
  """
@@ -54,6 +82,10 @@ def add_relation(self, relation, target, properties=None):
  :type target str
  :type properties dict
  """
+ # remove None values from properties
+ if properties:
+ properties = {k: v for k, v in properties.items() if v is not None}
+
  self.relations.append((relation, target, properties))
 
  def get_relation_targets(self, relation_type):

grapher/grapher/models/football.py

@@ -0,0 +1,22 @@
+"""
+Football (and other sports) related models
+"""
+from . import BaseModel
+
+
+class PersonModel(BaseModel):
+ """
+ Person model
+ """
+ def __init__(self, name):
+ # https://schema.org/Person
+ super(PersonModel, self).__init__(model_type='Person', name=name)
+
+
+class SportsTeamModel(BaseModel):
+ """
+ F.C. Model
+ """
+ def __init__(self, name):
+ # https://schema.org/SportsTeam
+ super(SportsTeamModel, self).__init__(model_type='SportsTeam', name=name)