update diffnet

Author: PeiJieSun

diffnet/class/DataModule.py

@@ -0,0 +1,263 @@
+'''
+ author: Peijie Sun
+ e-mail: sun.hfut@gmail.com 
+ released date: 04/18/2019
+'''
+
+from collections import defaultdict
+import numpy as np
+from time import time
+import random
+
+class DataModule():
+ def __init__(self, conf, filename):
+ self.conf = conf
+ self.data_dict = {}
+ self.terminal_flag = 1
+ self.filename = filename
+ self.index = 0
+
+########################################### Initalize Procedures ############################################
+ def prepareModelSupplement(self, model):
+ data_dict = {}
+ if 'CONSUMED_ITEMS_SPARSE_MATRIX' in model.supply_set:
+ self.generateConsumedItemsSparseMatrix()
+ data_dict['CONSUMED_ITEMS_INDICES_INPUT'] = self.consumed_items_indices_list
+ data_dict['CONSUMED_ITEMS_VALUES_INPUT'] = self.consumed_items_values_list
+ if 'SOCIAL_NEIGHBORS_SPARSE_MATRIX' in model.supply_set:
+ self.readSocialNeighbors()
+ self.generateSocialNeighborsSparseMatrix()
+ data_dict['SOCIAL_NEIGHBORS_INDICES_INPUT'] = self.social_neighbors_indices_list
+ data_dict['SOCIAL_NEIGHBORS_VALUES_INPUT'] = self.social_neighbors_values_list
+ return data_dict
+
+ def initializeRankingTrain(self):
+ self.readData()
+ self.arrangePositiveData()
+ self.generateTrainNegative()
+
+ def initializeRankingVT(self):
+ self.readData()
+ self.arrangePositiveData()
+ self.generateTrainNegative()
+
+ def initalizeRankingEva(self):
+ self.readData()
+ self.getEvaPositiveBatch()
+ self.generateEvaNegative()
+
+ def linkedMap(self):
+ self.data_dict['USER_LIST'] = self.user_list
+ self.data_dict['ITEM_LIST'] = self.item_list
+ self.data_dict['LABEL_LIST'] = self.labels_list
+ 
+ def linkedRankingEvaMap(self):
+ self.data_dict['EVA_USER_LIST'] = self.eva_user_list
+ self.data_dict['EVA_ITEM_LIST'] = self.eva_item_list
+
+########################################### Ranking ############################################
+ def readData(self):
+ f = open(self.filename) ## May should be specific for different subtasks
+ total_user_list = set()
+ hash_data = defaultdict(int)
+ for _, line in enumerate(f):
+ arr = line.split("\t")
+ hash_data[(int(arr[0]), int(arr[1]))] = 1
+ total_user_list.add(int(arr[0]))
+ self.total_user_list = list(total_user_list)
+ self.hash_data = hash_data
+ 
+ def arrangePositiveData(self):
+ positive_data = defaultdict(set)
+ total_data = set()
+ hash_data = self.hash_data
+ for (u, i) in hash_data:
+ total_data.add((u, i))
+ positive_data[u].add(i)
+ self.positive_data = positive_data
+ self.total_data = len(total_data)
+ 
+ '''
+ This function designes for the train/val/test negative generating section
+ '''
+ def generateTrainNegative(self):
+ num_items = self.conf.num_items
+ num_negatives = self.conf.num_negatives
+ negative_data = defaultdict(set)
+ total_data = set()
+ hash_data = self.hash_data
+ for (u, i) in hash_data:
+ total_data.add((u, i))
+ for _ in range(num_negatives):
+ j = np.random.randint(num_items)
+ while (u, j) in hash_data:
+ j = np.random.randint(num_items)
+ negative_data[u].add(j)
+ total_data.add((u, j))
+ self.negative_data = negative_data
+ self.terminal_flag = 1
+ 
+ '''
+ This function designes for the val/test section, compute loss
+ '''
+ def getVTRankingOneBatch(self):
+ positive_data = self.positive_data
+ negative_data = self.negative_data
+ total_user_list = self.total_user_list
+ user_list = []
+ item_list = []
+ labels_list = []
+ for u in total_user_list:
+ user_list.extend([u] * len(positive_data[u]))
+ item_list.extend(positive_data[u])
+ labels_list.extend([1] * len(positive_data[u]))
+ user_list.extend([u] * len(negative_data[u]))
+ item_list.extend(negative_data[u])
+ labels_list.extend([0] * len(negative_data[u]))
+ 
+ self.user_list = np.reshape(user_list, [-1, 1])
+ self.item_list = np.reshape(item_list, [-1, 1])
+ self.labels_list = np.reshape(labels_list, [-1, 1])
+ 
+ '''
+ This function designes for the training process
+ '''
+ def getTrainRankingBatch(self):
+ positive_data = self.positive_data
+ negative_data = self.negative_data
+ total_user_list = self.total_user_list
+ index = self.index
+ batch_size = self.conf.training_batch_size
+
+ user_list, item_list, labels_list = [], [], []
+ 
+ if index + batch_size < len(total_user_list):
+ target_user_list = total_user_list[index:index+batch_size]
+ self.index = index + batch_size
+ else:
+ target_user_list = total_user_list[index:len(total_user_list)]
+ self.index = 0
+ self.terminal_flag = 0
+
+ for u in target_user_list:
+ user_list.extend([u] * len(positive_data[u]))
+ item_list.extend(list(positive_data[u]))
+ labels_list.extend([1] * len(positive_data[u]))
+ user_list.extend([u] * len(negative_data[u]))
+ item_list.extend(list(negative_data[u]))
+ labels_list.extend([0] * len(negative_data[u]))
+ 
+ self.user_list = np.reshape(user_list, [-1, 1])
+ self.item_list = np.reshape(item_list, [-1, 1])
+ self.labels_list = np.reshape(labels_list, [-1, 1])
+ 
+ '''
+ This function designes for the positive data in rating evaluate section
+ '''
+ def getEvaPositiveBatch(self):
+ hash_data = self.hash_data
+ user_list = []
+ item_list = []
+ index_dict = defaultdict(list)
+ index = 0
+ for (u, i) in hash_data:
+ user_list.append(u)
+ item_list.append(i)
+ index_dict[u].append(index)
+ index = index + 1
+ self.eva_user_list = np.reshape(user_list, [-1, 1])
+ self.eva_item_list = np.reshape(item_list, [-1, 1])
+ self.eva_index_dict = index_dict
+
+ '''
+ This function designes for the negative data generation process in rating evaluate section
+ '''
+ def generateEvaNegative(self):
+ hash_data = self.hash_data
+ total_user_list = self.total_user_list
+ num_evaluate = self.conf.num_evaluate
+ num_items = self.conf.num_items
+ eva_negative_data = defaultdict(list)
+ for u in total_user_list:
+ for _ in range(num_evaluate):
+ j = np.random.randint(num_items)
+ while (u, j) in hash_data:
+ j = np.random.randint(num_items)
+ eva_negative_data[u].append(j)
+ self.eva_negative_data = eva_negative_data
+
+ '''
+ This function designs for the rating evaluate section, generate negative batch
+ '''
+ def getEvaRankingBatch(self):
+ batch_size = self.conf.evaluate_batch_size
+ num_evaluate = self.conf.num_evaluate
+ eva_negative_data = self.eva_negative_data
+ total_user_list = self.total_user_list
+ index = self.index
+ terminal_flag = 1
+ total_users = len(total_user_list)
+ user_list = []
+ item_list = []
+ if index + batch_size < total_users:
+ batch_user_list = total_user_list[index:index+batch_size]
+ self.index = index + batch_size
+ else:
+ terminal_flag = 0
+ batch_user_list = total_user_list[index:total_users]
+ self.index = 0
+ for u in batch_user_list:
+ user_list.extend([u]*num_evaluate)
+ item_list.extend(eva_negative_data[u])
+ self.eva_user_list = np.reshape(user_list, [-1, 1])
+ self.eva_item_list = np.reshape(item_list, [-1, 1])
+ return batch_user_list, terminal_flag
+
+##################################################### Supplement for Sparse Computation ############################################
+ def readSocialNeighbors(self, friends_flag=1):
+ social_neighbors = defaultdict(set)
+ links_file = open(self.conf.links_filename)
+ for _, line in enumerate(links_file):
+ tmp = line.split('\t')
+ u1, u2 = int(tmp[0]), int(tmp[1])
+ social_neighbors[u1].add(u2)
+ if friends_flag == 1:
+ social_neighbors[u2].add(u1)
+ self.social_neighbors = social_neighbors
+
+ '''
+ Generate Social Neighbors Sparse Matrix Indices and Values
+ '''
+ def generateSocialNeighborsSparseMatrix(self):
+ social_neighbors = self.social_neighbors
+ social_neighbors_indices_list = []
+ social_neighbors_values_list = []
+ social_neighbors_dict = defaultdict(list)
+ for u in social_neighbors:
+ social_neighbors_dict[u] = sorted(social_neighbors[u])
+ 
+ user_list = sorted(list(social_neighbors.keys()))
+ for user in user_list:
+ for friend in social_neighbors_dict[user]:
+ social_neighbors_indices_list.append([user, friend])
+ social_neighbors_values_list.append(1.0/len(social_neighbors_dict[user]))
+ self.social_neighbors_indices_list = np.array(social_neighbors_indices_list).astype(np.int64)
+ self.social_neighbors_values_list = np.array(social_neighbors_values_list).astype(np.float32)
+ 
+ '''
+ Generate Consumed Items Sparse Matrix Indices and Values
+ '''
+ def generateConsumedItemsSparseMatrix(self):
+ positive_data = self.positive_data 
+ consumed_items_indices_list = []
+ consumed_items_values_list = []
+ consumed_items_dict = defaultdict(list)
+ for u in positive_data:
+ consumed_items_dict[u] = sorted(positive_data[u])
+ user_list = sorted(list(positive_data.keys()))
+ for u in user_list:
+ for i in consumed_items_dict[u]:
+ consumed_items_indices_list.append([u, i])
+ consumed_items_values_list.append(1.0/len(consumed_items_dict[u]))
+ self.consumed_items_indices_list = np.array(consumed_items_indices_list).astype(np.int64)
+ self.consumed_items_values_list = np.array(consumed_items_values_list).astype(np.float32)

diffnet/class/DataUtil.py

@@ -0,0 +1,39 @@
+'''
+ author: Peijie Sun
+ e-mail: sun.hfut@gmail.com 
+ released date: 04/18/2019
+'''
+
+import os
+from time import time
+from DataModule import DataModule
+
+class DataUtil():
+ def __init__(self, conf):
+ self.conf = conf
+ #print('DataUtil, Line12, test- conf data_dir:%s' % self.conf.data_dir)
+
+ def initializeRankingHandle(self):
+ #t0 = time()
+ self.createTrainHandle()
+ self.createEvaluateHandle()
+ #t1 = time()
+ #print('Prepare data cost:%.4fs' % (t1 - t0))
+ 
+ def createTrainHandle(self):
+ data_dir = self.conf.data_dir
+ train_filename = "%s/%s.train.rating" % (data_dir, self.conf.data_name)
+ val_filename = "%s/%s.val.rating" % (data_dir, self.conf.data_name)
+ test_filename = "%s/%s.test.rating" % (data_dir, self.conf.data_name)
+
+ self.train = DataModule(self.conf, train_filename)
+ self.val = DataModule(self.conf, val_filename)
+ self.test = DataModule(self.conf, test_filename)
+
+ def createEvaluateHandle(self):
+ data_dir = self.conf.data_dir
+ val_filename = "%s/%s.val.rating" % (data_dir, self.conf.data_name)
+ test_filename = "%s/%s.test.rating" % (data_dir, self.conf.data_name)
+
+ self.val_eva = DataModule(self.conf, val_filename)
+ self.test_eva = DataModule(self.conf, test_filename)

diffnet/class/Evaluate.py

@@ -0,0 +1,84 @@
+'''
+ author: Peijie Sun
+ e-mail: sun.hfut@gmail.com 
+ released date: 04/18/2019
+'''
+
+import math
+import numpy as np
+
+class Evaluate():
+ def __init__(self, conf):
+ self.conf = conf
+
+ def getIdcg(self, length):
+ idcg = 0.0
+ for i in range(length):
+ idcg = idcg + math.log(2) / math.log(i + 2)
+ return idcg
+
+ def getDcg(self, value):
+ dcg = math.log(2) / math.log(value + 2)
+ return dcg
+
+ def getHr(self, value):
+ hit = 1.0
+ return hit
+
+ def evaluateRankingPerformance(self, evaluate_index_dict, evaluate_real_rating_matrix, \
+ evaluate_predict_rating_matrix, topK, num_procs, exp_flag=0, sp_name=None, result_file=None):
+ user_list = list(evaluate_index_dict.keys())
+ batch_size = len(user_list) / num_procs
+
+ hr_list, ndcg_list = [], []
+ index = 0
+ for _ in range(num_procs):
+ if index + batch_size < len(user_list):
+ batch_user_list = user_list[index:index+batch_size]
+ index = index + batch_size
+ else:
+ batch_user_list = user_list[index:len(user_list)]
+ tmp_hr_list, tmp_ndcg_list = self.getHrNdcgProc(evaluate_index_dict, evaluate_real_rating_matrix, \
+ evaluate_predict_rating_matrix, topK, batch_user_list)
+ hr_list.extend(tmp_hr_list)
+ ndcg_list.extend(tmp_ndcg_list)
+ return np.mean(hr_list), np.mean(ndcg_list)
+
+ def getHrNdcgProc(self, 
+ evaluate_index_dict, 
+ evaluate_real_rating_matrix,
+ evaluate_predict_rating_matrix, 
+ topK, 
+ user_list):
+
+ tmp_hr_list, tmp_ndcg_list = [], []
+
+ for u in user_list:
+ real_item_index_list = evaluate_index_dict[u]
+ real_item_rating_list = list(np.concatenate(evaluate_real_rating_matrix[real_item_index_list]))
+ positive_length = len(real_item_rating_list)
+ target_length = min(positive_length, topK)
+ 
+ predict_rating_list = evaluate_predict_rating_matrix[u]
+ real_item_rating_list.extend(predict_rating_list)
+ sort_index = np.argsort(real_item_rating_list)
+ sort_index = sort_index[::-1]
+
+ user_hr_list = []
+ user_ndcg_list = []
+ hits_num = 0
+ for idx in range(topK):
+ ranking = sort_index[idx]
+ if ranking < positive_length:
+ hits_num += 1
+ user_hr_list.append(self.getHr(idx))
+ user_ndcg_list.append(self.getDcg(idx))
+
+ idcg = self.getIdcg(target_length)
+
+ tmp_hr = np.sum(user_hr_list) / target_length
+ tmp_ndcg = np.sum(user_ndcg_list) / idcg
+ tmp_hr_list.append(tmp_hr)
+ tmp_ndcg_list.append(tmp_ndcg)
+
+ return tmp_hr_list, tmp_ndcg_list