Major update. Splitting code into several separate files

Author: smerdov

ChairProcessing.py

@@ -0,0 +1,52 @@
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import os
+import joblib
+from utils import normalize_MPU9250_data, split_df
+from ChairAnalyzer import ChairAnalyser
+
+plt.interactive(True)
+pd.options.display.max_columns = 15
+pic_prefix = 'pic/'
+# data_path = 'data/CSV'
+# data_path = 'Anonimised Data/Data'
+
+data_dict = joblib.load('data/data_dict')
+
+chair_features_list = []
+
+def get_chair_features(df_chair, player_id):
+ chair_analyser = ChairAnalyser(df_chair, 0.01, pic_prefix, name=player_id) # + f'_{n_chunk}')
+ nonstationary_values_portion = chair_analyser.get_nonstationary_values_portion()
+ lean_back_portion = chair_analyser.get_lean_back_portion()
+ oscillations = chair_analyser.get_oscillation_intensity()
+
+ chair_features = pd.concat([nonstationary_values_portion, lean_back_portion, oscillations])
+
+ return chair_features
+
+
+for player_id, player_data_dict in data_dict.items():
+ if 'schairlog' in player_data_dict:
+ df_chair = player_data_dict['schairlog']
+ else:
+ continue
+
+ ### Each chair log is splitted to small sessions
+ chunk_lenght = 100 * 600
+ df_chunks_list = split_df(df_chair, n_chunks=3, chunk_lenght=chunk_lenght)
+ # print(len(df_chunks_list))
+ # chair_analyser = ChairAnalyser(df_chair, 0.01, pic_prefix, name=player_id) # + f'_{n_chunk}')
+ # chair_analyser.plot_measurements_timeline(sensors=('acc', 'gyro'), plot_suptitle=False, fontsize=22)
+
+ for n_chunk, df_chunk in enumerate(df_chunks_list):
+ chair_features = get_chair_features(df_chunk, player_id)
+ chair_features_list.append(chair_features)
+
+
+df_chair_features = pd.DataFrame(chair_features_list)
+df_chair_features.reset_index(inplace=True)
+df_chair_features.rename(columns={'index': 'player_id'}, inplace=True)
+
+df_chair_features.to_csv('data/chair_features.csv', index=False)

Exploration.py

@@ -1,123 +1,39 @@
 import numpy as np
 import pandas as pd
 import matplotlib.pyplot as plt
+import seaborn as sns
 import os
 from utils import normalize_MPU9250_data, split_df
 from ChairAnalyzer import ChairAnalyser
 
 plt.interactive(True)
 pd.options.display.max_columns = 15
 pic_prefix = 'pic/'
-data_path = 'data/CSV'
 
-folders = os.listdir(data_path)
-folders = [f"{data_path}/{folder}" for folder in folders if not folder.startswith('.')]
 
-data_dict_dict = {}
+df_chair_features = pd.read_csv('data/chair_features.csv')
+df_players = pd.read_csv('data/players.csv')
+df_chair_features4players = pd.merge(df_chair_features, df_players, on='player_id')
 
-chair_data_columns = ['time', 'acc_x', 'acc_y', 'acc_z', 'gyro_x', 'gyro_y', 'gyro_z', 'mag_x', 'mag_y', 'mag_z']
 
-data_sources_list = ['schairlog'] # List sources for analysis here
-
-for folder in folders:
- data_dict = {}
- name = folder.split('/')[-1]
-
- files = os.listdir(folder)
- files = [file for file in files if not file.startswith('.')]
- data_sources = [file.split('_')[0] for file in files] # There are might be repetitions
- print(data_sources)
-
- for file, data_source in zip(files, data_sources):
- if data_source not in data_sources_list:
- continue
-
- try:
- df = pd.read_csv(folder + '/' + file)
-
- if data_source in data_dict: # If already in dict it's appended
- new_df = pd.concat([data_dict[data_source], df], axis=0).reset_index(drop=True)
- data_dict[data_source] = new_df
- else:
- data_dict[data_source] = df
- except:
- pass
-
- data_dict_dict[name] = data_dict
-
-chair_data_dict = {}
-
-for key, value in data_dict_dict.items():
- key = key.replace('\t', ' ')
- if 'schairlog' in value:
- df_chair = value['schairlog']
- chair_data_dict[key] = df_chair
- print(len(df_chair))
-
-# keys = list(data_dict_dict.keys())
-# data_dict_dict[keys[0]]['schairlog']
-
-
-nonstationary_values_portion_list = []
-# TODO: do not draw pictures
-
-for player_name, df_chair in chair_data_dict.items():
- chunk_lenght = 100 * 300
- df_chunks_list = split_df(df_chair, n_chunks=10, chunk_lenght=chunk_lenght)
- print(len(df_chunks_list))
- # chair_analyser = ChairAnalyser(df_chair, 0.01, pic_prefix, name=player_name) # + f'_{n_chunk}')
- # chair_analyser.plot_measurements_timeline(sensors=('acc', 'gyro'), plot_suptitle=False, fontsize=22)
-
- for n_chunk, df_chunk in enumerate(df_chunks_list):
- chair_analyser = ChairAnalyser(df_chunk, 0.01, pic_prefix, name=player_name)# + f'_{n_chunk}')
- # chair_analyser.plot_measurements_timeline(sensors=('acc', 'gyro'))
- nonstationary_values_portion = chair_analyser.get_nonstationary_values_portion()
- lean_back_portion = chair_analyser.get_lean_back_portion()
- oscillations = chair_analyser.get_oscillation_intensity()
-
- nonstationary_values_portion = nonstationary_values_portion.append(lean_back_portion)
- nonstationary_values_portion = nonstationary_values_portion.append(oscillations)
-
- nonstationary_values_portion_list.append(nonstationary_values_portion)
-
-
-df_nonstationary_values_portion = pd.DataFrame(nonstationary_values_portion_list)
-df_nonstationary_values_portion.reset_index(inplace=True)
-df_nonstationary_values_portion.rename(columns={'index': 'player_name'}, inplace=True)
-
-df_players = pd.read_csv('../data/participants2_fixed.csv', sep=';')
-df_players['player_name'] = df_players[['First Name', 'Last Name']].apply(lambda x: ' '.join(x), axis=1)
-
-df_players.rename(columns={
- ' What experience do u have in shooter games (Counter-Strike, Doom, Battlefield, etc.)?': 'Skill'
- },
- inplace=True,
+plt.close()
+plt.figure(figsize=(15, 15))
+cmap = sns.diverging_palette(220, 10, as_cmap=True)
+corr_data = df.drop(['player_id'], axis=1).corr()
+corr_data = corr_data.round(1)
+sns.heatmap(
+ corr_data, square=True, cmap=cmap, vmax=1,vmin=-1, linewidths=.2, cbar_kws={"shrink": .8},
+ annot=True, annot_kws={"size": 7},
+ # xticklabels=False, yticklabels=False,
 )
-
-df_players = df_players[['player_name', 'Skill']]
-skill_is_none = df_players['Skill'] == 'None'
-df_players.loc[skill_is_none, 'Skill'] = 'Small'
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+plt.title('Correlation between player skill and his behaviour on the chair', fontsize=15)
+plt.tight_layout()
+plt.savefig('pic/heatmap_10_3.png')
 
 
+df.shape
 
+# TODO: make hours binary for multiple thresholds
 
 
 

GameLogProcessing.py

@@ -0,0 +1,133 @@
+# -*- coding: utf-8 -*-
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import os
+import json
+import joblib
+from utils import normalize_MPU9250_data, split_df, string2json
+from ChairAnalyzer import ChairAnalyser
+
+plt.interactive(True)
+pd.options.display.max_columns = 15
+pic_prefix = 'pic/'
+data_path = 'Anonimised Data/Data'
+processed_data_path = 'data/players_data_processed'
+
+data_dict = joblib.load('data/data_dict')
+
+def check_player_is_killed(parameters_dict):
+ # First condition: event is that somebody dies
+ # Second condition: dying player is skoltech experimental rat
+ return ('userid' in parameters_dict) and (parameters_dict['userid'].find('koltech') != -1)
+
+
+gamedata_dict = {}
+
+for player_id, player_data_dict in data_dict.items():
+ if 'gamelog' not in player_data_dict:
+ continue
+
+ mask_player_events = player_data_dict['gamelog']['parameters'].apply(lambda x: x.find('koltech')) != -1
+ df_gamelog = player_data_dict['gamelog'].loc[mask_player_events, :]
+ df_gamelog['parameters'] = df_gamelog['parameters'].apply(string2json)
+ df_gamelog['health'] = df_gamelog['parameters'].apply(lambda x: int(x['health']) if 'health' in x else None)
+ mask_somebody_is_killed = df_gamelog['health'] == 0
+
+ mask_player_is_killed = mask_somebody_is_killed & df_gamelog.loc[:, 'parameters'].apply(check_player_is_killed)
+ mask_player_kills = mask_somebody_is_killed & ~mask_player_is_killed
+
+ times_is_killed = df_gamelog.loc[mask_player_is_killed, 'time'].values
+ times_kills = df_gamelog.loc[mask_player_kills, 'time'].values
+
+ player_gamedata_dict = {
+ 'times_is_killed': times_is_killed,
+ 'times_kills': times_kills,
+ }
+
+ gamedata_dict[player_id] = player_gamedata_dict
+
+
+# gamedata_dict['9'].keys()
+# gamedata_dict['9']['times_is_killed']
+# gamedata_dict['9']['times_kills']
+
+
+joblib.dump(gamedata_dict, 'data/gamedata_dict')
+
+
+
+
+
+data_dict['2']['gamelog']['parameters']
+
+
+
+
+
+
+# with open(gamelog_path, 'rb') as f:
+# gamelog = f.readlines()
+#
+# # gamelog = [string.decode() for string in gamelog]
+# gamelog_lenght_initial = len(gamelog)
+# gamelog = [string for string in gamelog if string.find(b'koltech') != -1]
+# gamelog_lenght_filtered = len(gamelog)
+# print(f'gamelog_lenght_initial = {gamelog_lenght_initial}, gamelog_lenght_filtered = {gamelog_lenght_filtered}')
+#
+# # with open('tmp/gamelog.csv', 'wb') as f:
+# with open(processed_data_path + '/gamelog.csv', 'wb') as f:
+# # for line in gamelog:
+# # f.write(line)
+# f.writelines(gamelog)
+#
+# df_gamelog = pd.read_csv('tmp/gamelog.csv', header=None)
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+times = pd.to_datetime(df_gamelog.loc[mask_killed, 'time'])
+np.diff(times.values) / 10 ** 9
+
+
+(times.iloc[1:] - times.iloc[:-1].values).iloc[10]
+
+
+
+(df_gamelog['health_is_0']).sum()
+
+
+plt.plot(df_gamelog['health_is_0'])
+
+
+# TODO: check player behaviour right after death
+
+
+df_gamelog['event'].value_counts()
+
+
+mask_fire = df_gamelog['event'] == 'weapon_fire'
+
+df_gamelog.loc[mask_fire, 'parameters']
+fire_times = pd.to_datetime(df_gamelog.loc[mask_fire, 'time'])
+(fire_times.values[1:] - fire_times.values[:-1]).min()
+
+df_fire = pd.DataFrame(list(df_gamelog.loc[mask_fire, 'parameters'].values))
+
+df_fire.info()
+
+
+
+

GeneralDataProcessing.py

@@ -0,0 +1,59 @@
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import os
+import joblib
+from utils import normalize_MPU9250_data
+
+plt.interactive(True)
+pd.options.display.max_columns = 15
+pic_prefix = 'pic/'
+# data_path = 'data/CSV'
+data_path = 'Anonimised Data/Data'
+
+player_folders = os.listdir(data_path)
+player_folders = [f"{data_path}/{folder}" for folder in player_folders if not folder.startswith('.')]
+
+data_dict = {}
+
+data_sources_list = ['schairlog', 'gamelog'] # List sources for analysis here
+
+# chair_data_columns = ['time', 'acc_x', 'acc_y', 'acc_z', 'gyro_x', 'gyro_y', 'gyro_z', 'mag_x', 'mag_y', 'mag_z']
+
+for player_folder in player_folders:
+ player_data_dict = {}
+ player_id = player_folder.split('/')[-1]
+
+ player_files = os.listdir(player_folder)
+ player_files = [file for file in player_files if not file.startswith('.')]
+ player_data_sources = [file.split('_')[0] for file in player_files] # There are might be repetitions
+ # print(player_data_sources)
+
+ for file, data_source in zip(player_files, player_data_sources):
+ if data_source not in data_sources_list:
+ continue
+
+ try:
+ df = pd.read_csv(player_folder + '/' + file)
+
+ if data_source in player_data_dict: # If already in dict it's appended
+ new_df = pd.concat([player_data_dict[data_source], df], axis=0).reset_index(drop=True)
+ player_data_dict[data_source] = new_df
+ else:
+ player_data_dict[data_source] = df
+ except:
+ pass
+
+ # Sorting by time and fixing naming
+ for data_source in player_data_dict.keys():
+ if data_source == 'gamelog':
+ player_data_dict[data_source].rename(columns={'Unnamed: 0': 'time'}, inplace=True)
+
+ player_data_dict[data_source].sort_values(by='time', inplace=True)
+ player_data_dict[data_source].reset_index(drop=True, inplace=True)
+
+ data_dict[player_id] = player_data_dict
+
+joblib.dump(data_dict, 'data/data_dict')
+
+