PCA and LDA

Author: phillity

DimensionalityReduction/LDA/LDA.py

@@ -0,0 +1,89 @@
+# Tyler Phillips
+# CSCI57300 Data Mining
+# Linear Discriminant Analysis (LDA)
+
+import numpy as np
+import sys
+
+# LDA function
+# Args:
+# D - nxd data matrix (last column is class labels)
+# num_components - parameter to set number of ouput dimensions
+def LDA(D, num_components):
+ # Get class information
+ Y = D[:,-1]
+ c = np.unique(Y).shape[0]
+
+ D = D[:,:-1]
+ n, d = D.shape
+
+ # Get class subsets
+ C = []
+ for i in range(c):
+ C.append(D[Y == i])
+
+ # Get class means
+ mu_c = np.zeros((c,d))
+ for i in range(c):
+ mu_c[i,:] = np.sum(C[i],axis=0) / len(C[i])
+
+ # Get mean of class means
+ mu = np.sum(mu_c,axis=0) / c
+
+ # Get between class scatter
+ B = np.zeros((d,d))
+ for i in range(c):
+ B = B + np.outer((mu_c[i,:] - mu).T,(mu_c[i,:] - mu))
+ B = B / c
+
+ # Center class matricies
+ for i in range(c):
+ C[i] = C[i] - mu_c[i,:]
+
+ # Get class scatter matricies
+ W = []
+ for i in range(c):
+ Z_i = C[i].T @ C[i]
+ W.append(Z_i)
+
+ # Compute within class scatter matrix
+ S = np.zeros((d,d))
+ for i in range(c):
+ S = S + W[i]
+
+ # Compute eigenvectors and values of S^-1 @ B
+ w, v = np.linalg.eig(np.linalg.inv(S) @ B)
+
+ # Sort descending eigenvalues and respective eigenvectors
+ idx = np.argsort(w)[::-1]
+ w = w[idx]
+ v = v[:,idx]
+
+ # Get num_components dominant eigenvalues and vectors
+ w = w[:num_components]
+ v = v[:,:num_components]
+
+ # Project data into eigenvector basis subspace 
+ A = D @ v
+
+ return A
+ 
+
+# Get the arguments list 
+argv = str(sys.argv)
+print(str(argv))
+
+# Get number of arguments
+argc = len(sys.argv)
+
+# Print error if not enough arguments
+if argc < 3:
+ sys.exit("Datafile and k arguments are required!")
+
+# Read in D data matrix
+D = np.loadtxt(sys.argv[1],delimiter=',')
+
+# Read in num_components
+num_components = int(sys.argv[2])
+
+A = LDA(D, num_components)

DimensionalityReduction/LDA/iris.data.txt

@@ -0,0 +1,150 @@
+5.1,3.5,1.4,0.2,0
+4.9,3.0,1.4,0.2,0
+4.7,3.2,1.3,0.2,0
+4.6,3.1,1.5,0.2,0
+5.0,3.6,1.4,0.2,0
+5.4,3.9,1.7,0.4,0
+4.6,3.4,1.4,0.3,0
+5.0,3.4,1.5,0.2,0
+4.4,2.9,1.4,0.2,0
+4.9,3.1,1.5,0.1,0
+5.4,3.7,1.5,0.2,0
+4.8,3.4,1.6,0.2,0
+4.8,3.0,1.4,0.1,0
+4.3,3.0,1.1,0.1,0
+5.8,4.0,1.2,0.2,0
+5.7,4.4,1.5,0.4,0
+5.4,3.9,1.3,0.4,0
+5.1,3.5,1.4,0.3,0
+5.7,3.8,1.7,0.3,0
+5.1,3.8,1.5,0.3,0
+5.4,3.4,1.7,0.2,0
+5.1,3.7,1.5,0.4,0
+4.6,3.6,1.0,0.2,0
+5.1,3.3,1.7,0.5,0
+4.8,3.4,1.9,0.2,0
+5.0,3.0,1.6,0.2,0
+5.0,3.4,1.6,0.4,0
+5.2,3.5,1.5,0.2,0
+5.2,3.4,1.4,0.2,0
+4.7,3.2,1.6,0.2,0
+4.8,3.1,1.6,0.2,0
+5.4,3.4,1.5,0.4,0
+5.2,4.1,1.5,0.1,0
+5.5,4.2,1.4,0.2,0
+4.9,3.1,1.5,0.1,0
+5.0,3.2,1.2,0.2,0
+5.5,3.5,1.3,0.2,0
+4.9,3.1,1.5,0.1,0
+4.4,3.0,1.3,0.2,0
+5.1,3.4,1.5,0.2,0
+5.0,3.5,1.3,0.3,0
+4.5,2.3,1.3,0.3,0
+4.4,3.2,1.3,0.2,0
+5.0,3.5,1.6,0.6,0
+5.1,3.8,1.9,0.4,0
+4.8,3.0,1.4,0.3,0
+5.1,3.8,1.6,0.2,0
+4.6,3.2,1.4,0.2,0
+5.3,3.7,1.5,0.2,0
+5.0,3.3,1.4,0.2,0
+7.0,3.2,4.7,1.4,1
+6.4,3.2,4.5,1.5,1
+6.9,3.1,4.9,1.5,1
+5.5,2.3,4.0,1.3,1
+6.5,2.8,4.6,1.5,1
+5.7,2.8,4.5,1.3,1
+6.3,3.3,4.7,1.6,1
+4.9,2.4,3.3,1.0,1
+6.6,2.9,4.6,1.3,1
+5.2,2.7,3.9,1.4,1
+5.0,2.0,3.5,1.0,1
+5.9,3.0,4.2,1.5,1
+6.0,2.2,4.0,1.0,1
+6.1,2.9,4.7,1.4,1
+5.6,2.9,3.6,1.3,1
+6.7,3.1,4.4,1.4,1
+5.6,3.0,4.5,1.5,1
+5.8,2.7,4.1,1.0,1
+6.2,2.2,4.5,1.5,1
+5.6,2.5,3.9,1.1,1
+5.9,3.2,4.8,1.8,1
+6.1,2.8,4.0,1.3,1
+6.3,2.5,4.9,1.5,1
+6.1,2.8,4.7,1.2,1
+6.4,2.9,4.3,1.3,1
+6.6,3.0,4.4,1.4,1
+6.8,2.8,4.8,1.4,1
+6.7,3.0,5.0,1.7,1
+6.0,2.9,4.5,1.5,1
+5.7,2.6,3.5,1.0,1
+5.5,2.4,3.8,1.1,1
+5.5,2.4,3.7,1.0,1
+5.8,2.7,3.9,1.2,1
+6.0,2.7,5.1,1.6,1
+5.4,3.0,4.5,1.5,1
+6.0,3.4,4.5,1.6,1
+6.7,3.1,4.7,1.5,1
+6.3,2.3,4.4,1.3,1
+5.6,3.0,4.1,1.3,1
+5.5,2.5,4.0,1.3,1
+5.5,2.6,4.4,1.2,1
+6.1,3.0,4.6,1.4,1
+5.8,2.6,4.0,1.2,1
+5.0,2.3,3.3,1.0,1
+5.6,2.7,4.2,1.3,1
+5.7,3.0,4.2,1.2,1
+5.7,2.9,4.2,1.3,1
+6.2,2.9,4.3,1.3,1
+5.1,2.5,3.0,1.1,1
+5.7,2.8,4.1,1.3,1
+6.3,3.3,6.0,2.5,2
+5.8,2.7,5.1,1.9,2
+7.1,3.0,5.9,2.1,2
+6.3,2.9,5.6,1.8,2
+6.5,3.0,5.8,2.2,2
+7.6,3.0,6.6,2.1,2
+4.9,2.5,4.5,1.7,2
+7.3,2.9,6.3,1.8,2
+6.7,2.5,5.8,1.8,2
+7.2,3.6,6.1,2.5,2
+6.5,3.2,5.1,2.0,2
+6.4,2.7,5.3,1.9,2
+6.8,3.0,5.5,2.1,2
+5.7,2.5,5.0,2.0,2
+5.8,2.8,5.1,2.4,2
+6.4,3.2,5.3,2.3,2
+6.5,3.0,5.5,1.8,2
+7.7,3.8,6.7,2.2,2
+7.7,2.6,6.9,2.3,2
+6.0,2.2,5.0,1.5,2
+6.9,3.2,5.7,2.3,2
+5.6,2.8,4.9,2.0,2
+7.7,2.8,6.7,2.0,2
+6.3,2.7,4.9,1.8,2
+6.7,3.3,5.7,2.1,2
+7.2,3.2,6.0,1.8,2
+6.2,2.8,4.8,1.8,2
+6.1,3.0,4.9,1.8,2
+6.4,2.8,5.6,2.1,2
+7.2,3.0,5.8,1.6,2
+7.4,2.8,6.1,1.9,2
+7.9,3.8,6.4,2.0,2
+6.4,2.8,5.6,2.2,2
+6.3,2.8,5.1,1.5,2
+6.1,2.6,5.6,1.4,2
+7.7,3.0,6.1,2.3,2
+6.3,3.4,5.6,2.4,2
+6.4,3.1,5.5,1.8,2
+6.0,3.0,4.8,1.8,2
+6.9,3.1,5.4,2.1,2
+6.7,3.1,5.6,2.4,2
+6.9,3.1,5.1,2.3,2
+5.8,2.7,5.1,1.9,2
+6.8,3.2,5.9,2.3,2
+6.7,3.3,5.7,2.5,2
+6.7,3.0,5.2,2.3,2
+6.3,2.5,5.0,1.9,2
+6.5,3.0,5.2,2.0,2
+6.2,3.4,5.4,2.3,2
+5.9,3.0,5.1,1.8,2

DimensionalityReduction/LDA/iris_2d.txt

@@ -0,0 +1,150 @@
+5.1,3.5,0
+4.9,3.0,0
+4.7,3.2,0
+4.6,3.1,0
+5.0,3.6,0
+5.4,3.9,0
+4.6,3.4,0
+5.0,3.4,0
+4.4,2.9,0
+4.9,3.1,0
+5.4,3.7,0
+4.8,3.4,0
+4.8,3.0,0
+4.3,3.0,0
+5.8,4.0,0
+5.7,4.4,0
+5.4,3.9,0
+5.1,3.5,0
+5.7,3.8,0
+5.1,3.8,0
+5.4,3.4,0
+5.1,3.7,0
+4.6,3.6,0
+5.1,3.3,0
+4.8,3.4,0
+5.0,3.0,0
+5.0,3.4,0
+5.2,3.5,0
+5.2,3.4,0
+4.7,3.2,0
+4.8,3.1,0
+5.4,3.4,0
+5.2,4.1,0
+5.5,4.2,0
+4.9,3.1,0
+5.0,3.2,0
+5.5,3.5,0
+4.9,3.1,0
+4.4,3.0,0
+5.1,3.4,0
+5.0,3.5,0
+4.5,2.3,0
+4.4,3.2,0
+5.0,3.5,0
+5.1,3.8,0
+4.8,3.0,0
+5.1,3.8,0
+4.6,3.2,0
+5.3,3.7,0
+5.0,3.3,0
+7.0,3.2,1
+6.4,3.2,1
+6.9,3.1,1
+5.5,2.3,1
+6.5,2.8,1
+5.7,2.8,1
+6.3,3.3,1
+4.9,2.4,1
+6.6,2.9,1
+5.2,2.7,1
+5.0,2.0,1
+5.9,3.0,1
+6.0,2.2,1
+6.1,2.9,1
+5.6,2.9,1
+6.7,3.1,1
+5.6,3.0,1
+5.8,2.7,1
+6.2,2.2,1
+5.6,2.5,1
+5.9,3.2,1
+6.1,2.8,1
+6.3,2.5,1
+6.1,2.8,1
+6.4,2.9,1
+6.6,3.0,1
+6.8,2.8,1
+6.7,3.0,1
+6.0,2.9,1
+5.7,2.6,1
+5.5,2.4,1
+5.5,2.4,1
+5.8,2.7,1
+6.0,2.7,1
+5.4,3.0,1
+6.0,3.4,1
+6.7,3.1,1
+6.3,2.3,1
+5.6,3.0,1
+5.5,2.5,1
+5.5,2.6,1
+6.1,3.0,1
+5.8,2.6,1
+5.0,2.3,1
+5.6,2.7,1
+5.7,3.0,1
+5.7,2.9,1
+6.2,2.9,1
+5.1,2.5,1
+5.7,2.8,1
+6.3,3.3,2
+5.8,2.7,2
+7.1,3.0,2
+6.3,2.9,2
+6.5,3.0,2
+7.6,3.0,2
+4.9,2.5,2
+7.3,2.9,2
+6.7,2.5,2
+7.2,3.6,2
+6.5,3.2,2
+6.4,2.7,2
+6.8,3.0,2
+5.7,2.5,2
+5.8,2.8,2
+6.4,3.2,2
+6.5,3.0,2
+7.7,3.8,2
+7.7,2.6,2
+6.0,2.2,2
+6.9,3.2,2
+5.6,2.8,2
+7.7,2.8,2
+6.3,2.7,2
+6.7,3.3,2
+7.2,3.2,2
+6.2,2.8,2
+6.1,3.0,2
+6.4,2.8,2
+7.2,3.0,2
+7.4,2.8,2
+7.9,3.8,2
+6.4,2.8,2
+6.3,2.8,2
+6.1,2.6,2
+7.7,3.0,2
+6.3,3.4,2
+6.4,3.1,2
+6.0,3.0,2
+6.9,3.1,2
+6.7,3.1,2
+6.9,3.1,2
+5.8,2.7,2
+6.8,3.2,2
+6.7,3.3,2
+6.7,3.0,2
+6.3,2.5,2
+6.5,3.0,2
+6.2,3.4,2
+5.9,3.0,2