# Analyzing Microscopic Residue

My goal is to analyze a set microscopic residue. I'd like to apply entropy discretization to attribute A1, segmentation by natural partitioning to A2, then correlate A1 and A3. The first attribute of my dataset represents the number of different microscopic residues of an organic element. The second attribute represents length in microns. The third attribute represents the saturation of the largest residue after being exposed to phosphoric acid. Then finally the last attribute is a class that represents a grouping of organic elements. I've attached the dataset to this question, as well.

My plan is the following:

1-Discretize the attribute A1 using Entropy-Based discretization When either the condition “a” or condition “b” is true for a partition, then that partition stops splitting:

a-The number of distinct classes within a partition is 1. b-The ratio of the minimum to maximum frequencies among the distinct values for the attribute Class in the partition is <0.5 and the number of distinct values within the attribute of Class in the partition is Floor(n/2),where n is the number of distinct values in the original dataset.

2-Discretize the attribute A2 using Segmentation by Natural Partitioning To get the attribute values between 5 to 95 percentiles, simply

(i) sort the data in ascending order

(ii) keep values from Floor(n0.05)to Floor(n0.95), where n is the number of values in the dataset.

3-Calculate the correlation between A1 and A3 and remove A3,if correlation is >0.6 or correlation is <-0.6

4-Finally apply Principal Component Analysis on the dataset, convert it into anew dataset, and save the new dataset

Sample data

A1,A2,A3,Class
2,0.4631338,1.5,3
8,0.7460648,3.0,3
6,0.264391038,2.5,2
5,0.4406713,2.3,1
2,0.410438159,1.5,3
2,0.302901816,1.5,2
6,0.275869396,2.5,3
8,0.084782428,3.0,3
2,0.53226533,1.5,2
8,0.070034818,2.9,1
2,0.668631847,1.5,2
2,0.215622639,1.5,2
2,0.148916231,1.5,3


Program

from numpy.core.defchararray import count
import pandas as pd
import numpy as np
import numpy as np
from math import ceil, floor, log2
from sklearn.decomposition import PCA
from numpy import linalg as LA

def print_full(x):
pd.set_option('display.max_rows', len(x))
print(x)
pd.reset_option('display.max_rows')

def main():
print("******************************************************")
print("Entropy Discretization                         STARTED")
s = entropy_discretization(s)
print("Entropy Discretization                         COMPLETED")
print(s)
print("******************************************************")
print("Segmentation By Natural Partitioning           STARTED")
s = segmentation_by_natural_partitioning(s)
print("Applying Segmentation By Natural Partitioning COMPLETED")
print(s)
print("*******************************************************")
print("Correlation Calculation                         STARTED")
s = calculate_correlation(s)
print("*******************************************************")
print("Correlation Calculation                       COMPLETED")
print(s)
print("*******************************************************")
print("PCA                                             STARTED")
s = pca(s)
print("PCA                                            COMPLETED")
print(s)
print("*******************************************************")
print("Writing to csv                                  STARTED")
compression_opts = dict(method='zip',archive_name='out.csv')
s.to_csv('out.zip', index=False,compression=compression_opts)
print("Writing to csv                                COMPLETED")
print("*******************************************************")

# This method discretizes attribute A1
# If the information gain is 0, i.e the number of
# distinct class is 1 or
# If min f/ max f < 0.5 and the number of distinct values is floor(n/2)
# Then that partition stops splitting.
# This method discretizes s A1
# If the information gain is 0, i.e the number of
# distinct class is 1 or
# If min f/ max f < 0.5 and the number of distinct values is floor(n/2)
# Then that partition stops splitting.
def entropy_discretization(s):

I = {}
i = 0
n = s.nunique()['Class']
s1 = pd.DataFrame()
s2 = pd.DataFrame()
distinct_values = s['A1'].value_counts().index
information_gain_indicies = []
print(f'The unique values for dataset s["A1"] are {distinct_values}')
for i in distinct_values:

# Step 1: pick a threshold
threshold = i
print(f'Using threshold {threshold}')

# Step 2: Partititon the data set into two parttitions
s1 = s[s['Class'] < threshold]
print("s1 after spitting")
print(s1)
print("******************")
s2 = s[s['Class'] >= threshold]
print("s2 after spitting")
print(s2)
print("******************")

print("******************")
print("calculating maxf")
print(f" maxf {maxf(s['Class'])}")
print("******************")

print("******************")
print("calculating minf")
print(f" maxf {minf(s['Class'])}")
print("******************")

# Step 3: calculate the information gain.
informationGain = information_gain(s1,s2,s)
I.update({f'informationGain_{i}':informationGain,f'threshold_{i}': threshold})
information_gain_indicies.append(i)

print(f"Checking condition a if {s1.nunique()['Class']} == {1}")
if (s1.nunique()['Class'] == 1):
break

print(f"Checking condition b  {maxf(s1['Class'])}/{minf(s1['Class'])} < {0.5} {s1.nunique()['Class']} == {floor(n/2)}")
if (maxf(s1['Class'])/minf(s1['Class']) < 0.5) and (s1.nunique()['Class'] == floor(n/2)):
print(f"Condition b is met{maxf(s1['Class'])}/{minf(s1['Class'])} < {0.5} {s1.nunique()['Class']} == {floor(n/2)}")
break

print("Elements in I")
print(I)
print("*****************************")

# Step 5: calculate the min information gain
n = int(((len(I)/2)-1))
print("Calculating maximum threshold")
print("*****************************")
maxInformationGain = 0
maxThreshold       = 0
for i in information_gain_indicies:
print(f"if({I[f'informationGain_{i}']} > {maxInformationGain})")
if(I[f'informationGain_{i}'] > maxInformationGain):
maxInformationGain = I[f'informationGain_{i}']
maxThreshold       = I[f'threshold_{i}']

print(f'maxThreshold: {maxThreshold}, maxInformationGain: {maxInformationGain}')

# replace values
print(f" {s1['A1'].value_counts().index}")
for i in s1['A1'].value_counts().index:
print(f"s1['A1'].replace({i},1)")
s1['A1'] = s1['A1'].replace(i,1)

print(f" {s2['A1'].value_counts().index}")
for i in s2['A1'].value_counts().index:
print(f"s2['A1'].replace({i},2)")
s2['A1'] = s2['A1'].replace(i,2)

print("s1 after replacing values")
print(s1)
print("******************")
print("s2 after replacing values")
print(s2)
print("******************")

partitions = [s1,s2]
s = pd.concat(partitions)

# Step 6: keep the partitions of S based on the value of threshold_i
return s #maxPartition(maxInformationGain,maxThreshold,s,s1,s2)

def maxf(s):
return s.max()

def minf(s):
return s.min()

def uniqueValue(s):
# are records in s the same? return true
if s.nunique()['Class'] == 1:
return False
# otherwise false
else:
return True

def maxPartition(maxInformationGain,maxThreshold,s,s1,s2):
print(f'informationGain: {maxInformationGain}, threshold: {maxThreshold}')
merged_partitions =  pd.merge(s1,s2)
merged_partitions =  pd.merge(merged_partitions,s)
print("Best Partition")
print("***************")
print(merged_partitions)
print("***************")
return merged_partitions

def information_gain(s1, s2, s):
# calculate cardinality for s1
cardinalityS1 = len(pd.Index(s1['Class']).value_counts())
print(f'The Cardinality of s1 is: {cardinalityS1}')
# calculate cardinality for s2
cardinalityS2 = len(pd.Index(s2['Class']).value_counts())
print(f'The Cardinality of s2 is: {cardinalityS2}')
# calculate cardinality of s
cardinalityS = len(pd.Index(s['Class']).value_counts())
print(f'The Cardinality of s is: {cardinalityS}')
# calculate informationGain
informationGain = (cardinalityS1/cardinalityS) * entropy(s1) + (cardinalityS2/cardinalityS) * entropy(s2)
print(f'The total informationGain is: {informationGain}')
return informationGain

def entropy(s):
print("calculating the entropy for s")
print("*****************************")
print(s)
print("*****************************")

# initialize ent
ent = 0

# calculate the number of classes in s
numberOfClasses = s['Class'].nunique()
print(f'Number of classes for dataset: {numberOfClasses}')
value_counts = s['Class'].value_counts()
p = []
for i in range(0,numberOfClasses):
n = s['Class'].count()
# calculate the frequency of class_i in S1
print(f'p{i} {value_counts.iloc[i]}/{n}')
f = value_counts.iloc[i]
pi = f/n
p.append(pi)

print(p)

for pi in p:
ent += -pi*log2(pi)

return ent

def segmentation_by_natural_partitioning(s):
# calculate 5th and 95th percentiles.
s_as_array = np.array(s)
fith_percentile = np.percentile(s_as_array, 5)
nienty_fith_percentile = np.percentile(s_as_array, 95)
print(f"range [{s['A2'].max()},{s['A2'].min()}]")
print()
print("*****************************")
print(f'fith_percentile {fith_percentile}')
print(f'nienty_fith_percentile {nienty_fith_percentile}')
print("*****************************")

# sort the data.
s['A2'] = s['A2'].sort_values()

n = s['A2'].count()
print("*****************************")
print(f'Total number of records {n}')
print("*****************************")
# keep the values from floor(n*0.05) to floor(n*0.95)
print("******************************")
print("Dataset attribute A2")
print(s['A2'])
print("******************************")
f1 = fith_percentile # np.math.floor(n*0.05)
f2 = nienty_fith_percentile # np.math.floor(n*0.95)
print("*****************************")
print(f'fith_percentile {f1}')
print(f'nienty_fith_percentile {f2}')
print("*****************************")
s = s[s['A2'] > f1]
s = s[s['A2'] < f2]

print(f"range after cleaning [{s['A2'].max()},{s['A2'].min()}]")

maximum = ceil(s['A2'].max())
minimum = floor(s['A2'].min())

print(f"[maximum,minimum] [{minimum},{maximum}]")
print(f"subtract {most_significant_while_floordiv(maximum)} - {most_significant_while_floordiv(minimum)}")
numberOfGaps =  most_significant_while_floordiv(maximum) - most_significant_while_floordiv(minimum)
print(f"numberOfGaps {numberOfGaps}")
print(f"The number of values that cover the range are [{minimum},{maximum}]")

return s

def most_significant_while_floordiv(i):
while i >= 10:
i //= 10
return i

def calculate_correlation(s):
s_temp = s[['A1','A3']]
correlation = s_temp.corr().iloc[1,0]
print("******************************")
print(f'Correlation between A1 & A3: {correlation}')
print("******************************")
# if correlation > 0.6 or correlation < 0.6 remove A3
if correlation > 0.6 or correlation < -0.6:
s = s.drop(['A3'], axis=1)
print(f'A3 was removed {correlation} > 0.6 or {correlation} < -0.6')
print("******************************")

return s

def df_to_array(df):
A1 = df[['A1']].to_numpy()
A2 = df[['A2']].to_numpy()
if 'A3' in df:
A3 = df[['A3']].to_numpy()

df_as_matrix = np.array(A1,A2)
if 'A3' in s:
df_as_matrix = np.concatenate(df_as_matrix,A3)

return df_as_matrix

def pca(s):
# Normalize each s
A1 = s[['A1']].to_numpy()
A2 = s[['A2']].to_numpy()

print(A1.ndim)
if 'A3' in s:
A3 = s[['A3']].to_numpy()
A3_norm = A3/np.linalg.norm(A3)

A1_norm = A1/np.linalg.norm(A1)
A2_norm = A2/np.linalg.norm(A2)

data = np.array([A1_norm,A2_norm])
if 'A3' in s:
data = np.array([A1_norm,A2_norm,A3_norm]).squeeze()

# determine covariance
covMatrix = np.cov(data,bias=True)
print(covMatrix)

# compute eigen vactors and eigenvalues
w, v = LA.eig(covMatrix)
print("eigen vectors")
print(v)

print("eigen values")
print(w)

varianceV = np.empty(3)

# calculate variances
varianceV[0] = w[0]/(w[0]+w[1]+w[2])
varianceV[1] = w[1]/(w[0]+w[1]+w[2])
varianceV[2] = w[2]/(w[0]+w[1]+w[2])

print(f' variance of v1 : {varianceV[0]}')
print(f' variance of v2 : {varianceV[1]}')
print(f' variance of v3 : {varianceV[2]}')

# calculate feature vector
v_initial = 0
featureVector = np.empty(3)
for i in range(0,3):
if varianceV[i] > v_initial:
featureVector = v[i]

print(f'feature vector: {featureVector}')
resolved_dataset = np.matmul(np.transpose(featureVector),data)
print(f'resolved_dataset.ndim = {resolved_dataset.ndim}')
print(f'dataset = {resolved_dataset}')
df = pd.DataFrame(resolved_dataset, columns = ['data'])
return df

main()


I have attached the dataset to this thread for reference. I would like to understand the results from the output of the PCA analysis step. Any help would be greatly appreciated.