# -*- coding: utf-8 -*-
###############################################################################
# KB_CAT KNOWLEDGE DISCOVERY IN DATA MINING (CATALOG PROGRAM) #
# by ROBERTO BELLO (COPYRIGHT MARCH 2011 ALL RIGHTS RESERVED) #
# Language used: PYTHON . #
###############################################################################
import os
import random
import copy
import datetime
def mean(x): # mean
n = len(x)
mean = sum(x) / n
return mean
def sd(x): # standard deviattion
n = len(x)
mean = sum(x) / n
sd = (sum((x-mean)**2 for x in x) / n) ** 0.5
return sd
def is_number(s):
try:
float(s)
return True
except ValueError:
return False
class ndim: # from 3D array to flat array
def __init__(self,x,y,z,d):
self.dimensions=[x,y,z]
self.numdimensions=d
self.gridsize=x*y*z
def getcellindex(self, location):
cindex = 0
cdrop = self.gridsize
for index in xrange(self.numdimensions):
cdrop /= self.dimensions[index]
cindex += cdrop * location[index]
return cindex
def getlocation(self, cellindex):
res = []
for size in reversed(self.dimensions):
res.append(cellindex % size)
cellindex /= size
return res[::-1]
""" how to use ndim class
n=ndim(4,4,5,3)
print n.getcellindex((0,0,0))
print n.getcellindex((0,0,1))
print n.getcellindex((0,1,0))
print n.getcellindex((1,0,0))
print n.getlocation(20)
print n.getlocation(5)
print n.getlocation(1)
print n.getlocation(0)
"""
print("###############################################################################")
print("# KB_CAT KNOWLEDGE DISCOVERY IN DATA MINING (CATALOG PROGRAM) #")
print("# by ROBERTO BELLO (COPYRIGHT MARCH 2011 ALL RIGHTS RESERVED) #")
print("# Language used: PYTHON #")
print("###############################################################################")
# input and run parameters
error = 0
while True:
arch_input = raw_input('InputFile : ')
if not os.path.isfile(arch_input):
print("Oops! File does not exist. Try again... or CTR/C to exit")
else:
break
while True:
try:
num_gruppi = int(raw_input('Number of Groups (3 - 20) : '))
except ValueError:
print("Oops! That was no valid number. Try again...")
else:
if(num_gruppi < 3):
print("Oops! Number of Groups too low. Try again...")
else:
if(num_gruppi > 20):
print("Oops! Number of Groups too big. Try again...")
else:
break
while True:
normaliz = raw_input('Normalization(Max, Std, None) : ')
normaliz = normaliz.upper()
normaliz = normaliz[0]
if(normaliz <> 'M' and normaliz <> 'S' and normaliz <> 'N'):
print("Oops! Input M, S or N. Try again...")
else:
break
while True:
try:
max_alpha = float(raw_input('Start value of alpha (1.8 - 0.9) : '))
except ValueError:
print("Oops! That was no valid number. Try again...")
else:
if(max_alpha > 1.8):
print("Oops! Start value of alpha too big. Try again...")
else:
if(max_alpha < 0.9):
print("Oops! Start value of alpha too low. Try again...")
else:
break
while True:
try:
min_alpha = float(raw_input('End value of alpha (0.5 - 0.0001) : '))
except ValueError:
print("Oops! That was no valid number. Try again...")
else:
if(min_alpha > 0.5):
print("Oops! alpha too big. Try again...")
else:
if(min_alpha < 0.0001):
print("Oops! alpha too low. Try again...")
else:
break
while True:
try:
step_alpha = float(raw_input('Decreasing step of alpha (0.1 - 0.001) : '))
except ValueError:
print("Oops! That was no valid number. Try again...")
else:
if(step_alpha > 0.1):
print("Oops! Decreasing step of alpha too big. Try again...")
else:
if(step_alpha < 0.001):
print("Oops! Decreasing step of alpha too low. Try again...")
else:
break
file_input = arch_input
gruppi_num = num_gruppi
tipo_norm = normaliz
alpha_min = min_alpha
alpha_max = max_alpha
alpha_step = step_alpha
# outputs files
file_input = arch_input
tipo_norm = normaliz
gruppi_num = num_gruppi
nome_input = file_input.split(".")
arch_output = nome_input[0] + "_" + tipo_norm + "_g" + str(gruppi_num) + "_out.txt"
arch_outsrt = nome_input[0] + "_" + tipo_norm + "_g" + str(gruppi_num) + "_outsrt.txt"
arch_sort = nome_input[0] + "_" + tipo_norm + "_g" + str(gruppi_num) + "_sort.txt"
arch_catal = nome_input[0] + "_" + tipo_norm + "_g" + str(gruppi_num) + "_catal.txt"
arch_medsd = nome_input[0] + "_" + tipo_norm + "_g" + str(gruppi_num) + "_medsd.txt"
arch_cv = nome_input[0] + "_" + tipo_norm + "_g" + str(gruppi_num) + "_cv.txt"
arch_grid = nome_input[0] + "_" + tipo_norm + "_g" + str(gruppi_num) + "_grid.txt"
arch_log = nome_input[0] + "_" + tipo_norm + "_g" + str(gruppi_num) + "_log.txt"
# start time
t0 = datetime.datetime.now()
# read input file
arr_r = []
arr_orig = []
arr_c = []
mtchx = []
mtchy = []
txt_col = []
xnomi = []
# the numbers of variables / columns in all record must be the same
n_rows = 0
n_cols = 0
err_cols = 0
index = 0
for line in open(file_input).readlines():
linea = line.split()
if(index == 0):
xnomi.append(linea)
n_cols = len(linea)
else:
arr_r.append(linea)
if(len(linea) != n_cols):
err_cols = 1
print("Different numbers of variables / columns in the record " + str(index)
+ " cols " + str(len(linea)))
index += 1
if(err_cols == 1):
print("File " + file_input + " contains errors. Exit ")
quit()
index = 0
while index < len(arr_r):
linea = arr_r[index]
index_c = 0
while index_c < len(linea):
if linea[index_c].isdigit():
linea[index_c] = float(linea[index_c])
index_c += 1
arr_r[index] = linea
index += 1
arr_orig = copy.deepcopy(arr_r) # original input file
testata_cat = copy.deepcopy(xnomi[0]) # original header row
# finding columns containing strings and columns containing numbers
testata = xnomi[0]
testata_orig = copy.deepcopy(xnomi[0])
n_cols = len(testata) - 1
n_rows = len(arr_r)
ind_c = 1
err_type = 0
while ind_c < len(testata):
ind_r = 1
tipo_num = 0
tipo_txt = 0
while ind_r < len(arr_r):
arr_c = arr_r[ind_r]
if is_number(arr_c[ind_c]):
tipo_num = 1
else:
tipo_txt = 1
ind_r += 1
if tipo_num == 1 and tipo_txt == 1:
print "The columns / variables " + testata[ind_c] + " contains both strings and numbers."
print arr_c
err_type = 1
ind_c += 1
if err_type == 1:
print "Oops! The columns / variables contains both strings and numbers. Exit. "
quit()
index_c = 1
while index_c <= n_cols:
txt_col = []
index = 0
while index < len(arr_r):
arr_c = arr_r[index]
if(isinstance(arr_c[index_c],str)):
txt_col.append(arr_c[index_c])
index += 1
set_txt_col = set(txt_col) # remove duplicates
txt_col = list(set(set_txt_col))
txt_col.sort()
# from strings to numbers
if(len(txt_col) > 0):
if(len(txt_col) > 1):
passo1 = 1.0 / (len(txt_col) - 1)
else:
passo1 = 0.0
index = 0
while index < len(arr_r):
arr_c = arr_r[index]
campo1 = arr_c[index_c]
indice1 = txt_col.index(campo1)
if(len(txt_col) == 1): # same values in the column
val_num1 = float(1)
else:
val_num1 = float(passo1 * indice1)
arr_c[index_c] = val_num1 + 0.00000001 # to avoid zero values in means
# (to prevent zero divide in CV)
index += 1
index_c += 1
# means, max & std
xmeans = []
xmaxs = []
xmins = [] ### aggiunto Roberto 4/03/2012
xsds = []
xcv = []
index_c = 0
while index_c <= n_cols:
xmeans.append(0.0)
xmaxs.append(-9999999999999999.9)
xmins.append(9999999999999999.9) ### aggiunto Roberto 4/03/2012
xsds.append(0.0)
xcv.append(0.0)
index_c += 1
# means & max
index = 0
while index < n_rows:
arr_c = arr_r[index]
index_c = 1
while index_c <= n_cols:
xmeans[index_c] += arr_c[index_c]
if(arr_c[index_c] > xmaxs[index_c]):
xmaxs[index_c] = arr_c[index_c]
index_c += 1
index += 1
index_c = 1
while index_c <= n_cols:
xmeans[index_c] = xmeans[index_c] / n_rows
index_c += 1
# std
index = 0
while index < n_rows:
arr_c = arr_r[index]
index_c = 1
while index_c <= n_cols:
xsds[index_c] += (arr_c[index_c] - xmeans[index_c])**2
index_c += 1
index += 1
index_c = 1
while index_c <= n_cols:
xsds[index_c] = (xsds[index_c] / (n_cols - 1)) ** 0.5
index_c += 1
# Means, Max, Std, CV output file
medsd_file = open(arch_medsd, 'w')
# columns names
medsd_file.write('%s %s ' % ('Function' , "\t"))
index_c = 1
while index_c <= n_cols:
medsd_file.write('%s %s ' % (testata[index_c], "\t"))
index_c += 1
medsd_file.write('%s' % ('\n'))
# means
medsd_file.write('%s %s ' % ('Mean' , "\t"))
index_c = 1
while index_c <= n_cols:
valore = str(xmeans[index_c])
valore = valore[0:6]
medsd_file.write('%s %s ' % (valore, "\t"))
index_c += 1
medsd_file.write('%s' % ('\n'))
# max
medsd_file.write('%s %s ' % ('Max' , "\t"))
index_c = 1
while index_c <= n_cols:
valore = str(xmaxs[index_c])
valore = valore[0:6]
medsd_file.write('%s %s ' % (valore, "\t"))
index_c += 1
medsd_file.write('%s' % ('\n'))
# std
medsd_file.write('%s %s ' % ('Std' , "\t"))
index_c = 1
while index_c <= n_cols:
valore = str(xsds[index_c])
valore = valore[0:6]
medsd_file.write('%s %s ' % (valore, "\t"))
index_c += 1
medsd_file.write('%s' % ('\n'))
# CV
medsd_file.write('%s %s ' % ('CV' , "\t"))
index_c = 1
med_cv_gen = 0.0 # cv average of all columns / variables
while index_c <= n_cols:
if xmeans[index_c] == 0:
media1 = 0.000001
else:
media1 = xmeans[index_c]
xcv[index_c] = xsds[index_c] / abs(media1)
valore = str(xcv[index_c])
med_cv_gen += xcv[index_c]
valore = valore[0:6]
medsd_file.write('%s %s ' % (valore, "\t"))
index_c += 1
med_cv_gen = med_cv_gen / n_cols
str_med_cv_gen = str(med_cv_gen)
str_med_cv_gen = str_med_cv_gen[0:6]
medsd_file.write('%s' % ('\n'))
medsd_file.close()
# input standardization
# standardization on max
if tipo_norm == 'M':
index = 0
while index < n_rows:
arr_c = arr_r[index]
index_c = 1
while index_c <= n_cols: ## aggiornare anche kb_cla.py
if xmaxs[index_c] == 0.0:
xmaxs[index_c] = 0.00001
arr_c[index_c] = arr_c[index_c] / xmaxs[index_c]
index_c += 1
index += 1
# standardization on std
if tipo_norm == 'S':
index = 0
while index < n_rows:
arr_c = arr_r[index]
index_c = 1
while index_c <= n_cols:
if xsds[index_c] == 0.0:
xsds[index_c] = 0.00001
arr_c[index_c] = (arr_c[index_c] - xmeans[index_c]) / xsds[index_c]
if arr_c[index_c] < xmins[index_c]: ### aggiunto Roberto 4/03/2012
xmins[index_c] = arr_c[index_c] ### aggiunto Roberto 4/03/2012
index_c += 1
index += 1
# aggiungo xmins per eliminare i valori negativi (aggiunto da Roberto 4/03/2012)
index = 0
while index < n_rows:
arr_c = arr_r[index]
index_c = 1
while index_c <= n_cols:
arr_c[index_c] = arr_c[index_c] - xmins[index_c]
print arr_c[index_c]
index_c += 1
index += 1
# fine aggiunta da Roberto 4/03/2012
# start of kohonen algorithm
# min and max vectors
vmaxs = []
vmins = []
index_c = 0
while index_c <= n_cols:
vmaxs.append(-10000000000000.0)
vmins.append( 10000000000000.0)
index_c += 1
# columns min & max
index = 0
while index < n_rows:
arr_c = arr_r[index]
index_c = 1
while index_c <= n_cols:
if arr_c[index_c] > vmaxs[index_c]:
vmaxs[index_c] = arr_c[index_c]
if arr_c[index_c] < vmins[index_c]:
vmins[index_c] = arr_c[index_c]
index_c += 1
index += 1
# run parameters and temp arrays
n = n_rows
m = n_cols
nx = gruppi_num
ny = gruppi_num
ix = 950041 # integer as random seed
nsteps = int(10000 * nx * ny) # number of steps
nepoks = int(nsteps / n ** 0.5) # number of epochs
unit_calc = int(n * m * nx * ny) # running units
passo = int(5000 / n) # step of visualization on monitor
rmax = nx - 1
rmin = 1.0
if passo < 1:
passo = 1
grid = [] # training grid
index = 0
while index < nx * ny * m:
grid.append(0.0)
index += 1
n=ndim(nx,ny,m,3)
random.seed(ix) # initial value of random seed to obtain the same sequences in new runs
index = 0
while index < nx:
index_c = 0
while index_c < ny:
index_k = 0
while index_k < m:
ig = n.getcellindex((index,index_c,index_k))
grid[ig] = random.random()
index_k += 1
index_c += 1
index += 1
gridp = copy.deepcopy(grid) # initial previous grid = current grid
gridm = copy.deepcopy(grid) # initial min grid = current grid
# for each record in each epoch
iter = 0
discrea = 1000000000000.0 # current error
discrep = 0.0 # previous error
if nepoks < 20:
nepoks = 20 # min epochs = 20
nepokx = 0
min_epok = 0 # epoch with min error
min_err = 1000000000.0 # min error
alpha = float(alpha_max) # initial value of alpha parameter
ir = 0.0 # initial value of ir parameter ir
ne = 1
print " "
print 'Record ' + str(n_rows) + ' Columns ' + str(n_cols)
# main loop
try:
while ne <= nepoks:
if (ne % passo == 0): # print running message when modulo division = zero
min_err_txt = "%14.5f" % min_err # format 8 integers and 3 decimals
alpha_txt = "%12.5f" % alpha # format 6 integers and 5 decimals
print ('Epoch ' + str(ne) + ' min err ' + min_err_txt + ' min epoch ' +
str(min_epok - 1) + " alpha " + alpha_txt)
if min_err < 1000000000.0:
nepokx += 1
if min_err > discrea and discrep > discrea and discrea > 0.0:
min_epok = ne # current epoch (min)
min_err = discrea
# copy current grid to min grid
gridm = copy.deepcopy(grid)
min_err_txt = "%12.3f" % min_err # format 8 integers and 3 decimals
alpha_txt = "%12.5f" % alpha # format 6 integer and 5 decimals
print ('**** Epoch ' + str(ne - 1) + ' WITH MIN ERROR ' + min_err_txt +
" alpha " + alpha_txt)
# cheking the current value of alpha
if alpha > alpha_min:
discrea = discrep
discrep = 0.0
# copy current grid to previous grid
gridp = copy.deepcopy(grid)
# from the starting row to the ending row
i = 0
while i < n_rows:
iter += 1
# find the best grid coefficient
ihit = 0
jhit = 0
dhit = 100000.0
igx = 0
igy = 0
while igx < nx:
igy = 0
while igy < ny:
d = 0.0
neff = 0
k = 0
arr_c = arr_r[i]
while k < m: # update the sum of squared deviation of input
# value from the grid coefficient
ig = n.getcellindex((igx,igy,k))
d = d + (arr_c[k+1] - grid[ig]) ** 2
k += 1
d = d / float(m)
# d = d / m
if d < dhit:
dhit = d
ihit = int(igx)
jhit = int(igy)
igy += 1
igx += 1
# update iteration error
discrep = discrep + dhit
# now we have the coordinates of the best grid coefficient
ir = max(rmax * float(1001 - iter) / 1000.0 + 0.9999999999 , 1)
ir = int(ir)
# new alpha value to increase the radius of groups proximity
alpha = max(alpha_max * float(1 - ne * alpha_step) , alpha_min)
# update the grid coefficients applying alpha parameter
inn0 = int(ihit) - int(ir)
inn9 = int(ihit) + int(ir)
jnn0 = int(jhit) - int(ir)
jnn9 = int(jhit) + int(ir)
while inn0 <= inn9:
jnn0 = int(jhit) - int(ir)
while jnn0 <= jnn9:
if not (inn0 < 0 or inn0 >= nx):
if not (jnn0 < 0 or jnn0 >= ny):
arr_c = arr_r[i]
k = 0
while k < m:
ig = n.getcellindex((inn0,jnn0,k))
grid[ig] += alpha * (arr_c[k+1] - grid[ig])
k += 1
jnn0 += 1
inn0 += 1
i += 1
else:
print
print "Min alpha reached "
print
break
ne += 1
except KeyboardInterrupt:
print
print "KeyboardInterrupt (Ctrl/C) "
print
pass
# computing results
# grid = grid min
grid = copy.deepcopy(gridm)
# write min grid file
arch_grid_file = open(arch_grid, 'w')
ii = 0
while ii < nx:
j = 0
while j < ny:
k = 0
while k < m:
ig = n.getcellindex((ii,j,k))
arch_grid_file.write('%6i %s %.6i %s %.6i %s %14.7f %s' % (ii,' ', j ,' ', k,' ', grid[ig], "\n"))
k += 1
j += 1
ii += 1
arch_grid_file.close()
# catalog input by min grid
ii = 0
while ii < n_rows:
ihit = 0
jhit = 0
dhit = 100000.0
# from 1 to numbers of groups
ir = 0
while ir < nx: # from 1 to numbers of groups
jc = 0
while jc < ny: # from 1 to numbers of groups
d = 0.0
neff = 0
k = 0
while k < n_cols: # update the sum of squared deviation of input
# value from the grid coefficient
arr_c = arr_r[ii]
ig = n.getcellindex((ir,jc,k))
d = d + (arr_c[k+1] - grid[ig]) ** 2
k += 1
d = d / m
if d < dhit: # save the coordinates of the best coefficient
dhit = d
ihit = ir
jhit = jc
jc += 1
ir += 1
mtchx.append(ihit)
mtchy.append(jhit)
ii += 1
# write arch_catal file
arch_catal_file = open(arch_catal, 'w')
ii = 0
while ii < n_rows:
arch_catal_file.write("%.6i %s %.6i %s %.6i %s" % (ii, ' ', mtchx[ii], ' ', mtchy[ii], "\n"))
ii += 1
arch_catal_file.close()
# matrix of statistics
arr_cv = [] # CV array of the Groups and Total
arr_med = [] # means array of the Groups
riga_cv = [] # CV row in arr_cv
arr_col = [] # group temporary array
arr_grsg = [] # input data array (normalized)
arr_grsg_c = [] # copy of arr_grsg (for file out sort)
# input matrix sort in group sequence
ii = 0
ix = 0
while ii < n_rows:
ix += 1
gr1 = str(mtchx[ii])
if mtchx[ii] < 10:
gr1 = '0' + str(mtchx[ii])
sg1 = str(mtchy[ii])
if mtchy[ii] < 10:
sg1 = '0' + str(mtchy[ii])
riga_norm = arr_r[ii]
im = 0
riga_norm1 = []
while im <= m:
riga_norm1.append(str(riga_norm[im]))
im += 1
riga_norm2 = " ".join(riga_norm1)
gr_sg_txt = "G_" + gr1 + "_" + sg1 + " " + str(ix) + " " + riga_norm2
arr_grsg.append(gr_sg_txt)
ii += 1
arr_grsg.sort()
ii = 0
while ii < n_rows:
arr_grsg_c.append(arr_grsg[ii])
ii += 1
# setup of arr_cv matrix
num_gr = 0
gruppo0 = ""
ir = 0
while ir < n_rows:
grsg_key = arr_grsg_c[ir].split()
if not grsg_key[0] == gruppo0:
gruppo0 = grsg_key[0]
num_gr +=1
ic = 1
riga1 = []
riga1.append(grsg_key[0])
while ic <= m + 2: # adding new columns for row mean and n° of records
riga1.append(0.0)
ic += 1
arr_cv.append(riga1) # cv row
ir += 1
riga1 = []
riga1.append("*Means*") # adding new row for cv mean
ic = 1
while ic <= m + 2: # adding new column for row mean and n° of records
riga1.append(0.0)
ic += 1
arr_cv.append(riga1)
def found(x):
ir = 0
while ir < len(arr_cv):
linea_cv = arr_cv[ir]
key_cv = linea_cv[0]
if key_cv == x:
return ir
ir += 1
ir = 0
irx = len(arr_grsg_c)
ic = 3
linea_cv = arr_cv[0]
icx = len(linea_cv)
val_col = []
while ic < icx:
ir = 0
gruppo = ""
val_col = []
while ir < irx:
linea = arr_grsg_c[ir].split()
if linea[0] == gruppo or gruppo == "":
gruppo = linea[0]
val_col.append(float(linea[ic]))
else:
i_gruppo = found(gruppo)
linea_cv = arr_cv[i_gruppo]
media_v = abs(mean(val_col))
if media_v == 0.0:
media_v = 0.0000000001
std_v = sd(val_col)
cv_v = std_v / media_v
linea_cv[ic-2] = cv_v # cv value
linea_cv[len(linea_cv)-1] = len(val_col) # number of records
val_col = []
val_col.append(float(linea[ic]))
gruppo = linea[0]
ir += 1
i_gruppo = found(gruppo)
linea_cv = arr_cv[i_gruppo]
media_v = abs(mean(val_col))
if media_v == 0.0:
media_v = 0.0000000001
std_v = sd(val_col)
cv_v = std_v / media_v
linea_cv[ic-2] = cv_v # cv value
linea_cv[len(linea_cv)-1] = len(val_col) # number of records
ic += 1
ir = 0
irx = len(arr_cv)
linea_cv = arr_cv[0]
icx = len(linea_cv) - 2
ic = 1
num_rec1 = 0
while ir < irx: # rows mean
media_riga = 0.0
ic = 1
num_col1 = 0
linea_cv = arr_cv[ir]
while ic < icx:
media_riga += float(linea_cv[ic])
num_col1 += 1
ic += 1
linea_cv[icx] = media_riga / num_col1
num_rec1 += linea_cv[icx + 1]
ir += 1
ir = 0
ic = 1
while ic < icx: # weighted mean of columns
media_col = 0.0
ir = 0
num_rec1 = 0
while ir < irx - 1:
linea_cv = arr_cv[ir]
media_col = media_col + linea_cv[ic] * linea_cv[icx+1] # linea_cv[icx+1] = number of records
num_rec1 = num_rec1 + linea_cv[icx+1]
ir += 1
linea_cv = arr_cv[irx - 1]
linea_cv[ic] = media_col / num_rec1
ic += 1
# updating mean of the row
linea_cv = arr_cv[irx - 1]
linea_means = linea_cv[1:icx]
media_riga = mean(linea_means)
linea_cv[icx] = media_riga # Total mean
linea_cv[icx + 1] = num_rec1 # n° of records
cv_media_gen_after = str(media_riga)
cv_media_gen_after = cv_media_gen_after[0:6]
# write cv file
testata_cv = testata
testata_cv[0] = "*Groups*"
testata_cv.append("*Mean*")
testata_cv.append("N_recs")
arch_cv_file = open(arch_cv, 'w')
ic = 0
while ic <= icx + 1:
arch_cv_file.write('%s %s ' % (testata_cv[ic], " "*(9-len(testata_cv[ic]))))
ic += 1
arch_cv_file.write('%s' % ('\n'))
ir = 0
while ir < irx:
ic = 0
linea_cv = arr_cv[ir]
while ic <= icx + 1:
if ic == 0:
arch_cv_file.write('%s %s ' % (linea_cv[0], " "))
else:
if ic <= icx:
arch_cv_file.write('%7.4f %s ' % (linea_cv[ic], " "))
else:
arch_cv_file.write('%6i %s ' % (linea_cv[ic], " "))
ic += 1
arch_cv_file.write('%s' % ("\n"))
ir += 1
ic = 0
media_xcv = mean(xcv[1:icx])
while ic <= icx : # print CV input (before catalogue)
if ic == 0:
arch_cv_file.write('%s %s ' % ("*CVinp*", " "))
else:
if ic < icx:
arch_cv_file.write('%7.4f %s ' % (xcv[ic], " "))
else:
arch_cv_file.write('%7.4f %s ' % (media_xcv, " "))
arch_cv_file.write('%6i %s ' % (linea_cv[ic+1], " "))
ic += 1
arch_cv_file.write('%s' % ("\n"))
#=========istruzioni aggiunte Roberto Bello 29/02/2012======================
#know_index = str(1.0 - float(cv_media_gen_after) / float(str_med_cv_gen))
#know_index = know_index[0:6]
#arch_cv_file.write('%s %s %s' % ('*KIndex* ', know_index, '\n'))
#=========fine istruzioni aggiunte da Roberto Bello 29/02/2012==============
arch_cv_file.close()
# writing out catalog file
testata_cat1 = []
testata_cat1.append("*Group*")
arch_output_file = open(arch_output, 'w')
ic= 0
while ic < icx:
testata_cat1.append(testata_cat[ic])
ic += 1
ic= 0
while ic < len(testata_cat1):
arch_output_file.write('%s %s ' % (testata_cat1[ic], " "*(15-len(testata_cat1[ic]))))
ic += 1
arch_output_file.write('%s' % ("\n"))
index = 0
while index < len(arr_orig):
riga_orig = arr_orig[index]
ic = 0
while ic < len(riga_orig):
if not(isinstance(riga_orig[ic],str)):
riga_orig[ic] = str(riga_orig[ic])
ic += 1
# place before 0 if gr / sg < 10
gr1 = str(mtchx[index])
if mtchx[index] < 10:
gr1 = '0' + str(mtchx[index])
sg1 = str(mtchy[index])
if mtchy[index] < 10:
sg1 = '0' + str(mtchy[index])
arr_rig0 = "G_" + gr1 + "_" + sg1 + " "*8
arch_output_file.write('%s ' % (arr_rig0))
ic= 0
while ic < len(riga_orig):
arch_output_file.write('%s %s ' % (riga_orig[ic], " "*(15-len(riga_orig[ic]))))
ic += 1
arch_output_file.write('%s' % ("\n"))
index += 1
testata_cat1 = []
testata_cat1.append("*Group*")
testata_cat1.append("*RecNum*")
arch_sort_file = open(arch_sort, 'w')
ic= 0
while ic < icx:
testata_cat1.append(testata_cat[ic])
ic += 1
ic= 0
while ic < len(testata_cat1):
arch_sort_file.write('%s %s ' % (testata_cat1[ic], " "*(15-len(testata_cat1[ic]))))
ic += 1
arch_sort_file.write('%s' % ("\n"))
index = 0
while index < len(arr_grsg_c):
riga_grsg = arr_grsg_c[index].split()
ic = 0
while ic < len(riga_grsg):
val_txt = riga_grsg[ic]
val_txt = val_txt[0:13]
arch_sort_file.write('%s %s ' % (val_txt, " "*(15-len(val_txt))))
ic += 1
if index < len(arr_grsg_c) - 1:
arch_sort_file.write('%s' % ("\n"))
index += 1
arch_sort_file.close()
# writing out catalog and sorted file
arr_outsrt = []
index = 0
while index < len(arr_orig):
riga_sort = []
# place before 0 if gr / sg < 10
gr1 = str(mtchx[index])
if mtchx[index] < 10:
gr1 = '0' + str(mtchx[index])
sg1 = str(mtchy[index])
if mtchy[index] < 10:
sg1 = '0' + str(mtchy[index])
riga_sort.append("G_" + gr1 + "_" + sg1)
ic = 0
riga_orig = arr_orig[index]
while ic < len(riga_orig):
val_riga = riga_orig[ic]
riga_sort.append(val_riga)
ic += 1
arr_outsrt.append(riga_sort)
index += 1
for line in arr_outsrt:
line = "".join(line)
arr_outsrt.sort()
testata_srt = []
testata_srt.append("*Group*")
arch_outsrt_file = open(arch_outsrt, 'w')
ic= 0
while ic < icx:
testata_srt.append(testata_orig[ic])
ic += 1
ic= 0
while ic < len(testata_srt):
arch_outsrt_file.write('%s %s' % (testata_srt[ic], " "*(15-len(testata_srt[ic]))))
ic += 1
arch_outsrt_file.write('%s' % ("\n"))
index = 0
key_gruppo = ""
while index < len(arr_outsrt):
riga_sort = arr_outsrt[index]
index_c = 0
while index_c < len(riga_sort):
if index_c == 0:
if riga_sort[0] != key_gruppo:
# arch_outsrt_file.write('%s ' % ("\n"))
key_gruppo = riga_sort[0]
valore = riga_sort[index_c]
arch_outsrt_file.write('%s %s' % (valore, " "*(15-len(valore))))
index_c += 1
if index < len(arr_grsg_c) - 1:
arch_outsrt_file.write('%s' % ("\n"))
index += 1
arch_outsrt_file.close()
print("###############################################################################")
print("# KB_CAT KNOWLEDGE DISCOVERY IN DATA MINING (CATALOG PROGRAM) #")
print("# by ROBERTO BELLO (COPYRIGHT MARCH 2011 ALL RIGHTS RESERVED) #")
print("# Language used: PYTHON #")
print("###############################################################################")
arch_log_file = open(arch_log, 'w')
arch_log_file.write("%s %s" % ("############################################################################", "\n"))
arch_log_file.write("%s %s" % ("# KB_CAT KNOWLEDGE DISCOVERY IN DATA MINING (CATALOG PROGRAM) #", "\n"))
arch_log_file.write("%s %s" % ("# by ROBERTO BELLO (COPYRIGHT MARCH 2011 ALL RIGHTS RESERVED) #", "\n"))
arch_log_file.write("%s %s" % ("# Language used: PYTHON . #", "\n"))
arch_log_file.write("%s %s" % ("############################################################################", "\n"))
arch_log_file.write("%s %s %s" % ("Input File -> ", file_input, "\n"))
arch_log_file.write("%s %s %s" % ("Numer of Groups (3 - 20) -> ", str(gruppi_num), "\n"))
arch_log_file.write("%s %s %s" % ("Normalization (Max, Std, None) -> ", tipo_norm, "\n"))
arch_log_file.write("%s %s %s" % ("Start Value of alpha (from 1.8 to 0.9) -> ", str(alpha_max), "\n"))
arch_log_file.write("%s %s %s" % ("End Value of alpha (from 0.5 to 0.0001) -> ", str(alpha_min), "\n"))
arch_log_file.write("%s %s %s" % ("Decreasing step of alpha (from 0.1 to 0.001) -> ", str(alpha_step), "\n"))
arch_log_file.write("%s" % ("=========================OUTPUT=======================================================\n"))
arch_log_file.write("%s %s %s" % ("Output File Catalog.original ", arch_output, "\n"))
arch_log_file.write("%s %s %s" % ("Output File Catalog.sort ", arch_outsrt, "\n"))
arch_log_file.write("%s %s %s" % ("Output File Summary sort ", arch_sort, "\n"))
arch_log_file.write("%s %s %s" % ("Output File Matrix Catal. ", arch_catal, "\n"))
arch_log_file.write("%s %s %s" % ("Output File Means, STD, CV. ", arch_medsd, "\n"))
arch_log_file.write("%s %s %s" % ("Output File CV of the Groups ", arch_cv, "\n"))
arch_log_file.write("%s %s %s" % ("Output File Training Grid ", arch_grid, "\n"))
arch_log_file.write("%s %s %s" % ("Output File Run Parameters ", arch_log, "\n"))
#=========istruzioni aggiunte Roberto Bello 29/02/2012======================
know_index = str(1.0 - float(cv_media_gen_after) / float(str_med_cv_gen))
know_index = know_index[0:6]
arch_log_file.write('%s %s %s' % ('*KIndex* ', know_index, '\n'))
#=========fine istruzioni aggiunte da Roberto Bello 29/02/2012==============
min_err_txt = "%12.3f" % min_err # format 8 integer and 3 decimals
alpha_txt = "%12.5f" % alpha # format 6 integer and 5 decimals
alpha_min_txt = "%12.5f" % alpha_min # format 6 integer and 5 decimals
print
if min_err == 1000000000.000:
print("Oops! No result. Try again with new alpha parameters")
print
print ("EPOCH " + str(min_epok -1) + " WITH MIN ERROR " + min_err_txt +
" starting alpha " + alpha_min_txt + " ending alpha " + alpha_txt +
" Iterations " + str(iter) + " Total Epochs " + str(ne - 1))
print
print 'Output File Catalog.original ' + arch_output
print 'Output File Catalog.sort ' + arch_outsrt
print 'Output File Summary sort ' + arch_sort
print 'Output File Matrix Catal. ' + arch_catal
print 'Output File Means, STD, CV. ' + arch_medsd
print 'Output File CV of the Groups ' + arch_cv
print 'Output File Training Grid ' + arch_grid
print 'Output File Run Parameters ' + arch_log
print 'CV before Catalog ' + str_med_cv_gen
print 'CV after Catalog ' + cv_media_gen_after
know_index = str(1.0 - float(cv_media_gen_after) / float(str_med_cv_gen))
know_index = know_index[0:6]
print 'Knowledge Index ' + know_index
print
# Elapsed time
t1 = datetime.datetime.now()
elapsed_time = t1 - t0
print "Elapsed time (seconds) : " + str(elapsed_time.seconds)
print
