#############################################################

# DATA MINING LAB SESSION 1 (FEATURE SELECTION)

# irrelevant features are modeled as pure independent noise!
# redundant features are modeled as simple functions of relevant ones! WARNING!

#############################################################


require(FSelector)
require(mlbench)
require(MASS)

n <- 100
p <- 7
sigma <- 1

# generate 5 relevant and 5 irrelevant features

set.seed(1)
sim <- mlbench.friedman1 (n, sd = sigma)
colnames(sim$x) <- c(paste("Rel.", 1:5, sep = ""), paste("Irrel.U.", 1:5, sep = ""))

# generate p more irrelevant features

irrelevant <- matrix(rnorm(n * p), nrow = n)
colnames(irrelevant) <- paste("Irrel.N.", 1:ncol(irrelevant), sep = "")

# generate 3 redundant features

redundant1 <- sim$x[,1] * sim$x[,2] + rnorm(n)
redundant2 <- sim$x[,1] * sim$x[,2] - rnorm(n) # this is the same as above
redundant3 <- (sim$x[,3] + sim$x[,4] + sim$x[,5])/3

redundant <- as.data.frame (cbind(redundant1,redundant2,redundant3))
colnames(redundant) <- paste("Red.", 1:3, sep = "")

# prepare for data frame

x <- cbind(sim$x, irrelevant, redundant)
y <- sim$y

# We have now 13+p predictors:
#
# 	5 (1 to 5) relevant features
# 	5 (6 to 10) irrelevant features distributed as U(0,1) 
# 	p (11 to 11+p-1) irrelevant features distributed as N(0,sigma^2)
# 	3 (11+p-1 to 11+p+4) redundant features

# The predictors are centered and scaled:

x <- as.data.frame(scale(x))

# form final data frame

fullset.formula <- as.simple.formula(colnames(x), "Target")

dataforFSS <- cbind (x,y)
colnames(dataforFSS)[dim(dataforFSS)[2]] <- "Target"

#############################################################
# Filters
#############################################################

# 1. Correlation filter

subset.CFS <- cfs (Target~., dataforFSS)
(subset.CFS.formula <- as.simple.formula(subset.CFS,"Target"))


# 2. Consistency filter

subset.Consistency <- consistency (Target~., dataforFSS)
(subset.Consistency.formula <- as.simple.formula(subset.Consistency,"Target"))


# 3. Relief
# Warning: both 'neighbours.count' and 'sample.size' are arbitrary choices

weights.relief <- relief (Target~., dataforFSS, neighbours.count = round(n/20), sample.size = n)
print(weights.relief)

subset.relief <- cutoff.k (weights.relief, 5) # this is tricky!
(subset.relief.formula <- as.simple.formula(subset.relief,"Target"))

#############################################################
# Wrappers
#############################################################

# 4. Linear Regression

LinearRegression <- glm (Target~., family = gaussian, data = dataforFSS)
subset.LinearRegression.formula <- step(LinearRegression)$formula

# 5. Random Forest

weights.randomForest <- random.forest.importance (Target~., dataforFSS, importance.type = 1)
print(weights.randomForest)

(subset.randomForest.formula <- cutoff.k(weights.randomForest, 5)) # this is tricky!

##########################################################

# Comparison

print(subset.randomForest.formula)
print(subset.LinearRegression.formula)
print(subset.relief.formula)
print(subset.Consistency.formula)
print(subset.CFS.formula)

# a. Does any method include all the relevant features?
# b. Does any method exclude all the irrelevant features?
# c. Does any method 'hit' the right answer? How do they treat the redundant features?

# Which -in your opinion- is the best method overall?


##########################################################


##########################################################

# Laboratory EXERCISES

##########################################################

# EXERCISE 1: change n from 100 to 1000. Redo everything, including the final comparison. Does any of the methods yield far better results? How do the methods fare compared to the case n = 100? Why? Do we need a separate test set?

# EXERCISE 2: answer the questions:
# 	a. How to solve the tricky part? Could we use a wrapper?
#     b. Do the wrapper methods seem to work better? If so, why? Do they need to be 'tuned'?
#     c. In conclusion, what is the best 'option'?

# EXERCISE 3: work with a real-world data set: n cannot be increased, the knowledge on variable importance is (completely?) unknown. Devise a feature selection solution. 

# use the 'Pima*' data sets: 
#     The training set ‘Pima.tr’ contains a randomly selected set of
#     200 subjects, and ‘Pima.te’ contains the remaining 322
#     subjects.  ‘Pima.tr2’ contains ‘Pima.tr’ plus 100 subjects
#     with missing values in the explanatory variables.

# Use the training set ‘Pima.tr’ for feature selection. Then apply the subset of selected features to predict the test set ‘Pima.te’. Compare to the results obtained using the full set of features.

# Note 1:   This is a different problem in that it is a classification problem.
# 		This means you have a set of extra tools: 
#			* FOR CONTINUOUS VARIABLES  FISHER's F (against the class variable). Use 'oneway.test'
# 			* FOR CATEGORICAL VARIABLES  CHI-SQUARE (against the class variable). Use 'chisq.test'
#           And you should use Logistic Regression instead of Linear Regression

# Note 2:   If you have time, try the same thing using the training set ‘Pima.tr2’. You shall need a method that accepts missing values!