|
|
@@ -0,0 +1,153 @@ |
|
|
# Quiz 4 |
|
|
|
|
|
# Question 1. |
|
|
library(ElemStatLearn) |
|
|
library(randomForest) |
|
|
library(caret) |
|
|
|
|
|
data(vowel.train) |
|
|
data(vowel.test) |
|
|
|
|
|
# Set the variable y to be a factor variable in both the training and test set. Then set the seed to 33833. Fit (1) a random forest predictor relating the factor variable y to the remaining variables and (2) a boosted predictor using the "gbm" method. Fit these both with the train() command in the caret package. What are the accuracies for the two approaches on the test data set? What is the accuracy among the test set samples where the two methods agree? |
|
|
vowel.train$y <- as.factor(vowel.train$y) |
|
|
vowel.test$y <- as.factor(vowel.test$y) |
|
|
|
|
|
set.seed(33833) |
|
|
|
|
|
fit1 <- train(y ~., data=vowel.train, method='rf') |
|
|
fit2 <- train(y ~., data=vowel.train, method='gbm') |
|
|
|
|
|
results1 <- predict(fit1, newdata=vowel.test) |
|
|
results2 <- predict(fit2, newdata=vowel.test) |
|
|
|
|
|
combo <- data.frame(results1, results2, y = vowel.test$y) |
|
|
fit3 <- train(y ~ ., data = combo, method = "rf") |
|
|
results3 <- predict(fit3, newdata = vowel.test) |
|
|
|
|
|
c1 <- confusionMatrix(results1, vowel.test$y) |
|
|
c2 <- confusionMatrix(results2, vowel.test$y) |
|
|
c3 <- confusionMatrix(results3, combo$y) |
|
|
|
|
|
# Question 2. |
|
|
library(caret) |
|
|
library(gbm) |
|
|
|
|
|
set.seed(3433) |
|
|
library(AppliedPredictiveModeling) |
|
|
|
|
|
# Load data. |
|
|
data(AlzheimerDisease) |
|
|
adData = data.frame(diagnosis,predictors) |
|
|
inTrain = createDataPartition(adData$diagnosis, p = 3/4)[[1]] |
|
|
training = adData[ inTrain,] |
|
|
testing = adData[-inTrain,] |
|
|
|
|
|
# Set the seed to 62433 and predict diagnosis with all the other variables using a random forest ("rf"), boosted trees ("gbm") and linear discriminant analysis ("lda") model. Stack the predictions together using random forests ("rf"). What is the resulting accuracy on the test set? Is it better or worse than each of the individual predictions? |
|
|
set.seed(62433) |
|
|
|
|
|
# Train using 3 different models. |
|
|
fit1 <- train(diagnosis ~., data=training, method='rf') |
|
|
fit2 <- train(diagnosis ~., data=training, method='gbm') |
|
|
fit3 <- train(diagnosis ~., data=training, method='lda') |
|
|
|
|
|
# Run models on testing data. |
|
|
results1 <- predict(fit1, newdata=testing) |
|
|
results2 <- predict(fit2, newdata=testing) |
|
|
results3 <- predict(fit3, newdata=testing) |
|
|
|
|
|
# Stack models together and combine with random forests. |
|
|
combo <- data.frame(results1, results2, results3, diagnosis = testing$diagnosis) |
|
|
fit4 <- train(diagnosis ~ ., data = combo, method = "rf") |
|
|
|
|
|
# Run stacked model on testing data. |
|
|
results4 <- predict(fit4, newdata = testing) |
|
|
|
|
|
# random forests = 0.78 |
|
|
c1 <- confusionMatrix(results1, testing$diagnosis) |
|
|
# boosting = 0.80 |
|
|
c2 <- confusionMatrix(results2, testing$diagnosis) |
|
|
# lda = 0.77 |
|
|
c3 <- confusionMatrix(results3, testing$diagnosis) |
|
|
# Stacked models = 0.82 |
|
|
c4 <- confusionMatrix(results4, combo$diagnosis) |
|
|
|
|
|
# Question 3. |
|
|
set.seed(3523) |
|
|
library(AppliedPredictiveModeling) |
|
|
data(concrete) |
|
|
inTrain = createDataPartition(concrete$CompressiveStrength, p = 3/4)[[1]] |
|
|
training = concrete[ inTrain,] |
|
|
testing = concrete[-inTrain,] |
|
|
|
|
|
# Set the seed to 233 and fit a lasso model to predict Compressive Strength. Which variable is the last coefficient to be set to zero as the penalty increases? (Hint: it may be useful to look up ?plot.enet). |
|
|
set.seed(233) |
|
|
|
|
|
fit1 <- train(CompressiveStrength ~., data=training, method='lasso') |
|
|
|
|
|
plot.enet(fit1$finalModel, use.color=TRUE) |
|
|
|
|
|
# Question 4. |
|
|
# Fit a model using the bats() function in the forecast package to the training time series. Then forecast this model for the remaining time points. For how many of the testing points is the true value within the 95% prediction interval bounds? |
|
|
library(lubridate) # For year() function below |
|
|
library(forecast) |
|
|
library(quantmod) |
|
|
|
|
|
url <- 'https://d396qusza40orc.cloudfront.net/predmachlearn/gaData.csv' |
|
|
fileName <- basename(url); |
|
|
if (!file.exists(fileName)) { |
|
|
download.file(url, fileName, method='curl') |
|
|
} |
|
|
|
|
|
dat <- read.csv(fileName, na.strings=c('', 'NA')) |
|
|
training = dat[year(dat$date) < 2012,] |
|
|
testing = dat[(year(dat$date)) > 2011,] |
|
|
|
|
|
# Create a time-series. |
|
|
tstrain = ts(training$visitsTumblr) |
|
|
|
|
|
# Create a model using bats. |
|
|
fit <- bats(tstrain) |
|
|
|
|
|
# Count the length of the test set, so we can predict for this many points beyond the training data. |
|
|
start <- dim(testing)[1] |
|
|
|
|
|
# Create forecast model for the remaining points beyond training (up to the testing count), use a 95% prediction interval bound. |
|
|
fcast <- forecast(fit, level = 95, h = start) |
|
|
|
|
|
# Check accuracy. |
|
|
accuracy(fcast, testing$visitsTumblr) |
|
|
|
|
|
# For how many of the testing points is the true value within the 95% prediction interval bounds? |
|
|
result <- c() |
|
|
l <- length(fcast$lower) |
|
|
|
|
|
for (i in 1:l){ |
|
|
x <- testing$visitsTumblr[i] |
|
|
a <- fcast$lower[i] < x & x < fcast$upper[i] |
|
|
result <- c(result, a) |
|
|
} |
|
|
|
|
|
sum(result)/l * 100 |
|
|
|
|
|
# Question 5. |
|
|
set.seed(3523) |
|
|
library(AppliedPredictiveModeling) |
|
|
data(concrete) |
|
|
inTrain = createDataPartition(concrete$CompressiveStrength, p = 3/4)[[1]] |
|
|
training = concrete[ inTrain,] |
|
|
testing = concrete[-inTrain,] |
|
|
|
|
|
# Set the seed to 325 and fit a support vector machine using the e1071 package to predict Compressive Strength using the default settings. Predict on the testing set. What is the RMSE? |
|
|
set.seed(325) |
|
|
|
|
|
library(e1071) |
|
|
library(caret) |
|
|
|
|
|
# Train an svm. |
|
|
fit <- svm(CompressiveStrength ~ ., data=training) |
|
|
|
|
|
# Run svm on test set. |
|
|
results <- predict(fit, testing) |
|
|
|
|
|
# Check accuracy. |
|
|
accuracy(results, testing$CompressiveStrength) |
primaryobjects revised this gist