Decision Trees

R Code for Data Generation and Analysis

# Generate synthetic data with complex relationships
set.seed(123)
library(dplyr)

n <- 1000

# Continuous features
x1 <- runif(n, -5, 5)
x2 <- rnorm(n, mean=0, sd=3)

# Categorical feature
x3 <- factor(sample(c("A","B","C"), n, replace=TRUE, prob=c(0.3,0.5,0.2)))

# Binary feature
x4 <- rbinom(n, 1, plogis(0.5*x1))

# Create non-linear relationships for regression
# Piecewise linear with interaction effects
y_reg <- ifelse(x1 < 0, 
               2 + 0.8*x1 - 1.2*x2,
               4 - 0.5*x1 + 0.3*x1*x2) +
  ifelse(x3 == "A", 1.5, ifelse(x3 == "B", -0.5, 0)) +
  rnorm(n, sd=1)

# Create classification outcome with complex decision boundaries
y_class <- factor(ifelse(
  (x1 > -2 & x1 < 3 & x2 < 1) | 
  (x3 %in% c("A","C") & x4 == 1 & x2 > -2),
  "Class1", "Class2"))

# Combine into data frames
df <- data.frame(y_reg, y_class, x1, x2, x3, x4)

# Split into train and test
set.seed(456)
train_idx <- sample(1:n, 0.7*n)
train_data <- df[train_idx, ]
test_data <- df[-train_idx, ]

# Visualize relationships
par(mfrow=c(2,2))
plot(x1, y_reg, main="Regression Outcome vs X1")
boxplot(y_reg ~ x3, main="Regression Outcome by X3")
plot(jitter(as.numeric(y_class)) ~ x1, col=as.numeric(y_class), 
     main="Classification Outcome vs X1")
plot(x2, x1, col=as.numeric(y_class), pch=19, 
     main="Classification Decision Boundary")
par(mfrow=c(1,1))

# Fit and evaluate regression tree
library(rpart)
reg_tree <- rpart(y_reg ~ x1 + x2 + x3 + x4, data=train_data, method="anova",
                 control=rpart.control(cp=0.01, maxdepth=4))

# Visualize tree
library(rpart.plot)
rpart.plot(reg_tree, main="Regression Tree")

# Evaluate performance
pred_reg <- predict(reg_tree, test_data)
rmse <- sqrt(mean((test_data$y_reg - pred_reg)^2))
r_squared <- 1 - sum((test_data$y_reg - pred_reg)^2) / 
              sum((test_data$y_reg - mean(test_data$y_reg))^2)

# Fit and evaluate classification tree
class_tree <- rpart(y_class ~ x1 + x2 + x3 + x4, data=train_data, method="class",
                   control=rpart.control(cp=0.01, maxdepth=4))

# Visualize tree
rpart.plot(class_tree, main="Classification Tree")

# Evaluate performance
pred_class <- predict(class_tree, test_data, type="class")
conf_matrix <- table(Predicted=pred_class, Actual=test_data$y_class)
accuracy <- sum(diag(conf_matrix))/sum(conf_matrix)

# Print performance metrics
cat("Regression Performance:\n")
cat("RMSE:", rmse, "\n")
cat("R-squared:", r_squared, "\n\n")

cat("Classification Performance:\n")
print(conf_matrix)
cat("Accuracy:", accuracy, "\n")

# Variable importance
cat("\nRegression Tree Variable Importance:\n")
print(reg_tree$variable.importance)

cat("\nClassification Tree Variable Importance:\n")
print(class_tree$variable.importance)