Terraform for MLOps Infrastructure Guide
=========================================

## Overview
Terraform is an open-source Infrastructure as Code (IaC) tool by HashiCorp that enables managing cloud infrastructure through declarative configuration files. For MLOps, Terraform provisions and manages compute clusters, storage, networking, and ML platform services consistently and reproducibly.

## Core Terraform Concepts

### HCL (HashiCorp Configuration Language) Syntax
```hcl
# Variables
variable "environment" {
  description = "Deployment environment"
  type        = string
  default     = "dev"
  validation {
    condition     = contains(["dev", "staging", "prod"], var.environment)
    error_message = "Environment must be dev, staging, or prod."
  }
}

variable "ml_instance_type" {
  description = "EC2 instance type for ML workloads"
  type        = string
  default     = "ml.p3.2xlarge"
}

# Local values
locals {
  common_tags = {
    Project     = "MLPlatform"
    Environment = var.environment
    ManagedBy   = "Terraform"
    CostCenter  = "ML-Team"
  }
  resource_prefix = "${var.project_name}-${var.environment}"
}

# Data sources
data "aws_caller_identity" "current" {}
data "aws_region" "current" {}
```

### Resources and Providers
```hcl
terraform {
  required_version = ">= 1.5.0"
  required_providers {
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.0"
    }
    kubernetes = {
      source  = "hashicorp/kubernetes"
      version = "~> 2.20"
    }
    helm = {
      source  = "hashicorp/helm"
      version = "~> 2.10"
    }
  }
}

provider "aws" {
  region = "us-east-1"
  default_tags {
    tags = local.common_tags
  }
}
```

## Remote State Management

### S3 Backend with DynamoDB Locking
```hcl
# backend.tf
terraform {
  backend "s3" {
    bucket         = "my-terraform-state-bucket"
    key            = "mlops/terraform.tfstate"
    region         = "us-east-1"
    encrypt        = true
    kms_key_id     = "arn:aws:kms:us-east-1:123456:key/my-key"
    dynamodb_table = "terraform-state-lock"
  }
}

# Create the S3 bucket (one-time bootstrap)
resource "aws_s3_bucket" "terraform_state" {
  bucket = "my-terraform-state-bucket"
}

resource "aws_s3_bucket_versioning" "terraform_state" {
  bucket = aws_s3_bucket.terraform_state.id
  versioning_configuration {
    status = "Enabled"
  }
}

resource "aws_dynamodb_table" "terraform_lock" {
  name         = "terraform-state-lock"
  billing_mode = "PAY_PER_REQUEST"
  hash_key     = "LockID"

  attribute {
    name = "LockID"
    type = "S"
  }
}
```

## Terraform Modules for MLOps

### VPC Module for ML Workloads
```hcl
# modules/ml-vpc/main.tf
module "vpc" {
  source = "terraform-aws-modules/vpc/aws"
  version = "5.0.0"

  name = "${var.name}-vpc"
  cidr = var.vpc_cidr

  azs             = data.aws_availability_zones.available.names
  private_subnets = var.private_subnet_cidrs
  public_subnets  = var.public_subnet_cidrs
  intra_subnets   = var.intra_subnet_cidrs  # For EKS node groups

  enable_nat_gateway     = true
  single_nat_gateway     = var.environment != "prod"
  enable_dns_hostnames   = true
  enable_dns_support     = true

  # S3 VPC endpoint for fast data access
  enable_s3_endpoint = true

  # SageMaker VPC endpoints
  enable_sagemaker_api_endpoint      = true
  enable_sagemaker_runtime_endpoint  = true

  private_subnet_tags = {
    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
    "kubernetes.io/role/internal-elb"           = "1"
  }

  public_subnet_tags = {
    "kubernetes.io/cluster/${var.cluster_name}" = "shared"
    "kubernetes.io/role/elb"                    = "1"
  }
}
```

### EKS Cluster for ML Workloads
```hcl
# modules/eks-ml/main.tf
module "eks" {
  source  = "terraform-aws-modules/eks/aws"
  version = "~> 19.0"

  cluster_name    = var.cluster_name
  cluster_version = "1.28"

  vpc_id                         = var.vpc_id
  subnet_ids                     = var.private_subnet_ids
  cluster_endpoint_public_access = false  # Private cluster

  eks_managed_node_groups = {
    # General ML workloads
    general = {
      name           = "general"
      instance_types = ["m5.xlarge", "m5.2xlarge"]
      min_size       = 2
      max_size       = 10
      desired_size   = 3
      disk_size      = 100
    }

    # GPU nodes for training
    gpu_training = {
      name           = "gpu-training"
      instance_types = ["p3.2xlarge", "p3.8xlarge"]
      min_size       = 0
      max_size       = 4
      desired_size   = 0
      disk_size      = 200
      capacity_type  = "SPOT"  # Use spot for training

      taints = [{
        key    = "nvidia.com/gpu"
        value  = "true"
        effect = "NO_SCHEDULE"
      }]

      labels = {
        workload_type = "gpu-training"
      }
    }

    # GPU nodes for inference
    gpu_inference = {
      name           = "gpu-inference"
      instance_types = ["g4dn.xlarge", "g4dn.2xlarge"]
      min_size       = 1
      max_size       = 8
      desired_size   = 2
      disk_size      = 100
      capacity_type  = "ON_DEMAND"  # On-demand for serving reliability
    }
  }

  cluster_addons = {
    coredns    = { most_recent = true }
    kube-proxy = { most_recent = true }
    vpc-cni    = { most_recent = true }
    aws-ebs-csi-driver = { most_recent = true }
    aws-efs-csi-driver = { most_recent = true }
  }
}
```

### S3 Buckets for ML Data Lake
```hcl
# modules/ml-storage/main.tf
resource "aws_s3_bucket" "ml_data_lake" {
  for_each = {
    raw       = "Raw ingested data"
    processed = "Processed features"
    models    = "Trained model artifacts"
    outputs   = "Model inference outputs"
  }

  bucket = "${local.resource_prefix}-${each.key}-${data.aws_caller_identity.current.account_id}"
}

resource "aws_s3_bucket_versioning" "ml_data_lake" {
  for_each = aws_s3_bucket.ml_data_lake

  bucket = each.value.id
  versioning_configuration {
    status = "Enabled"
  }
}

resource "aws_s3_bucket_lifecycle_configuration" "ml_data_lake" {
  bucket = aws_s3_bucket.ml_data_lake["raw"].id

  rule {
    id     = "transition-to-ia"
    status = "Enabled"
    transition {
      days          = 90
      storage_class = "STANDARD_IA"
    }
    transition {
      days          = 180
      storage_class = "GLACIER"
    }
  }
}
```

### SageMaker Infrastructure
```hcl
# SageMaker Domain for Studio
resource "aws_sagemaker_domain" "ml_domain" {
  domain_name = "${local.resource_prefix}-studio"
  auth_mode   = "SSO"
  vpc_id      = module.vpc.vpc_id
  subnet_ids  = module.vpc.private_subnets

  default_user_settings {
    execution_role = aws_iam_role.sagemaker_execution_role.arn

    jupyter_server_app_settings {
      default_resource_spec {
        instance_type       = "system"
        sagemaker_image_arn = data.aws_sagemaker_prebuilt_ecr_image.jupyterserver.registry_path
      }
    }

    kernel_gateway_app_settings {
      default_resource_spec {
        instance_type = "ml.t3.medium"
      }
    }
  }

  kms_key_id = aws_kms_key.sagemaker_kms.arn
}

# SageMaker Model Registry
resource "aws_sagemaker_model_package_group" "ml_models" {
  model_package_group_name        = "${local.resource_prefix}-models"
  model_package_group_description = "Production ML model registry"
}
```

## Terraform Workspaces for Multi-Environment

```bash
# Create workspaces for different environments
terraform workspace new dev
terraform workspace new staging
terraform workspace new prod

# Switch between workspaces
terraform workspace select staging
terraform plan

# Use workspace in configuration
resource "aws_instance" "ml_server" {
  instance_type = terraform.workspace == "prod" ? "ml.p3.8xlarge" : "ml.t3.medium"

  tags = {
    Environment = terraform.workspace
  }
}
```

## Terragrunt for DRY Configuration

```hcl
# terragrunt.hcl (root)
remote_state {
  backend = "s3"
  config = {
    bucket         = "my-terraform-state"
    key            = "${path_relative_to_include()}/terraform.tfstate"
    region         = "us-east-1"
    encrypt        = true
    dynamodb_table = "terraform-state-lock"
  }
}

inputs = {
  project_name = "mlops-platform"
  aws_region   = "us-east-1"
}

# environments/prod/eks/terragrunt.hcl
include "root" {
  path = find_in_parent_folders()
}

terraform {
  source = "../../../modules//eks-ml"
}

inputs = {
  environment  = "prod"
  cluster_name = "mlops-prod"
  min_size     = 3
  max_size     = 20
}
```

## Terraform for Monitoring Infrastructure

```hcl
# Prometheus and Grafana on EKS via Helm
resource "helm_release" "prometheus_stack" {
  name       = "kube-prometheus-stack"
  repository = "https://prometheus-community.github.io/helm-charts"
  chart      = "kube-prometheus-stack"
  namespace  = "monitoring"

  create_namespace = true

  values = [
    yamlencode({
      prometheus = {
        prometheusSpec = {
          retention       = "30d"
          storageSpec = {
            volumeClaimTemplate = {
              spec = {
                accessModes = ["ReadWriteOnce"]
                resources = {
                  requests = { storage = "50Gi" }
                }
              }
            }
          }
        }
      }
      grafana = {
        persistence = { enabled = true, size = "10Gi" }
        adminPassword = var.grafana_admin_password
      }
    })
  ]
}
```

## Outputs and Data Sources

```hcl
# outputs.tf
output "eks_cluster_endpoint" {
  description = "EKS cluster API server endpoint"
  value       = module.eks.cluster_endpoint
  sensitive   = false
}

output "ml_data_lake_bucket_arns" {
  description = "ARNs of ML data lake S3 buckets"
  value       = { for k, v in aws_s3_bucket.ml_data_lake : k => v.arn }
}

output "sagemaker_execution_role_arn" {
  description = "IAM role ARN for SageMaker executions"
  value       = aws_iam_role.sagemaker_execution_role.arn
}
```

## Best Practices

1. Store Terraform state remotely (S3 + DynamoDB) — never commit state files to Git
2. Use workspaces or directory-based separation for environment isolation
3. Lock provider versions with `~>` constraints to prevent unexpected upgrades
4. Use modules to avoid code repetition across environments
5. Tag all resources with project, environment, and cost center for attribution
6. Enable versioning on S3 state bucket and encrypt with KMS
7. Use Terragrunt for DRY multi-environment setups at scale
8. Run `terraform plan` in CI before every `apply`
9. Use `terraform import` to bring existing resources under IaC management
10. Implement RBAC in your CI system — `plan` permissions are separate from `apply` permissions