Terraform State Independence: Why Path-Based Isolation Beats Terraform Cloud for Complex Deployments

1# The classic mistake - one state for everything
2terraform {
3  backend "s3" {
4    bucket = "my-terraform-state"
5    key    = "terraform.tfstate"  # One file to rule them all
6    region = "ap-southeast-2"
7  }
8}

1# infrastructure/network/main.tf
2terraform {
3  backend "s3" {
4    bucket         = "company-terraform-state"
5    key            = "network/prod/terraform.tfstate"
6    region         = "ap-southeast-2"
7    encrypt        = true
8    dynamodb_table = "terraform-locks"
9  }
10}
11
12# infrastructure/compute/vm-01/main.tf
13terraform {
14  backend "s3" {
15    bucket         = "company-terraform-state"
16    key            = "compute/vm-01/terraform.tfstate"
17    region         = "ap-southeast-2"
18    encrypt        = true
19    dynamodb_table = "terraform-locks"
20  }
21}
22
23# infrastructure/compute/vm-02/main.tf
24terraform {
25  backend "s3" {
26    bucket         = "company-terraform-state"
27    key            = "compute/vm-02/terraform.tfstate"
28    region         = "ap-southeast-2"
29    encrypt        = true
30    dynamodb_table = "terraform-locks"
31  }
32}
33
34# infrastructure/database/prod/main.tf
35terraform {
36  backend "s3" {
37    bucket         = "company-terraform-state"
38    key            = "database/prod/terraform.tfstate"
39    region         = "ap-southeast-2"
40    encrypt        = true
41    dynamodb_table = "terraform-locks"
42  }
43}

1# infrastructure/network/main.tf
2terraform {
3  backend "azurerm" {
4    resource_group_name  = "rg-terraform-state"
5    storage_account_name = "stterraformstate"
6    container_name       = "tfstate"
7    key                  = "network/prod/terraform.tfstate"
8  }
9}
10
11# infrastructure/compute/vm-01/main.tf
12terraform {
13  backend "azurerm" {
14    resource_group_name  = "rg-terraform-state"
15    storage_account_name = "stterraformstate"
16    container_name       = "tfstate"
17    key                  = "compute/vm-01/terraform.tfstate"
18  }
19}

1s3://company-terraform-state/
2├── network/
3│   ├── prod/terraform.tfstate
4│   ├── staging/terraform.tfstate
5│   └── dev/terraform.tfstate
6├── compute/
7│   ├── vm-01/terraform.tfstate
8│   ├── vm-02/terraform.tfstate
9│   ├── vm-03/terraform.tfstate
10│   └── vm-web-cluster/terraform.tfstate
11├── database/
12│   ├── prod/terraform.tfstate
13│   ├── staging/terraform.tfstate
14│   └── dev/terraform.tfstate
15├── dns/
16│   └── terraform.tfstate
17└── cdn/
18    └── terraform.tfstate

1infrastructure/
2├── _modules/                    # Shared modules (no state)
3│   ├── vpc/
4│   ├── ec2/
5│   └── rds/
6├── network/
7│   ├── prod/
8│   │   ├── main.tf
9│   │   ├── variables.tf
10│   │   ├── outputs.tf
11│   │   └── backend.tf          # key = "network/prod/terraform.tfstate"
12│   ├── staging/
13│   │   └── ...                 # key = "network/staging/terraform.tfstate"
14│   └── dev/
15│       └── ...                 # key = "network/dev/terraform.tfstate"
16├── compute/
17│   ├── vm-web-01/
18│   │   └── ...                 # key = "compute/vm-web-01/terraform.tfstate"
19│   ├── vm-web-02/
20│   │   └── ...                 # key = "compute/vm-web-02/terraform.tfstate"
21│   ├── vm-api-01/
22│   │   └── ...                 # key = "compute/vm-api-01/terraform.tfstate"
23│   └── eks-prod/
24│       └── ...                 # key = "compute/eks-prod/terraform.tfstate"
25├── database/
26│   ├── prod/
27│   │   ├── postgres-primary/
28│   │   │   └── ...             # key = "database/prod/postgres-primary/terraform.tfstate"
29│   │   └── postgres-replica/
30│   │       └── ...             # key = "database/prod/postgres-replica/terraform.tfstate"
31│   └── staging/
32│       └── postgres/
33│           └── ...             # key = "database/staging/postgres/terraform.tfstate"
34└── shared/
35    ├── dns/
36    │   └── ...                 # key = "shared/dns/terraform.tfstate"
37    └── iam/
38        └── ...                 # key = "shared/iam/terraform.tfstate"

1# backend.tf (in each component)
2terraform {
3  backend "s3" {
4    # Partial configuration - key injected via CLI
5    bucket         = "company-terraform-state"
6    region         = "ap-southeast-2"
7    encrypt        = true
8    dynamodb_table = "terraform-locks"
9  }
10}

1# Initialize with dynamic key
2terraform init -backend-config="key=compute/vm-01/terraform.tfstate"

1#!/bin/bash
2# deploy.sh - Reusable deployment script
3
4COMPONENT="${1:-compute/vm-01}"  # Default or passed as argument
5ENVIRONMENT="${2:-prod}"
6
7# Build the state key dynamically
8STATE_KEY="${COMPONENT}/${ENVIRONMENT}/terraform.tfstate"
9
10terraform init \
11  -backend-config="key=${STATE_KEY}" \
12  -reconfigure
13
14terraform apply -auto-approve

1# Deploy different components with same script
2./deploy.sh compute/vm-01 prod    # key = compute/vm-01/prod/terraform.tfstate
3./deploy.sh compute/vm-02 prod    # key = compute/vm-02/prod/terraform.tfstate
4./deploy.sh database/postgres dev # key = database/postgres/dev/terraform.tfstate

1# backends/prod.hcl
2bucket         = "company-terraform-state-prod"
3region         = "ap-southeast-2"
4encrypt        = true
5dynamodb_table = "terraform-locks-prod"

1# backends/staging.hcl
2bucket         = "company-terraform-state-staging"
3region         = "ap-southeast-2"
4encrypt        = true
5dynamodb_table = "terraform-locks-staging"

1# Initialize for prod with dynamic key
2terraform init \
3  -backend-config=backends/prod.hcl \
4  -backend-config="key=compute/vm-01/terraform.tfstate"
5
6# Initialize for staging with same key structure
7terraform init \
8  -backend-config=backends/staging.hcl \
9  -backend-config="key=compute/vm-01/terraform.tfstate"

1# .github/workflows/terraform.yml
2env:
3  TF_STATE_BUCKET: company-terraform-state
4  TF_STATE_REGION: ap-southeast-2
5
6jobs:
7  deploy:
8    runs-on: ubuntu-latest
9    strategy:
10      matrix:
11        component: [compute/vm-01, compute/vm-02, database/prod]
12    steps:
13      - uses: actions/checkout@v4
14
15      - name: Terraform Init with Dynamic Key
16        run: |
17          terraform init \
18            -backend-config="bucket=${{ env.TF_STATE_BUCKET }}" \
19            -backend-config="region=${{ env.TF_STATE_REGION }}" \
20            -backend-config="key=${{ matrix.component }}/terraform.tfstate" \
21            -backend-config="encrypt=true"
22
23      - name: Terraform Apply
24        run: terraform apply -auto-approve

1# backend.tf
2terraform {
3  backend "azurerm" {
4    resource_group_name  = "rg-terraform-state"
5    storage_account_name = "stterraformstate"
6    container_name       = "tfstate"
7    # key injected via -backend-config
8  }
9}

1# Dynamic key injection for Azure
2terraform init -backend-config="key=compute/vm-01/terraform.tfstate"

1# compute/vm-01/main.tf
2
3# Read VPC outputs from network state
4data "terraform_remote_state" "network" {
5  backend = "s3"
6  config = {
7    bucket = "company-terraform-state"
8    key    = "network/prod/terraform.tfstate"
9    region = "ap-southeast-2"
10  }
11}
12
13# Use network outputs
14resource "aws_instance" "vm" {
15  ami           = var.ami_id
16  instance_type = var.instance_type
17
18  subnet_id              = data.terraform_remote_state.network.outputs.private_subnet_ids[0]
19  vpc_security_group_ids = [data.terraform_remote_state.network.outputs.default_sg_id]
20
21  tags = {
22    Name = "vm-01"
23  }
24}

1# network/prod/outputs.tf
2
3output "vpc_id" {
4  description = "VPC ID for compute and database resources"
5  value       = aws_vpc.main.id
6}
7
8output "private_subnet_ids" {
9  description = "Private subnet IDs for internal resources"
10  value       = aws_subnet.private[*].id
11}
12
13output "public_subnet_ids" {
14  description = "Public subnet IDs for load balancers"
15  value       = aws_subnet.public[*].id
16}
17
18output "default_sg_id" {
19  description = "Default security group allowing internal traffic"
20  value       = aws_security_group.default.id
21}
22
23output "nat_gateway_ips" {
24  description = "NAT Gateway public IPs for allowlisting"
25  value       = aws_nat_gateway.main[*].public_ip
26}

1# terragrunt.hcl (root)
2remote_state {
3  backend = "s3"
4  generate = {
5    path      = "backend.tf"
6    if_exists = "overwrite_terragrunt"
7  }
8  config = {
9    bucket         = "company-terraform-state"
10    key            = "${path_relative_to_include()}/terraform.tfstate"
11    region         = "ap-southeast-2"
12    encrypt        = true
13    dynamodb_table = "terraform-locks"
14  }
15}

1# infrastructure/compute/vm-01/terragrunt.hcl
2include "root" {
3  path = find_in_parent_folders()
4}
5
6# State will be: compute/vm-01/terraform.tfstate
7terraform {
8  source = "../../../_modules/ec2"
9}
10
11inputs = {
12  instance_name = "vm-01"
13  instance_type = "t3.medium"
14}
15
16dependency "network" {
17  config_path = "../../network/prod"
18}
19
20inputs = {
21  vpc_id    = dependency.network.outputs.vpc_id
22  subnet_id = dependency.network.outputs.private_subnet_ids[0]
23}

1# Deploy all components in dependency order
2terragrunt run-all apply
3
4# Plan across all components
5terragrunt run-all plan
6
7# Destroy in reverse dependency order
8terragrunt run-all destroy

1terraform {
2  backend "s3" {
3    bucket       = "company-terraform-state"
4    key          = "compute/vm-01/terraform.tfstate"
5    region       = "ap-southeast-2"
6    encrypt      = true
7    use_lockfile = true  # Native S3 locking (Terraform 1.10+)
8  }
9}

1# .github/workflows/terraform.yml
2name: Terraform Deploy
3
4on:
5  push:
6    paths:
7      - 'infrastructure/**'
8
9jobs:
10  detect-changes:
11    runs-on: ubuntu-latest
12    outputs:
13      matrix: ${{ steps.changes.outputs.matrix }}
14    steps:
15      - uses: actions/checkout@v4
16      - id: changes
17        run: |
18          # Detect which components changed
19          changed_dirs=$(git diff --name-only HEAD~1 | grep '^infrastructure/' | cut -d'/' -f1-3 | sort -u)
20          matrix=$(echo "$changed_dirs" | jq -R -s -c 'split("\n") | map(select(length > 0))')
21          echo "matrix=$matrix" >> $GITHUB_OUTPUT
22
23  terraform:
24    needs: detect-changes
25    runs-on: ubuntu-latest
26    strategy:
27      matrix:
28        component: ${{ fromJson(needs.detect-changes.outputs.matrix) }}
29      fail-fast: false  # Don't fail all if one component fails
30    steps:
31      - uses: actions/checkout@v4
32
33      - name: Setup Terraform
34        uses: hashicorp/setup-terraform@v3
35
36      - name: Terraform Init
37        working-directory: ${{ matrix.component }}
38        run: terraform init
39
40      - name: Terraform Plan
41        working-directory: ${{ matrix.component }}
42        run: terraform plan -out=tfplan
43
44      - name: Terraform Apply
45        if: github.ref == 'refs/heads/main'
46        working-directory: ${{ matrix.component }}
47        run: terraform apply -auto-approve tfplan