Cell Operations & Deployment Strategy

# deployment-pipeline.yaml (Step Functions or similar)
states:
  DeployToCanary:
    cell: canary-us-internal
    action: deploy
    next: ValidateCanary

  ValidateCanary:
    checks:
      - error_rate < 0.1%
      - p99_latency < 500ms
      - no_pagerduty_alerts: 30m
    on_failure: RollbackCanary
    on_success: DeployToSMBWave1

  DeployToSMBWave1:
    cells: [smb-us-1, smb-us-2, smb-eu-1]  # 25% of SMB
    parallel: true
    next: ValidateSMBWave1

  ValidateSMBWave1:
    bake_time: 4h
    checks: [error_rate, latency, customer_complaints]
    on_failure: RollbackSMBWave1
    on_success: DeployToSMBWave2

  # ... continues through all waves

  DeployToEnterprise:
    cells: [enterprise-us-hipaa]
    requires_approval: true  # Human gate for enterprise
    notification: slack://enterprise-releases

// Feature flags let you deploy code without activating it
const features = {
  newVoiceCodec: {
    enabled: false,           // Deployed everywhere, activated nowhere
    rolloutPercentage: 0,
    allowlist: ['enterprise-customer-123']  // Beta testers
  }
};

// Gradual activation AFTER deployment is complete
// Day 1: Deploy to all cells (feature off)
// Day 2: Enable for 5% of traffic
// Day 3: Enable for 25%
// Day 7: Enable for 100%

infrastructure-repo/
├── modules/
│   └── cell/                    # Shared cell template
│       ├── sms-service.tf
│       ├── voice-service.tf
│       ├── video-service.tf
│       └── variables.tf
│
├── cells/
│   ├── enterprise-us-hipaa/
│   │   └── terragrunt.hcl       # Just variables, includes module
│   ├── enterprise-eu-standard/
│   │   └── terragrunt.hcl
│   ├── smb-us-standard/
│   │   └── terragrunt.hcl
│   └── ...
│
└── argocd/
    └── applications/            # One ArgoCD app per cell

# Datadog dashboard hierarchy
Global View:
  - All cells health heatmap (green/yellow/red)
  - Cross-cell comparison: latency, error rates, CPU
  - Deployment status: which cells on which version

Cell View (drill down):
  - Per-service metrics within cell
  - Customer impact (affected accounts)
  - Recent deployments and config changes

Service View (drill down further):
  - Individual pod/container metrics
  - Traces, logs, spans

// Cells are version-agnostic - they run ALL supported versions
// The edge determines which version handler to invoke

// In Landing Zone API Gateway:
const routes = {
  'POST /v1/Messages': 'sms-service:handleV1Message',
  'POST /v2/Messages': 'sms-service:handleV2Message',  // Breaking change
  'POST /v1/Calls':    'voice-service:handleV1Call',
  'POST /v2/Calls':    'voice-service:handleV2Call',
};

// Inside each cell - BOTH handlers exist simultaneously
class SmsService {
  handleV1Message(req) {
    // Old contract: { To, From, Body }
    return this.sendMessage(req.To, req.From, req.Body);
  }

  handleV2Message(req) {
    // New contract: { recipient, sender, content, metadata }
    return this.sendMessage(req.recipient, req.sender, req.content);
  }

  // Shared internal logic - only contract differs
  sendMessage(to, from, body) { /* ... */ }
}

# api-gateway-config.yaml (AWS API Gateway / Kong / Envoy)
routes:
  - path: /v1/Messages
    status: active
    backend: ${cell_router}/sms/v1/messages
    deprecation_date: null

  - path: /v2/Messages
    status: active
    backend: ${cell_router}/sms/v2/messages
    introduced: 2025-01-15

  - path: /v0/Messages
    status: deprecated
    backend: ${cell_router}/sms/v1/messages  # Internally aliased
    sunset_date: 2025-06-01
    response_headers:
      Deprecation: "true"
      Sunset: "Sat, 01 Jun 2025 00:00:00 GMT"

// Control Plane tracks cell capabilities
const cellMetadata = {
  'enterprise-us-hipaa': {
    versions: {
      sms: ['v1', 'v2'],
      voice: ['v1', 'v2'],
      video: ['v1']  // v2 not yet deployed to this cell
    },
    lastDeployed: '2025-01-15T10:30:00Z'
  }
};

// During v2 rollout, Control Plane can:
// 1. Track which cells have v2 code
// 2. Gate v2 enablement until all cells ready
// 3. Provide rollout status dashboard

// During migration window, code must be "schema-bilingual"
class MessageRepository {
  async getMessage(id) {
    const row = await db.query('SELECT * FROM messages WHERE id = ?', [id]);

    // Handle both old and new schema
    return {
      id: row.id,
      // New column with fallback to old
      recipientId: row.recipient_id || row.to_phone,
      senderId: row.sender_id || row.from_phone,
      // ... rest of fields
    };
  }

  async createMessage(msg) {
    // Write to BOTH during migration (dual-write)
    await db.query(`
      INSERT INTO messages (id, to_phone, from_phone, recipient_id, sender_id, body)
      VALUES (?, ?, ?, ?, ?, ?)
    `, [msg.id, msg.recipientId, msg.senderId, msg.recipientId, msg.senderId, msg.body]);
    //         ↑ old columns        ↑ new columns (same values)
  }
}

# migration-manifest.yaml
migration:
  id: "2025-01-15-recipient-id-column"
  description: "Rename to_phone → recipient_id"

  phases:
    - name: expand
      type: schema
      script: migrations/001_add_recipient_id.sql
      rollout:
        - cells: [canary-internal]
          wait: 1h
        - cells: [smb-us-1, smb-us-2]
          wait: 4h
        - cells: [smb-*]
          wait: 24h
        - cells: [enterprise-*]
          wait: 48h
          requires_approval: true

    - name: migrate-code
      type: deployment
      version: "v2.5.0"  # Code with dual-write
      rollout: standard  # Normal wave-based deployment

    - name: backfill
      type: data
      script: migrations/001_backfill_recipient_id.py
      parallel_cells: 3  # Max 3 cells backfilling at once
      checkpoint_interval: 10000  # Rows between checkpoints

    - name: migrate-code-readonly
      type: deployment
      version: "v2.6.0"  # Code reads only new column
      rollout: standard

    - name: contract
      type: schema
      script: migrations/001_drop_to_phone.sql
      rollout:
        - cells: [canary-internal]
          wait: 24h
        - cells: [smb-*]
          wait: 48h
        - cells: [enterprise-*]
          wait: 168h  # 1 week bake time
          requires_approval: true

# pt-online-schema-change (Percona) - for MySQL
# Creates shadow table, syncs with triggers, atomic swap
pt-online-schema-change \
  --alter "ADD COLUMN recipient_id VARCHAR(50)" \
  --execute \
  D=twilio,t=messages

# gh-ost (GitHub) - for MySQL
# Binlog-based replication, no triggers
gh-ost \
  --alter="ADD COLUMN recipient_id VARCHAR(50)" \
  --database=twilio \
  --table=messages \
  --execute

# For PostgreSQL - native online DDL is better
# But for large tables, use pg_repack or similar

// Control Plane tracks migration state per cell
const migrationState = {
  'migration-2025-01-15-recipient-id': {
    status: 'in_progress',
    currentPhase: 'backfill',
    cells: {
      'canary-internal':     { phase: 'contract', status: 'complete' },
      'smb-us-1':            { phase: 'backfill', status: 'complete', rows: 1500000 },
      'smb-us-2':            { phase: 'backfill', status: 'running', rows: 850000, total: 2000000 },
      'smb-eu-1':            { phase: 'migrate-code', status: 'complete' },
      'enterprise-us-hipaa': { phase: 'expand', status: 'pending' },
    },
    canRollback: true,  // Until contract phase
    startedAt: '2025-01-15T10:00:00Z',
  }
};

// Dashboard shows:
// ████████████░░░░░░░░ 60% complete (12/20 cells)
// Current: Backfilling smb-us-2 (42% of rows)
// Next: smb-eu-2 (waiting for smb-us-2)