🏢 DataCenter NVMe Storage Architecture

Transport Protocols

• RDMA (InfiniBand/RoCE): Ultra-low latency, CPU offload, 100/200/400 Gbps
• TCP: Standard Ethernet, software implementation, 10/25/40/100 Gbps
• Fibre Channel (FC-NVMe): Enterprise SAN integration, 32/64/128 Gbps

Fabric Discovery & Management

# NVMe Discovery Controller Configuration
nvme discover -t rdma -a 192.168.100.100 -s 4420

# Subsystem Connection
nvme connect -t rdma -n nqn.1988-11.com.dell:PowerScale-cluster1 \
             -a 192.168.100.100 -s 4420

# Multipath Configuration for HA
echo "device {
    vendor \"NVME\"
    product \"Dell PowerScale\"
    path_grouping_policy multibus
    path_selector \"round-robin 0\"
    failback immediate
}" >> /etc/multipath.conf
                    

Kubernetes CSI (Container Storage Interface)

# StorageClass for High-Performance NVMe
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: nvme-ultra-fast
provisioner: nvme.csi.dell.com
parameters:
  # Performance tier selection
  storagePool: "tier0-nvme-pcie5"
  # QoS constraints
  minIOPS: "100000"
  maxIOPS: "1000000"
  # Security requirements
  encryptionEnabled: "true"
  # Placement constraints
  rackAffinity: "high-performance"
allowVolumeExpansion: true
volumeBindingMode: WaitForFirstConsumer
                    

Persistent Volume Claim Example

apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  name: database-storage
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 1Ti
  storageClassName: nvme-ultra-fast
  # Performance annotations
  annotations:
    volume.beta.kubernetes.io/storage-provisioner: nvme.csi.dell.com
    # Security annotations
    volume.beta.kubernetes.io/mount-options: "rsize=1048576,wsize=1048576,hard,timeo=600"
                    

SMART Monitoring & Alerts

# SMART attribute monitoring
smartctl -a /dev/nvme0n1

# Key NVMe SMART attributes to monitor:
# - Critical Warning (0x01)
# - Temperature (0x02)
# - Available Spare (0x03)
# - Available Spare Threshold (0x04)
# - Percentage Used (0x05)
# - Data Units Read/Written (0x06/0x07)
# - Host Read/Write Commands (0x08/0x09)
# - Power Cycles (0x0C)
# - Unsafe Shutdowns (0x0E)

# Automated health checking
#!/bin/bash
for drive in /dev/nvme*n1; do
    temp=$(nvme smart-log $drive | grep "^temperature" | awk '{print $3}')
    spare=$(nvme smart-log $drive | grep "^available_spare" | awk '{print $3}')

    if [ $temp -gt 70 ]; then
        echo "WARNING: $drive temperature ${temp}°C exceeds threshold"
    fi

    if [ $spare -lt 10 ]; then
        echo "CRITICAL: $drive available spare ${spare}% below threshold"
    fi
done
                    

Drive Provisioning Workflow

Ansible Storage Automation

---
# NVMe storage cluster configuration
- name: Configure NVMe storage cluster
  hosts: storage_nodes
  vars:
    nvme_devices:
      - device: /dev/nvme0n1
        tier: tier0
        capacity: 7.68TB
        encryption: enabled
      - device: /dev/nvme1n1
        tier: tier1
        capacity: 15.36TB
        encryption: enabled

  tasks:
    - name: Initialize NVMe devices
      shell: |
        # Format with 4K sectors for optimal performance
        nvme format {{ item.device }} --ses=1 --lbaf=1

        # Enable TCG Opal security if supported
        sedutil-cli --initialSetup {{ ansible_vault_opal_password }} {{ item.device }}

      loop: "{{ nvme_devices }}"

    - name: Configure multipath
      template:
        src: multipath.conf.j2
        dest: /etc/multipath.conf
      notify: restart multipath

    - name: Setup monitoring
      copy:
        content: |
          #!/bin/bash
          # NVMe health monitoring script
          for device in {{ nvme_devices | map(attribute='device') | join(' ') }}; do
            nvme smart-log $device | logger -t nvme-health
          done
        dest: /usr/local/bin/nvme-health-check.sh
        mode: '0755'

    - name: Configure cron for health monitoring
      cron:
        name: "NVMe health monitoring"
        minute: "*/5"
        job: "/usr/local/bin/nvme-health-check.sh"
                    

Docker Volume Plugin Configuration

# Docker daemon configuration for NVMe storage
{
  "storage-driver": "overlay2",
  "storage-opts": [
    "overlay2.override_kernel_check=true"
  ],
  "data-root": "/data/docker",
  "log-driver": "json-file",
  "log-opts": {
    "max-size": "100m",
    "max-file": "3"
  },
  # Direct NVMe device mounting for performance
  "graph": "/nvme-fast/docker"
}

# High-performance container deployment
docker run -d \
  --name database-server \
  --mount type=bind,source=/nvme-tier0/db,target=/var/lib/mysql \
  --mount type=tmpfs,destination=/tmp,tmpfs-mode=1777,tmpfs-size=1G \
  --cpuset-cpus="0-7" \
  --memory=32g \
  --restart=unless-stopped \
  mysql:8.0
                    

Prometheus NVMe Exporter Configuration

# prometheus.yml configuration
global:
  scrape_interval: 15s

scrape_configs:
  - job_name: 'nvme-exporter'
    static_configs:
      - targets: ['storage-node1:9101', 'storage-node2:9101']
    scrape_interval: 30s
    metrics_path: /metrics

# Key NVMe metrics to monitor:
# - nvme_device_temperature_celsius
# - nvme_available_spare_percent
# - nvme_percentage_used
# - nvme_data_units_read_total
# - nvme_data_units_written_total
# - nvme_host_read_commands_total
# - nvme_host_write_commands_total
# - nvme_critical_warning

# Grafana Dashboard Query Examples:
# Temperature: avg(nvme_device_temperature_celsius) by (device)
# IOPS: rate(nvme_host_read_commands_total[5m]) + rate(nvme_host_write_commands_total[5m])
# Throughput: rate(nvme_data_units_read_total[5m]) * 512 + rate(nvme_data_units_written_total[5m]) * 512