Configure Ansible dynamic inventory for AWS, Azure, and GCP with automated discovery

Install Ansible and required dependencies

Install Ansible and Python libraries needed for cloud provider plugins.

sudo apt update
sudo apt install -y python3-pip python3-venv ansible
python3 -m pip install --user pipx
pipx ensurepath

sudo dnf update -y
sudo dnf install -y python3-pip python3-virtualenv ansible
python3 -m pip install --user pipx
pipx ensurepath

Install cloud provider libraries

Install Python libraries for AWS, Azure, and GCP integration.

pip3 install --user boto3 botocore azure-identity azure-mgmt-compute google-auth google-cloud-compute
ansible-galaxy collection install amazon.aws azure.azcollection google.cloud

Create Ansible project structure

Set up directory structure for dynamic inventory configurations.

mkdir -p ~/ansible-multi-cloud/{inventories,group_vars,host_vars,playbooks}
cd ~/ansible-multi-cloud
mkdir -p inventories/{aws,azure,gcp,unified}

Configure AWS dynamic inventory

Create AWS EC2 plugin configuration with automated discovery and filtering.

plugin: amazon.aws.aws_ec2
regions:
  - us-east-1
  - us-west-2
  - eu-west-1
strict: false
keyed_groups:
  - key: tags
    prefix: tag
  - key: instance_type
    prefix: instance_type
  - key: placement.region
    prefix: aws_region
  - key: state.name
    prefix: state
groups:
  web_servers: "'web' in (tags.Role|list)"
  database_servers: "'database' in (tags.Role|list)"
  production: "tags.Environment == 'production'"
  staging: "tags.Environment == 'staging'"
compose:
  ansible_host: public_ip_address | default(private_ip_address)
  ec2_state: state.name
  ec2_architecture: architecture
  ec2_instance_type: instance_type
  ec2_region: placement.region
  ec2_availability_zone: placement.availability_zone
cache: true
cache_plugin: jsonfile
cache_timeout: 3600
cache_connection: /tmp/ansible-aws-cache
use_contrib_script_compatible_sanitization: true

Configure Azure dynamic inventory

Set up Azure plugin with resource group filtering and custom grouping.

plugin: azure.azcollection.azure_rm
include_vm_resource_groups:
  - "*"
authentication_source: auto
strict: false
keyed_groups:
  - key: tags
    prefix: tag
  - key: location
    prefix: location
  - key: resource_group
    prefix: rg
  - key: os_profile.admin_username
    prefix: user
groups:
  web_servers: "'web' in tags.get('role', '')"
  database_servers: "'database' in tags.get('role', '')"
  production: "tags.get('environment') == 'production'"
  staging: "tags.get('environment') == 'staging'
  linux: "os_disk.os_type == 'Linux'"
  windows: "os_disk.os_type == 'Windows'"
compose:
  ansible_host: public_ips[0] | default(private_ips[0])
  azure_location: location
  azure_resource_group: resource_group
  azure_vm_size: vm_size
  azure_os_type: os_disk.os_type
  azure_power_state: powerstate
cache: true
cache_plugin: jsonfile
cache_timeout: 3600
cache_connection: /tmp/ansible-azure-cache
use_contrib_script_compatible_sanitization: true

Configure GCP dynamic inventory

Create Google Cloud Compute plugin configuration with zone-based grouping.

plugin: google.cloud.gcp_compute
projects:
  - your-gcp-project-id
auth_kind: serviceaccount
service_account_file: ~/.config/gcloud/application_default_credentials.json
strict: false
keyed_groups:
  - key: labels
    prefix: label
  - key: machineType
    prefix: machine_type
  - key: zone
    prefix: zone
  - key: status
    prefix: status
groups:
  web_servers: "labels.get('role') == 'web'"
  database_servers: "labels.get('role') == 'database'"
  production: "labels.get('environment') == 'production'"
  staging: "labels.get('environment') == 'staging'"
  preemptible: "scheduling.preemptible == true"
compose:
  ansible_host: networkInterfaces[0].accessConfigs[0].natIP | default(networkInterfaces[0].networkIP)
  gcp_machine_type: machineType.split('/')[-1]
  gcp_zone: zone.split('/')[-1]
  gcp_project: selfLink.split('/')[6]
  gcp_status: status
  gcp_preemptible: scheduling.preemptible
cache: true
cache_plugin: jsonfile
cache_timeout: 3600
cache_connection: /tmp/ansible-gcp-cache
use_contrib_script_compatible_sanitization: true

Create unified multi-cloud inventory

Configure a unified inventory that combines all cloud providers with consistent grouping.

plugin: advanced_host_list
compose:
  cloud_provider: |
    "aws" if "ec2_" in group_names[0] else
    "azure" if "azure_" in group_names[0] else
    "gcp" if "gcp_" in group_names[0] else
    "unknown"
  normalized_hostname: inventory_hostname.split('.')[0]
keyed_groups:
  - key: cloud_provider
    prefix: cloud
  - key: hostvars[inventory_hostname].get('ec2_instance_type', hostvars[inventory_hostname].get('azure_vm_size', hostvars[inventory_hostname].get('gcp_machine_type', 'unknown')))
    prefix: size
groups:
  all_web_servers: "'web_servers' in group_names"
  all_database_servers: "'database_servers' in group_names"
  all_production: "'production' in group_names"
  all_staging: "'staging' in group_names"
  multi_cloud_web: "'web_servers' in group_names and (cloud_provider == 'aws' or cloud_provider == 'azure' or cloud_provider == 'gcp')"
  hybrid_infrastructure: "groups['aws'] | default([]) | length > 0 and groups['azure'] | default([]) | length > 0"

Configure AWS credentials

Set up AWS credentials using IAM roles or access keys with minimal required permissions.

mkdir -p ~/.aws
aws configure

[default]
aws_access_key_id = YOUR_ACCESS_KEY
aws_secret_access_key = YOUR_SECRET_KEY
region = us-east-1

[default]
region = us-east-1
output = json

[profile ansible]
region = us-east-1
role_arn = arn:aws:iam::123456789012:role/AnsibleInventoryRole
source_profile = default

Configure Azure authentication

Set up Azure service principal authentication for automated inventory discovery.

az login
az account set --subscription "your-subscription-id"
az ad sp create-for-rbac --name "ansible-inventory" --role "Reader" --scopes "/subscriptions/your-subscription-id"

[default]
client_id = your-client-id
secret = your-client-secret
subscription_id = your-subscription-id
tenant = your-tenant-id

Configure GCP authentication

Set up Google Cloud service account for inventory access with minimal permissions.

gcloud auth application-default login
gcloud config set project your-gcp-project-id

gcloud iam service-accounts create ansible-inventory \
  --display-name="Ansible Inventory Service Account"

gcloud projects add-iam-policy-binding your-gcp-project-id \
  --member="serviceAccount:ansible-inventory@your-gcp-project-id.iam.gserviceaccount.com" \
  --role="roles/compute.viewer"

gcloud iam service-accounts keys create ~/.config/gcloud/ansible-sa-key.json \
  --iam-account=ansible-inventory@your-gcp-project-id.iam.gserviceaccount.com

Create Ansible configuration

Configure Ansible with optimized settings for dynamic inventory performance.

[defaults]
inventory = inventories/
host_key_checking = False
gathering = smart
fact_caching = jsonfile
fact_caching_connection = /tmp/ansible-facts-cache
fact_caching_timeout = 86400
retry_files_enabled = False
stdout_callback = yaml
bin_ansible_callbacks = True
interpreter_python = auto_silent

[inventory]
enable_plugins = aws_ec2, azure_rm, gcp_compute, advanced_host_list, auto
cache = True
cache_plugin = jsonfile
cache_timeout = 3600
cache_connection = ~/.ansible/tmp/inventory_cache

[ssh_connection]
ssh_args = -o ControlMaster=auto -o ControlPersist=60s -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no
pipelining = True
control_path_dir = ~/.ansible/tmp/cp
control_path = %(directory)s/%%h-%%p-%%r

Configure inventory caching optimization

Set up advanced caching strategies for improved performance with large inventories.

---
Inventory caching configuration
ansible_cache_plugin: jsonfile
ansible_cache_plugin_connection: ~/.ansible/tmp/fact_cache
ansible_cache_plugin_timeout: 86400

Performance optimization
ansible_python_interpreter: /usr/bin/python3
ansible_ssh_pipelining: true
ansible_ssh_connection_timeout: 30
ansible_ssh_retries: 3

Cloud-specific optimizations
ansible_host_key_checking: false
ansible_ssh_common_args: '-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null'

Create inventory refresh script

Automate inventory cache refresh with intelligent scheduling and error handling.

#!/bin/bash
set -euo pipefail

INVENTORY_DIR="$(dirname "$0")/inventories"
CACHE_DIR="~/.ansible/tmp/inventory_cache"
LOG_FILE="/var/log/ansible-inventory-refresh.log"

Function to log messages
log_message() {
    echo "$(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "$LOG_FILE"
}

Function to refresh specific cloud inventory
refresh_cloud_inventory() {
    local cloud="$1"
    local inventory_file="$2"
    
    log_message "Refreshing $cloud inventory..."
    
    if ansible-inventory -i "$inventory_file" --list > /dev/null 2>&1; then
        log_message "$cloud inventory refresh successful"
        return 0
    else
        log_message "ERROR: $cloud inventory refresh failed"
        return 1
    fi
}

Clear old cache
log_message "Starting inventory refresh process"
rm -rf "$CACHE_DIR"
mkdir -p "$CACHE_DIR"

Refresh each cloud provider inventory
refresh_cloud_inventory "AWS" "$INVENTORY_DIR/aws/aws_ec2.yml" &
refresh_cloud_inventory "Azure" "$INVENTORY_DIR/azure/azure_rm.yml" &
refresh_cloud_inventory "GCP" "$INVENTORY_DIR/gcp/gcp_compute.yml" &

Wait for all background jobs
wait

log_message "Inventory refresh process completed"

Test unified inventory
log_message "Testing unified inventory..."
if ansible-inventory -i "$INVENTORY_DIR" --list > /dev/null 2>&1; then
    log_message "Unified inventory test successful"
else
    log_message "ERROR: Unified inventory test failed"
    exit 1
fi

chmod +x ~/ansible-multi-cloud/refresh-inventory.sh

Set up automated credential rotation

Configure credential management with automated rotation and secure storage.

#!/bin/bash
set -euo pipefail

Credential rotation script
CRED_DIR="$HOME/.ansible-credentials"
BACKUP_DIR="$CRED_DIR/backup"
LOG_FILE="/var/log/ansible-credential-rotation.log"

Create secure credential directory
mkdir -p "$CRED_DIR" "$BACKUP_DIR"
chmod 700 "$CRED_DIR" "$BACKUP_DIR"

Function to rotate AWS credentials
rotate_aws_credentials() {
    echo "$(date): Rotating AWS credentials" >> "$LOG_FILE"
    
    # Backup current credentials
    cp ~/.aws/credentials "$BACKUP_DIR/aws-credentials-$(date +%Y%m%d-%H%M%S)"
    
    # Create new access key (implement your rotation logic)
    # aws iam create-access-key --user-name ansible-user
    
    echo "AWS credential rotation completed" >> "$LOG_FILE"
}

Function to rotate Azure credentials
rotate_azure_credentials() {
    echo "$(date): Rotating Azure credentials" >> "$LOG_FILE"
    
    # Backup current credentials
    cp ~/.azure/credentials "$BACKUP_DIR/azure-credentials-$(date +%Y%m%d-%H%M%S)"
    
    # Rotate service principal secret
    # az ad sp credential reset --name ansible-inventory
    
    echo "Azure credential rotation completed" >> "$LOG_FILE"
}

Function to rotate GCP credentials
rotate_gcp_credentials() {
    echo "$(date): Rotating GCP credentials" >> "$LOG_FILE"
    
    # Backup current service account key
    cp ~/.config/gcloud/ansible-sa-key.json "$BACKUP_DIR/gcp-key-$(date +%Y%m%d-%H%M%S).json"
    
    # Create new service account key
    # gcloud iam service-accounts keys create ~/.config/gcloud/ansible-sa-key-new.json
    
    echo "GCP credential rotation completed" >> "$LOG_FILE"
}

Schedule rotation based on environment variable
case "${ROTATE_CLOUD:-}" in
    aws) rotate_aws_credentials ;;
    azure) rotate_azure_credentials ;;
    gcp) rotate_gcp_credentials ;;
    all) 
        rotate_aws_credentials
        rotate_azure_credentials
        rotate_gcp_credentials
        ;;
    *) echo "Usage: ROTATE_CLOUD=[aws|azure|gcp|all] $0" ;;
esac

chmod 700 ~/ansible-multi-cloud/scripts/credential-manager.sh

Configure systemd timer for automated refresh

Set up systemd service and timer for automated inventory refresh.

[Unit]
Description=Ansible Dynamic Inventory Refresh
After=network.target

[Service]
Type=oneshot
User=ansible
ExecStart=/home/ansible/ansible-multi-cloud/refresh-inventory.sh
StandardOutput=journal
StandardError=journal

[Install]
WantedBy=multi-user.target

[Unit]
Description=Run Ansible inventory refresh every 30 minutes
Requires=ansible-inventory-refresh.service

[Timer]
OnCalendar=*:0/30
Persistent=true

[Install]
WantedBy=timers.target

sudo systemctl daemon-reload
sudo systemctl enable ansible-inventory-refresh.timer
sudo systemctl start ansible-inventory-refresh.timer

Test individual cloud inventories

Verify each cloud provider inventory is working correctly.

# Test AWS inventory
ansible-inventory -i inventories/aws/aws_ec2.yml --list
ansible-inventory -i inventories/aws/aws_ec2.yml --graph

Test Azure inventory
ansible-inventory -i inventories/azure/azure_rm.yml --list
ansible-inventory -i inventories/azure/azure_rm.yml --graph

Test GCP inventory
ansible-inventory -i inventories/gcp/gcp_compute.yml --list
ansible-inventory -i inventories/gcp/gcp_compute.yml --graph

Test unified multi-cloud inventory

Verify the unified inventory combines all cloud providers correctly.

# List all discovered hosts
ansible-inventory --list | jq '.'

Show inventory graph
ansible-inventory --graph

Test specific groups
ansible all_web_servers --list-hosts
ansible production --list-hosts
ansible cloud_aws --list-hosts

Test inventory filtering and grouping

Verify custom groups and filtering are working as expected.

# Test environment-based grouping
ansible production -m ping --limit 5
ansible staging -m ping --limit 3

Test role-based grouping
ansible web_servers -m shell -a "uptime" --limit 3
ansible database_servers -m shell -a "df -h" --limit 2

Test cloud-specific groups
ansible cloud_aws -m setup -a "filter=ansible_ec2_*" --limit 2
ansible cloud_azure -m setup -a "filter=ansible_azure_*" --limit 2