data "openstack_images_image_v2" "os" {
  name        = var.image_name
  most_recent = true
}

data "openstack_networking_network_v2" "external" {
  name = var.external_network
}

resource "openstack_networking_network_v2" "private" {
  name           = "${var.app_name}-net"
  admin_state_up = true
}

resource "openstack_networking_subnet_v2" "private" {
  name            = "${var.app_name}-subnet"
  network_id      = openstack_networking_network_v2.private.id
  cidr            = var.private_cidr
  ip_version      = 4
  dns_nameservers = ["1.1.1.1", "8.8.8.8"]
}

resource "openstack_networking_router_v2" "main" {
  name                = "${var.app_name}-router"
  external_network_id = data.openstack_networking_network_v2.external.id
}

resource "openstack_networking_router_interface_v2" "private" {
  router_id = openstack_networking_router_v2.main.id
  subnet_id = openstack_networking_subnet_v2.private.id
}

resource "openstack_networking_secgroup_v2" "airflow" {
  name        = "${var.app_name}-sg"
  description = "SSH and HTTP/HTTPS for a reverse proxy; web UI port 8080 restricted"
}

resource "openstack_networking_secgroup_rule_v2" "ssh" {
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 22
  port_range_max    = 22
  remote_ip_prefix  = "0.0.0.0/0"
  security_group_id = openstack_networking_secgroup_v2.airflow.id
}

# 80 and 443 carry the web UI when it is served over a domain with automatic
# TLS through a reverse proxy (Caddy or Nginx). They are not used until you put
# a proxy in front of Airflow; see the reference page.
resource "openstack_networking_secgroup_rule_v2" "http" {
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 80
  port_range_max    = 80
  remote_ip_prefix  = "0.0.0.0/0"
  security_group_id = openstack_networking_secgroup_v2.airflow.id
}

resource "openstack_networking_secgroup_rule_v2" "https" {
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 443
  port_range_max    = 443
  remote_ip_prefix  = "0.0.0.0/0"
  security_group_id = openstack_networking_secgroup_v2.airflow.id
}

# Raw web UI HTTP on 8080 is restricted to ui_allowed_cidr (the private
# network by default). Prefer a domain with TLS on 443 for routine access.
# Airflow's own login still gates the UI; the port stays off the public
# internet by default.
resource "openstack_networking_secgroup_rule_v2" "web_ui" {
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 8080
  port_range_max    = 8080
  remote_ip_prefix  = var.ui_allowed_cidr
  security_group_id = openstack_networking_secgroup_v2.airflow.id
}

resource "openstack_networking_port_v2" "airflow" {
  name               = "${var.app_name}-port"
  network_id         = openstack_networking_network_v2.private.id
  security_group_ids = [openstack_networking_secgroup_v2.airflow.id]

  fixed_ip {
    subnet_id = openstack_networking_subnet_v2.private.id
  }

  depends_on = [openstack_networking_router_interface_v2.private]
}

resource "openstack_blockstorage_volume_v3" "data" {
  name = "${var.app_name}-data"
  size = var.volume_size
}

resource "openstack_compute_instance_v2" "airflow" {
  name        = var.app_name
  flavor_name = var.flavor_name
  key_pair    = var.key_name

  user_data = templatefile("${path.module}/cloud-init/airflow.yaml.tftpl", {
    app_name = var.app_name
  })

  block_device {
    uuid                  = data.openstack_images_image_v2.os.id
    source_type           = "image"
    destination_type      = "volume"
    volume_size           = 30
    boot_index            = 0
    delete_on_termination = true
  }

  network {
    port = openstack_networking_port_v2.airflow.id
  }
}

resource "openstack_compute_volume_attach_v2" "data" {
  instance_id = openstack_compute_instance_v2.airflow.id
  volume_id   = openstack_blockstorage_volume_v3.data.id
}

resource "openstack_networking_floatingip_v2" "airflow" {
  pool = var.external_network
}

resource "openstack_networking_floatingip_associate_v2" "airflow" {
  floating_ip = openstack_networking_floatingip_v2.airflow.address
  port_id     = openstack_networking_port_v2.airflow.id
}

variable "key_name" {
  description = "SSH keypair name (must already exist in your project)"
  type        = string
}

variable "flavor_name" {
  description = "Instance size. Airflow LocalExecutor with bundled metadata PostgreSQL runs comfortably on 4 vCPU and 4 GiB RAM. Size up for heavier DAG schedules or larger task concurrency."
  type        = string
  default     = "s1a.medium"
}

variable "image_name" {
  description = "Operating system image. Ubuntu 24.04 is the recommended base."
  type        = string
  default     = "Ubuntu-24.04"
}

variable "app_name" {
  description = "Display name prefix for compute and network resources"
  type        = string
  default     = "airflow"
}

variable "volume_size" {
  description = "Block volume size in GiB, mounted at /var/lib/docker so the metadata PostgreSQL database, DAG logs, and Docker named volumes live on a volume you can grow rather than on the boot disk."
  type        = number
  default     = 40
}

variable "external_network" {
  description = "Shared external network for router gateway and floating IPs; defaults to PublicStatic (persisted FIP / production pattern). Override with PublicEphemeral for ephemeral demos."
  type        = string
  default     = "PublicStatic"
}

variable "private_cidr" {
  description = "CIDR for the private tenant network the instance lives in"
  type        = string
  default     = "10.40.0.0/24"
}

variable "ui_allowed_cidr" {
  description = "CIDR allowed to reach the Airflow web UI on port 8080. Defaults to the private network only, so the raw UI stays off the public internet. Reach it over an SSH tunnel, or (recommended) serve it over a domain with HTTPS on 443 behind a reverse proxy. To allow direct access from your workstation, set this to YOUR_IP/32."
  type        = string
  default     = "10.40.0.0/24"
}

#cloud-config
package_update: true
packages:
  - ca-certificates
  - curl
write_files:
  - path: /opt/airflow/docker-compose.yml
    permissions: "0644"
    content: |
      services:
        postgres:
          image: postgres:16-alpine
          restart: unless-stopped
          env_file:
            - /opt/airflow/.env
          volumes:
            - postgres-data:/var/lib/postgresql/data
          healthcheck:
            test: ["CMD-SHELL", "pg_isready -U airflow"]
            interval: 10s
            timeout: 5s
            retries: 5
        airflow-init:
          image: apache/airflow:2.10.5
          env_file:
            - /opt/airflow/.env
          volumes:
            - /opt/airflow/dags:/opt/airflow/dags
            - /opt/airflow/logs:/opt/airflow/logs
            - /opt/airflow/plugins:/opt/airflow/plugins
          user: "$${AIRFLOW_UID}:0"
          depends_on:
            postgres:
              condition: service_healthy
          entrypoint: /bin/bash
          command:
            - -c
            - |
              airflow db migrate
              airflow users create \
                --username "$${_AIRFLOW_WWW_USER_USERNAME}" \
                --password "$${_AIRFLOW_WWW_USER_PASSWORD}" \
                --firstname Admin \
                --lastname User \
                --role Admin \
                --email [email protected] || true
        airflow-webserver:
          image: apache/airflow:2.10.5
          restart: unless-stopped
          env_file:
            - /opt/airflow/.env
          volumes:
            - /opt/airflow/dags:/opt/airflow/dags
            - /opt/airflow/logs:/opt/airflow/logs
            - /opt/airflow/plugins:/opt/airflow/plugins
          user: "$${AIRFLOW_UID}:0"
          ports:
            - "8080:8080"
          depends_on:
            airflow-init:
              condition: service_completed_successfully
          command: webserver
        airflow-scheduler:
          image: apache/airflow:2.10.5
          restart: unless-stopped
          env_file:
            - /opt/airflow/.env
          volumes:
            - /opt/airflow/dags:/opt/airflow/dags
            - /opt/airflow/logs:/opt/airflow/logs
            - /opt/airflow/plugins:/opt/airflow/plugins
          user: "$${AIRFLOW_UID}:0"
          depends_on:
            airflow-init:
              condition: service_completed_successfully
          command: scheduler
      volumes:
        postgres-data:
  - path: /opt/airflow/dags/hello_quake.py
    permissions: "0644"
    content: |
      from datetime import datetime

      from airflow import DAG
      from airflow.operators.bash import BashOperator

      with DAG(
          dag_id="hello_quake",
          start_date=datetime(2024, 1, 1),
          schedule="@once",
          catchup=False,
          tags=["example"],
      ) as dag:
          BashOperator(
              task_id="hello",
              bash_command='echo "Hello from Quake AI Airflow"',
          )
runcmd:
  - |
    set -e
    # The data volume attaches as /dev/sdb on this platform (not /dev/vdb).
    # Mount it at /var/lib/docker before Docker is installed so the metadata
    # PostgreSQL data, DAG logs, and Docker named volumes live on the resizable
    # volume rather than the boot disk.
    DEV=/dev/sdb
    for i in $(seq 1 30); do [ -b "$DEV" ] && break; sleep 5; done
    if ! blkid "$DEV" >/dev/null 2>&1; then mkfs.ext4 -F -L airflowdata "$DEV"; fi
    mkdir -p /var/lib/docker /opt/airflow/dags /opt/airflow/logs /opt/airflow/plugins
    mount "$DEV" /var/lib/docker
    grep -q "$DEV" /etc/fstab || echo "$DEV /var/lib/docker ext4 defaults,nofail 0 2" >> /etc/fstab
    # Install Docker Engine plus the compose plugin from Docker's convenience
    # script.
    curl -fsSL https://get.docker.com | sh
    # Generate secrets on first boot. No credential ships with this template.
    POSTGRES_PASS=$(openssl rand -hex 24)
    ADMIN_PASS=$(openssl rand -hex 16)
    FERNET_KEY=$(docker run --rm apache/airflow:2.10.5 python -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())")
    umask 077
    {
      echo "POSTGRES_USER=airflow"
      echo "POSTGRES_PASSWORD=$POSTGRES_PASS"
      echo "POSTGRES_DB=airflow"
      echo "_AIRFLOW_WWW_USER_USERNAME=admin"
      echo "_AIRFLOW_WWW_USER_PASSWORD=$ADMIN_PASS"
      echo "AIRFLOW_UID=50000"
      echo "AIRFLOW__CORE__EXECUTOR=LocalExecutor"
      echo "AIRFLOW__DATABASE__SQL_ALCHEMY_CONN=postgresql+psycopg2://airflow:$POSTGRES_PASS@postgres/airflow"
      echo "AIRFLOW__CORE__FERNET_KEY=$FERNET_KEY"
      echo "AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=true"
      echo "AIRFLOW__CORE__LOAD_EXAMPLES=false"
      echo "AIRFLOW__WEBSERVER__EXPOSE_CONFIG=false"
    } > /opt/airflow/.env
    chmod 600 /opt/airflow/.env
    {
      echo "Airflow web UI login (read once, then store in your password manager):"
      echo "  Username: admin"
      echo "  Password: $ADMIN_PASS"
    } > /opt/airflow/credentials.txt
    chmod 600 /opt/airflow/credentials.txt
    chown -R ubuntu:ubuntu /opt/airflow/dags /opt/airflow/logs /opt/airflow/plugins
    cd /opt/airflow
    docker compose up airflow-init
    docker compose up -d

# Apache Airflow orchestration

Single compute instance running [Apache Airflow](https://airflow.apache.org), a self-hosted workflow orchestration platform, on infrastructure you control. After apply, you sign in to the web UI, deploy DAGs, and schedule data pipelines on a VM you operate.


**Network class:** production — `external_network` defaults to `PublicStatic` for persisted floating IPs and multi-tier stacks; override with `PublicEphemeral` for ephemeral demos.

The instance provisions a private network, a floating IP, and a block volume mounted at `/var/lib/docker` so the metadata PostgreSQL database, DAG logs, and Docker named volumes live on a resizable volume. cloud-init installs Docker Engine and starts Airflow from a compose file on first boot.

## Where this fits

Airflow is the orchestration layer for data pipelines: it schedules tasks, tracks dependencies, and retries failures on infrastructure you own rather than on a managed orchestration cloud. It pairs with warehouse and transform tools you deploy separately (for example [self-managed PostgreSQL](/resources/iac-templates/self-managed-postgres) and the dbt deployment pattern).

## Prerequisites

- OpenTofu >= 1.6.0 or Terraform >= 1.6.0
- Quake AI account with OpenStack credentials
- An existing SSH keypair in your project (the value of `key_name` must match that keypair)

## Resource baseline

Airflow LocalExecutor with bundled metadata PostgreSQL runs on 4 vCPU and 4 GiB RAM. The default `s1a.medium` flavor leaves headroom for Docker plus moderate DAG volume. Size up for heavier schedules or higher task concurrency.

## Usage

1. Clone or copy this template directory
2. Copy `terraform.tfvars.example` to `terraform.tfvars` and fill in your values
3. Source your OpenStack credentials: `source openrc.sh`
4. Initialize: `tofu init`
5. Preview: `tofu plan`
6. Apply: `tofu apply`

After apply, cloud-init takes a few minutes to install Docker and start Airflow on first boot. Read the admin password from `/opt/airflow/credentials.txt` over SSH, then open `web_ui_url` from the outputs. No credential ships with this template.

## Web UI access and security

The web UI listens on port 8080 over plain HTTP. The security group restricts 8080 to `ui_allowed_cidr`, which defaults to the private network only, so the raw UI is not exposed to the public internet. Airflow's login still gates the UI. Choose one of:

- **Recommended:** put a reverse proxy (Caddy or Nginx) in front of Airflow and serve the UI over HTTPS on 443. Point the domain's DNS A record at `floating_ip`, then set `AIRFLOW__WEBSERVER__BASE_URL` in `/opt/airflow/.env`.
- **SSH tunnel:** `ssh -L 8080:localhost:8080 user@<floating_ip>`, then open `http://localhost:8080`.
- **Direct, scoped:** set `ui_allowed_cidr` to your workstation IP (`YOUR_IP/32`) to reach 8080 directly from one address.

Ports 80 and 443 stay open for the reverse proxy you put in front; they carry no traffic until you add one.

## How the instance is provisioned

cloud-init:

1. Mounts the data volume at `/var/lib/docker` (formatting it on first boot) and adds an `/etc/fstab` entry so it persists across reboots.
2. Writes `/opt/airflow/docker-compose.yml`, a sample DAG at `/opt/airflow/dags/hello_quake.py`, and generates `/opt/airflow/.env` with a Fernet key, metadata database password, and admin password on first boot.
3. Installs Docker Engine from `https://get.docker.com`, runs the init container (`airflow db migrate` and admin user creation), then starts the webserver and scheduler with LocalExecutor and bundled PostgreSQL.

## Variables

| Name | Type | Required | Default | Description |
| --- | --- | --- | --- | --- |
| `key_name` | string | yes | n/a | SSH keypair name (must already exist in your project) |
| `flavor_name` | string | no | `s1a.medium` | Instance size (LocalExecutor plus metadata PostgreSQL runs on 4 vCPU / 4 GiB) |
| `image_name` | string | no | `Ubuntu-24.04` | Operating system image |
| `app_name` | string | no | `airflow` | Display name prefix for resources |
| `volume_size` | number | no | `40` | Block volume size in GiB, mounted at `/var/lib/docker` |
| `external_network` | string | no | `PublicStatic` | Persisted FIP / production default; override with `PublicEphemeral` for demos |
| `private_cidr` | string | no | `10.40.0.0/24` | CIDR for the private subnet |
| `ui_allowed_cidr` | string | no | `10.40.0.0/24` | CIDR allowed to reach the web UI on port 8080 |

## Outputs

| Name | Description |
| --- | --- |
| `floating_ip` | Public floating IP assigned to the instance |
| `private_ip` | Private IP address of the instance |
| `web_ui_url` | Airflow web UI URL on port 8080 |
| `instance_id` | Compute instance ID |

## Scope

This is a single-VM Airflow host with LocalExecutor that you operate, not a managed orchestration cloud. It is CPU-only and runs in one region. For scale-out execution, move to the Kubernetes executor and reuse the [Kubernetes cluster template](/resources/iac-templates/k8s-cluster); the deployment page walks through that path in prose.

## Documentation

See also: [self-managed PostgreSQL](/resources/iac-templates/self-managed-postgres), [Kubernetes cluster](/resources/iac-templates/k8s-cluster)