data "openstack_images_image_v2" "os" {
  name        = var.image_name
  most_recent = true
}

data "openstack_networking_network_v2" "external" {
  name = var.external_network
}

resource "openstack_networking_network_v2" "private" {
  name           = "${var.app_name}-net"
  admin_state_up = true
}

resource "openstack_networking_subnet_v2" "private" {
  name            = "${var.app_name}-subnet"
  network_id      = openstack_networking_network_v2.private.id
  cidr            = var.private_cidr
  ip_version      = 4
  dns_nameservers = ["1.1.1.1", "8.8.8.8"]
}

resource "openstack_networking_router_v2" "main" {
  name                = "${var.app_name}-router"
  external_network_id = data.openstack_networking_network_v2.external.id
}

resource "openstack_networking_router_interface_v2" "private" {
  router_id = openstack_networking_router_v2.main.id
  subnet_id = openstack_networking_subnet_v2.private.id
}

resource "openstack_networking_secgroup_v2" "airbyte" {
  name        = "${var.app_name}-sg"
  description = "SSH and HTTP/HTTPS for a reverse proxy; Airbyte UI port 8000 restricted"
}

resource "openstack_networking_secgroup_rule_v2" "ssh" {
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 22
  port_range_max    = 22
  remote_ip_prefix  = "0.0.0.0/0"
  security_group_id = openstack_networking_secgroup_v2.airbyte.id
}

resource "openstack_networking_secgroup_rule_v2" "http" {
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 80
  port_range_max    = 80
  remote_ip_prefix  = "0.0.0.0/0"
  security_group_id = openstack_networking_secgroup_v2.airbyte.id
}

resource "openstack_networking_secgroup_rule_v2" "https" {
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 443
  port_range_max    = 443
  remote_ip_prefix  = "0.0.0.0/0"
  security_group_id = openstack_networking_secgroup_v2.airbyte.id
}

resource "openstack_networking_secgroup_rule_v2" "ui" {
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 8000
  port_range_max    = 8000
  remote_ip_prefix  = var.ui_allowed_cidr
  security_group_id = openstack_networking_secgroup_v2.airbyte.id
}

resource "openstack_networking_port_v2" "airbyte" {
  name               = "${var.app_name}-port"
  network_id         = openstack_networking_network_v2.private.id
  security_group_ids = [openstack_networking_secgroup_v2.airbyte.id]

  fixed_ip {
    subnet_id = openstack_networking_subnet_v2.private.id
  }

  depends_on = [openstack_networking_router_interface_v2.private]
}

resource "openstack_blockstorage_volume_v3" "data" {
  name = "${var.app_name}-data"
  size = var.volume_size
}

resource "openstack_compute_instance_v2" "airbyte" {
  name        = var.app_name
  flavor_name = var.flavor_name
  key_pair    = var.key_name

  user_data = templatefile("${path.module}/cloud-init/airbyte.yaml.tftpl", {
    app_name        = var.app_name
    airbyte_version = var.airbyte_version
  })

  block_device {
    uuid                  = data.openstack_images_image_v2.os.id
    source_type           = "image"
    destination_type      = "volume"
    volume_size           = 40
    boot_index            = 0
    delete_on_termination = true
  }

  network {
    port = openstack_networking_port_v2.airbyte.id
  }
}

resource "openstack_compute_volume_attach_v2" "data" {
  instance_id = openstack_compute_instance_v2.airbyte.id
  volume_id   = openstack_blockstorage_volume_v3.data.id
}

resource "openstack_networking_floatingip_v2" "airbyte" {
  pool = var.external_network
}

resource "openstack_networking_floatingip_associate_v2" "airbyte" {
  floating_ip = openstack_networking_floatingip_v2.airbyte.address
  port_id     = openstack_networking_port_v2.airbyte.id
}

variable "key_name" {
  description = "SSH keypair name (must already exist in your project)"
  type        = string
}

variable "flavor_name" {
  description = "Instance size. The Airbyte stack (web app, worker, metadata PostgreSQL, Temporal) runs on 2 vCPU and 8 GiB RAM. Size up for heavy sync jobs or many concurrent connections."
  type        = string
  default     = "m2a.large"
}

variable "image_name" {
  description = "Operating system image. Ubuntu 24.04 is the recommended base."
  type        = string
  default     = "Ubuntu-24.04"
}

variable "app_name" {
  description = "Display name prefix for compute and network resources"
  type        = string
  default     = "airbyte"
}

variable "volume_size" {
  description = "Block volume size in GiB, mounted at /var/lib/docker so connector workspaces, metadata, and sync logs live on a volume you can grow rather than on the boot disk."
  type        = number
  default     = 50
}

variable "external_network" {
  description = "Shared external network for router gateway and floating IPs; defaults to PublicStatic (persisted FIP / production pattern). Override with PublicEphemeral for ephemeral demos."
  type        = string
  default     = "PublicStatic"
}

variable "private_cidr" {
  description = "CIDR for the private tenant network the instance lives in"
  type        = string
  default     = "10.49.0.0/24"
}

variable "ui_allowed_cidr" {
  description = "CIDR allowed to reach the Airbyte web UI on port 8000. Defaults to the private network only, so the raw UI stays off the public internet. Reach it over an SSH tunnel, or (recommended) serve it over a domain with HTTPS on 443 behind a reverse proxy. To allow direct access from your workstation, set this to YOUR_IP/32."
  type        = string
  default     = "10.49.0.0/24"
}

variable "airbyte_version" {
  description = "Pinned Airbyte platform release used by run-ab-platform.sh on first boot."
  type        = string
  default     = "0.63.15"
}

#cloud-config
package_update: true
packages:
  - ca-certificates
  - curl
write_files:
  - path: /opt/airbyte/install.sh
    permissions: "0755"
    content: |
      #!/bin/bash
      set -euo pipefail
      cd /opt/airbyte
      export DO_NOT_TRACK=1
      curl -fsSL -o run-ab-platform.sh \
        "https://raw.githubusercontent.com/airbytehq/airbyte/v${airbyte_version}/run-ab-platform.sh"
      chmod +x run-ab-platform.sh
      ./run-ab-platform.sh -b
runcmd:
  - |
    set -e
    DEV=/dev/sdb
    for i in $(seq 1 30); do [ -b "$DEV" ] && break; sleep 5; done
    if ! blkid "$DEV" >/dev/null 2>&1; then mkfs.ext4 -F -L airbytedata "$DEV"; fi
    mkdir -p /var/lib/docker
    mount "$DEV" /var/lib/docker
    grep -q "$DEV" /etc/fstab || echo "$DEV /var/lib/docker ext4 defaults,nofail 0 2" >> /etc/fstab
    curl -fsSL https://get.docker.com | sh
    /opt/airbyte/install.sh

# Airbyte ingestion (ELT)

Single compute instance running [Airbyte](https://airbyte.com), an open-source extract-and-load platform (a self-hosted alternative to Fivetran or Stitch), on infrastructure you control. After apply, you open the web UI, configure sources and destinations, and run replication jobs that land raw data in Object Storage and/or a PostgreSQL warehouse.


**Network class:** production — `external_network` defaults to `PublicStatic` for persisted floating IPs and multi-tier stacks; override with `PublicEphemeral` for ephemeral demos.

The instance provisions a private network, a floating IP, and a block volume mounted at `/var/lib/docker` so connector workspaces, metadata, and sync logs live on a resizable volume. cloud-init installs Docker Engine and starts the Airbyte stack from docker compose on first boot.

## Where this fits

Airbyte covers the extract-and-load stage of a self-operated data stack: pull data from SaaS APIs, databases, and files, then write it to destinations you control. You run the replication engine on a VM you own, which keeps connection credentials and sync history on your infrastructure.

## Prerequisites

- OpenTofu >= 1.6.0 or Terraform >= 1.6.0
- Quake AI account with OpenStack credentials
- An existing SSH keypair in your project (the value of `key_name` must match that keypair)

## Resource baseline

The Airbyte stack runs multiple containers (web app, worker, metadata PostgreSQL, Temporal). The default `m2a.large` flavor (2 vCPU / 8 GiB RAM) and 50 GiB data volume suit moderate connector counts. Size up for heavy sync jobs or many concurrent replications.

## Usage

1. Clone or copy this template directory
2. Copy `terraform.tfvars.example` to `terraform.tfvars` and fill in your values
3. Source your OpenStack credentials: `source openrc.sh`
4. Initialize: `tofu init`
5. Preview: `tofu plan`
6. Apply: `tofu apply`

After apply, cloud-init takes several minutes to install Docker and start the Airbyte stack on first boot. Then reach `ui_url` from the outputs and configure sources and destinations in the web UI. No credential ships with this template.

## Web UI access and security

The web UI listens on port 8000 over plain HTTP. The security group restricts 8000 to `ui_allowed_cidr`, which defaults to the private network only, so the raw UI stays off the public internet. Choose one of:

- **Recommended:** put a reverse proxy (Caddy or Nginx) in front of Airbyte and serve the UI over HTTPS on 443. Point the domain's DNS A record at `floating_ip`.
- **SSH tunnel:** `ssh -L 8000:localhost:8000 user@<floating_ip>`, then open `http://localhost:8000`.
- **Direct, scoped:** set `ui_allowed_cidr` to your workstation IP (`YOUR_IP/32`) to reach 8000 directly from one address.

Ports 80 and 443 stay open for the reverse proxy you put in front; they carry no traffic until you add one.

## Destinations

Configure destinations in the Airbyte UI after first boot:

- **Object Storage:** point a destination at your Quake AI Object Storage bucket (S3-compatible endpoint and access keys you create in the Console).
- **PostgreSQL warehouse:** point a destination at a [self-managed PostgreSQL](/resources/iac-templates/self-managed-postgres) instance on the same private network.

Connection secrets stay in the Airbyte UI and on the data volume; they are never set in tfvars or committed to the repo.

## Variables

| Name | Type | Required | Default | Description |
| --- | --- | --- | --- | --- |
| `key_name` | string | yes | n/a | SSH keypair name (must already exist in your project) |
| `flavor_name` | string | no | `m2a.large` | Instance size (Airbyte stack runs on 2 vCPU / 8 GiB) |
| `image_name` | string | no | `Ubuntu-24.04` | Operating system image |
| `app_name` | string | no | `airbyte` | Display name prefix for resources |
| `volume_size` | number | no | `50` | Block volume size in GiB, mounted at `/var/lib/docker` |
| `external_network` | string | no | `PublicStatic` | Persisted FIP / production default; override with `PublicEphemeral` for demos |
| `private_cidr` | string | no | `10.49.0.0/24` | CIDR for the private subnet |
| `ui_allowed_cidr` | string | no | `10.49.0.0/24` | CIDR allowed to reach the web UI on port 8000 |
| `airbyte_version` | string | no | `0.63.15` | Pinned Airbyte platform release for first boot |

## Outputs

| Name | Description |
| --- | --- |
| `floating_ip` | Public floating IP assigned to the instance |
| `private_ip` | Private IP address of the instance |
| `ui_url` | Airbyte web UI URL on port 8000 |
| `instance_id` | Compute instance ID |

## Scope

This is a single-VM Airbyte host that you operate, not a hosted ELT cloud. The instance is CPU-only and runs in one region. You operate the instance, Docker, Airbyte, and the data volume yourself: back them up, patch them, and watch resource use as sync volume grows. Snapshot the volume before you resize or rebuild the host.

## Documentation

See also: [self-managed PostgreSQL](/resources/iac-templates/self-managed-postgres), [S3 storage ACL template](/resources/iac-templates/s3-storage-acl), [data pipelines and analytics brief](/resources/solutions/data-pipelines-and-analytics)