data "openstack_images_image_v2" "os" {
  name        = var.image_name
  most_recent = true
}

data "openstack_networking_network_v2" "external" {
  name = var.external_network
}

resource "openstack_networking_network_v2" "private" {
  name           = "${var.app_name}-net"
  admin_state_up = true
}

resource "openstack_networking_subnet_v2" "private" {
  name            = "${var.app_name}-subnet"
  network_id      = openstack_networking_network_v2.private.id
  cidr            = var.private_cidr
  ip_version      = 4
  dns_nameservers = ["1.1.1.1", "8.8.8.8"]
}

resource "openstack_networking_router_v2" "main" {
  name                = "${var.app_name}-router"
  external_network_id = data.openstack_networking_network_v2.external.id
}

resource "openstack_networking_router_interface_v2" "private" {
  router_id = openstack_networking_router_v2.main.id
  subnet_id = openstack_networking_subnet_v2.private.id
}

resource "openstack_networking_secgroup_v2" "lakehouse" {
  name        = "${var.app_name}-sg"
  description = "SSH; MinIO S3 API, console, and Iceberg REST catalog restricted to api_allowed_cidr"
}

resource "openstack_networking_secgroup_rule_v2" "ssh" {
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 22
  port_range_max    = 22
  remote_ip_prefix  = "0.0.0.0/0"
  security_group_id = openstack_networking_secgroup_v2.lakehouse.id
}

resource "openstack_networking_secgroup_rule_v2" "minio_api" {
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 9000
  port_range_max    = 9000
  remote_ip_prefix  = var.api_allowed_cidr
  security_group_id = openstack_networking_secgroup_v2.lakehouse.id
}

resource "openstack_networking_secgroup_rule_v2" "minio_console" {
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 9001
  port_range_max    = 9001
  remote_ip_prefix  = var.api_allowed_cidr
  security_group_id = openstack_networking_secgroup_v2.lakehouse.id
}

resource "openstack_networking_secgroup_rule_v2" "iceberg_rest" {
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 8181
  port_range_max    = 8181
  remote_ip_prefix  = var.api_allowed_cidr
  security_group_id = openstack_networking_secgroup_v2.lakehouse.id
}

resource "openstack_networking_port_v2" "lakehouse" {
  name               = "${var.app_name}-port"
  network_id         = openstack_networking_network_v2.private.id
  security_group_ids = [openstack_networking_secgroup_v2.lakehouse.id]

  fixed_ip {
    subnet_id = openstack_networking_subnet_v2.private.id
  }

  depends_on = [openstack_networking_router_interface_v2.private]
}

resource "openstack_blockstorage_volume_v3" "data" {
  name = "${var.app_name}-data"
  size = var.volume_size
}

resource "openstack_compute_instance_v2" "lakehouse" {
  name        = var.app_name
  flavor_name = var.flavor_name
  key_pair    = var.key_name

  user_data = templatefile("${path.module}/cloud-init/minio-iceberg.yaml.tftpl", {
    app_name         = var.app_name
    warehouse_bucket = var.warehouse_bucket
  })

  block_device {
    uuid                  = data.openstack_images_image_v2.os.id
    source_type           = "image"
    destination_type      = "volume"
    volume_size           = 30
    boot_index            = 0
    delete_on_termination = true
  }

  network {
    port = openstack_networking_port_v2.lakehouse.id
  }
}

resource "openstack_compute_volume_attach_v2" "data" {
  instance_id = openstack_compute_instance_v2.lakehouse.id
  volume_id   = openstack_blockstorage_volume_v3.data.id
}

resource "openstack_networking_floatingip_v2" "lakehouse" {
  pool = var.external_network
}

resource "openstack_networking_floatingip_associate_v2" "lakehouse" {
  floating_ip = openstack_networking_floatingip_v2.lakehouse.address
  port_id     = openstack_networking_port_v2.lakehouse.id
}

variable "key_name" {
  description = "SSH keypair name (must already exist in your project)"
  type        = string
}

variable "flavor_name" {
  description = "Instance size. MinIO plus the Iceberg REST catalog run on 2 vCPU and 8 GiB RAM; size up for heavy ingest or many concurrent catalog clients."
  type        = string
  default     = "m2a.large"
}

variable "image_name" {
  description = "Operating system image. Ubuntu 24.04 is the recommended base."
  type        = string
  default     = "Ubuntu-24.04"
}

variable "app_name" {
  description = "Display name prefix for compute and network resources"
  type        = string
  default     = "lakehouse"
}

variable "volume_size" {
  description = "Block volume size in GiB, mounted at /var/lib/docker so MinIO object data and catalog metadata live on a volume you can grow rather than on the boot disk."
  type        = number
  default     = 50
}

variable "external_network" {
  description = "Shared external network for router gateway and floating IPs; defaults to PublicStatic (persisted FIP / production pattern). Override with PublicEphemeral for ephemeral demos."
  type        = string
  default     = "PublicStatic"
}

variable "private_cidr" {
  description = "CIDR for the private tenant network the instance lives in"
  type        = string
  default     = "10.52.0.0/24"
}

variable "api_allowed_cidr" {
  description = "CIDR allowed to reach the MinIO S3 API (9000), MinIO console (9001), and Iceberg REST catalog (8181). Defaults to the private network only. Set to YOUR_IP/32 for workstation access during setup, or keep the default and reach services over an SSH tunnel from a peer in the same private network."
  type        = string
  default     = "10.52.0.0/24"
}

variable "warehouse_bucket" {
  description = "S3 bucket name the Iceberg REST catalog uses as its warehouse root. Created on first boot if it does not exist."
  type        = string
  default     = "warehouse"
}

output "instance_id" {
  description = "ID of the compute instance running MinIO and the Iceberg REST catalog"
  value       = openstack_compute_instance_v2.lakehouse.id
}

output "floating_ip" {
  description = "Public floating IP address of the lakehouse host"
  value       = openstack_networking_floatingip_v2.lakehouse.address
}

output "private_ip" {
  description = "Private IP address of the instance"
  value       = openstack_compute_instance_v2.lakehouse.access_ip_v4
}

output "minio_api_url" {
  description = "MinIO S3-compatible API endpoint. Reachable from api_allowed_cidr."
  value       = "http://${openstack_networking_floatingip_v2.lakehouse.address}:9000"
}

output "minio_console_url" {
  description = "MinIO web console URL. Reachable from api_allowed_cidr."
  value       = "http://${openstack_networking_floatingip_v2.lakehouse.address}:9001"
}

output "iceberg_rest_url" {
  description = "Apache Iceberg REST catalog base URL for table registration and commits."
  value       = "http://${openstack_networking_floatingip_v2.lakehouse.address}:8181"
}

output "warehouse_uri" {
  description = "Iceberg warehouse URI rooted at the MinIO bucket created on first boot."
  value       = "s3://${var.warehouse_bucket}/"
}

#cloud-config
package_update: true
packages:
  - ca-certificates
  - curl
  - openssl
write_files:
  - path: /opt/lakehouse/docker-compose.yml
    permissions: "0644"
    content: |
      services:
        minio:
          image: minio/minio:latest
          restart: unless-stopped
          command: server /data --console-address ":9001"
          ports:
            - "9000:9000"
            - "9001:9001"
          env_file:
            - /opt/lakehouse/.env
          volumes:
            - minio_data:/data
          healthcheck:
            test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
            interval: 10s
            timeout: 5s
            retries: 12
        iceberg-rest:
          image: tabulario/iceberg-rest:latest
          restart: unless-stopped
          ports:
            - "8181:8181"
          env_file:
            - /opt/lakehouse/.env
          depends_on:
            minio:
              condition: service_healthy
      volumes:
        minio_data:
  - path: /opt/lakehouse/init.sh
    permissions: "0755"
    content: |
      #!/bin/bash
      set -euo pipefail
      cd /opt/lakehouse
      # shellcheck disable=SC1091
      source /opt/lakehouse/.env
      docker compose up -d minio
      for i in $(seq 1 60); do
        curl -fsS "http://127.0.0.1:9000/minio/health/live" >/dev/null 2>&1 && break
        sleep 5
      done
      docker run --rm --network host minio/mc:latest \
        alias set local "http://127.0.0.1:9000" "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
      docker run --rm --network host minio/mc:latest \
        mb --ignore-existing "local/${warehouse_bucket}"
      docker compose up -d
runcmd:
  - |
    set -e
    DEV=/dev/sdb
    for i in $(seq 1 30); do [ -b "$DEV" ] && break; sleep 5; done
    if ! blkid "$DEV" >/dev/null 2>&1; then mkfs.ext4 -F -L lakehousedata "$DEV"; fi
    mkdir -p /var/lib/docker
    mount "$DEV" /var/lib/docker
    grep -q "$DEV" /etc/fstab || echo "$DEV /var/lib/docker ext4 defaults,nofail 0 2" >> /etc/fstab
    curl -fsSL https://get.docker.com | sh
    MINIO_ROOT_USER=lakehouse
    MINIO_ROOT_PASSWORD=$(openssl rand -hex 24)
    umask 077
    cat > /opt/lakehouse/.env <<EOF
    MINIO_ROOT_USER=$MINIO_ROOT_USER
    MINIO_ROOT_PASSWORD=$MINIO_ROOT_PASSWORD
    AWS_ACCESS_KEY_ID=$MINIO_ROOT_USER
    AWS_SECRET_ACCESS_KEY=$MINIO_ROOT_PASSWORD
    AWS_REGION=us-east-1
    CATALOG_WAREHOUSE=s3://${warehouse_bucket}/
    CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
    CATALOG_S3_ENDPOINT=http://minio:9000
    EOF
    chmod 600 /opt/lakehouse/.env
    printf 'minio_root_user: %s\nminio_root_password: %s\nwarehouse_bucket: %s\n' \
      "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" "${warehouse_bucket}" \
      > /opt/lakehouse/.bootstrap-credentials
    chmod 600 /opt/lakehouse/.bootstrap-credentials
    /opt/lakehouse/init.sh

# MinIO + Apache Iceberg lakehouse

Single compute instance running [MinIO](https://min.io) (S3-compatible object storage) and an [Apache Iceberg](https://iceberg.apache.org) REST catalog on infrastructure you control. After apply, clients register Iceberg tables against the REST endpoint and read or write table data through MinIO.


**Network class:** production — `external_network` defaults to `PublicStatic` for persisted floating IPs and multi-tier stacks; override with `PublicEphemeral` for ephemeral demos.

The instance provisions a private network, a floating IP, and a block volume mounted at `/var/lib/docker` so object data lives on a resizable volume. cloud-init installs Docker Engine, starts MinIO and the REST catalog from a compose file, and creates the warehouse bucket on first boot.

For a durable lake that outlives this VM, point ingest and curated layers at Quake AI [Object Storage](/docs/storage/object) instead of the bundled MinIO instance. This template keeps the catalog and a local object store on one host for development and integration testing.

## Prerequisites

- OpenTofu >= 1.6.0 or Terraform >= 1.6.0
- Quake AI account with OpenStack credentials
- An existing SSH keypair in your project (the value of `key_name` must match that keypair)

## Resource baseline

MinIO plus the Iceberg REST catalog run on 2 vCPU and 8 GiB RAM. The default `m2a.large` flavor and 50 GiB data volume suit development and light ingest. Size up for heavy write volume or large curated layers.

## Usage

1. Clone or copy this template directory
2. Copy `terraform.tfvars.example` to `terraform.tfvars` and fill in your values
3. Source your OpenStack credentials: `source openrc.sh`
4. Initialize: `tofu init`
5. Preview: `tofu plan`
6. Apply: `tofu apply`

After apply, cloud-init takes a few minutes to install Docker, start MinIO, create the warehouse bucket, and bring up the REST catalog. Read bootstrap credentials from `/opt/lakehouse/.bootstrap-credentials` on the instance. No credential ships with this template.

## API access and security

MinIO listens on port 9000 (S3 API) and 9001 (console). The Iceberg REST catalog listens on port 8181. The security group restricts all three to `api_allowed_cidr`, which defaults to the private network only.

Reach the services one of two ways:

- Set `api_allowed_cidr` to your workstation IP (`YOUR_IP/32`) during setup.
- Tunnel over SSH: `ssh -L 9000:localhost:9000 -L 9001:localhost:9001 -L 8181:localhost:8181 ubuntu@FLOATING_IP`, then use `http://localhost:9000` and `http://localhost:8181`.

## Object lake target

The bundled MinIO instance stores table files on the attached block volume. For production lake workloads, use Quake AI Object Storage as the durable target: create S3 credentials in the Console, point the REST catalog warehouse at your bucket URI, and keep this host as a catalog-only node or replace it with a [Trino](/resources/iac-templates/trino) query layer over the same bucket.

## Variables

| Name | Type | Required | Default | Description |
| --- | --- | --- | --- | --- |
| `key_name` | string | yes | n/a | SSH keypair name (must already exist in your project) |
| `flavor_name` | string | no | `m2a.large` | Instance size (2 vCPU / 8 GiB) |
| `image_name` | string | no | `Ubuntu-24.04` | Operating system image |
| `app_name` | string | no | `lakehouse` | Display name prefix for resources |
| `volume_size` | number | no | `50` | Block volume size in GiB, mounted at `/var/lib/docker` |
| `external_network` | string | no | `PublicStatic` | Persisted FIP / production default; override with `PublicEphemeral` for demos |
| `private_cidr` | string | no | `10.52.0.0/24` | CIDR for the private subnet |
| `api_allowed_cidr` | string | no | `10.52.0.0/24` | CIDR allowed to reach MinIO and the REST catalog |
| `warehouse_bucket` | string | no | `warehouse` | S3 bucket name for the Iceberg warehouse root |

## Outputs

| Name | Description |
| --- | --- |
| `floating_ip` | Public floating IP assigned to the instance |
| `private_ip` | Private IP address of the instance |
| `minio_api_url` | MinIO S3-compatible API endpoint |
| `minio_console_url` | MinIO web console URL |
| `iceberg_rest_url` | Iceberg REST catalog base URL |
| `warehouse_uri` | Iceberg warehouse URI (`s3://warehouse/` by default) |
| `instance_id` | Compute instance ID |

## Scope

This is a single-VM lakehouse catalog and object store that you operate, not a managed warehouse. It is CPU-only and runs in one region. Pair it with [Trino](/resources/iac-templates/trino) for federated SQL over Iceberg tables, or repoint the warehouse at Quake AI Object Storage for durable multi-host access.

## Documentation

See also: [S3 storage with ACLs](/resources/iac-templates/s3-storage-acl), [Trino query engine](/resources/iac-templates/trino)