data "openstack_images_image_v2" "os" {
  name        = var.image_name
  most_recent = true
}

data "openstack_networking_network_v2" "external" {
  name = var.external_network
}

resource "aws_s3_bucket" "jobs" {
  bucket = var.jobs_bucket_name
}

resource "aws_s3_bucket_acl" "jobs" {
  bucket = aws_s3_bucket.jobs.id
  acl    = "private"
}

resource "aws_s3_bucket" "output" {
  bucket = var.output_bucket_name
}

resource "aws_s3_bucket_acl" "output" {
  bucket = aws_s3_bucket.output.id
  acl    = "private"
}


resource "openstack_networking_network_v2" "private" {
  name           = "${var.instance_prefix}-private"
  admin_state_up = true
}

resource "openstack_networking_subnet_v2" "private" {
  name            = "${var.instance_prefix}-private-sn"
  network_id      = openstack_networking_network_v2.private.id
  cidr            = var.private_cidr
  ip_version      = 4
  enable_dhcp     = true
  dns_nameservers = ["8.8.8.8", "8.8.4.4"]
}

resource "openstack_networking_router_v2" "router" {
  name                = "${var.instance_prefix}-router"
  external_network_id = data.openstack_networking_network_v2.external.id
  admin_state_up      = true
}

resource "openstack_networking_router_interface_v2" "private" {
  router_id = openstack_networking_router_v2.router.id
  subnet_id = openstack_networking_subnet_v2.private.id
}

resource "openstack_networking_secgroup_v2" "dispatcher" {
  name        = "${var.instance_prefix}-dispatcher-sg"
  description = "Dispatcher queue API from workers; SSH from admin CIDR"
}

resource "openstack_networking_secgroup_rule_v2" "dispatcher_ssh" {
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 22
  port_range_max    = 22
  remote_ip_prefix  = var.admin_cidr
  security_group_id = openstack_networking_secgroup_v2.dispatcher.id
}

resource "openstack_networking_secgroup_rule_v2" "dispatcher_queue_workers" {
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 8080
  port_range_max    = 8080
  remote_ip_prefix  = var.private_cidr
  security_group_id = openstack_networking_secgroup_v2.dispatcher.id
}

resource "openstack_networking_secgroup_rule_v2" "dispatcher_queue_public" {
  count             = var.enable_dispatcher_fip ? 1 : 0
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 8080
  port_range_max    = 8080
  remote_ip_prefix  = var.submit_cidr
  security_group_id = openstack_networking_secgroup_v2.dispatcher.id
}

resource "openstack_networking_secgroup_v2" "worker" {
  name        = "${var.instance_prefix}-worker-sg"
  description = "Render workers: SSH from admin CIDR; egress to dispatcher and Object Storage"
}

resource "openstack_networking_secgroup_rule_v2" "worker_ssh" {
  direction         = "ingress"
  ethertype         = "IPv4"
  protocol          = "tcp"
  port_range_min    = 22
  port_range_max    = 22
  remote_ip_prefix  = var.admin_cidr
  security_group_id = openstack_networking_secgroup_v2.worker.id
}

resource "openstack_networking_port_v2" "dispatcher" {
  name               = "${var.instance_prefix}-dispatcher-port"
  network_id         = openstack_networking_network_v2.private.id
  security_group_ids = [openstack_networking_secgroup_v2.dispatcher.id]

  fixed_ip {
    subnet_id = openstack_networking_subnet_v2.private.id
  }

  depends_on = [openstack_networking_router_interface_v2.private]
}

resource "openstack_compute_instance_v2" "dispatcher" {
  name        = "${var.instance_prefix}-dispatcher"
  flavor_name = var.dispatcher_flavor
  key_pair    = var.key_name

  user_data = templatefile("${path.module}/cloud-init/dispatcher.yaml", {
    s3_endpoint           = var.s3_endpoint
    s3_region             = var.s3_region
    s3_access_key         = var.s3_access_key
    s3_secret_key         = var.s3_secret_key
    jobs_bucket           = aws_s3_bucket.jobs.id
    output_bucket         = aws_s3_bucket.output.id
    worker_mode           = var.worker_mode
    enable_monitoring     = var.enable_monitoring ? "true" : "false"
  })

  block_device {
    uuid                  = data.openstack_images_image_v2.os.id
    source_type           = "image"
    destination_type      = "volume"
    volume_size           = 40
    boot_index            = 0
    delete_on_termination = true
  }

  network {
    port = openstack_networking_port_v2.dispatcher.id
  }

  depends_on = [openstack_networking_router_interface_v2.private]
}

resource "openstack_networking_port_v2" "worker" {
  count = var.worker_count

  name               = "${var.instance_prefix}-worker-${count.index + 1}-port"
  network_id         = openstack_networking_network_v2.private.id
  security_group_ids = [openstack_networking_secgroup_v2.worker.id]

  fixed_ip {
    subnet_id = openstack_networking_subnet_v2.private.id
  }

  depends_on = [openstack_networking_router_interface_v2.private]
}

resource "openstack_compute_instance_v2" "worker" {
  count       = var.worker_count
  name        = "${var.instance_prefix}-worker-${count.index + 1}"
  flavor_name = var.worker_flavor
  key_pair    = var.key_name

  user_data = templatefile("${path.module}/cloud-init/worker.yaml", {
    s3_endpoint           = var.s3_endpoint
    s3_region             = var.s3_region
    s3_access_key         = var.s3_access_key
    s3_secret_key         = var.s3_secret_key
    jobs_bucket           = aws_s3_bucket.jobs.id
    output_bucket         = aws_s3_bucket.output.id
    dispatcher_ip         = openstack_networking_port_v2.dispatcher.all_fixed_ips[0]
    worker_mode           = var.worker_mode
    worker_index          = count.index + 1
    poll_interval_seconds = var.poll_interval_seconds
    enable_monitoring     = var.enable_monitoring ? "true" : "false"
  })

  block_device {
    uuid                  = data.openstack_images_image_v2.os.id
    source_type           = "image"
    destination_type      = "volume"
    volume_size           = 80
    boot_index            = 0
    delete_on_termination = true
  }

  network {
    port = openstack_networking_port_v2.worker[count.index].id
  }

  depends_on = [
    openstack_networking_router_interface_v2.private,
    openstack_compute_instance_v2.dispatcher,
  ]
}

resource "openstack_networking_floatingip_v2" "dispatcher" {
  count = var.enable_dispatcher_fip ? 1 : 0
  pool  = var.external_network
}

resource "openstack_networking_floatingip_associate_v2" "dispatcher" {
  count       = var.enable_dispatcher_fip ? 1 : 0
  floating_ip = openstack_networking_floatingip_v2.dispatcher[0].address
  port_id     = openstack_networking_port_v2.dispatcher.id
}

variable "key_name" {
  description = "Existing SSH keypair name in the project for compute instances"
  type        = string
}

variable "s3_access_key" {
  description = "EC2-compatible access key for Object Storage (Keystone ec2 credentials create)"
  type        = string
  sensitive   = true
}

variable "s3_secret_key" {
  description = "EC2-compatible secret key paired with s3_access_key"
  type        = string
  sensitive   = true
}

variable "jobs_bucket_name" {
  description = "S3 bucket name for job manifests and input assets"
  type        = string
}

variable "output_bucket_name" {
  description = "S3 bucket name for rendered output frames and clips"
  type        = string
}

variable "worker_count" {
  description = "Number of CPU render worker instances in the pool"
  type        = number
  default     = 2

  validation {
    condition     = var.worker_count >= 1 && var.worker_count <= 20
    error_message = "worker_count must be between 1 and 20."
  }
}

variable "worker_flavor" {
  description = "Compute flavor for each render worker (CPU-only; no GPU on Quake AI)"
  type        = string
  default     = "c2a.large"
}

variable "dispatcher_flavor" {
  description = "Compute flavor for the job dispatcher VM"
  type        = string
  default     = "c2a.large"
}

variable "image_name" {
  description = "Boot image name"
  type        = string
  default     = "Ubuntu-24.04"
}

variable "external_network" {
  description = "Shared external network for router gateway and floating IPs; defaults to PublicStatic (persisted FIP / production pattern). Override with PublicEphemeral for ephemeral demos."
  type        = string
  default     = "PublicStatic"
}

variable "private_cidr" {
  description = "Private subnet CIDR for dispatcher and workers"
  type        = string
  default     = "192.168.80.0/24"
}

variable "instance_prefix" {
  description = "Prefix for dispatcher and worker instance names"
  type        = string
  default     = "render-farm"
}

variable "worker_mode" {
  description = "Worker software profile: queue-puller (default), opencue-rqd, or tractor-blade stub"
  type        = string
  default     = "queue-puller"

  validation {
    condition     = contains(["queue-puller", "opencue-rqd", "tractor-blade"], var.worker_mode)
    error_message = "worker_mode must be queue-puller, opencue-rqd, or tractor-blade."
  }
}

variable "enable_dispatcher_fip" {
  description = "When true, attach one floating IP to the dispatcher for remote job submission over HTTPS"
  type        = bool
  default     = false
}

variable "admin_cidr" {
  description = "Source CIDR allowed for SSH (22/tcp) on dispatcher and workers"
  type        = string
  default     = "0.0.0.0/0"
}

variable "submit_cidr" {
  description = "Source CIDR allowed to POST jobs to the dispatcher API when enable_dispatcher_fip is true"
  type        = string
  default     = "0.0.0.0/0"
}

variable "enable_monitoring" {
  description = "When true, install node_exporter on dispatcher and workers for Prometheus scrape from a monitoring-stack deployment"
  type        = bool
  default     = false
}

variable "poll_interval_seconds" {
  description = "Seconds between worker polls of the dispatcher queue"
  type        = number
  default     = 30
}

variable "s3_endpoint" {
  description = "Quake AI S3-compatible endpoint URL"
  type        = string
  default     = "https://object.us-east-1.rumble.cloud"
}

variable "s3_region" {
  description = "S3 region identifier passed to the AWS provider"
  type        = string
  default     = "us-east-1"
}

output "dispatcher_instance_id" {
  description = "Nova instance ID for the job dispatcher"
  value       = openstack_compute_instance_v2.dispatcher.id
}

output "dispatcher_private_ip" {
  description = "Private IPv4 address workers use to reach the queue API"
  value       = openstack_networking_port_v2.dispatcher.all_fixed_ips[0]
}

output "dispatcher_floating_ip" {
  description = "Public floating IP for remote job submission (empty when enable_dispatcher_fip is false)"
  value       = var.enable_dispatcher_fip ? openstack_networking_floatingip_v2.dispatcher[0].address : ""
}

output "job_submit_url" {
  description = "HTTP endpoint for submitting render jobs when a floating IP is enabled"
  value       = var.enable_dispatcher_fip ? "http://${openstack_networking_floatingip_v2.dispatcher[0].address}:8080/jobs" : "http://${openstack_networking_port_v2.dispatcher.all_fixed_ips[0]}:8080/jobs"
}

output "worker_instance_ids" {
  description = "Nova instance IDs for the render worker pool"
  value       = openstack_compute_instance_v2.worker[*].id
}

output "jobs_bucket" {
  description = "Object Storage bucket for job manifests and input assets"
  value       = aws_s3_bucket.jobs.id
}

output "output_bucket" {
  description = "Object Storage bucket for rendered frames and clips"
  value       = aws_s3_bucket.output.id
}

output "floating_ip_count" {
  description = "Floating IPs this template allocates (zero by default; one when enable_dispatcher_fip is true)"
  value       = var.enable_dispatcher_fip ? 1 : 0
}

terraform {
  required_version = ">= 1.6.0"

  required_providers {
    openstack = {
      source  = "terraform-provider-openstack/openstack"
      version = "~> 2.0"
    }
    aws = {
      source  = "hashicorp/aws"
      version = "~> 5.0"
    }
  }
}

provider "openstack" {}

provider "aws" {
  region                      = var.s3_region
  access_key                  = var.s3_access_key
  secret_key                  = var.s3_secret_key
  skip_credentials_validation = true
  skip_metadata_api_check     = true
  skip_requesting_account_id  = true

  endpoints {
    s3 = var.s3_endpoint
  }
}

#cloud-config
package_update: true
write_files:
  - path: /etc/render-dispatcher.env
    owner: root:root
    permissions: "0600"
    content: |
      AWS_ACCESS_KEY_ID=${s3_access_key}
      AWS_SECRET_ACCESS_KEY=${s3_secret_key}
      AWS_DEFAULT_REGION=${s3_region}
      AWS_ENDPOINT_URL_S3=${s3_endpoint}
      JOBS_BUCKET=${jobs_bucket}
      OUTPUT_BUCKET=${output_bucket}
      WORKER_MODE=${worker_mode}
      ENABLE_MONITORING=${enable_monitoring}
  - path: /usr/local/bin/render-dispatcher.py
    owner: root:root
    permissions: "0755"
    content: |
      #!/usr/bin/env python3
      import json, os, subprocess, tempfile, uuid
      from http.server import BaseHTTPRequestHandler, HTTPServer

      ENV = {}
      for line in open("/etc/render-dispatcher.env"):
          line = line.strip()
          if line and "=" in line:
              k, v = line.split("=", 1)
              ENV[k] = v

      def s3_ls(prefix):
          cmd = [
              "aws", "s3", "ls", f"s3://{ENV['JOBS_BUCKET']}/{prefix}",
              "--endpoint-url", ENV["AWS_ENDPOINT_URL_S3"],
          ]
          try:
              out = subprocess.check_output(cmd, text=True, stderr=subprocess.STDOUT, env={**os.environ, **ENV})
          except subprocess.CalledProcessError:
              return []
          return [ln.split()[-1] for ln in out.splitlines() if ln.strip()]

      class Handler(BaseHTTPRequestHandler):
          def _json(self, code, body):
              data = json.dumps(body).encode()
              self.send_response(code)
              self.send_header("Content-Type", "application/json")
              self.send_header("Content-Length", str(len(data)))
              self.end_headers()
              self.wfile.write(data)

          def do_GET(self):
              if self.path == "/health":
                  return self._json(200, {"status": "ok", "mode": ENV["WORKER_MODE"]})
              if self.path == "/jobs/next":
                  pending = s3_ls("pending/")
                  if not pending:
                      return self._json(204, {})
                  job_key = pending[0]
                  job_id = job_key[:-5] if job_key.endswith(".json") else job_key
                  claimed = f"claimed/{job_key}"
                  subprocess.check_call([
                      "aws", "s3", "mv",
                      f"s3://{ENV['JOBS_BUCKET']}/pending/{job_key}",
                      f"s3://{ENV['JOBS_BUCKET']}/{claimed}",
                      "--endpoint-url", ENV["AWS_ENDPOINT_URL_S3"],
                  ], env={**os.environ, **ENV})
                  return self._json(200, {"job_id": job_id, "manifest_key": claimed})
              return self._json(404, {"error": "not found"})

          def do_POST(self):
              if self.path != "/jobs":
                  return self._json(404, {"error": "not found"})
              length = int(self.headers.get("Content-Length", 0))
              body = self.rfile.read(length) if length else b"{}"
              manifest = json.loads(body.decode() or "{}")
              job_id = manifest.get("job_id") or str(uuid.uuid4())
              manifest.setdefault("job_id", job_id)
              manifest.setdefault("engine", "ffmpeg")
              with tempfile.NamedTemporaryFile("w", delete=False) as fh:
                  json.dump(manifest, fh)
                  tmp = fh.name
              subprocess.check_call([
                  "aws", "s3", "cp", tmp,
                  f"s3://{ENV['JOBS_BUCKET']}/pending/{job_id}.json",
                  "--endpoint-url", ENV["AWS_ENDPOINT_URL_S3"],
              ], env={**os.environ, **ENV})
              os.unlink(tmp)
              return self._json(201, {"job_id": job_id, "status": "queued"})

          def log_message(self, fmt, *args):
              return

      if __name__ == "__main__":
          HTTPServer(("0.0.0.0", 8080), Handler).serve_forever()
  - path: /etc/systemd/system/render-dispatcher.service
    owner: root:root
    permissions: "0644"
    content: |
      [Unit]
      Description=CPU render farm job dispatcher
      After=network-online.target
      Wants=network-online.target

      [Service]
      Type=simple
      ExecStart=/usr/local/bin/render-dispatcher.py
      Restart=always
      RestartSec=5

      [Install]
      WantedBy=multi-user.target
runcmd:
  - |
    set -e
    export DEBIAN_FRONTEND=noninteractive
    apt-get update
    apt-get install -y python3 curl python3-pip
    pip3 install --break-system-packages awscli
    if [ "${enable_monitoring}" = "true" ]; then
      useradd --no-create-home --shell /bin/false node_exporter || true
      curl -fsSL -o /usr/local/bin/node_exporter \
        https://github.com/prometheus/node_exporter/releases/download/v1.8.2/node_exporter-1.8.2.linux-amd64.tar.gz
      tar -xzf /usr/local/bin/node_exporter -C /tmp
      install /tmp/node_exporter-1.8.2.linux-amd64/node_exporter /usr/local/bin/node_exporter
      rm -rf /tmp/node_exporter-1.8.2.linux-amd64 /usr/local/bin/node_exporter
    fi
    systemctl daemon-reload
    systemctl enable render-dispatcher.service
    systemctl start render-dispatcher.service

#cloud-config
package_update: true
write_files:
  - path: /etc/render-worker.env
    owner: root:root
    permissions: "0600"
    content: |
      AWS_ACCESS_KEY_ID=${s3_access_key}
      AWS_SECRET_ACCESS_KEY=${s3_secret_key}
      AWS_DEFAULT_REGION=${s3_region}
      AWS_ENDPOINT_URL_S3=${s3_endpoint}
      JOBS_BUCKET=${jobs_bucket}
      OUTPUT_BUCKET=${output_bucket}
      DISPATCHER_URL=http://${dispatcher_ip}:8080
      WORKER_MODE=${worker_mode}
      WORKER_INDEX=${worker_index}
      POLL_INTERVAL=${poll_interval_seconds}
      ENABLE_MONITORING=${enable_monitoring}
  - path: /usr/local/bin/render-worker.sh
    owner: root:root
    permissions: "0755"
    content: |
      #!/bin/bash
      set -euo pipefail
      source /etc/render-worker.env
      WORK_DIR=/var/lib/render-worker
      mkdir -p "$WORK_DIR"
      while true; do
        resp=$(curl -fsS "$DISPATCHER_URL/jobs/next" || true)
        if [ -z "$resp" ] || [ "$resp" = "{}" ] || [ "$resp" = "null" ]; then
          sleep "$POLL_INTERVAL"
          continue
        fi
        job_id=$(echo "$resp" | python3 -c "import sys,json; print(json.load(sys.stdin).get('job_id',''))")
        manifest_key=$(echo "$resp" | python3 -c "import sys,json; print(json.load(sys.stdin).get('manifest_key',''))")
        [ -z "$job_id" ] && sleep "$POLL_INTERVAL" && continue
        manifest="$WORK_DIR/$job_id.json"
        aws s3 cp "s3://$JOBS_BUCKET/$manifest_key" "$manifest"
        engine=$(python3 -c "import json; print(json.load(open('$manifest')).get('engine','ffmpeg'))")
        input_key=$(python3 -c "import json; print(json.load(open('$manifest')).get('input_key',''))")
        outfile="$WORK_DIR/$job_id-out.mp4"
        infile="$WORK_DIR/$job_id-in"
        aws s3 cp "s3://$JOBS_BUCKET/$input_key" "$infile"
        case "$engine" in
          blender)
            apt-get install -y blender >/dev/null 2>&1 || true
            blender -b -noaudio -E CYCLES -o "$WORK_DIR/frame_" -F PNG -f 1 "$infile" || \
              ffmpeg -hide_banner -loglevel error -i "$infile" -c:v libx264 -preset veryfast "$outfile"
            ;;
          *)
            ffmpeg -hide_banner -loglevel error -i "$infile" -c:v libx264 -preset veryfast -crf 23 "$outfile"
            ;;
        esac
        aws s3 cp "$outfile" "s3://$OUTPUT_BUCKET/$job_id/render.mp4"
        aws s3 mv "s3://$JOBS_BUCKET/$manifest_key" "s3://$JOBS_BUCKET/done/$job_id.json"
        rm -f "$manifest" "$infile" "$outfile"
        sleep 2
      done
  - path: /etc/systemd/system/render-worker.service
    owner: root:root
    permissions: "0644"
    content: |
      [Unit]
      Description=CPU render farm worker ${worker_index}
      After=network-online.target
      Wants=network-online.target

      [Service]
      Type=simple
      EnvironmentFile=/etc/render-worker.env
      ExecStart=/usr/local/bin/render-worker.sh
      Restart=always
      RestartSec=10

      [Install]
      WantedBy=multi-user.target
runcmd:
  - |
    set -e
    export DEBIAN_FRONTEND=noninteractive
    apt-get update
    apt-get install -y ffmpeg curl python3 python3-pip
    pip3 install --break-system-packages awscli
    if [ "${enable_monitoring}" = "true" ]; then
      curl -fsSL https://github.com/prometheus/node_exporter/releases/download/v1.8.2/node_exporter-1.8.2.linux-amd64.tar.gz -o /tmp/ne.tar.gz
      tar -xzf /tmp/ne.tar.gz -C /tmp
      install /tmp/node_exporter-1.8.2.linux-amd64/node_exporter /usr/local/bin/node_exporter
      rm -rf /tmp/ne.tar.gz /tmp/node_exporter-1.8.2.linux-amd64
    fi
    systemctl daemon-reload
    systemctl enable render-worker.service
    systemctl start render-worker.service

# CPU render-farm worker pool

Dispatcher VM plus a parameterized pool of CPU render workers that pull batch jobs from Object Storage, render on CPU (Blender Cycles CPU, FFmpeg, or batch audio), and write outputs back to Object Storage.


**Network class:** production — `external_network` defaults to `PublicStatic` for persisted floating IPs and multi-tier stacks; override with `PublicEphemeral` for ephemeral demos.

## Prerequisites

- OpenTofu >= 1.6.0 or Terraform >= 1.6.0
- Quake AI account with OpenStack credentials
- EC2-compatible Object Storage credentials
- Optional: one floating IP quota slot when `enable_dispatcher_fip` is true

## Usage

1. Clone or copy this template directory
2. Copy `terraform.tfvars.example` to `terraform.tfvars` and fill in your values
3. Source your OpenStack credentials: `source openrc.sh`
4. Initialize: `tofu init`
5. Preview: `tofu plan`
6. Apply: `tofu apply`

Submit jobs by POSTing a JSON manifest to the dispatcher (`engine`, `input_key`) or uploading manifests to `s3://jobs_bucket/pending/`.

## Variables

| Name | Type | Required | Default | Description |
| --- | --- | --- | --- | --- |
| `key_name` | string | yes | n/a | Existing SSH keypair name in the project for compute instances |
| `jobs_bucket_name` | string | yes | n/a | Job manifests and input assets |
| `output_bucket_name` | string | yes | n/a | Rendered output |
| `worker_count` | number | no | `2` | CPU worker pool size |
| `worker_flavor` | string | no | `c2a.large` | Flavor per worker |
| `enable_dispatcher_fip` | bool | no | `false` | Public job submission endpoint |
| `enable_monitoring` | bool | no | `false` | Install node_exporter for Prometheus |

## Documentation

Full documentation: [CPU render-farm worker template](/docs/automation/templates/render-farm-worker)

## Honesty note

Quake AI offers CPU compute only. This template targets hobbyist, motion-design preview, overnight batch, and audio-render workloads. It does not provision GPU render nodes.