macOS (Apple M3)
└─ UTM (Apple Virtualization, 중첩 가상화)
   └─ Ubuntu 24.04 ARM64  (호스트 yoosung-vm, 강의의 "Ubuntu 호스트") ← Vagrant·Ansible 실행
      └─ KVM + libvirt   (VirtualBox 대체, vagrant-libvirt 플러그인)
         ├─ main   Rocky Linux 10.2 aarch64  192.168.56.101  main.cloud.org   (central manager + submit)
         ├─ work1  Rocky Linux 10.2 aarch64  192.168.56.102  work1.cloud.org  (execute)
         └─ work2  Rocky Linux 10.2 aarch64  192.168.56.103  work2.cloud.org  (execute)

virsh -c qemu:///system list --all
virsh -c qemu:///system undefine main  --nvram --remove-all-storage
virsh -c qemu:///system undefine work1 --nvram --remove-all-storage
virsh -c qemu:///system undefine work2 --nvram --remove-all-storage
virsh -c qemu:///system list --all
virsh -c qemu:///system vol-list images

wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
echo "deb [arch=amd64 signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com noble main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
sudo apt update
sudo apt install vagrant

sudo rm /etc/apt/sources.list.d/hashicorp.list && sudo apt clean
sudo apt install -y ruby-full ruby-dev build-essential libarchive-dev libarchive-tools zlib1g-dev libssl-dev pkg-config libvirt-dev
ruby --version; gem --version; bsdtar --version
sudo gem install vagrant --no-document
vagrant --version

vagrant plugin install vagrant-libvirt
vagrant plugin list

mkdir vrepo && cd vrepo && vagrant init
ls
grep -v "^ *#" Vagrantfile | grep -v "^ *$"

Call to virStoragePoolDefineXML failed: ... Storage source conflict with pool: 'images'

vagrant box add rockylinux/10 --provider libvirt        # → 404

vagrant box add --name rockylinux/10 https://dl.rockylinux.org/pub/rocky/10/images/aarch64/Rocky-10-Vagrant-Libvirt.latest.aarch64.vagrant.libvirt.box
vagrant box list

#!/bin/bash
# box-nomsi.sh <box.img>
# qcow2 박스 이미지의 커널 명령줄에 pci=nomsi 추가 (qemu-nbd로 오프라인 수정)
set -e
IMG="$1"; [ -f "$IMG" ] || { echo "usage: $0 <box.img>"; exit 1; }
sudo modprobe nbd max_part=16
sudo qemu-nbd --disconnect /dev/nbd0 >/dev/null 2>&1 || true
sudo qemu-nbd --connect=/dev/nbd0 "$IMG"
sleep 2; sudo partprobe /dev/nbd0 2>/dev/null || true; sleep 1
lsblk -o NAME,SIZE,FSTYPE,LABEL /dev/nbd0
M=$(mktemp -d)
cleanup() { sudo umount "$M/boot" 2>/dev/null; sudo umount "$M" 2>/dev/null; rmdir "$M" 2>/dev/null; sudo qemu-nbd --disconnect /dev/nbd0 >/dev/null 2>&1; }
trap cleanup EXIT
ROOT=""; BOOT=""
for p in /dev/nbd0p*; do
  fs=$(lsblk -no FSTYPE "$p"); [ -z "$fs" ] && continue
  [ "$fs" = vfat ] && continue
  sudo mount -o ro "$p" "$M"
  if [ -d "$M/etc" ] && [ -d "$M/boot" ]; then ROOT="$p"; fi
  if [ -d "$M/loader/entries" ]; then BOOT="$p"; fi
  sudo umount "$M"
done
echo "root=$ROOT boot=$BOOT"
sudo mount "$ROOT" "$M"
[ -n "$BOOT" ] && sudo mount "$BOOT" "$M/boot"
# /boot/loader/entries는 root 전용(0700)이라 glob을 sudo 안에서 펼침
show() { sudo bash -c "grep -h '^options' '$M'/boot/loader/entries/*.conf; cat '$M/etc/kernel/cmdline' 2>/dev/null; grep '^GRUB_CMDLINE_LINUX=' '$M/etc/default/grub'; true"; }
echo "--- before"; show
sudo bash -c "sed -i '/^options/{/pci=nomsi/!s/\$/ pci=nomsi/}' '$M'/boot/loader/entries/*.conf"
sudo bash -c "[ -f '$M/etc/kernel/cmdline' ] && sed -i '/pci=nomsi/!s/\$/ pci=nomsi/' '$M/etc/kernel/cmdline' || true"
sudo sed -i '/^GRUB_CMDLINE_LINUX=/{/pci=nomsi/!s/"$/ pci=nomsi"/}' "$M/etc/default/grub" 2>/dev/null || true
echo "--- after"; show
sync; cleanup; trap - EXIT
echo "done: pci=nomsi added to $IMG"

~/bin/box-nomsi.sh ~/.vagrant.d/boxes/rockylinux-VAGRANTSLASH-10/0/arm64/libvirt/box.img

NODE_COUNT=2

Vagrant.configure("2") do |config|
  config.vm.box = "rockylinux/10"
  config.vm.synced_folder ".", "/vagrant", disabled: true

  config.vm.provider "libvirt" do |v|
    v.storage_pool_name = "images"
    v.machine_virtual_size = 10
    v.machine_type = "virt"
    v.machine_arch = "aarch64"
    v.cpu_mode = "host-passthrough"
    v.loader = "/usr/share/AAVMF/AAVMF_CODE.no-secboot.fd"
    v.graphics_type = "none"
    v.input type: "keyboard", bus: "virtio"
    v.cpus = 1
    v.memory = 1024
  end

  config.vm.define "main" do |main|
    main.vm.hostname = "main.cloud.org"
    main.vm.network "private_network", ip: "192.168.56.101", libvirt__network_name: "cloud56"
    main.vm.provider "libvirt" do |v|
      v.title = "Main(vagrant)"
      v.nvram = "/var/lib/libvirt/qemu/nvram/vrepo_main_VARS.fd"
    end
  end

  NODE_COUNT.times do |i|
    node_id = "work#{i+1}"
    ip_address = "192.168.56.10#{i+2}"
    config.vm.define node_id do |node|
      node.vm.hostname = "#{node_id}.cloud.org"
      node.vm.network "private_network", ip: ip_address, libvirt__network_name: "cloud56"
      node.vm.provider "libvirt" do |v|
        v.title = "Work#{i+1}(vagrant)"
        v.nvram = "/var/lib/libvirt/qemu/nvram/vrepo_#{node_id}_VARS.fd"
      end
    end
  end
end

vagrant up                      # 1차: 비상 셸
virsh -c qemu:///system console vrepo_main
vagrant destroy -f
grep -n machine_virtual_size Vagrantfile
vagrant up                      # 2차: 성공

vagrant status
vagrant ssh main      # hostname; ip -br addr; exit   (work1, work2도 같음)

tree .vagrant/ -L 4 -a
vagrant ssh-config

sudo apt install -y ansible
ansible --version
sudo mkdir -p /etc/ansible
sudo cp /mnt/share/VM/share/ansible-hosts /etc/ansible/hosts   # 공유 폴더에 미리 써 둔 파일
cat /etc/ansible/hosts

[cluster]
main.cloud.org ansible_user=vagrant ansible_ssh_private_key_file=/home/yoosung/vrepo/.vagrant/machines/main/libvirt/private_key
work1.cloud.org ansible_user=vagrant ansible_ssh_private_key_file=/home/yoosung/vrepo/.vagrant/machines/work1/libvirt/private_key
work2.cloud.org ansible_user=vagrant ansible_ssh_private_key_file=/home/yoosung/vrepo/.vagrant/machines/work2/libvirt/private_key

[main]
main.cloud.org ansible_user=vagrant ansible_ssh_private_key_file=/home/yoosung/vrepo/.vagrant/machines/main/libvirt/private_key

[worker]
work1.cloud.org ansible_user=vagrant ansible_ssh_private_key_file=/home/yoosung/vrepo/.vagrant/machines/work1/libvirt/private_key
work2.cloud.org ansible_user=vagrant ansible_ssh_private_key_file=/home/yoosung/vrepo/.vagrant/machines/work2/libvirt/private_key

printf "%s\n" "192.168.56.101 main.cloud.org main" "192.168.56.102 work1.cloud.org work1" "192.168.56.103 work2.cloud.org work2" | sudo tee -a /etc/hosts
ping -c 2 main.cloud.org; ping -c 2 work1.cloud.org; ping -c 2 work2.cloud.org

ssh-keyscan main.cloud.org work1.cloud.org work2.cloud.org >> ~/.ssh/known_hosts
ansible cluster -m ping

---
- name: HTCondor Installation
  hosts: cluster
  become: yes
  become_user: root

  tasks:
    - name: Disable SELinux
      ansible.posix.selinux:
        state: permissive
        policy: targeted

    - name: Stop and disable firewalld
      ansible.builtin.service:
        name: firewalld
        state: stopped
        enabled: no

    - name: Modify /etc/hosts file
      ansible.builtin.blockinfile:
        path: /etc/hosts
        block: |
          192.168.56.101 main.cloud.org     main
          192.168.56.102 work1.cloud.org    work1
          192.168.56.103 work2.cloud.org    work2

    - name: Ensure OpenSSH fully update
      ansible.builtin.dnf:
        name:
          - openssh-server
          - openssh-clients
          - openssl
          - openssl-libs
        state: latest

    - name: Install wget package
      ansible.builtin.dnf:
        name: wget
        state: present

    - name: Install EPEL
      ansible.builtin.dnf:
        name: epel-release
        state: present

    - name: Install HTCondor Repository for EL10
      ansible.builtin.dnf:
        name: https://research.cs.wisc.edu/htcondor/repo/25.0/htcondor-release-current.el10.noarch.rpm
        state: present
        disable_gpg_check: yes

    - name: Install condor
      ansible.builtin.dnf:
        name: condor
        state: present

- name: HTCondor Main Node Setup
  hosts: main
  become: yes
  become_user: root

  tasks:
    - name: Remove old configuration file
      ansible.builtin.file:
        path: /etc/condor/config.d/condor_config.local
        state: absent

    - name: Creating a configuration file
      ansible.builtin.copy:
        dest: /etc/condor/config.d/condor_config.local
        content: |
          UID_DOMAIN = cloud.org
          ALLOW_READ = *
          ALLOW_WRITE = 192.168.56.*
          CONDOR_ADMIN = root@$(FULL_HOSTNAME)
          CONDOR_HOST = main.cloud.org
          IN_HIGHPORT = 9999
          IN_LOWPORT = 9000
          REQUIRE_LOCAL_CONFIG_FILE = False
          NETWORK_INTERFACE = enp4s0
          DAEMON_LIST = MASTER, SCHEDD, COLLECTOR, NEGOTIATOR
          SEC_DEFAULT_AUTHENTICATION_METHODS = CLAIMTOBE
          SEC_CLIENT_AUTHENTICATION_METHODS = CLAIMTOBE
        mode: '0644'

    - name: Restart HTCondor
      ansible.builtin.service:
        name: condor
        state: restarted
        enabled: yes

- name: HTCondor Worker Node Setup
  hosts: worker
  become: yes
  become_user: root

  tasks:
    - name: Remove old configuration file
      ansible.builtin.file:
        path: /etc/condor/config.d/condor_config.local
        state: absent

    - name: Creating a configuration file
      ansible.builtin.copy:
        dest: /etc/condor/config.d/condor_config.local
        content: |
          UID_DOMAIN = cloud.org
          ALLOW_READ = *
          ALLOW_WRITE = 192.168.56.*
          CONDOR_ADMIN = root@$(FULL_HOSTNAME)
          CONDOR_HOST = main.cloud.org
          IN_HIGHPORT = 9999
          IN_LOWPORT = 9000
          REQUIRE_LOCAL_CONFIG_FILE = False
          NETWORK_INTERFACE = enp4s0
          DAEMON_LIST = MASTER, STARTD
          SEC_DEFAULT_AUTHENTICATION_METHODS = CLAIMTOBE
          SEC_CLIENT_AUTHENTICATION_METHODS = CLAIMTOBE
        mode: '0644'

    - name: Restart HTCondor
      ansible.builtin.service:
        name: condor
        state: restarted
        enabled: yes

ansible-playbook --syntax-check htcondor.yaml
ansible-playbook htcondor.yaml

attempt to connect to <192.168.56.101:9618> failed: No route to host
TCP connection to collector main.cloud.org failed

vagrant ssh main
condor_status                                                   # 빈 출력
ansible worker  -b -m shell -a "tail -3 /var/log/condor/StartLog"
ansible cluster -b -m shell -a "firewall-cmd --state; firewall-cmd --list-services"
ansible-playbook htcondor.yaml                                  # firewalld 태스크 추가 후

vagrant ssh main
condor_status