From 5561b08e6ab58209cb1a5febd57c355e01530f67 Mon Sep 17 00:00:00 2001 From: peter Date: Thu, 21 Jan 2021 19:48:05 -0800 Subject: [PATCH] add gpu passthrough support as well as dynamic OpenCore regeneration --- helm/Dockerfile | 72 ++++ helm/INSTALL-QEMU-AND-GPU-IOMMU.md | 283 +++++++++++++++ helm/README.md | 19 +- helm/templates/configmap.yaml | 543 +++++++++++++++-------------- helm/templates/deployment.yaml | 45 ++- helm/templates/service.yaml | 10 +- helm/values.yaml | 117 ++++++- 7 files changed, 790 insertions(+), 299 deletions(-) create mode 100644 helm/Dockerfile create mode 100644 helm/INSTALL-QEMU-AND-GPU-IOMMU.md diff --git a/helm/Dockerfile b/helm/Dockerfile new file mode 100644 index 0000000..55c0e52 --- /dev/null +++ b/helm/Dockerfile @@ -0,0 +1,72 @@ +#!/usr/bin/docker +# +# This Dockerfile is to be consumed with the docker_osx helm templates. It consumes the +# Ubuntu image so that OpenCore.qcow2 can be re-generated (which happens in Kube itself), +# not to mention that OSX-KVM was written for Ubuntu. This was not designed to be run in Docker +# by itself.. very well anyway. +# + +FROM ubuntu:21.04 + +SHELL ["/bin/bash", "-c"] + +# this has to match .Values.image.userName in helm template +ARG USER=ubuntu +# this installs the kvm linux kernel in the docker container so that OpenCore.qcow2 boot images +# can be built. +ARG DOCKER_KERNEL_VERSION=linux-image-kvm + +ENV TZ=America/Los_Angeles +ARG DEBIAN_FRONTEND=noninteractive + +RUN DEBCONF_FRONTEND=noninteractive apt update \ + && apt install \ + bridge-utils \ + fish \ + git wget \ + libguestfs-tools \ + libvirt-daemon-system \ + $DOCKER_KERNEL_VERSION \ + p7zip-full \ + qemu \ + sudo \ + uml-utilities \ + virt-manager \ + -y + +# Configure SSH +RUN apt install git vim nano alsa-utils openssh-server -y + +# Create user and grant sudo privledges +RUN adduser --disabled-password \ + --gecos '' $USER \ + && echo "$USER ALL=(ALL) NOPASSWD: ALL" > /etc/sudoers.d/$USER \ + && chmod 0440 /etc/sudoers.d/$USER + +# Configure VNC for user +RUN apt install \ + dbus-x11 \ + openbox \ + tigervnc-common \ + tigervnc-standalone-server \ + xfce4 \ + xfce4-goodies \ + x11-xserver-utils \ + xdotool \ + xorg \ + xterm \ + ufw \ + -y + +USER $USER + +# only create ~/.vnc as helm will build out ~/.vnc/config +RUN mkdir -p ${HOME}/.vnc + +RUN git clone --depth 1 https://github.com/kholia/OSX-KVM.git /home/$USER/OSX-KVM + +VOLUME ["/tmp/.X11-unix"] + +WORKDIR /home/$USER/OSX-KVM +# helm will build out ./Launch_custom.sh +CMD envsubst < ./Launch_custom.sh | bash diff --git a/helm/INSTALL-QEMU-AND-GPU-IOMMU.md b/helm/INSTALL-QEMU-AND-GPU-IOMMU.md new file mode 100644 index 0000000..2b2fd50 --- /dev/null +++ b/helm/INSTALL-QEMU-AND-GPU-IOMMU.md @@ -0,0 +1,283 @@ +# Install macOS Docker Virtualization +## Setup +This walks through setting up QEMU virtualization for running macOS in Docker & Kubernetes + +Please note, this guide assumes the host operation system is running Centos 7 (or ClearOS 7 more specifically). These commands can mostly be transferred to other distros, but there are a few areas that need commands (i.e. updating ) + +### Host configuration + +## Build QEMU and libvirt from source + +Since there is no official QEMU 5.X repo it appears, build from source. + +### QEMU Requirements + +Python, glib2-devel, and pixman + +``` +sudo yum install python glib2-devel cairo-devel -y +``` + +Ninja + +``` +pip3 install ninja +``` + +### Build QEMU from source steps + +Clone the offical QEMU repo and build from source: + +``` +git clone git://git.qemu-project.org/qemu.git +cd qemu +mkdir -p bin/debug/native +cd bin/debug/native +../../../configure --enable-debug +make -j24 +make install +``` + +_Note: adjust make to use the desired number of threads avaliable on your system_ + +### libvirt Requirements + +Configure repo: + +``` +yum-config-manager --nogpgcheck --add-repo http://mirror.centos.org/centos/7/virt/x86_64/libvirt-latest/ +``` + +### Install libvirt + +``` +yum install libvirt -y +``` + +### Update permissions + +``` +chmod 660 -R /dev/kvm && chown 1000:1000 /dev/kvm +usermod -a -G kvm root +``` + +_Note: these may not be required_ + +### Verification + +Ensure latest version installed + +``` +virsh -c qemu:///system version --daemon +``` + +* For example, should output something like: + + ``` + [root@server repos]# virsh -c qemu:///system version --daemon + Compiled against library: libvirt 5.0.0 + Using library: libvirt 5.0.0 + Using API: QEMU 5.0.0 + Running hypervisor: QEMU 5.2.50 + Running against daemon: 5.0.0 + ``` + +## Install IMMO for GPU passthrough + +1. Modify GRUB boot args: + + Add the following to `/etc/default/grub` to the end of the `GRUB_CMDLINE_LINUX` parameter: + + ``` + GRUB_CMDLINE_LINUX="... iommu=pt intel_iommu=on" + ``` + +1. Update GRUB2: + + ``` + grub2-mkconfig -o /boot/efi/EFI/clearos/grub.cfg + ``` + + _Note: this command may vary based on location of the grub.cfg for the boot entry_ + +1. Reboot system + +1. Ensure that the kernel parameter changes worked: + + ``` + cat /proc/cmdline + ``` + +1. Find GPU hardware ids with `lspci` + + Example: + ``` + lspci -nn | grep -i nvidia + ``` + +1. Add the hardware ids to `/etc/modprobe.d/vfio.conf` + + Example: + ``` + options vfio-pci ids=10de:1b81,10de:10f0 + ``` + + _Note: this is for the NVIDIA GTX 1070_ + +1. Enable `vfio-pci` + + ``` + echo 'vfio-pci' > /etc/modules-load.d/vfio-pci.conf + ``` + + Make backup and rebuild `initramfs`: + + ``` + cp -p /boot/initramfs-$(uname -r).img /boot/initramfs-$(uname -r).img.bak + dracut -f + ``` + + _Note: `dracut -f` may take awhile.._ + +1. Increase ulimits + + _This is done to avoid memory issues like `VFIO_MAP_DMA: -12` and etc_ + + Append the following to `/etc/security/limits.conf`: + + ``` + @kvm soft memlock unlimited + @kvm hard memlock unlimited + ``` + + Append the following to `/etc/docker/daemon.json`: + + ``` + { + "default-ulimits": { + "nofile": { + "name": "nofile", + "hard": 65536, + "soft": 1024 + }, + "memlock": + { + "name": "memlock", + "soft": -1, + "hard": -1 + } + } + } + ``` + + Add `LimitMEMLOCK` to `/etc/systemd/system/multi-user.target.wants/libvirtd.service` like: + + ``` + [Unit] + Description=Virtualization daemon + ... + + [Service] + ... + LimitMEMLOCK=infinity + ``` + +1. Reload systemd after changing config + + ``` + systemctl daemon-reload + ``` + +1. Reboot system + +1. Ensure that `vfio` worked + + ``` + dmesg | grep -i vfio + ``` + +# Issues + +Many issues can rise up as a result of adding the complexity layers involved here. Some of the main areas are improperly loading the `vfio-pci` driver for the GPU and permission issues. + +## Modules for vfio not loading + +When `vfio` does not load, errors such as the following can be seen: + +``` +error getting device from group *: No such device +Verify all devices in group * are bound to vfio- or pci-stub and not already in use +``` + +This can show up when `vfio-pci` driver is not loaded for the peripheral. Ensure that `vfio-pci` is loaded. + +``` +dmesg | grep -i vfio +``` + +If so, explicitly tell `vfio` modules to start + +``` +echo 'vfio +vfio_iommu_type1 +vfio_pci +vfio_virqfd' > /etc/modules +``` + +Make backup and rebuild `initramfs`: + +``` +cp -p /boot/initramfs-$(uname -r).img /boot/initramfs-$(uname -r).img.bak +dracut -f +``` + +_Note: `dracut -f` may take awhile.._ + +Do a system reboot + +After rebooting, check on the gpu with `lspci` utilizing your gpu hardware id: + +I.E. + +``` +[root@server docker-docker-osx]# lspci -vvv -s 09:00.0 +09:00.0 VGA compatible controller: Advanced Micro Devices, Inc. [AMD/ATI] Ellesmere [Radeon RX 470/480/570/570X/580/580X/590] (rev c7) (prog-if 00 [VGA controller]) + Subsystem: Advanced Micro Devices, Inc. [AMD/ATI] Radeon RX 480 + Physical Slot: 5 + Control: I/O- Mem- BusMaster- SpecCycle- MemWINV- VGASnoop- ParErr+ Stepping- SERR+ FastB2B- DisINTx- + Status: Cap+ 66MHz- UDF- FastB2B- ParErr- DEVSEL=fast >TAbort- SERR- `. If permission errors are seen, try the following commands: + +``` +chmod 660 -R /dev/kvm && chown 1000:1000 /dev/kvm +chmod 777 -R /dev/vfio && chown 1000:1000 -R /dev/vfio +``` + +# References + +https://gist.github.com/dghubble/c2dc319249b156db06aff1d49c15272e + +`Configure IOMMU and vfio` +https://www.server-world.info/en/note?os=CentOS_7&p=kvm&f=10 + +`Configuring GPU driver with vfio-pci binding` +https://github.com/intel/nemu/wiki/Testing-VFIO-with-GPU + +`IOMMU Interrupt Mapping` +https://pve.proxmox.com/wiki/Pci_passthrough#IOMMU_Interrupt_Remapping + +`Manual Graphics Driver Binding` +https://lwn.net/Articles/143397/ + +`QEMU Stdio Example` +https://lists.gnu.org/archive/html/qemu-devel/2017-08/msg04521.html \ No newline at end of file diff --git a/helm/README.md b/helm/README.md index 28fc026..5728202 100644 --- a/helm/README.md +++ b/helm/README.md @@ -17,15 +17,16 @@ This installs `docker-osx` in Kubernetes. 1) Kubernetes resource requests/limits 1) Defining version of macOS to install 1) Defining install partition size - -### What doesn't/isn't defined 1) Defining a different version of macOS 1) Additional QEMU parameters 1) GPU support +### What doesn't +1) Simultaneous VNC + GPU IOMMU (this is a limitation of QEMU :( unfortunately) + ## Requirements -*) Install [host machine requirements](https://github.com/cephasara/Docker-OSX#requirements-kvm-on-the-host) +*) Install [host machine requirements](#INSTALL-QEMU-AND-GPU-IOMMU.md) *) Ensure you are running QEMU 5.X *) Kubernetes *) Helm v2 @@ -39,13 +40,17 @@ This installs `docker-osx` in Kubernetes. ``` docker build \ -t sickcodes/docker-osx-vnc:latest \ - -f vnc-version/Dockerfile . + -f helm/Dockerfile . ``` + _Please ensure you are using the Dockerfile in the `helm` folder_ + _Do not worry about passing `CPU`, `RAM`, etc as they are handled in `values.yaml` now._ ### Installation +If planning on using a GPU with IOMMU passthrough it is recommended to configure it first and install macOS--otherwise installing may take a very long time depending on your hardware. Please see `qemu.systemInstaller.downloadDelay`, `qemu.systemInstaller.cache`, `qemu.systemDisk.downloadDelay`, and `qemu.systemDisk.cache` for possibly reducing installation time. It has taken me over three hours to install on some occasions with a NVMe secondary disk without GPU passthrough configured.. + In `values.yaml`.. 1) Set a unique password for `vnc.password`. @@ -56,6 +61,12 @@ In `values.yaml`.. 1) Set `service.ip` to reflect an IP address of your choice, or use ingress. 1) Update `extraVolumes.hostPath.path` to something useful for you. +Optionally.. +1) Install kexts to `kexts.path` and enable. +1) Adjust `openCore.boot.timeout` if desire for macOS to load automatically. +1) Add usb devices with `qemu.usb` or `qemu.extraArgs` if desired. +1) Add more ports for portforwarding services if needed. + Afterwards.. 1) Launch your VNC viewer of choice and connect to the IP/hostname you defined + the port `8888` with the password specified diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml index 81aeb23..b862798 100644 --- a/helm/templates/configmap.yaml +++ b/helm/templates/configmap.yaml @@ -215,6 +215,24 @@ data: Add + {{- if .Values.qemu.hardwareGpu.enabled }} + + BundlePath + mXHCD.kext + Comment + Hello There + Enabled + + ExecutablePath + Contents/MacOS/mXHCD + MaxKernel + + MinKernel + + PlistPath + Contents/Info.plist + + {{- end }} Arch x86_64 @@ -222,7 +240,11 @@ data: VoodooHDA.kext Comment Patch engine + {{- if .Values.qemu.audio.enabled }} Enabled + {{- else -}} + Disabled + {{- end }} ExecutablePath Contents/MacOS/VoodooHDA @@ -355,6 +377,28 @@ data: PlistPath Contents/Info.plist + {{- if .Values.kexts.add }} + {{- range .Values.kexts.kextsToAdd }} + + Arch + Any + BundlePath + {{ .name }} + Comment + + Enabled + + ExecutablePath + {{ .executablePath }} + MaxKernel + + MinKernel + + PlistPath + {{ .plistPath }} + + {{- end }} + {{- end }} Block @@ -375,6 +419,16 @@ data: Emulate + {{- if .Values.qemu.hardwareGpu.enabled }} + Cpuid1Data + + VwYFAAAAAAAAAAAAAAAAAA== + + Cpuid1Mask + + /////wAAAAAAAAAAAAAAAA== + + {{- else -}} Cpuid1Data VAYFAAAAAAAAAAAAAAAAAA== @@ -383,9 +437,40 @@ data: ////AAAAAAAAAAAAAAAAAA== + {{- end }} Force + {{- if .Values.qemu.hardwareGpu.enabled }} + + Base + + Comment + algrey - cpuid_set_generic_info - disable check to allow leaf7 + Count + 1 + Enabled + + Find + ADoPgg== + Identifier + kernel + Limit + 0 + Mask + + MaxKernel + 19.99.99 + MinKernel + 17.0.0 + Replace + AAAPgg== + ReplaceMask + + Skip + 0 + + {{- else -}} Arch Any @@ -406,9 +491,40 @@ data: PlistPath Contents/Info.plist + {{- end }} Patch + {{- if .Values.qemu.hardwareGpu.enabled }} + + Base + _cpu_topology_sort + Comment + algrey - cpu_topology_sort -disable _x86_validate_topology + Count + 1 + Enabled + + Find + 6AAA//8= + Identifier + kernel + Limit + 0 + Mask + /wAA//8= + MaxKernel + 19.99.99 + MinKernel + 17.0.0 + Replace + Dx9EAAA= + ReplaceMask + + Skip + 0 + + {{- else -}} Base _cpu_topology_sort @@ -444,6 +560,7 @@ data: Skip 0 + {{- end }} Base @@ -453,6 +570,7 @@ data: 1 Enabled + {{- if .Values.qemu.hardwareGpu.enabled }} Find MduAPQAAAAAGdQA= @@ -465,6 +583,20 @@ data: /////wAAAP///wA= + {{- else -}} + Find + + MduAPQAAAAAGdQA= + + Identifier + kernel + Limit + 0 + Mask + + /////wAAAP///wA= + + {{- end }} MaxKernel 20.99.99 MinKernel @@ -479,6 +611,64 @@ data: Skip 0 + {{- if .Values.qemu.hardwareGpu.enabled }} + + Base + + Comment + algrey - - skip cpuid_cores_per_package test -10.15 + Count + 0 + Enabled + + Find + gz0AAAAAAA8AAAAAAItdvA== + Identifier + kernel + Limit + 0 + Mask + //8AAAD///8AAAAA/////w== + MaxKernel + 19.99.99 + MinKernel + 19.0.0 + Replace + AAAAAAAAAQAAAAAAAAAAAA== + ReplaceMask + AAAAAAAADwAAAAAAAAAAAA== + Skip + 0 + + + Base + + Comment + algrey - - skip cpuid_cores_per_package test + Count + 0 + Enabled + + Find + gz0AAAAAAHQAi128 + Identifier + kernel + Limit + 0 + Mask + //8AAAD///8A//// + MaxKernel + 18.99.99 + MinKernel + 17.0.0 + Replace + AAAAAAAAAQAAAAAA + ReplaceMask + AAAAAAAADwAAAAAA + Skip + 0 + + {{- end }} Quirks @@ -516,6 +706,7 @@ data: Scheme + {{- if not .Values.qemu.hardwareGpu.enabled }} FuzzyMatch @@ -524,6 +715,7 @@ data: KernelCache Auto + {{- end }} Misc @@ -539,6 +731,8 @@ data: PickerAttributes 1 + PickerVariant + Modern PickerAudioAssist PickerMode @@ -550,7 +744,7 @@ data: TakeoffDelay 0 Timeout - 0 + {{ .Values.openCore.boot.timeout }} Debug @@ -657,7 +851,7 @@ data: SystemAudioVolume Rg== boot-args - -v keepsyms=1 tlbto_us=0 vti=9 + {{ .Values.configPlist.bootArgs }} run-efi-updater No csr-active-config @@ -838,7 +1032,7 @@ data: ReplaceTabWithSpace Resolution - 1920x1080@32 + {{ .Values.vnc.resolution }} SanitiseClearScreen TextRenderer @@ -905,232 +1099,37 @@ data: - macOS-libvirt-Catalina.xml: |- - - - - macOS - 2aca0dd6-cec9-4717-9ab2-0b7b13d111c3 - macOS - {{ .Values.resources.requests.memory | trimSuffix "Mi" }} - {{ .Values.resources.requests.memory | trimSuffix "Mi" }} - {{ .Values.resources.requests.cpu }} - - hvm - - /home/CHANGEME/OSX-KVM/OVMF_CODE.fd - /home/CHANGEME/OSX-KVM/OVMF_VARS-1024x768.fd - - - - - - - - - - - destroy - restart - restart - - /usr/bin/qemu-system-x86_64 - - - - - -
- - - - - - -
- - -
- - - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - - - -
- - -
- - -
- - - -
- - - -
- - - -
- - - - - - - - - - - - - - - - - - -
- - - -
- - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Launch_custom.sh: |- #/bin/sh - if ! [ -d "/system_image/installers" ]; then - mkdir -p /system_image/installers + # Add extra kexts to EFI/OC/kexts + {{- if .Values.kexts.add }} + {{- range .Values.kexts.kextsToAdd }} + {{- $rangeItem := . -}} + {{- with $ }} + echo 'Installing kext {{ $rangeItem.name }}..' + cp -r "{{ .Values.kexts.path }}/{{ $rangeItem.name }}" /home/{{ .Values.image.userName }}/OSX-KVM/OpenCore-Catalina/EFI/OC/Kexts/ + sudo chmod 755 /home/{{ .Values.image.userName }}/OSX-KVM/OpenCore-Catalina/EFI/OC/Kexts/{{ $rangeItem.name }} + {{- end }} + {{- end }} + {{- end }} + + # Rebuild Opencore.qcow2 after making changes to config.plist and etc.. + {{- if .Values.openCore.rebuild }} + echo 'Building new Opencore.qcow2..' + sudo apt install {{ .Values.openCore.kernel }} -y + pushd OpenCore-Catalina/ + mkdir -p EFI/OC/Resources + rm -f OpenCore.qcow2 + sudo ./opencore-image-ng.sh \ + --cfg config.plist \ + --img OpenCore.qcow2 + sudo chown {{ .Values.image.userName }}:{{ .Values.image.userName }} OpenCore.qcow2 + popd + {{- end }} + + if ! [ -d "{{ .Values.qemu.systemInstaller.path }}" ]; then + mkdir -p {{ .Values.qemu.systemInstaller.path }} fi if ! [ -d "/system_image/{{ .Values.serverName }}" ]; then @@ -1138,70 +1137,102 @@ data: fi # Download and build installer image if no system drive found.. - if ! [ -f "/system_image/installers/BaseSystem{{ .Values.qemu.systemInstaller.version }}.img" ]; then + if ! [ -f "{{ .Values.qemu.systemInstaller.path }}/BaseSystem{{ .Values.qemu.systemInstaller.version }}.img" ]; then echo "Downloading {{ .Values.qemu.systemInstaller.version }} base image.." python fetch-macOS.py --version {{ .Values.qemu.systemInstaller.version }} - echo 'Converting downloaded BaseSystem.dmg into BaseSystem.img' - qemu-img convert BaseSystem.dmg -O qcow2 -p -c /system_image/installers/BaseSystem{{ .Values.qemu.systemInstaller.version }}.img + echo 'Converting downloaded BaseSystem.dmg into BaseSystem{{ .Values.qemu.systemInstaller.version }}.img and saving in {{ .Values.qemu.systemInstaller.path }}' + qemu-img convert BaseSystem.dmg -O qcow2 -p -c {{ .Values.qemu.systemInstaller.path }}/BaseSystem{{ .Values.qemu.systemInstaller.version }}.img rm -f BaseSystem.dmg else echo 'Base Image downloaded and converted into img already..' fi if ! [ -f "/system_image/{{ .Values.serverName }}/mac_hdd_ng.img" ]; then - echo "Creating a {{ .Values.qemu.diskSize }} /system_image/{{ .Values.serverName }}/mac_hdd_ng.img for system partition.." - qemu-img create -f qcow2 /system_image/{{ .Values.serverName }}/mac_hdd_ng.img "{{ .Values.qemu.diskSize }}" + echo "Creating a {{ .Values.qemu.systemDisk.size }} /system_image/{{ .Values.serverName }}/mac_hdd_ng.img for system partition.." + qemu-img create -f qcow2 /system_image/{{ .Values.serverName }}/mac_hdd_ng.img "{{ .Values.qemu.systemDisk.size }}" echo 'Finished creating system partition!' else echo 'Image already created. Skipping creation..' fi + # # Fix permissions on usb devices.. + # {{- if .Values.qemu.usb }} + # {{- range .Values.qemu.usb }} + # echo "Updating permissions to r/w for /dev/bus/usb/$(lsusb | grep {{ .vendorId}}:{{ .productId }} | grep -o -P 'Bus.{0,4}' | tail -c 4)/$(lsusb | grep {{ .vendorId}}:{{ .productId }} | grep -o -P 'Device.{0,4}' | tail -c 4).." + # sudo chmod 666 /dev/bus/usb/$(lsusb | grep {{ .vendorId}}:{{ .productId }} | grep -o -P 'Bus.{0,4}' | tail -c 4)/$(lsusb | grep {{ .vendorId}}:{{ .productId }} | grep -o -P 'Device.{0,4}' | tail -c 4) + # {{- end }} + # {{- end }} + # Start VNC.. + {{- if .Values.vnc.enabled }} + echo 'geometry={{ .Values.vnc.resolution }} + localhost + alwaysshared' > ~/.vnc/config + sudo rm -f /tmp/.X99-lock export DISPLAY=:99 vncpasswd -f < vncpasswd_file > ${HOME}/.vnc/passwd - /usr/bin/Xvnc -geometry 1920x1080 -rfbauth "${HOME}/.vnc/passwd" :99 &\ + /usr/bin/Xvnc -geometry {{ .Values.vnc.resolution }} -rfbauth "${HOME}/.vnc/passwd" :99 &\ + sudo chmod 600 ~/.vnc/passwd + {{- end }} - {{- if .Values.qemu.gpu.enabled }} - ulimit -l $(( 8*1048576+100000 )) - user hard memlock $(( 8*1048576+100000 )) - user soft memlock $(( 8*1048576+100000 )) + sudo chown {{ .Values.image.userName }}:{{ .Values.image.userName }} /dev/kvm + {{- if .Values.qemu.hardwareGpu.enabled }} + sudo chown {{ .Values.image.userName }}:{{ .Values.image.userName }} -R /dev/vfio {{- end }} # Start QEMU.. + echo 'Starting QEMU..' set -eu sudo chown $(id -u):$(id -g) /dev/kvm 2>/dev/null || true sudo chown -R $(id -u):$(id -g) /dev/snd 2>/dev/null || true exec qemu-system-x86_64 -m {{ .Values.resources.requests.memory | trimSuffix "i" }} \ -cpu {{ .Values.qemu.cpu }} \ -machine q35,accel=kvm:tcg \ - {{- if .Values.qemu.gpu.enabled }} - -vga none \ - -device pcie-root-port,bus=pcie.0,multifunction=on,port=1,chassis=1,id=port.1 \ - -device vfio-pci,host={{ .Values.qemu.gpu.hardwareId }}.0,multifunction=on,x-vga=on,rombar=1 \ - -device vfio-pci,host={{ .Values.qemu.gpu.hardwareId }}.1,bus=port.1 \ - -display none \ - {{- else -}} - -vga {{ .Values.qemu.softwareGpu }} \ + {{- if .Values.qemu.hardwareGpu.enabled }} + -device pcie-root-port,bus=pcie.0,multifunction=on,port=1,chassis=1,id=port.1 \ + {{- if .Values.qemu.hardwareGpu.romfile }} + -device vfio-pci,host={{ .Values.qemu.hardwareGpu.hardwareId }}.0,bus=port.1,multifunction=on,romfile={{ .Values.qemu.hardwareGpu.romfile}} \ + {{- else -}} + -device vfio-pci,host={{ .Values.qemu.hardwareGpu.hardwareId }}.0,multifunction=on \ + {{- end }} + -device vfio-pci,host={{ .Values.qemu.hardwareGpu.hardwareId }}.1,bus=port.1 \ + {{- else -}} + -vga {{ .Values.qemu.softwareGpu }} \ {{- end }} -smp {{ .Values.resources.requests.cpu }},cores={{ .Values.resources.requests.cpu }} \ - -usb -device usb-kbd -device usb-tablet \ + {{- if .Values.vnc.enabled }} + -usb -device usb-kbd -device usb-tablet \ + {{- else -}} + -vga none \ + -display none \ + {{- end }} -device isa-applesmc,osk=ourhardworkbythesewordsguardedpleasedontsteal\(c\)AppleComputerInc \ - -drive if=pflash,format=raw,readonly,file=/home/arch/OSX-KVM/OVMF_CODE.fd \ - -drive if=pflash,format=raw,file=/home/arch/OSX-KVM/OVMF_VARS-1024x768.fd \ + -drive if=pflash,format=raw,readonly,file=/home/{{ .Values.image.userName }}/OSX-KVM/OVMF_CODE.fd \ + -drive if=pflash,format=raw,file=/home/{{ .Values.image.userName }}/OSX-KVM/OVMF_VARS-1024x768.fd \ -smbios type=2 \ {{- if .Values.qemu.audio.enabled }} - -audiodev {{ .Values.qemu.audo.driver }},id=hda -device ich9-intel-hda -device hda-duplex,audiodev=hda \ \ + -audiodev {{ .Values.qemu.audio.driver }},id=hda \ + -device ich9-intel-hda \ + -device hda-duplex,audiodev=hda \ {{- end }} -device ich9-ahci,id=sata \ - -drive id=OpenCoreBoot,if=none,snapshot=on,format=qcow2,file=/home/arch/OSX-KVM/OpenCore-Catalina/OpenCore.qcow2 \ + -drive id=OpenCoreBoot,if=none,snapshot=on,format=qcow2,file=/home/{{ .Values.image.userName }}/OSX-KVM/OpenCore-Catalina/OpenCore.qcow2 \ -device ide-hd,bus=sata.2,drive=OpenCoreBoot \ - -device ide-hd,bus=sata.3,drive=InstallMedia \ - -drive id=InstallMedia,if=none,file=/system_image/installers/BaseSystem{{ .Values.qemu.systemInstaller.version }}.img,format=qcow2 \ + {{- if .Values.qemu.systemInstaller.enabled }} + -device ide-hd,bus=sata.3,drive=InstallMedia \ + -drive id=InstallMedia,if=none,file={{ .Values.qemu.systemInstaller.path }}/BaseSystem{{ .Values.qemu.systemInstaller.version }}.img,format=qcow2 \ + {{- end }} -drive id=MacHDD,if=none,file=/system_image/{{ .Values.serverName }}/mac_hdd_ng.img,format=qcow2 \ -device ide-hd,bus=sata.4,drive=MacHDD \ - -netdev user,id=net0,hostfwd=tcp::${INTERNAL_SSH_PORT:-10022}-:22,hostfwd=tcp::${SCREEN_SHARE_PORT:-5900}-:5900,{{ .Values.qemu.netdev.extraArgs }} -device e1000-82545em,netdev=net0,id=net0,mac=52:54:00:09:49:17 \ - -monitor stdio \ + -netdev user,id=net0,hostfwd=tcp::${INTERNAL_SSH_PORT:-10022}-:22,hostfwd=tcp::${SCREEN_SHARE_PORT:-5900}-:5900,{{ .Values.qemu.netdev.extraPortForwarding }} \ + -device e1000-82545em,netdev=net0,id=net0,mac=52:54:00:09:49:17 \ + {{- range .Values.qemu.usb }} + -usb -device usb-host,productid=0x{{ .productId }},vendorid=0x{{ .vendorId }} \ + {{- end }} + {{- range .Values.qemu.extraArgs }} + {{ . }} \ + {{- end }} ${EXTRA:-} vncpasswd_file: |- {{ .Values.vnc.password }} @@ -1263,7 +1294,7 @@ data: #@faculty hard nproc 50 #ftp hard nproc 0 #@student - maxlogins 4 - @arch soft memlock unlimited - @arch hard memlock unlimited + @{{ .Values.image.userName }} soft memlock unlimited + @{{ .Values.image.userName }} hard memlock unlimited - # End of file \ No newline at end of file + # End of file diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml index 87c33d1..4062414 100644 --- a/helm/templates/deployment.yaml +++ b/helm/templates/deployment.yaml @@ -56,30 +56,35 @@ spec: value: "{{ .Values.resources.requests.memory | trimSuffix "Mi" }}" - name: TZ value: "{{ .Values.tz }}" + - name: DISPLAY + value: ':0.0' resources: {{ toYaml .Values.resources | indent 10 }} volumeMounts: - - mountPath: /home/arch/OSX-KVM/config.plist + - mountPath: /home/{{ .Values.image.userName }}/OSX-KVM/OpenCore-Catalina/config.plist subPath: config.plist name: boot-components - - mountPath: /home/arch/OSX-KVM/macOS-libvirt-Catalina.xml - subPath: macOS-libvirt-Catalina.xml - name: boot-components - - mountPath: /home/arch/OSX-KVM/Launch_custom.sh + - mountPath: /home/{{ .Values.image.userName }}/OSX-KVM/Launch_custom.sh subPath: Launch_custom.sh name: boot-components - - mountPath: /home/arch/OSX-KVM/vncpasswd_file + - mountPath: /home/{{ .Values.image.userName }}/OSX-KVM/vncpasswd_file subPath: vncpasswd_file name: boot-components - - mountPath: /etc/security/limits.conf - subPath: limits.conf - name: boot-components - mountPath: /dev/kvm name: kvm - mountPath: /dev/net/tun name: tun - - mountPath: /dev/vfio + {{- if .Values.qemu.hardwareGpu.enabled }} + - mountPath: /etc/security/limits.conf + subPath: limits.conf + name: boot-components + - mountPath: /dev/vfio/vfio name: vfio + - mountPath: /dev/vfio/{{ .Values.qemu.hardwareGpu.vfioGroup }} + name: vfio-group + - mountPath: /lib/modules + name: lib-modules + {{- end }} - mountPath: /dev/snd name: snd - mountPath: /tmp/.X11-unix @@ -98,23 +103,31 @@ spec: items: - key: config.plist path: config.plist - - key: macOS-libvirt-Catalina.xml - path: macOS-libvirt-Catalina.xml - key: Launch_custom.sh path: Launch_custom.sh - key: vncpasswd_file path: vncpasswd_file + {{- if .Values.qemu.hardwareGpu.enabled }} - key: limits.conf path: limits.conf - - name: kvm - hostPath: - path: /dev/kvm + {{- end }} - name: tun hostPath: path: /dev/net/tun + - name: kvm + hostPath: + path: /dev/kvm + {{- if .Values.qemu.hardwareGpu.enabled }} - name: vfio hostPath: - path: /dev/vfio + path: /dev/vfio/vfio + - name: vfio-group + hostPath: + path: /dev/vfio/{{ .Values.qemu.hardwareGpu.vfioGroup }} + - name: lib-modules + hostPath: + path: /lib/modules + {{- end }} - name: snd hostPath: path: /dev/snd diff --git a/helm/templates/service.yaml b/helm/templates/service.yaml index 3161a34..a6383e3 100644 --- a/helm/templates/service.yaml +++ b/helm/templates/service.yaml @@ -18,10 +18,12 @@ spec: targetPort: 5999 protocol: TCP name: vnc - - port: 1359 - targetPort: 1359 - protocol: TCP - name: airmessage + {{- range .Values.service.extraPorts }} + - port: {{ .port }} + targetPort: {{ .targetPort }} + protocol: {{ .protocol }} + name: {{ .name }} + {{- end }} selector: app.kubernetes.io/name: {{ include "docker-osx.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} diff --git a/helm/values.yaml b/helm/values.yaml index 29509d5..984d266 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -7,46 +7,118 @@ image: repository: sickcodes/docker-osx-vnc tag: latest pullPolicy: IfNotPresent + userName: ubuntu # Please note, this must be a directory name within `/system_image` mounted at the bottom in extraVolumeMounts serverName: server +# add kexts to EFI/OC/kexts and update config.plist +# make sure your kexts are in kexts.path +kexts: + add: false + path: /system_image/kexts + kextsToAdd: {} + # - name: BrcmBluetoothInjector.kext + # executablePath: Contents/MacOS/BrcmBluetoothInjector + # plistPath: Contents/Info.plist + # - name: BrcmFirmwareData.kext + # executablePath: Contents/MacOS/BrcmFirmwareData + # plistPath: Contents/Info.plist + # - name: BrcmPatchRAM3.kext + # executablePath: Contents/MacOS/BrcmPatchRAM3 + # plistPath: Contents/Info.plist + # SMBIOS settings configPlist: SystemProductName: iMacPro1,1 MLB: D25338500GUF8YLJA SystemSerialNumber: D25LF7ZEF8JC SystemUUID: 139C94D6-A533-47D2-874F-D365BFD8B047 + bootArgs: '-v keepsyms=1 tlbto_us=0 vti=9 -wegoff agdpmod=pikera' -# This defines QEMU and virtlo parameters +openCore: + # Rebuild OpenCore.qcow2: this can be disabled if none of the above parameters in kexts, configPlist changes, and changing the + # resolution are not desired, utilizing gpu passthrough and a few other things. Disabling is not recommended + rebuild: true + boot: + # set to zero to have OpenCore stay at boot menu + timeout: 0 + +# This section defines QEMU and virtlo parameters # # Note: # *) Increase downloadDelay if the pod gets killed for readiness/liveliness check. The first time the pod is started, # it will download the BaseSystem image and create a data partition to install the OS. If this value is really long, # the pod will take very long to allow VNC connections if it is rebooted/killed. -# *) Big Sur (11.X) is currently broken, as it seems to need the BaseSystem extracted from the InstallAssistant.pkg file it downloads -# *) GPU support is considered broken still, but WIP +# *) Big Sur (11.X) is currently broken, as it seems Apple reworked the packaging so that will have to be something fixed in +# https://github.com/kholia/OSX-KVM/blob/master/fetch-macOS-v2.py +# *) VNC and IOMMU GPU passthrough do not play together well. Disable one if using the other. This is a limitation of QEMU +# unfortunately, so this means VNC from macOS will have to be used to view the VM when not using a physical GPU. +# *) If using GPU passthrough, it is recommended to configure it first, as the installer will take way longer (several hours) +# to install macOS with a software GPU configured. # qemu: - cpu: Penryn,vendor=GenuineIntel,+hypervisor,+invtsc,kvm=on,+fma,+avx,+avx2,+aes,+ssse3,+sse4_2,+popcnt,+sse4a,+bmi1,+bmi2 + cpu: Penryn,kvm=on,vendor=GenuineIntel,+invtsc,vmware-cpuid-freq=on,+pcid,+ssse3,+sse4.2,+popcnt,+avx,+avx2,+aes,+xsave,+xsaveopt,check softwareGpu: vmware - gpu: - # if disabled, will rely on softwareGpu instead + hardwareGpu: + # when enabled, qemu.softwareGpu will be disabled automatically. disabling this will rely on softwareGpu instead enabled: false - hardwareId: '03:00' + hardwareId: 09:00 + # slot can be found with `lspci -v -s ` + slot: 05 + # vfioGroup can be found with lsgroup.sh: + # https://github.com/kholia/OSX-KVM/blob/master/scripts/lsgroup.sh + vfioGroup: 50 + # leave romfile blank if not using one to disable + romfile: /system_image/vbios/Sapphire.RX480.8192.160603.rom systemInstaller: + # if using more than one deployment, a write lock will be put on the system installer dmg, so this will need to be disabled + # for other pods + enabled: false version: 10.15.7 - downloadDelay: 300 - diskSize: 128G + path: /system_image/installers + downloadDelay: 15 + # can be `writethrough`, `writeback`, or `none` + cache: none + # can be `native` or `threads` + io: threads + systemDisk: + size: 128G + # can be `writethrough`, `writeback`, or `none` + cache: writeback + # can be `native` or `threads` + io: threads audio: - enabled: true + # this enables onboard audio, hdmi audio is handled with qemu.hardwareGpu + enabled: false driver: alsa netdev: - extraArgs: + extraPortForwarding: hostfwd=tcp::5901-:5900 + # for usb, pass host adapters like such: + # usb: + # - vendorId: 0a5c + # productId: 21e8 + # + # please use lsgroup.sh to find your host ids: + # https://github.com/kholia/OSX-KVM/blob/master/scripts/lsgroup.sh + # + # for usb, may need to fix permissions: + # sudo chmod 666 /dev/bus/usb// + # + # if need to add a usb controller via vfio-pci, use qemu.extraArgs + # + usb: {} + # use the following formatting + # extraArgs: + # - -parallel none + # - -boot order=dc + extraArgs: {} # Password for accessing vm over vnc vnc: - password: updateme + enabled: true + resolution: 1920x1080 + password: a5aeQbaPd4$jR80Q43 nameOverride: "" fullnameOverride: "" @@ -56,6 +128,11 @@ service: ip: 192.168.1.10 targetPort: 50922 port: 10022 + extraPorts: + - port: 5901 + targetPort: 5901 + protocol: TCP + name: os-level-vnc ingress: enabled: false @@ -71,19 +148,21 @@ ingress: # hosts: # - chart-example.local -# Note: seems that host needs around x5 the cpu and x9 memory limits allocated to MacOS -# when under load. This may be due to my personal hardware or inefficencies such as -# software GPU rendering. Otherwise the pod will be killed due to OOMing. +# Note: Resources can vary dramatically depending on disk caching and software GPU rendering. With disk +# caching and software rendering up to x5 the cpu and x9 memory can be consumed. With disk cache off +# and GPU passthrough enabled up to x3 the cpu and x1.5 memory can be consumed. Therefore, these settings +# really depend on hardware and configuration choices. Note, these values used a 1080p video resolution +# as well. # # Warning: do not perform unit conversion on cpu and memory requests, as these units # are tied qemu and virtio settings. Also, only use intergers for cpu requests. resources: limits: - cpu: 10 - memory: 36864Mi + cpu: 15 + memory: 32768Mi requests: - cpu: 2 - memory: 4096Mi + cpu: 4 + memory: 16384Mi nodeSelector: {}