Add Vm.update_spdk_version(...)

Previously to change SPDK version of a VM we needed to do multiple
weird manual steps. This patch automates some of them and what we need
to do is:

1. Install the new SPDK version on the host:

```
> Prog::Storage::SetupSpdk.assemble(
     vm_host.id, "some-spdk-version",
     start_service: true, allocation_weight: 100)
```

2. Update all VMs we want to migrate to this SPDK version. Note that
this will just update related configs and won't take effect until host
reboot:

```
> vm.update_spdk_version("some-spdk-version")
```

3. Reboot host:

```
> vm_host.incr_reboot
```

4. Optionally, remove the old SPDK installation:

```
> Prog::Storage::RemoveSpdk.assemble(old_spdk_installation.id)
```

model/vm.rb

@@ -21,7 +21,7 @@ class Vm < Sequel::Model
   include ResourceMethods
   include SemaphoreMethods
   include HealthMonitorMethods
-  semaphore :destroy, :start_after_host_reboot, :prevent_destroy, :update_firewall_rules, :checkup
+  semaphore :destroy, :start_after_host_reboot, :prevent_destroy, :update_firewall_rules, :checkup, :update_spdk_dependency
 
   include Authorization::HyperTagMethods
 
@@ -184,6 +184,13 @@ def check_pulse(session:, previous_pulse:)
     pulse
   end
 
+  def update_spdk_version(version)
+    spdk_installation = vm_host.spdk_installations_dataset[version: version]
+    fail "SPDK version #{version} not found on host" unless spdk_installation
+    vm_storage_volumes_dataset.update(spdk_installation_id: spdk_installation.id)
+    incr_update_spdk_dependency
+  end
+
   def self.redacted_columns
     super + [:public_key]
   end

prog/vm/nexus.rb

@@ -8,7 +8,7 @@
 
 class Prog::Vm::Nexus < Prog::Base
   subject_is :vm
-  semaphore :destroy, :start_after_host_reboot, :prevent_destroy, :update_firewall_rules, :checkup
+  semaphore :destroy, :start_after_host_reboot, :prevent_destroy, :update_firewall_rules, :checkup, :update_spdk_dependency
 
   def self.assemble(public_key, project_id, name: nil, size: "standard-2",
     unix_user: "ubi", location: "hetzner-hel1", boot_image: "ubuntu-jammy",
@@ -503,6 +503,11 @@ def write_params_json
       hop_update_firewall_rules
     end
 
+    when_update_spdk_dependency_set? do
+      register_deadline(:wait, 5 * 60)
+      hop_update_spdk_dependency
+    end
+
     when_checkup_set? do
       hop_unavailable if !available?
       decr_checkup
@@ -524,6 +529,13 @@ def write_params_json
     push Prog::Vnet::UpdateFirewallRules, {}, :update_firewall_rules
   end
 
+  label def update_spdk_dependency
+    decr_update_spdk_dependency
+    write_params_json
+    host.sshable.cmd("sudo host/bin/setup-vm reinstall-systemd-units #{q_vm}")
+    hop_wait
+  end
+
   label def unavailable
     # If the VM become unavailable due to host unavailability, it first needs to
     # go through start_after_host_reboot state to be able to recover.

spec/model/vm_spec.rb

@@ -95,6 +95,31 @@
     end
   end
 
+  describe "#update_spdk_version" do
+    let (:vmh) {
+      sshable = Sshable.create_with_id
+      VmHost.create(location: "a") { _1.id = sshable.id }
+    }
+
+    before do
+      expect(vm).to receive(:vm_host).and_return(vmh)
+    end
+
+    it "can update spdk version" do
+      spdk_installation = SpdkInstallation.create(version: "b", allocation_weight: 100, vm_host_id: vmh.id) { _1.id = vmh.id }
+      volume_dataset = instance_double(Sequel::Dataset)
+      expect(vm).to receive(:vm_storage_volumes_dataset).and_return(volume_dataset)
+      expect(volume_dataset).to receive(:update).with(spdk_installation_id: spdk_installation.id)
+      expect(vm).to receive(:incr_update_spdk_dependency)
+
+      vm.update_spdk_version("b")
+    end
+
+    it "fails if spdk installation not found" do
+      expect { vm.update_spdk_version("b") }.to raise_error RuntimeError, "SPDK version b not found on host"
+    end
+  end
+
   describe "#utility functions" do
     it "can compute the ipv4 addresses" do
       as_ad = instance_double(AssignedVmAddress, ip: NetAddr::IPv4Net.new(NetAddr.parse_ip("1.1.1.0"), NetAddr::Mask32.new(32)))

spec/prog/vm/nexus_spec.rb

@@ -792,6 +792,11 @@ def new_host(**args)
       expect { nx.wait }.to hop("start_after_host_reboot")
     end
 
+    it "hops to update_spdk_dependency when needed" do
+      expect(nx).to receive(:when_update_spdk_dependency_set?).and_yield
+      expect { nx.wait }.to hop("update_spdk_dependency")
+    end
+
     it "hops to update_firewall_rules when needed" do
       expect(nx).to receive(:when_update_firewall_rules_set?).and_yield
       expect { nx.wait }.to hop("update_firewall_rules")
@@ -825,6 +830,19 @@ def new_host(**args)
     end
   end
 
+  describe "#update_spdk_dependency" do
+    it "hops to wait after doing the work" do
+      sshable = instance_double(Sshable)
+      vm_host = instance_double(VmHost, sshable: sshable)
+      allow(vm).to receive(:vm_host).and_return(vm_host)
+
+      expect(nx).to receive(:decr_update_spdk_dependency)
+      expect(nx).to receive(:write_params_json)
+      expect(sshable).to receive(:cmd).with("sudo host/bin/setup-vm reinstall-systemd-units #{vm.inhost_name}")
+      expect { nx.update_spdk_dependency }.to hop("wait")
+    end
+  end
+
   describe "#unavailable" do
     it "hops to start_after_host_reboot when needed" do
       expect(nx).to receive(:when_start_after_host_reboot_set?).and_yield