Skip to content

Commit

Permalink
Register deadlines before hopping to the new label
Browse files Browse the repository at this point in the history
We have a known issue #718. If the label that we register deadline
fails, we can't persist the deadline information to the strand's stack.

If destroy or start_after_host_reboot fails for some reasons, we will
not get a page.
  • Loading branch information
enescakir committed Oct 21, 2023
1 parent ef9ad48 commit 7675548
Show file tree
Hide file tree
Showing 3 changed files with 4 additions and 8 deletions.
3 changes: 1 addition & 2 deletions prog/vm/github_runner.rb
Expand Up @@ -96,6 +96,7 @@ def github_client
def before_run
when_destroy_set? do
unless ["destroy", "wait_vm_destroy"].include?(strand.label)
register_deadline(nil, 10 * 60)
hop_destroy
end
end
Expand Down Expand Up @@ -216,8 +217,6 @@ def before_run
end

label def destroy
register_deadline(nil, 10 * 60)

decr_destroy

# Waiting 404 Not Found response for get runner request
Expand Down
6 changes: 2 additions & 4 deletions prog/vm/nexus.rb
Expand Up @@ -189,6 +189,7 @@ def before_run
if strand.label != "destroy"
vm.active_billing_record&.finalize
vm.assigned_vm_address&.active_billing_record&.finalize
register_deadline(nil, 5 * 60)
hop_destroy
end
end
Expand Down Expand Up @@ -307,15 +308,14 @@ def before_run

label def wait
when_start_after_host_reboot_set? do
register_deadline(:wait, 5 * 60)
hop_start_after_host_reboot
end

nap 30
end

label def destroy
register_deadline(nil, 5 * 60)

decr_destroy

vm.update(display_state: "deleting")
Expand Down Expand Up @@ -358,8 +358,6 @@ def before_run
end

label def start_after_host_reboot
register_deadline(:wait, 5 * 60)

vm.update(display_state: "starting")

secrets_json = JSON.generate({
Expand Down
3 changes: 1 addition & 2 deletions spec/prog/vm/github_runner_spec.rb
Expand Up @@ -123,6 +123,7 @@
describe "#before_run" do
it "hops to destroy when needed" do
expect(nx).to receive(:when_destroy_set?).and_yield
expect(nx).to receive(:register_deadline)
expect { nx.before_run }.to hop("destroy")
end

Expand Down Expand Up @@ -277,7 +278,6 @@
end

it "destroys resources and hops if runner deregistered" do
expect(nx).to receive(:register_deadline)
expect(nx).to receive(:decr_destroy)
expect(client).to receive(:get).and_raise(Octokit::NotFound)
expect(client).not_to receive(:delete)
Expand All @@ -288,7 +288,6 @@
end

it "does not destroy vm if it's already destroyed" do
expect(nx).to receive(:register_deadline)
expect(nx).to receive(:decr_destroy)
expect(client).to receive(:get).and_raise(Octokit::NotFound)
expect(client).not_to receive(:delete)
Expand Down

0 comments on commit 7675548

Please sign in to comment.