Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions src/main.ts
Original file line number Diff line number Diff line change
Expand Up @@ -676,6 +676,7 @@ void actionsToolkit.run(
let cleanupError: Error | null = null;
let fsDiskUsageBytes: number | null = null;
let integrityCheckPassed: boolean | null = null;
let buildkitdShutdownCleanly: boolean | null = null;

try {
// Step 1: Check if buildkitd is running and shut it down
Expand All @@ -700,18 +701,22 @@ void actionsToolkit.run(

// Critical: Shutdown buildkitd
const buildkitdShutdownStartTime = Date.now();
await shutdownBuildkitd();
buildkitdShutdownCleanly = await shutdownBuildkitd();
const buildkitdShutdownDurationMs =
Date.now() - buildkitdShutdownStartTime;
await reporter.reportMetric(
Metric_MetricType.BPA_BUILDKITD_SHUTDOWN_DURATION_MS,
buildkitdShutdownDurationMs,
);
core.info("Shutdown buildkitd gracefully");
core.info(
`Shutdown buildkitd ${buildkitdShutdownCleanly ? "gracefully" : "forcefully"}`,
);
} else {
// Check if buildkitd was expected to be running (we have state indicating it was started)
const buildkitdAddr = stateHelper.getBuildkitdAddr();
if (buildkitdAddr) {
buildkitdShutdownCleanly = false;

core.warning(
"buildkitd process has crashed - process not found but was expected to be running",
);
Expand All @@ -729,6 +734,8 @@ void actionsToolkit.run(
);
}
} else {
buildkitdShutdownCleanly = true;

core.debug(
"No buildkitd process found running and none was expected",
);
Expand All @@ -745,6 +752,8 @@ void actionsToolkit.run(
// Check if buildkitd was expected to be running (we have state indicating it was started)
const buildkitdAddr = stateHelper.getBuildkitdAddr();
if (buildkitdAddr) {
buildkitdShutdownCleanly = false;

core.warning(
"buildkitd process has crashed - pgrep failed but buildkitd was expected to be running",
);
Expand All @@ -762,6 +771,8 @@ void actionsToolkit.run(
);
}
} else {
buildkitdShutdownCleanly = true;

core.debug(
"No buildkitd process found (pgrep returned 1) and none was expected",
);
Expand Down Expand Up @@ -932,9 +943,9 @@ void actionsToolkit.run(
core.warning(
"Skipping sticky disk commit due to previous step failures",
);
} else if (stateHelper.getSigkillUsed()) {
} else if (!buildkitdShutdownCleanly) {
core.warning(
"Skipping sticky disk commit because SIGKILL was used to terminate buildkitd - disk may be in a bad state",
"Skipping sticky disk commit because buildkitd did not shut down cleanly - disk may be in a bad state",
);
} else {
// No failures detected and cleanup was successful
Expand Down
8 changes: 3 additions & 5 deletions src/shutdown.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
import * as core from "@actions/core";
import { promisify } from "util";
import { exec } from "child_process";
import * as stateHelper from "./state-helper";

const execAsync = promisify(exec);

export async function shutdownBuildkitd(): Promise<void> {
export async function shutdownBuildkitd(): Promise<boolean> {
const gracefulTimeout = 30000; // 30 seconds for graceful shutdown.
const forceTimeout = 5000; // 5 seconds for forced shutdown.
const backoff = 300; // 300ms.
Expand All @@ -28,7 +27,7 @@ export async function shutdownBuildkitd(): Promise<void> {
if ((error as { code?: number }).code === 1) {
// pgrep returns exit code 1 when no process is found, which means shutdown successful.
core.info("buildkitd successfully shutdown gracefully");
return;
return true;
}
// Some other error occurred.
throw error;
Expand All @@ -42,7 +41,6 @@ export async function shutdownBuildkitd(): Promise<void> {
core.warning(
"Disk may be in a bad state after SIGKILL - will prevent sticky disk commit",
);
stateHelper.setSigkillUsed(true);
await execAsync(`sudo pkill -KILL buildkitd`);

// Wait for forced shutdown.
Expand All @@ -58,7 +56,7 @@ export async function shutdownBuildkitd(): Promise<void> {
if ((error as { code?: number }).code === 1) {
// Process is gone.
core.warning("buildkitd was forcefully terminated with SIGKILL");
return;
return false;
}
// Some other error occurred.
throw error;
Expand Down
8 changes: 0 additions & 8 deletions src/state-helper.ts
Original file line number Diff line number Diff line change
Expand Up @@ -37,11 +37,3 @@ export function setBuilderName(name: string) {
export function getBuilderName(): string {
return core.getState("builderName");
}

export function setSigkillUsed(used: boolean) {
core.saveState("sigkillUsed", used.toString());
}

export function getSigkillUsed(): boolean {
return core.getState("sigkillUsed") === "true";
}