From a529a8e37161b151f14507771a48926652b49d25 Mon Sep 17 00:00:00 2001 From: Jan Wozniak Date: Thu, 12 Jul 2018 14:59:03 +0200 Subject: [PATCH] Bug 1596440 - surface OOMKilled pod to build --- pkg/build/apis/build/types.go | 4 ++++ .../controller/build/build_controller.go | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/pkg/build/apis/build/types.go b/pkg/build/apis/build/types.go index 3a92c2b262d1..e6b7dacb9495 100644 --- a/pkg/build/apis/build/types.go +++ b/pkg/build/apis/build/types.go @@ -514,6 +514,9 @@ const ( // range of build failures. StatusReasonGenericBuildFailed StatusReason = "GenericBuildFailed" + // StatusReasonOOMKilled indicates that the build pod was killed for its memory consumption + StatusReasonOOMKilled StatusReason = "OOMKilled" + // StatusCannotRetrieveServiceAccount is the reason associated with a failure // to look up the service account associated with the BuildConfig. StatusReasonCannotRetrieveServiceAccount StatusReason = "CannotRetrieveServiceAccount" @@ -540,6 +543,7 @@ const ( StatusMessageNoBuildContainerStatus = "The pod for this build has no container statuses indicating success or failure." StatusMessageFailedContainer = "The pod for this build has at least one container with a non-zero exit status." StatusMessageGenericBuildFailed = "Generic Build failure - check logs for details." + StatusMessageOOMKilled = "The build pod was killed due to an out of memory condition." StatusMessageUnresolvableEnvironmentVariable = "Unable to resolve build environment variable reference." StatusMessageCannotRetrieveServiceAccount = "Unable to look up the service account secrets for this build." ) diff --git a/pkg/build/controller/build/build_controller.go b/pkg/build/controller/build/build_controller.go index c57fde2c11dc..3e326d48e4c0 100644 --- a/pkg/build/controller/build/build_controller.go +++ b/pkg/build/controller/build/build_controller.go @@ -1006,6 +1006,8 @@ func (bc *BuildController) handleActiveBuild(build *buildapi.Build, pod *v1.Pod) // soon be deleted. The build should be transitioned to the Error phase. if pod.DeletionTimestamp != nil { update = transitionToPhase(buildapi.BuildPhaseError, buildapi.StatusReasonBuildPodDeleted, buildapi.StatusMessageBuildPodDeleted) + } else if isOOMKilled(pod) { + update = transitionToPhase(buildapi.BuildPhaseFailed, buildapi.StatusReasonOOMKilled, buildapi.StatusMessageOOMKilled) } else { update = transitionToPhase(buildapi.BuildPhaseFailed, buildapi.StatusReasonGenericBuildFailed, buildapi.StatusMessageGenericBuildFailed) } @@ -1014,6 +1016,25 @@ func (bc *BuildController) handleActiveBuild(build *buildapi.Build, pod *v1.Pod) return update, nil } +func isOOMKilled(pod *v1.Pod) bool { + if pod.Status.Reason == "OOMKilled" { + return true + } + for _, c := range pod.Status.InitContainerStatuses { + terminated := c.State.Terminated + if terminated != nil && terminated.Reason == "OOMKilled" { + return true + } + } + for _, c := range pod.Status.ContainerStatuses { + terminated := c.State.Terminated + if terminated != nil && terminated.Reason == "OOMKilled" { + return true + } + } + return false +} + // handleCompletedBuild will only be called on builds that are already in a terminal phase. It is used to setup the // completion timestamp and failure logsnippet as needed. func (bc *BuildController) handleCompletedBuild(build *buildapi.Build, pod *v1.Pod) (*buildUpdate, error) {