Pause the deployment config before deleting

pkg/deploy/reaper/reaper.go

@@ -9,8 +9,11 @@ import (
 	kerrors "k8s.io/kubernetes/pkg/api/errors"
 	kclient "k8s.io/kubernetes/pkg/client/unversioned"
 	"k8s.io/kubernetes/pkg/kubectl"
+	kutil "k8s.io/kubernetes/pkg/util"
+	"k8s.io/kubernetes/pkg/util/wait"
 
 	"github.com/openshift/origin/pkg/client"
+	deployapi "github.com/openshift/origin/pkg/deploy/api"
 	"github.com/openshift/origin/pkg/deploy/util"
 )
 
@@ -26,26 +29,44 @@ type DeploymentConfigReaper struct {
 	pollInterval, timeout time.Duration
 }
 
+// pause marks the deployment configuration as paused to avoid triggering new
+// deployments.
+func (reaper *DeploymentConfigReaper) pause(namespace, name string) (*deployapi.DeploymentConfig, error) {
+	return reaper.updateDeploymentWithRetries(namespace, name, func(d *deployapi.DeploymentConfig) {
+		d.Spec.RevisionHistoryLimit = kutil.Int32Ptr(0)
+		d.Spec.Replicas = 0
+		d.Spec.Paused = true
+	})
+}
+
 // Stop scales a replication controller via its deployment configuration down to
 // zero replicas, waits for all of them to get deleted and then deletes both the
 // replication controller and its deployment configuration.
 func (reaper *DeploymentConfigReaper) Stop(namespace, name string, timeout time.Duration, gracePeriod *kapi.DeleteOptions) error {
-	// If the config is already deleted, it may still have associated
-	// deployments which didn't get cleaned up during prior calls to Stop. If
-	// the config can't be found, still make an attempt to clean up the
-	// deployments.
-	//
-	// It's important to delete the config first to avoid an undesirable side
-	// effect which can cause the deployment to be re-triggered upon the
-	// config's deletion. See https://github.com/openshift/origin/issues/2721
-	// for more details.
-	err := reaper.oc.DeploymentConfigs(namespace).Delete(name)
+	// Pause the deployment configuration to prevent the new deployments from
+	// being triggered.
+	err := reaper.pause(namespace, name)
 	configNotFound := kerrors.IsNotFound(err)
 	if err != nil && !configNotFound {
 		return err
 	}
 
-	// Clean up deployments related to the config.
+	// Determine if the deployment config controller noticed the pause.
+	if !configNotFound {
+		if err := wait.Poll(1*time.Second, 1*time.Minute, func() (bool, error) {
+			dc, err := reaper.oc.DeploymentConfigs(namespace).Get(name)
+			if err != nil {
+				return false, err
+			}
+			return dc.Status.ObservedGeneration >= config.Generation, nil
+		}); err != nil {
+			return err
+		}
+	}
+
+	// Clean up deployments related to the config. Even if the deployment
+	// configuration has been deleted, we want to sweep the existing replication
+	// controllers and clean them up.
 	options := kapi.ListOptions{LabelSelector: util.ConfigSelector(name)}
 	rcList, err := reaper.kc.ReplicationControllers(namespace).List(options)
 	if err != nil {
@@ -62,10 +83,11 @@ func (reaper *DeploymentConfigReaper) Stop(namespace, name string, timeout time.
 	if configNotFound && len(deployments) == 0 {
 		return kerrors.NewNotFound(kapi.Resource("deploymentconfig"), name)
 	}
+
 	for _, rc := range deployments {
 		if err = rcReaper.Stop(rc.Namespace, rc.Name, timeout, gracePeriod); err != nil {
 			// Better not error out here...
-			glog.Infof("Cannot delete ReplicationController %s/%s: %v", rc.Namespace, rc.Name, err)
+			glog.Infof("Cannot delete ReplicationController %s/%s for deployment %s/%s: %v", rc.Namespace, rc.Name, namespace, name, err)
 		}
 
 		// Only remove deployer pods when the deployment was failed. For completed
@@ -84,10 +106,34 @@ func (reaper *DeploymentConfigReaper) Stop(namespace, name string, timeout time.
 			err := reaper.kc.Pods(pod.Namespace).Delete(pod.Name, gracePeriod)
 			if err != nil {
 				// Better not error out here...
-				glog.Infof("Cannot delete Pod %s/%s: %v", pod.Namespace, pod.Name, err)
+				glog.Infof("Cannot delete lifecycle Pod %s/%s for deployment %s/%s: %v", pod.Namespace, pod.Name, namespace, name, err)
 			}
 		}
 	}
 
-	return nil
+	return reaper.oc.DeploymentConfigs(namespace).Delete(name)
+}
+
+type updateConfigFunc func(d *deployapi.DeploymentConfig)
+
+func (reaper *DeploymentConfigReaper) updateDeploymentWithRetries(namespace, name string, applyUpdate updateConfigFunc) (*deployapi.DeploymentConfig, error) {
+	var (
+		config *deployapi.DeploymentConfig
+		err    error
+	)
+	deploymentConfigs := reaper.oc.DeploymentConfigs(namespace)
+	resultErr := wait.Poll(10*time.Millisecond, 1*time.Minute, func() (bool, error) {
+		config, err = deploymentConfigs.Get(name)
+		if err != nil {
+			return false, err
+		}
+		// Apply the update, then attempt to push it to the apiserver.
+		applyUpdate(config)
+		config, err = deploymentConfigs.Update(config)
+		if err != nil {
+			return false, nil
+		}
+		return true, nil
+	})
+	return config, resultErr
 }