From 711bdc851d6691ed9cc81dfb706a05f1269811ee Mon Sep 17 00:00:00 2001 From: Hemant Kumar Date: Tue, 6 Feb 2018 22:27:19 -0500 Subject: [PATCH] UPSTREAM: 58794: Resize mounted volumes --- .../k8s.io/kubernetes/pkg/util/resizefs/BUILD | 1 - .../pkg/util/resizefs/resizefs_linux.go | 75 +------- .../pkg/util/resizefs/resizefs_unsupported.go | 2 +- .../operationexecutor/operation_generator.go | 21 ++- .../test/e2e/framework/deployment_util.go | 9 +- .../k8s.io/kubernetes/test/e2e/storage/BUILD | 2 + .../test/e2e/storage/mounted_volume_resize.go | 172 ++++++++++++++++++ 7 files changed, 201 insertions(+), 81 deletions(-) create mode 100644 vendor/k8s.io/kubernetes/test/e2e/storage/mounted_volume_resize.go diff --git a/vendor/k8s.io/kubernetes/pkg/util/resizefs/BUILD b/vendor/k8s.io/kubernetes/pkg/util/resizefs/BUILD index aaa1f9aa5f49..17a44272025d 100644 --- a/vendor/k8s.io/kubernetes/pkg/util/resizefs/BUILD +++ b/vendor/k8s.io/kubernetes/pkg/util/resizefs/BUILD @@ -17,7 +17,6 @@ go_library( ] + select({ "@io_bazel_rules_go//go/platform:linux_amd64": [ "//vendor/github.com/golang/glog:go_default_library", - "//vendor/k8s.io/utils/exec:go_default_library", ], "//conditions:default": [], }), diff --git a/vendor/k8s.io/kubernetes/pkg/util/resizefs/resizefs_linux.go b/vendor/k8s.io/kubernetes/pkg/util/resizefs/resizefs_linux.go index 6a4d82d6c037..518eba2bb603 100644 --- a/vendor/k8s.io/kubernetes/pkg/util/resizefs/resizefs_linux.go +++ b/vendor/k8s.io/kubernetes/pkg/util/resizefs/resizefs_linux.go @@ -23,14 +23,6 @@ import ( "github.com/golang/glog" "k8s.io/kubernetes/pkg/util/mount" - utilexec "k8s.io/utils/exec" -) - -const ( - // 'fsck' found errors and corrected them - fsckErrorsCorrected = 1 - // 'fsck' found errors but exited without correcting them - fsckErrorsUncorrected = 4 ) // ResizeFs Provides support for resizing file systems @@ -44,11 +36,11 @@ func NewResizeFs(mounter *mount.SafeFormatAndMount) *ResizeFs { } // Resize perform resize of file system -func (resizefs *ResizeFs) Resize(devicePath string) (bool, error) { +func (resizefs *ResizeFs) Resize(devicePath string, deviceMountPath string) (bool, error) { format, err := resizefs.mounter.GetDiskFormat(devicePath) if err != nil { - formatErr := fmt.Errorf("error checking format for device %s: %v", devicePath, err) + formatErr := fmt.Errorf("ResizeFS.Resize - error checking format for device %s: %v", devicePath, err) return false, formatErr } @@ -58,63 +50,14 @@ func (resizefs *ResizeFs) Resize(devicePath string) (bool, error) { return false, nil } - deviceOpened, err := resizefs.mounter.DeviceOpened(devicePath) - - if err != nil { - deviceOpenErr := fmt.Errorf("error verifying if device %s is open: %v", devicePath, err) - return false, deviceOpenErr - } - - if deviceOpened { - deviceAlreadyOpenErr := fmt.Errorf("the device %s is already in use", devicePath) - return false, deviceAlreadyOpenErr - } - + glog.V(3).Infof("ResizeFS.Resize - Expanding mounted volume %s", devicePath) switch format { case "ext3", "ext4": - fsckErr := resizefs.extFsck(devicePath, format) - if fsckErr != nil { - return false, fsckErr - } return resizefs.extResize(devicePath) case "xfs": - fsckErr := resizefs.fsckDevice(devicePath) - if fsckErr != nil { - return false, fsckErr - } - return resizefs.xfsResize(devicePath) + return resizefs.xfsResize(deviceMountPath) } - return false, fmt.Errorf("resize of format %s is not supported for device %s", format, devicePath) -} - -func (resizefs *ResizeFs) fsckDevice(devicePath string) error { - glog.V(4).Infof("Checking for issues with fsck on device: %s", devicePath) - args := []string{"-a", devicePath} - out, err := resizefs.mounter.Exec.Run("fsck", args...) - if err != nil { - ee, isExitError := err.(utilexec.ExitError) - switch { - case err == utilexec.ErrExecutableNotFound: - glog.Warningf("'fsck' not found on system; continuing resizing without running 'fsck'.") - case isExitError && ee.ExitStatus() == fsckErrorsCorrected: - glog.V(2).Infof("Device %s has errors which were corrected by fsck: %s", devicePath, string(out)) - case isExitError && ee.ExitStatus() == fsckErrorsUncorrected: - return fmt.Errorf("'fsck' found errors on device %s but could not correct them: %s", devicePath, string(out)) - case isExitError && ee.ExitStatus() > fsckErrorsUncorrected: - glog.Infof("`fsck` error %s", string(out)) - } - } - return nil -} - -func (resizefs *ResizeFs) extFsck(devicePath string, fsType string) error { - glog.V(4).Infof("Checking for issues with fsck.%s on device: %s", fsType, devicePath) - args := []string{"-f", "-y", devicePath} - out, err := resizefs.mounter.Run("fsck."+fsType, args...) - if err != nil { - return fmt.Errorf("running fsck.%s failed on %s with error: %v\n Output: %s", fsType, devicePath, err, string(out)) - } - return nil + return false, fmt.Errorf("ResizeFS.Resize - resize of format %s is not supported for device %s mounted at %s", format, devicePath, deviceMountPath) } func (resizefs *ResizeFs) extResize(devicePath string) (bool, error) { @@ -129,15 +72,15 @@ func (resizefs *ResizeFs) extResize(devicePath string) (bool, error) { } -func (resizefs *ResizeFs) xfsResize(devicePath string) (bool, error) { - args := []string{"-d", devicePath} +func (resizefs *ResizeFs) xfsResize(deviceMountPath string) (bool, error) { + args := []string{"-d", deviceMountPath} output, err := resizefs.mounter.Exec.Run("xfs_growfs", args...) if err == nil { - glog.V(2).Infof("Device %s resized successfully", devicePath) + glog.V(2).Infof("Device %s resized successfully", deviceMountPath) return true, nil } - resizeError := fmt.Errorf("resize of device %s failed: %v. xfs_growfs output: %s", devicePath, err, string(output)) + resizeError := fmt.Errorf("resize of device %s failed: %v. xfs_growfs output: %s", deviceMountPath, err, string(output)) return false, resizeError } diff --git a/vendor/k8s.io/kubernetes/pkg/util/resizefs/resizefs_unsupported.go b/vendor/k8s.io/kubernetes/pkg/util/resizefs/resizefs_unsupported.go index 9241d7d5b27d..dd4dd017e829 100644 --- a/vendor/k8s.io/kubernetes/pkg/util/resizefs/resizefs_unsupported.go +++ b/vendor/k8s.io/kubernetes/pkg/util/resizefs/resizefs_unsupported.go @@ -35,6 +35,6 @@ func NewResizeFs(mounter *mount.SafeFormatAndMount) *ResizeFs { } // Resize perform resize of file system -func (resizefs *ResizeFs) Resize(devicePath string) (bool, error) { +func (resizefs *ResizeFs) Resize(devicePath string, deviceMountPath string) (bool, error) { return false, fmt.Errorf("Resize is not supported for this build") } diff --git a/vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_generator.go b/vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_generator.go index a322ed4ca3d8..5a0b45e0b67f 100644 --- a/vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_generator.go +++ b/vendor/k8s.io/kubernetes/pkg/volume/util/operationexecutor/operation_generator.go @@ -490,14 +490,6 @@ func (og *operationGenerator) GenerateMountVolumeFunc( glog.Infof(volumeToMount.GenerateMsgDetailed("MountVolume.WaitForAttach succeeded", fmt.Sprintf("DevicePath %q", devicePath))) - // resizeFileSystem will resize the file system if user has requested a resize of - // underlying persistent volume and is allowed to do so. - resizeError := og.resizeFileSystem(volumeToMount, devicePath, volumePlugin.GetPluginName()) - - if resizeError != nil { - return volumeToMount.GenerateError("MountVolume.Resize failed", resizeError) - } - deviceMountPath, err := volumeAttacher.GetDeviceMountPath(volumeToMount.VolumeSpec) if err != nil { @@ -524,6 +516,15 @@ func (og *operationGenerator) GenerateMountVolumeFunc( // On failure, return error. Caller will log and retry. return volumeToMount.GenerateError("MountVolume.MarkDeviceAsMounted failed", markDeviceMountedErr) } + + // resizeFileSystem will resize the file system if user has requested a resize of + // underlying persistent volume and is allowed to do so. + resizeError := og.resizeFileSystem(volumeToMount, devicePath, deviceMountPath, volumePlugin.GetPluginName()) + + if resizeError != nil { + return volumeToMount.GenerateError("MountVolume.Resize failed", resizeError) + } + } if og.checkNodeCapabilitiesBeforeMount { @@ -581,7 +582,7 @@ func (og *operationGenerator) GenerateMountVolumeFunc( }, nil } -func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devicePath string, pluginName string) error { +func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devicePath, deviceMountPath, pluginName string) error { if !utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) { glog.V(6).Infof("Resizing is not enabled for this volume %s", volumeToMount.VolumeName) return nil @@ -621,7 +622,7 @@ func (og *operationGenerator) resizeFileSystem(volumeToMount VolumeToMount, devi } resizer := resizefs.NewResizeFs(diskFormatter) - resizeStatus, resizeErr := resizer.Resize(devicePath) + resizeStatus, resizeErr := resizer.Resize(devicePath, deviceMountPath) if resizeErr != nil { resizeDetailedError := volumeToMount.GenerateErrorDetailed("MountVolume.resizeFileSystem failed", resizeErr) diff --git a/vendor/k8s.io/kubernetes/test/e2e/framework/deployment_util.go b/vendor/k8s.io/kubernetes/test/e2e/framework/deployment_util.go index 8a249d1e1319..99251282532a 100644 --- a/vendor/k8s.io/kubernetes/test/e2e/framework/deployment_util.go +++ b/vendor/k8s.io/kubernetes/test/e2e/framework/deployment_util.go @@ -213,8 +213,8 @@ func CheckDeploymentRevisionAndImage(c clientset.Interface, ns, deploymentName, return testutils.CheckDeploymentRevisionAndImage(c, ns, deploymentName, revision, image) } -func CreateDeployment(client clientset.Interface, replicas int32, podLabels map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, command string) (*extensions.Deployment, error) { - deploymentSpec := MakeDeployment(replicas, podLabels, namespace, pvclaims, false, command) +func CreateDeployment(client clientset.Interface, replicas int32, podLabels map[string]string, nodeSelector map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, command string) (*extensions.Deployment, error) { + deploymentSpec := MakeDeployment(replicas, podLabels, nodeSelector, namespace, pvclaims, false, command) deployment, err := client.Extensions().Deployments(namespace).Create(deploymentSpec) if err != nil { return nil, fmt.Errorf("deployment %q Create API error: %v", deploymentSpec.Name, err) @@ -229,7 +229,7 @@ func CreateDeployment(client clientset.Interface, replicas int32, podLabels map[ // MakeDeployment creates a deployment definition based on the namespace. The deployment references the PVC's // name. A slice of BASH commands can be supplied as args to be run by the pod -func MakeDeployment(replicas int32, podLabels map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string) *extensions.Deployment { +func MakeDeployment(replicas int32, podLabels map[string]string, nodeSelector map[string]string, namespace string, pvclaims []*v1.PersistentVolumeClaim, isPrivileged bool, command string) *extensions.Deployment { if len(command) == 0 { command = "while true; do sleep 1; done" } @@ -273,6 +273,9 @@ func MakeDeployment(replicas int32, podLabels map[string]string, namespace strin } deploymentSpec.Spec.Template.Spec.Containers[0].VolumeMounts = volumeMounts deploymentSpec.Spec.Template.Spec.Volumes = volumes + if nodeSelector != nil { + deploymentSpec.Spec.Template.Spec.NodeSelector = nodeSelector + } return deploymentSpec } diff --git a/vendor/k8s.io/kubernetes/test/e2e/storage/BUILD b/vendor/k8s.io/kubernetes/test/e2e/storage/BUILD index c65c4413f9be..4f678c6ca3eb 100644 --- a/vendor/k8s.io/kubernetes/test/e2e/storage/BUILD +++ b/vendor/k8s.io/kubernetes/test/e2e/storage/BUILD @@ -10,6 +10,7 @@ go_library( srcs = [ "empty_dir_wrapper.go", "flexvolume.go", + "mounted_volume_resize.go", "framework.go", "pd.go", "persistent_volumes.go", @@ -44,6 +45,7 @@ go_library( "//pkg/api/testapi:go_default_library", "//pkg/apis/core/v1/helper:go_default_library", "//pkg/apis/storage/v1/util:go_default_library", + "//pkg/client/conditions:go_default_library", "//pkg/cloudprovider/providers/vsphere:go_default_library", "//pkg/cloudprovider/providers/vsphere/vclib:go_default_library", "//pkg/kubelet/apis:go_default_library", diff --git a/vendor/k8s.io/kubernetes/test/e2e/storage/mounted_volume_resize.go b/vendor/k8s.io/kubernetes/test/e2e/storage/mounted_volume_resize.go new file mode 100644 index 000000000000..895e561cfde9 --- /dev/null +++ b/vendor/k8s.io/kubernetes/test/e2e/storage/mounted_volume_resize.go @@ -0,0 +1,172 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package storage + +import ( + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + storage "k8s.io/api/storage/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/pkg/client/conditions" + "k8s.io/kubernetes/test/e2e/framework" + "k8s.io/kubernetes/test/e2e/storage/utils" +) + +var _ = utils.SIGDescribe("Mounted volume expand [Feature:ExpandPersistentVolumes] [Slow]", func() { + var ( + c clientset.Interface + ns string + err error + pvc *v1.PersistentVolumeClaim + resizableSc *storage.StorageClass + nodeName string + isNodeLabeled bool + nodeKeyValueLabel map[string]string + nodeLabelValue string + nodeKey string + ) + + f := framework.NewDefaultFramework("mounted-volume-expand") + BeforeEach(func() { + framework.SkipUnlessProviderIs("aws", "gce") + c = f.ClientSet + ns = f.Namespace.Name + framework.ExpectNoError(framework.WaitForAllNodesSchedulable(c, framework.TestContext.NodeSchedulableTimeout)) + + nodeList := framework.GetReadySchedulableNodesOrDie(f.ClientSet) + if len(nodeList.Items) != 0 { + nodeName = nodeList.Items[0].Name + } else { + framework.Failf("Unable to find ready and schedulable Node") + } + + nodeKey = "mounted_volume_expand" + + if !isNodeLabeled { + nodeLabelValue = ns + nodeKeyValueLabel = make(map[string]string) + nodeKeyValueLabel[nodeKey] = nodeLabelValue + framework.AddOrUpdateLabelOnNode(c, nodeName, nodeKey, nodeLabelValue) + isNodeLabeled = true + } + + test := storageClassTest{ + name: "default", + claimSize: "2Gi", + } + resizableSc, err = createResizableStorageClass(test, ns, "resizing", c) + Expect(err).NotTo(HaveOccurred(), "Error creating resizable storage class") + Expect(*resizableSc.AllowVolumeExpansion).To(BeTrue()) + + pvc = newClaim(test, ns, "default") + pvc.Spec.StorageClassName = &resizableSc.Name + pvc, err = c.CoreV1().PersistentVolumeClaims(pvc.Namespace).Create(pvc) + Expect(err).NotTo(HaveOccurred(), "Error creating pvc") + }) + + framework.AddCleanupAction(func() { + if len(nodeLabelValue) > 0 { + framework.RemoveLabelOffNode(c, nodeName, nodeKey) + } + }) + + AfterEach(func() { + framework.Logf("AfterEach: Cleaning up resources for mounted volume resize") + + if c != nil { + if errs := framework.PVPVCCleanup(c, ns, nil, pvc); len(errs) > 0 { + framework.Failf("AfterEach: Failed to delete PVC and/or PV. Errors: %v", utilerrors.NewAggregate(errs)) + } + pvc, nodeName, isNodeLabeled, nodeLabelValue = nil, "", false, "" + nodeKeyValueLabel = make(map[string]string) + } + }) + + It("Should verify mounted devices can be resized", func() { + By("Waiting for PVC to be in bound phase") + pvcClaims := []*v1.PersistentVolumeClaim{pvc} + pvs, err := framework.WaitForPVClaimBoundPhase(c, pvcClaims, framework.ClaimProvisionTimeout) + Expect(err).NotTo(HaveOccurred(), "Failed waiting for PVC to be bound %v", err) + Expect(len(pvs)).To(Equal(1)) + + By("Creating a deployment with the provisioned volume") + deployment, err := framework.CreateDeployment(c, int32(1), map[string]string{"test": "app"}, nodeKeyValueLabel, ns, pvcClaims, "") + defer c.ExtensionsV1beta1().Deployments(ns).Delete(deployment.Name, &metav1.DeleteOptions{}) + + By("Expanding current pvc") + newSize := resource.MustParse("6Gi") + pvc, err = expandPVCSize(pvc, newSize, c) + Expect(err).NotTo(HaveOccurred(), "While updating pvc for more size") + Expect(pvc).NotTo(BeNil()) + + pvcSize := pvc.Spec.Resources.Requests[v1.ResourceStorage] + if pvcSize.Cmp(newSize) != 0 { + framework.Failf("error updating pvc size %q", pvc.Name) + } + + By("Waiting for cloudprovider resize to finish") + err = waitForControllerVolumeResize(pvc, c) + Expect(err).NotTo(HaveOccurred(), "While waiting for pvc resize to finish") + + By("Getting a pod from deployment") + podList, err := framework.GetPodsForDeployment(c, deployment) + Expect(podList.Items).NotTo(BeEmpty()) + pod := podList.Items[0] + + By("Deleting the pod from deployment") + err = framework.DeletePodWithWait(f, c, &pod) + Expect(err).NotTo(HaveOccurred(), "while deleting pod for resizing") + + By("Waiting for deployment to create new pod") + pod, err = waitForDeploymentToRecreatePod(c, deployment) + Expect(err).NotTo(HaveOccurred(), "While waiting for pod to be recreated") + + By("Waiting for file system resize to finish") + pvc, err = waitForFSResize(pvc, c) + Expect(err).NotTo(HaveOccurred(), "while waiting for fs resize to finish") + + pvcConditions := pvc.Status.Conditions + Expect(len(pvcConditions)).To(Equal(0), "pvc should not have conditions") + }) +}) + +func waitForDeploymentToRecreatePod(client clientset.Interface, deployment *extensions.Deployment) (v1.Pod, error) { + var runningPod v1.Pod + waitErr := wait.PollImmediate(10*time.Second, 5*time.Minute, func() (bool, error) { + podList, err := framework.GetPodsForDeployment(client, deployment) + for _, pod := range podList.Items { + switch pod.Status.Phase { + case v1.PodRunning: + runningPod = pod + return true, nil + case v1.PodFailed, v1.PodSucceeded: + return false, conditions.ErrPodCompleted + } + return false, nil + } + return false, err + }) + return runningPod, waitErr +}