cluster up: add persistent volumes on startup

contrib/completions/bash/oc

@@ -5610,6 +5610,8 @@ _oc_cluster_up()
     local_nonpersistent_flags+=("--host-config-dir=")
     flags+=("--host-data-dir=")
     local_nonpersistent_flags+=("--host-data-dir=")
+    flags+=("--host-pv-dir=")
+    local_nonpersistent_flags+=("--host-pv-dir=")
     flags+=("--host-volumes-dir=")
     local_nonpersistent_flags+=("--host-volumes-dir=")
     flags+=("--image=")

contrib/completions/bash/openshift

@@ -10231,6 +10231,8 @@ _openshift_cli_cluster_up()
     local_nonpersistent_flags+=("--host-config-dir=")
     flags+=("--host-data-dir=")
     local_nonpersistent_flags+=("--host-data-dir=")
+    flags+=("--host-pv-dir=")
+    local_nonpersistent_flags+=("--host-pv-dir=")
     flags+=("--host-volumes-dir=")
     local_nonpersistent_flags+=("--host-volumes-dir=")
     flags+=("--image=")

contrib/completions/zsh/oc

@@ -5758,6 +5758,8 @@ _oc_cluster_up()
     local_nonpersistent_flags+=("--host-config-dir=")
     flags+=("--host-data-dir=")
     local_nonpersistent_flags+=("--host-data-dir=")
+    flags+=("--host-pv-dir=")
+    local_nonpersistent_flags+=("--host-pv-dir=")
     flags+=("--host-volumes-dir=")
     local_nonpersistent_flags+=("--host-volumes-dir=")
     flags+=("--image=")

contrib/completions/zsh/openshift

@@ -10379,6 +10379,8 @@ _openshift_cli_cluster_up()
     local_nonpersistent_flags+=("--host-config-dir=")
     flags+=("--host-data-dir=")
     local_nonpersistent_flags+=("--host-data-dir=")
+    flags+=("--host-pv-dir=")
+    local_nonpersistent_flags+=("--host-pv-dir=")
     flags+=("--host-volumes-dir=")
     local_nonpersistent_flags+=("--host-volumes-dir=")
     flags+=("--image=")

docs/man/man1/oc-cluster-up.1

@@ -53,6 +53,10 @@ A public hostname can also be specified for the server with the \-\-public\-host
 \fB\-\-host\-data\-dir\fP=""
     Directory on Docker host for OpenShift data. If not specified, etcd data will not be persisted on the host.
 
+.PP
+\fB\-\-host\-pv\-dir\fP="/var/lib/origin/openshift.local.pv"
+    Directory on host for OpenShift persistent volumes
+
 .PP
 \fB\-\-host\-volumes\-dir\fP="/var/lib/origin/openshift.local.volumes"
     Directory on Docker host for OpenShift volumes

docs/man/man1/openshift-cli-cluster-up.1

@@ -53,6 +53,10 @@ A public hostname can also be specified for the server with the \-\-public\-host
 \fB\-\-host\-data\-dir\fP=""
     Directory on Docker host for OpenShift data. If not specified, etcd data will not be persisted on the host.
 
+.PP
+\fB\-\-host\-pv\-dir\fP="/var/lib/origin/openshift.local.pv"
+    Directory on host for OpenShift persistent volumes
+
 .PP
 \fB\-\-host\-volumes\-dir\fP="/var/lib/origin/openshift.local.volumes"
     Directory on Docker host for OpenShift volumes

pkg/bootstrap/docker/dockerhelper/filetransfer.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"io/ioutil"
+	"path"
 	"path/filepath"
 	"strings"
 
@@ -123,10 +124,10 @@ func DownloadDirFromContainer(client *docker.Client, container, src, dst string)
 
 // UploadFileToContainer uploads a file to a remote container.
 func UploadFileToContainer(client *docker.Client, container, src, dest string) error {
-	uploader, errch := newContainerUploader(client, container, filepath.Dir(dest))
+	uploader, errch := newContainerUploader(client, container, path.Dir(dest))
 
 	t := s2itar.New(s2iutil.NewFileSystem())
-	tarWriter := s2itar.RenameAdapter{Writer: tar.NewWriter(uploader), Old: filepath.Base(src), New: filepath.Base(dest)}
+	tarWriter := s2itar.RenameAdapter{Writer: tar.NewWriter(uploader), Old: filepath.Base(src), New: path.Base(dest)}
 
 	err := t.CreateTarStreamToTarWriter(src, true, tarWriter, nil)
 	if err == nil {

pkg/bootstrap/docker/host/host.go

@@ -14,37 +14,46 @@ import (
 )
 
 const (
-	cmdTestNsenterMount          = "nsenter --mount=/rootfs/proc/1/ns/mnt findmnt"
-	cmdEnsureHostDirs            = "for dir in %s; do if [ ! -d \"${dir}\" ]; then mkdir -p \"${dir}\"; fi; done"
+	cmdTestNsenterMount = "nsenter --mount=/rootfs/proc/1/ns/mnt findmnt"
+	cmdEnsureHostDirs   = `#/bin/bash
+for dir in %s; do
+  if [ ! -d "${dir}" ]; then
+    mkdir -p "${dir}"
+  fi
+done
+`
 	cmdCreateVolumesDirBindMount = "cat /rootfs/proc/1/mountinfo | grep /var/lib/origin || " +
 		"nsenter --mount=/rootfs/proc/1/ns/mnt mount -o bind %[1]s %[1]s"
 	cmdCreateVolumesDirShare = "cat /rootfs/proc/1/mountinfo | grep %[1]s | grep shared || " +
 		"nsenter --mount=/rootfs/proc/1/ns/mnt mount --make-shared %[1]s"
 
-	DefaultVolumesDir = "/var/lib/origin/openshift.local.volumes"
-	DefaultConfigDir  = "/var/lib/origin/openshift.local.config"
+	DefaultVolumesDir           = "/var/lib/origin/openshift.local.volumes"
+	DefaultConfigDir            = "/var/lib/origin/openshift.local.config"
+	DefaultPersistentVolumesDir = "/var/lib/origin/openshift.local.pv"
 )
 
 // HostHelper contains methods to help check settings on a Docker host machine
 // using a privileged container
 type HostHelper struct {
-	runHelper  *run.RunHelper
-	client     *docker.Client
-	image      string
-	volumesDir string
-	configDir  string
-	dataDir    string
+	runHelper            *run.RunHelper
+	client               *docker.Client
+	image                string
+	volumesDir           string
+	configDir            string
+	dataDir              string
+	persistentVolumesDir string
 }
 
 // NewHostHelper creates a new HostHelper
-func NewHostHelper(client *docker.Client, image, volumesDir, configDir, dataDir string) *HostHelper {
+func NewHostHelper(client *docker.Client, image, volumesDir, configDir, dataDir, pvDir string) *HostHelper {
 	return &HostHelper{
-		runHelper:  run.NewRunHelper(client),
-		client:     client,
-		image:      image,
-		volumesDir: volumesDir,
-		configDir:  configDir,
-		dataDir:    dataDir,
+		runHelper:            run.NewRunHelper(client),
+		client:               client,
+		image:                image,
+		volumesDir:           volumesDir,
+		configDir:            configDir,
+		dataDir:              dataDir,
+		persistentVolumesDir: pvDir,
 	}
 }
 
@@ -147,13 +156,16 @@ func (h *HostHelper) EnsureHostDirectories() error {
 	if h.volumesDir == DefaultVolumesDir {
 		dirs = append(dirs, path.Join("/rootfs", h.volumesDir))
 	}
+	if h.persistentVolumesDir == DefaultPersistentVolumesDir {
+		dirs = append(dirs, path.Join("/rootfs", h.persistentVolumesDir))
+	}
 	if len(dirs) > 0 {
 		cmd := fmt.Sprintf(cmdEnsureHostDirs, strings.Join(dirs, " "))
 		rc, err := h.runner().
 			Image(h.image).
 			DiscardContainer().
 			Privileged().
-			Bind("/var:/rootfs/var").
+			Bind("/var:/rootfs/var:z").
 			Entrypoint("/bin/bash").
 			Command("-c", cmd).Run()
 		if err != nil || rc != 0 {

pkg/bootstrap/docker/openshift/admin.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
 
 	"github.com/golang/glog"
@@ -31,7 +32,7 @@ const (
 )
 
 // InstallRegistry checks whether a registry is installed and installs one if not already installed
-func (h *Helper) InstallRegistry(kubeClient kclientset.Interface, f *clientcmd.Factory, configDir, images string, out, errout io.Writer) error {
+func (h *Helper) InstallRegistry(kubeClient kclientset.Interface, f *clientcmd.Factory, configDir, images, pvDir string, out, errout io.Writer) error {
 	_, err := kubeClient.Core().Services(DefaultNamespace).Get(SvcDockerRegistry)
 	if err == nil {
 		// If there's no error, the registry already exists
@@ -40,6 +41,12 @@ func (h *Helper) InstallRegistry(kubeClient kclientset.Interface, f *clientcmd.F
 	if !apierrors.IsNotFound(err) {
 		return errors.NewError("error retrieving docker registry service").WithCause(err).WithDetails(h.OriginLog())
 	}
+
+	err = AddSCCToServiceAccount(kubeClient, "privileged", "registry", "default")
+	if err != nil {
+		return errors.NewError("cannot add privileged SCC to registry service account").WithCause(err).WithDetails(h.OriginLog())
+	}
+
 	imageTemplate := variable.NewDefaultImageTemplate()
 	imageTemplate.Format = images
 	opts := &registry.RegistryOptions{
@@ -52,6 +59,7 @@ func (h *Helper) InstallRegistry(kubeClient kclientset.Interface, f *clientcmd.F
 			Labels:         "docker-registry=default",
 			Volume:         "/registry",
 			ServiceAccount: "registry",
+			HostMount:      path.Join(pvDir, "registry"),
 		},
 	}
 	cmd := registry.NewCmdRegistry(f, "", "registry", out, errout)

pkg/bootstrap/docker/openshift/helper.go

@@ -68,21 +68,22 @@ type Helper struct {
 
 // StartOptions represent the parameters sent to the start command
 type StartOptions struct {
-	ServerIP           string
-	RouterIP           string
-	DNSPort            int
-	UseSharedVolume    bool
-	SetPropagationMode bool
-	Images             string
-	HostVolumesDir     string
-	HostConfigDir      string
-	HostDataDir        string
-	UseExistingConfig  bool
-	Environment        []string
-	LogLevel           int
-	MetricsHost        string
-	LoggingHost        string
-	PortForwarding     bool
+	ServerIP                 string
+	RouterIP                 string
+	DNSPort                  int
+	UseSharedVolume          bool
+	SetPropagationMode       bool
+	Images                   string
+	HostVolumesDir           string
+	HostConfigDir            string
+	HostDataDir              string
+	HostPersistentVolumesDir string
+	UseExistingConfig        bool
+	Environment              []string
+	LogLevel                 int
+	MetricsHost              string
+	LoggingHost              string
+	PortForwarding           bool
 }
 
 // NewHelper creates a new OpenShift helper
@@ -355,6 +356,10 @@ func (h *Helper) Start(opt *StartOptions, out io.Writer) (string, error) {
 	if len(opt.HostDataDir) > 0 {
 		binds = append(binds, fmt.Sprintf("%s:/var/lib/origin/openshift.local.etcd:z", opt.HostDataDir))
 	}
+	if len(opt.HostPersistentVolumesDir) > 0 {
+		binds = append(binds, fmt.Sprintf("%[1]s:%[1]s", opt.HostPersistentVolumesDir))
+		env = append(env, fmt.Sprintf("OPENSHIFT_PV_DIR=%s", opt.HostPersistentVolumesDir))
+	}
 	_, err = h.runHelper.New().Image(h.image).
 		Name(h.containerName).
 		Privileged().
@@ -509,6 +514,8 @@ func (h *Helper) updateConfig(configDir, routerIP, metricsHost, loggingHost stri
 		cfg.AssetConfig.LoggingPublicURL = fmt.Sprintf("https://%s", loggingHost)
 	}
 
+	cfg.JenkinsPipelineConfig.TemplateName = "jenkins-persistent"
+
 	cfgBytes, err := configapilatest.WriteYAML(cfg)
 	if err != nil {
 		return err

pkg/bootstrap/docker/openshift/logging.go

@@ -112,6 +112,7 @@ func (h *Helper) InstallLogging(f *clientcmd.Factory, publicHostname, loggerHost
 		"public-master-url": fmt.Sprintf("https://%s:8443", publicHostname),
 		"es-cluster-size":   "1",
 		"es-instance-ram":   "1024M",
+		"es-pvc-size":       "100G",
 	}
 	kubeClient.Core().ConfigMaps(loggingNamespace).Create(loggingConfig)
 

pkg/bootstrap/docker/openshift/metrics.go

@@ -124,7 +124,7 @@ func metricsDeployerJob(hostName, imagePrefix, imageVersion string) *kbatch.Job
 		},
 		{
 			Name:  "USE_PERSISTENT_STORAGE",
-			Value: "false",
+			Value: "true",
 		},
 		{
 			Name:  "CASSANDRA_NODES",