From c4ecd5fd190da93f87d1e2fe29f1d64bdcfd5d17 Mon Sep 17 00:00:00 2001 From: David Eads Date: Fri, 16 Mar 2018 12:34:19 -0400 Subject: [PATCH 1/2] UPSTREAM: 61287: provide easy methods for direct kubeconfig loading from bytes --- .../tools/clientcmd/client_config.go | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/tools/clientcmd/client_config.go b/vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/tools/clientcmd/client_config.go index a8698af24320..8f41cb35c8b5 100644 --- a/vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/tools/clientcmd/client_config.go +++ b/vendor/k8s.io/kubernetes/staging/src/k8s.io/client-go/tools/clientcmd/client_config.go @@ -100,6 +100,26 @@ func NewInteractiveClientConfig(config clientcmdapi.Config, contextName string, return &DirectClientConfig{config, contextName, overrides, fallbackReader, configAccess, promptedCredentials{}} } +// NewClientConfigFromBytes takes your kubeconfig and gives you back a ClientConfig +func NewClientConfigFromBytes(configBytes []byte) (ClientConfig, error) { + config, err := Load(configBytes) + if err != nil { + return nil, err + } + + return &DirectClientConfig{*config, "", &ConfigOverrides{}, nil, nil, promptedCredentials{}}, nil +} + +// RESTConfigFromKubeConfig is a convenience method to give back a restconfig from your kubeconfig bytes. +// For programmatic access, this is what you want 80% of the time +func RESTConfigFromKubeConfig(configBytes []byte) (*restclient.Config, error) { + clientConfig, err := NewClientConfigFromBytes(configBytes) + if err != nil { + return nil, err + } + return clientConfig.ClientConfig() +} + func (config *DirectClientConfig) RawConfig() (clientcmdapi.Config, error) { return config.config, nil } From be6dc5eaa8feeee2d2b71c1adcef5ea4b78cf152 Mon Sep 17 00:00:00 2001 From: David Eads Date: Mon, 12 Mar 2018 18:06:02 -0400 Subject: [PATCH 2/2] make registry installation a component --- .../components/registry/registry_install.go | 102 ++++++++++++++++++ pkg/oc/bootstrap/docker/openshift/admin.go | 68 +----------- pkg/oc/bootstrap/docker/up.go | 69 ++++++------ 3 files changed, 140 insertions(+), 99 deletions(-) create mode 100644 pkg/oc/bootstrap/clusterup/components/registry/registry_install.go diff --git a/pkg/oc/bootstrap/clusterup/components/registry/registry_install.go b/pkg/oc/bootstrap/clusterup/components/registry/registry_install.go new file mode 100644 index 000000000000..a5a787d20234 --- /dev/null +++ b/pkg/oc/bootstrap/clusterup/components/registry/registry_install.go @@ -0,0 +1,102 @@ +package registry + +import ( + "fmt" + "os" + "path" + + "github.com/golang/glog" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "github.com/openshift/origin/pkg/oc/bootstrap/clusterup/componentinstall" + "github.com/openshift/origin/pkg/oc/bootstrap/docker/dockerhelper" + "github.com/openshift/origin/pkg/oc/bootstrap/docker/openshift" + "github.com/openshift/origin/pkg/oc/bootstrap/docker/run" + "github.com/openshift/origin/pkg/oc/errors" + securityclient "github.com/openshift/origin/pkg/security/generated/internalclientset/typed/security/internalversion" +) + +const ( + DefaultNamespace = "default" + SvcDockerRegistry = "docker-registry" + masterConfigDir = "/var/lib/origin/openshift.local.config/master" + // This is needed because of NO_PROXY cannot handle the CIDR range + RegistryServiceClusterIP = "172.30.1.1" +) + +type RegistryComponentOptions struct { + ClusterAdminKubeConfig *rest.Config + + OCImage string + MasterConfigDir string + Images string + PVDir string +} + +func (r *RegistryComponentOptions) Name() string { + return "openshift-image-registry" +} + +func (r *RegistryComponentOptions) Install(dockerClient dockerhelper.Interface, logdir string) error { + kubeClient, err := kubernetes.NewForConfig(r.ClusterAdminKubeConfig) + _, err = kubeClient.Core().Services(DefaultNamespace).Get(SvcDockerRegistry, metav1.GetOptions{}) + if err == nil { + // If there's no error, the registry already exists + return nil + } + if !apierrors.IsNotFound(err) { + return errors.NewError("error retrieving docker registry service").WithCause(err) + } + + imageRunHelper := run.NewRunHelper(dockerhelper.NewHelper(dockerClient)).New() + glog.Infof("Running %q", r.Name()) + + securityClient, err := securityclient.NewForConfig(r.ClusterAdminKubeConfig) + if err != nil { + return err + } + err = openshift.AddSCCToServiceAccount(securityClient, "privileged", "registry", "default", os.Stdout) + if err != nil { + return errors.NewError("cannot add privileged SCC to registry service account").WithCause(err) + } + + // Obtain registry markup. The reason it is not created outright is because + // we need to modify the ClusterIP of the registry service. The command doesn't + // have an option to set it. + flags := []string{ + "adm", + "registry", + "--loglevel=8", + // We need to set the ClusterIP for registry in order to be able to set the NO_PROXY no predicable + // IP address as NO_PROXY does not support CIDR format. + // TODO: We should switch the cluster up registry to use DNS. + "--cluster-ip=" + RegistryServiceClusterIP, + "--config=" + masterConfigDir + "/admin.kubeconfig", + fmt.Sprintf("--images=%s", r.Images), + fmt.Sprintf("--mount-host=%s", path.Join(r.PVDir, "registry")), + } + _, stdout, stderr, rc, err := imageRunHelper.Image(r.OCImage). + Privileged(). + DiscardContainer(). + HostNetwork(). + HostPid(). + Bind(r.MasterConfigDir + ":" + masterConfigDir). + Entrypoint("oc"). + Command(flags...).Output() + + if err := componentinstall.LogContainer(logdir, r.Name(), stdout, stderr); err != nil { + glog.Errorf("error logging %q: %v", r.Name(), err) + } + if err != nil { + return errors.NewError("could not run %q: %v", r.Name(), err).WithCause(err) + } + if rc != 0 { + return errors.NewError("could not run %q: rc==%v", r.Name(), rc) + } + + return nil +} diff --git a/pkg/oc/bootstrap/docker/openshift/admin.go b/pkg/oc/bootstrap/docker/openshift/admin.go index f4000c2ea44b..71e680c4dacc 100644 --- a/pkg/oc/bootstrap/docker/openshift/admin.go +++ b/pkg/oc/bootstrap/docker/openshift/admin.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "os" - "path" "path/filepath" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -27,11 +26,7 @@ import ( ) const ( - DefaultNamespace = "default" - RegistryServiceName = "docker-registry" - RegistryServiceAccountName = "registry" - // This is needed because of NO_PROXY cannot handle the CIDR range - RegistryServiceClusterIP = "172.30.1.1" + DefaultNamespace = "default" RouterServiceAccountName = "router" RouterServiceName = "router" @@ -39,67 +34,6 @@ const ( routerCertPath = masterConfigDir + "/router.pem" ) -// InstallRegistry checks whether a registry is installed and installs one if not already installed -func (h *Helper) InstallRegistry(dockerClient dockerhelper.Interface, ocImage string, kubeClient kclientset.Interface, f *clientcmd.Factory, configDir, logdir, images, pvDir string, out, errout io.Writer) error { - _, err := kubeClient.Core().Services(DefaultNamespace).Get(RegistryServiceName, metav1.GetOptions{}) - if err == nil { - glog.V(3).Infof("The %q service is already present, skipping installation", RegistryServiceName) - // If there's no error, the registry already exists - return nil - } - if !apierrors.IsNotFound(err) { - return errors.NewError("error retrieving docker-registry service").WithCause(err).WithDetails(h.OriginLog()) - } - - componentName := "install-registry" - imageRunHelper := run.NewRunHelper(dockerhelper.NewHelper(dockerClient)).New() - glog.Infof("Running %q", componentName) - - securityClient, err := f.OpenshiftInternalSecurityClient() - if err != nil { - return err - } - err = AddSCCToServiceAccount(securityClient.Security(), "privileged", RegistryServiceAccountName, "default", out) - if err != nil { - return errors.NewError("cannot add privileged SCC to registry service account").WithCause(err).WithDetails(h.OriginLog()) - } - - masterDir := filepath.Join(configDir, "master") - - // Obtain registry markup. The reason it is not created outright is because - // we need to modify the ClusterIP of the registry service. The command doesn't - // have an option to set it. - flags := []string{ - "adm", - "registry", - "--loglevel=8", - // We need to set the ClusterIP for registry in order to be able to set the NO_PROXY no predicable - // IP address as NO_PROXY does not support CIDR format. - // TODO: We should switch the cluster up registry to use DNS. - "--cluster-ip=" + RegistryServiceClusterIP, - "--config=" + masterConfigDir + "/admin.kubeconfig", - fmt.Sprintf("--images=%s", images), - fmt.Sprintf("--mount-host=%s", path.Join(pvDir, "registry")), - } - _, stdout, stderr, rc, err := imageRunHelper.Image(ocImage). - DiscardContainer(). - HostNetwork(). - Bind(masterDir + ":" + masterConfigDir). - Entrypoint("oc"). - Command(flags...).Output() - - if err := componentinstall.LogContainer(logdir, componentName, stdout, stderr); err != nil { - glog.Errorf("error logging %q: %v", componentName, err) - } - if err != nil { - return errors.NewError("could not run %q: %v", componentName, err).WithCause(err) - } - if rc != 0 { - return errors.NewError("could not run %q: rc==%v", componentName, rc) - } - return err -} - // InstallRouter installs a default router on the OpenShift server func (h *Helper) InstallRouter(dockerClient dockerhelper.Interface, ocImage string, kubeClient kclientset.Interface, f *clientcmd.Factory, configDir, logdir, images, hostIP string, portForwarding bool, out, errout io.Writer) error { _, err := kubeClient.Core().Services(DefaultNamespace).Get(RouterServiceName, metav1.GetOptions{}) diff --git a/pkg/oc/bootstrap/docker/up.go b/pkg/oc/bootstrap/docker/up.go index e97038bd458e..4e1e1e408c4a 100644 --- a/pkg/oc/bootstrap/docker/up.go +++ b/pkg/oc/bootstrap/docker/up.go @@ -17,7 +17,6 @@ import ( cliconfig "github.com/docker/docker/cli/config" dockerclient "github.com/docker/docker/client" "github.com/golang/glog" - "github.com/openshift/origin/pkg/oc/bootstrap/clusterup/tmpformac" "github.com/spf13/cobra" "github.com/spf13/pflag" "golang.org/x/net/context" @@ -37,6 +36,8 @@ import ( "github.com/openshift/origin/pkg/cmd/util/variable" "github.com/openshift/origin/pkg/oc/bootstrap" "github.com/openshift/origin/pkg/oc/bootstrap/clusterup/componentinstall" + "github.com/openshift/origin/pkg/oc/bootstrap/clusterup/components/registry" + "github.com/openshift/origin/pkg/oc/bootstrap/clusterup/tmpformac" "github.com/openshift/origin/pkg/oc/bootstrap/docker/dockerhelper" "github.com/openshift/origin/pkg/oc/bootstrap/docker/dockermachine" "github.com/openshift/origin/pkg/oc/bootstrap/docker/errors" @@ -481,12 +482,35 @@ func (c *ClusterUpConfig) Start(out io.Writer) error { } taskPrinter.Success() - // Install a registry - taskPrinter.StartTask("Installing registry") - if err := c.InstallRegistry(out); err != nil { - return taskPrinter.ToError(err) + clusterAdminKubeConfigBytes, err := ioutil.ReadFile(path.Join(c.LocalConfigDir, "master", "admin.kubeconfig")) + if err != nil { + return err + } + clusterAdminKubeConfig, err := kclientcmd.RESTConfigFromKubeConfig(clusterAdminKubeConfigBytes) + if err != nil { + return err + } + + // TODO, now we build up a set of things to install here. We build the list so that we can install everything in + // TODO parallel to avoid anyone accidentally introducing dependencies. We'll start with migrating what we have + // TODO and then we'll try to clean it up. + registryInstall := ®istry.RegistryComponentOptions{ + ClusterAdminKubeConfig: clusterAdminKubeConfig, + + OCImage: c.openshiftImage(), + MasterConfigDir: path.Join(c.LocalConfigDir, "master"), + Images: c.imageFormat(), + PVDir: c.HostPersistentVolumesDir, + } + + componentsToInstall := []componentinstall.Component{} + componentsToInstall = append(componentsToInstall, c.ImportInitialObjectsComponents(c.Out)...) + componentsToInstall = append(componentsToInstall, registryInstall) + + err = componentinstall.InstallComponents(componentsToInstall, c.GetDockerClient(), path.Join(c.BaseTempDir, "logs")) + if err != nil { + return err } - taskPrinter.Success() // Install a router taskPrinter.StartTask("Installing router") @@ -504,13 +528,6 @@ func (c *ClusterUpConfig) Start(out io.Writer) error { taskPrinter.Success() } - // Import default image streams - taskPrinter.StartTask("Importing default data router") - if err := c.ImportInitialObjects(out); err != nil { - return taskPrinter.ToError(err) - } - taskPrinter.Success() - // Install logging if c.ShouldInstallLogging { taskPrinter.StartTask("Installing logging") @@ -832,7 +849,7 @@ func (c *ClusterUpConfig) determineServerIP(out io.Writer) (string, []string, er // updateNoProxy will add some default values to the NO_PROXY setting if they are not present func (c *ClusterUpConfig) updateNoProxy() { - values := []string{"127.0.0.1", c.ServerIP, "localhost", openshift.ServiceCatalogServiceIP, openshift.RegistryServiceClusterIP} + values := []string{"127.0.0.1", c.ServerIP, "localhost", openshift.ServiceCatalogServiceIP, registry.RegistryServiceClusterIP} ipFromServer, err := c.OpenShiftHelper().ServerIP() if err == nil { values = append(values, ipFromServer) @@ -882,19 +899,6 @@ func (c *ClusterUpConfig) imageFormat() string { return fmt.Sprintf("%s-${component}:%s", c.Image, c.ImageVersion) } -// InstallRegistry installs the OpenShift registry on the server -func (c *ClusterUpConfig) InstallRegistry(out io.Writer) error { - _, kubeClient, err := c.Clients() - if err != nil { - return err - } - f, err := c.Factory() - if err != nil { - return err - } - return c.OpenShiftHelper().InstallRegistry(c.GetDockerClient(), c.openshiftImage(), kubeClient, f, c.LocalConfigDir, path.Join(c.BaseTempDir, "logs"), c.imageFormat(), c.HostPersistentVolumesDir, out, os.Stderr) -} - // InstallRouter installs a default router on the server func (c *ClusterUpConfig) InstallRouter(out io.Writer) error { _, kubeClient, err := c.Clients() @@ -935,7 +939,8 @@ func (c *ClusterUpConfig) InstallWebConsole(out io.Writer) error { return c.OpenShiftHelper().InstallWebConsole(f, c.imageFormat(), c.ServerLogLevel, publicURL, masterURL, loggingURL, metricsURL) } -func (c *ClusterUpConfig) ImportInitialObjects(out io.Writer) error { +// TODO this should become a separate thing we can install, like registry +func (c *ClusterUpConfig) ImportInitialObjectsComponents(out io.Writer) []componentinstall.Component { componentsToInstall := []componentinstall.Component{} componentsToInstall = append(componentsToInstall, c.makeObjectImportInstallationComponentsOrDie(out, openshift.Namespace, map[string]string{ @@ -950,7 +955,7 @@ func (c *ClusterUpConfig) ImportInitialObjects(out io.Writer) error { componentsToInstall = append(componentsToInstall, c.makeObjectImportInstallationComponentsOrDie(out, openshift.InfraNamespace, internalCurrentTemplateLocations)...) - return componentinstall.InstallComponents(componentsToInstall, c.GetDockerClient(), path.Join(c.BaseTempDir, "logs")) + return componentsToInstall } // InstallLogging will start the installation of logging components @@ -1117,9 +1122,9 @@ func (c *ClusterUpConfig) checkProxySettings() string { if len(dockerHTTPProxy) > 0 || len(dockerHTTPSProxy) > 0 { dockerNoProxyList := strings.Split(dockerNoProxy, ",") dockerNoProxySet := sets.NewString(dockerNoProxyList...) - if !dockerNoProxySet.Has(openshift.RegistryServiceClusterIP) { + if !dockerNoProxySet.Has(registry.RegistryServiceClusterIP) { warnings = append(warnings, fmt.Sprintf("A proxy is configured for Docker, however %[1]s is not included in its NO_PROXY list.\n"+ - " %[1]s needs to be included in the Docker daemon's NO_PROXY environment variable so pushes to the local OpenShift registry can succeed.", openshift.RegistryServiceClusterIP)) + " %[1]s needs to be included in the Docker daemon's NO_PROXY environment variable so pushes to the local OpenShift registry can succeed.", registry.RegistryServiceClusterIP)) } } @@ -1358,7 +1363,7 @@ func (c *ClusterUpConfig) ShouldInitializeData() bool { return true } - if _, err = kclient.Core().Services(openshift.DefaultNamespace).Get(openshift.RegistryServiceName, metav1.GetOptions{}); err != nil { + if _, err = kclient.Core().Services(openshift.DefaultNamespace).Get(registry.SvcDockerRegistry, metav1.GetOptions{}); err != nil { return true }