Skip to content

Commit

Permalink
make registry installation a component
Browse files Browse the repository at this point in the history
  • Loading branch information
deads2k committed Mar 20, 2018
1 parent c4ecd5f commit be6dc5e
Show file tree
Hide file tree
Showing 3 changed files with 140 additions and 99 deletions.
102 changes: 102 additions & 0 deletions pkg/oc/bootstrap/clusterup/components/registry/registry_install.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package registry

import (
"fmt"
"os"
"path"

"github.com/golang/glog"

apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"

"github.com/openshift/origin/pkg/oc/bootstrap/clusterup/componentinstall"
"github.com/openshift/origin/pkg/oc/bootstrap/docker/dockerhelper"
"github.com/openshift/origin/pkg/oc/bootstrap/docker/openshift"
"github.com/openshift/origin/pkg/oc/bootstrap/docker/run"
"github.com/openshift/origin/pkg/oc/errors"
securityclient "github.com/openshift/origin/pkg/security/generated/internalclientset/typed/security/internalversion"
)

const (
DefaultNamespace = "default"
SvcDockerRegistry = "docker-registry"
masterConfigDir = "/var/lib/origin/openshift.local.config/master"
// This is needed because of NO_PROXY cannot handle the CIDR range
RegistryServiceClusterIP = "172.30.1.1"
)

type RegistryComponentOptions struct {
ClusterAdminKubeConfig *rest.Config

OCImage string
MasterConfigDir string
Images string
PVDir string
}

func (r *RegistryComponentOptions) Name() string {
return "openshift-image-registry"
}

func (r *RegistryComponentOptions) Install(dockerClient dockerhelper.Interface, logdir string) error {
kubeClient, err := kubernetes.NewForConfig(r.ClusterAdminKubeConfig)
_, err = kubeClient.Core().Services(DefaultNamespace).Get(SvcDockerRegistry, metav1.GetOptions{})
if err == nil {
// If there's no error, the registry already exists
return nil
}
if !apierrors.IsNotFound(err) {
return errors.NewError("error retrieving docker registry service").WithCause(err)
}

imageRunHelper := run.NewRunHelper(dockerhelper.NewHelper(dockerClient)).New()
glog.Infof("Running %q", r.Name())

securityClient, err := securityclient.NewForConfig(r.ClusterAdminKubeConfig)
if err != nil {
return err
}
err = openshift.AddSCCToServiceAccount(securityClient, "privileged", "registry", "default", os.Stdout)
if err != nil {
return errors.NewError("cannot add privileged SCC to registry service account").WithCause(err)
}

// Obtain registry markup. The reason it is not created outright is because
// we need to modify the ClusterIP of the registry service. The command doesn't
// have an option to set it.
flags := []string{
"adm",
"registry",
"--loglevel=8",
// We need to set the ClusterIP for registry in order to be able to set the NO_PROXY no predicable
// IP address as NO_PROXY does not support CIDR format.
// TODO: We should switch the cluster up registry to use DNS.
"--cluster-ip=" + RegistryServiceClusterIP,
"--config=" + masterConfigDir + "/admin.kubeconfig",
fmt.Sprintf("--images=%s", r.Images),
fmt.Sprintf("--mount-host=%s", path.Join(r.PVDir, "registry")),
}
_, stdout, stderr, rc, err := imageRunHelper.Image(r.OCImage).
Privileged().
DiscardContainer().
HostNetwork().
HostPid().
Bind(r.MasterConfigDir + ":" + masterConfigDir).
Entrypoint("oc").
Command(flags...).Output()

if err := componentinstall.LogContainer(logdir, r.Name(), stdout, stderr); err != nil {
glog.Errorf("error logging %q: %v", r.Name(), err)
}
if err != nil {
return errors.NewError("could not run %q: %v", r.Name(), err).WithCause(err)
}
if rc != 0 {
return errors.NewError("could not run %q: rc==%v", r.Name(), rc)
}

return nil
}
68 changes: 1 addition & 67 deletions pkg/oc/bootstrap/docker/openshift/admin.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"fmt"
"io"
"os"
"path"
"path/filepath"

apierrors "k8s.io/apimachinery/pkg/api/errors"
Expand All @@ -27,79 +26,14 @@ import (
)

const (
DefaultNamespace = "default"
RegistryServiceName = "docker-registry"
RegistryServiceAccountName = "registry"
// This is needed because of NO_PROXY cannot handle the CIDR range
RegistryServiceClusterIP = "172.30.1.1"
DefaultNamespace = "default"
RouterServiceAccountName = "router"
RouterServiceName = "router"

masterConfigDir = "/var/lib/origin/openshift.local.config/master"
routerCertPath = masterConfigDir + "/router.pem"
)

// InstallRegistry checks whether a registry is installed and installs one if not already installed
func (h *Helper) InstallRegistry(dockerClient dockerhelper.Interface, ocImage string, kubeClient kclientset.Interface, f *clientcmd.Factory, configDir, logdir, images, pvDir string, out, errout io.Writer) error {
_, err := kubeClient.Core().Services(DefaultNamespace).Get(RegistryServiceName, metav1.GetOptions{})
if err == nil {
glog.V(3).Infof("The %q service is already present, skipping installation", RegistryServiceName)
// If there's no error, the registry already exists
return nil
}
if !apierrors.IsNotFound(err) {
return errors.NewError("error retrieving docker-registry service").WithCause(err).WithDetails(h.OriginLog())
}

componentName := "install-registry"
imageRunHelper := run.NewRunHelper(dockerhelper.NewHelper(dockerClient)).New()
glog.Infof("Running %q", componentName)

securityClient, err := f.OpenshiftInternalSecurityClient()
if err != nil {
return err
}
err = AddSCCToServiceAccount(securityClient.Security(), "privileged", RegistryServiceAccountName, "default", out)
if err != nil {
return errors.NewError("cannot add privileged SCC to registry service account").WithCause(err).WithDetails(h.OriginLog())
}

masterDir := filepath.Join(configDir, "master")

// Obtain registry markup. The reason it is not created outright is because
// we need to modify the ClusterIP of the registry service. The command doesn't
// have an option to set it.
flags := []string{
"adm",
"registry",
"--loglevel=8",
// We need to set the ClusterIP for registry in order to be able to set the NO_PROXY no predicable
// IP address as NO_PROXY does not support CIDR format.
// TODO: We should switch the cluster up registry to use DNS.
"--cluster-ip=" + RegistryServiceClusterIP,
"--config=" + masterConfigDir + "/admin.kubeconfig",
fmt.Sprintf("--images=%s", images),
fmt.Sprintf("--mount-host=%s", path.Join(pvDir, "registry")),
}
_, stdout, stderr, rc, err := imageRunHelper.Image(ocImage).
DiscardContainer().
HostNetwork().
Bind(masterDir + ":" + masterConfigDir).
Entrypoint("oc").
Command(flags...).Output()

if err := componentinstall.LogContainer(logdir, componentName, stdout, stderr); err != nil {
glog.Errorf("error logging %q: %v", componentName, err)
}
if err != nil {
return errors.NewError("could not run %q: %v", componentName, err).WithCause(err)
}
if rc != 0 {
return errors.NewError("could not run %q: rc==%v", componentName, rc)
}
return err
}

// InstallRouter installs a default router on the OpenShift server
func (h *Helper) InstallRouter(dockerClient dockerhelper.Interface, ocImage string, kubeClient kclientset.Interface, f *clientcmd.Factory, configDir, logdir, images, hostIP string, portForwarding bool, out, errout io.Writer) error {
_, err := kubeClient.Core().Services(DefaultNamespace).Get(RouterServiceName, metav1.GetOptions{})
Expand Down
69 changes: 37 additions & 32 deletions pkg/oc/bootstrap/docker/up.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@ import (
cliconfig "github.com/docker/docker/cli/config"
dockerclient "github.com/docker/docker/client"
"github.com/golang/glog"
"github.com/openshift/origin/pkg/oc/bootstrap/clusterup/tmpformac"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
"golang.org/x/net/context"
Expand All @@ -37,6 +36,8 @@ import (
"github.com/openshift/origin/pkg/cmd/util/variable"
"github.com/openshift/origin/pkg/oc/bootstrap"
"github.com/openshift/origin/pkg/oc/bootstrap/clusterup/componentinstall"
"github.com/openshift/origin/pkg/oc/bootstrap/clusterup/components/registry"
"github.com/openshift/origin/pkg/oc/bootstrap/clusterup/tmpformac"
"github.com/openshift/origin/pkg/oc/bootstrap/docker/dockerhelper"
"github.com/openshift/origin/pkg/oc/bootstrap/docker/dockermachine"
"github.com/openshift/origin/pkg/oc/bootstrap/docker/errors"
Expand Down Expand Up @@ -481,12 +482,35 @@ func (c *ClusterUpConfig) Start(out io.Writer) error {
}
taskPrinter.Success()

// Install a registry
taskPrinter.StartTask("Installing registry")
if err := c.InstallRegistry(out); err != nil {
return taskPrinter.ToError(err)
clusterAdminKubeConfigBytes, err := ioutil.ReadFile(path.Join(c.LocalConfigDir, "master", "admin.kubeconfig"))
if err != nil {
return err
}
clusterAdminKubeConfig, err := kclientcmd.RESTConfigFromKubeConfig(clusterAdminKubeConfigBytes)
if err != nil {
return err
}

// TODO, now we build up a set of things to install here. We build the list so that we can install everything in
// TODO parallel to avoid anyone accidentally introducing dependencies. We'll start with migrating what we have
// TODO and then we'll try to clean it up.
registryInstall := &registry.RegistryComponentOptions{
ClusterAdminKubeConfig: clusterAdminKubeConfig,

OCImage: c.openshiftImage(),
MasterConfigDir: path.Join(c.LocalConfigDir, "master"),
Images: c.imageFormat(),
PVDir: c.HostPersistentVolumesDir,
}

componentsToInstall := []componentinstall.Component{}
componentsToInstall = append(componentsToInstall, c.ImportInitialObjectsComponents(c.Out)...)
componentsToInstall = append(componentsToInstall, registryInstall)

err = componentinstall.InstallComponents(componentsToInstall, c.GetDockerClient(), path.Join(c.BaseTempDir, "logs"))
if err != nil {
return err
}
taskPrinter.Success()

// Install a router
taskPrinter.StartTask("Installing router")
Expand All @@ -504,13 +528,6 @@ func (c *ClusterUpConfig) Start(out io.Writer) error {
taskPrinter.Success()
}

// Import default image streams
taskPrinter.StartTask("Importing default data router")
if err := c.ImportInitialObjects(out); err != nil {
return taskPrinter.ToError(err)
}
taskPrinter.Success()

// Install logging
if c.ShouldInstallLogging {
taskPrinter.StartTask("Installing logging")
Expand Down Expand Up @@ -832,7 +849,7 @@ func (c *ClusterUpConfig) determineServerIP(out io.Writer) (string, []string, er

// updateNoProxy will add some default values to the NO_PROXY setting if they are not present
func (c *ClusterUpConfig) updateNoProxy() {
values := []string{"127.0.0.1", c.ServerIP, "localhost", openshift.ServiceCatalogServiceIP, openshift.RegistryServiceClusterIP}
values := []string{"127.0.0.1", c.ServerIP, "localhost", openshift.ServiceCatalogServiceIP, registry.RegistryServiceClusterIP}
ipFromServer, err := c.OpenShiftHelper().ServerIP()
if err == nil {
values = append(values, ipFromServer)
Expand Down Expand Up @@ -882,19 +899,6 @@ func (c *ClusterUpConfig) imageFormat() string {
return fmt.Sprintf("%s-${component}:%s", c.Image, c.ImageVersion)
}

// InstallRegistry installs the OpenShift registry on the server
func (c *ClusterUpConfig) InstallRegistry(out io.Writer) error {
_, kubeClient, err := c.Clients()
if err != nil {
return err
}
f, err := c.Factory()
if err != nil {
return err
}
return c.OpenShiftHelper().InstallRegistry(c.GetDockerClient(), c.openshiftImage(), kubeClient, f, c.LocalConfigDir, path.Join(c.BaseTempDir, "logs"), c.imageFormat(), c.HostPersistentVolumesDir, out, os.Stderr)
}

// InstallRouter installs a default router on the server
func (c *ClusterUpConfig) InstallRouter(out io.Writer) error {
_, kubeClient, err := c.Clients()
Expand Down Expand Up @@ -935,7 +939,8 @@ func (c *ClusterUpConfig) InstallWebConsole(out io.Writer) error {
return c.OpenShiftHelper().InstallWebConsole(f, c.imageFormat(), c.ServerLogLevel, publicURL, masterURL, loggingURL, metricsURL)
}

func (c *ClusterUpConfig) ImportInitialObjects(out io.Writer) error {
// TODO this should become a separate thing we can install, like registry
func (c *ClusterUpConfig) ImportInitialObjectsComponents(out io.Writer) []componentinstall.Component {
componentsToInstall := []componentinstall.Component{}
componentsToInstall = append(componentsToInstall,
c.makeObjectImportInstallationComponentsOrDie(out, openshift.Namespace, map[string]string{
Expand All @@ -950,7 +955,7 @@ func (c *ClusterUpConfig) ImportInitialObjects(out io.Writer) error {
componentsToInstall = append(componentsToInstall,
c.makeObjectImportInstallationComponentsOrDie(out, openshift.InfraNamespace, internalCurrentTemplateLocations)...)

return componentinstall.InstallComponents(componentsToInstall, c.GetDockerClient(), path.Join(c.BaseTempDir, "logs"))
return componentsToInstall
}

// InstallLogging will start the installation of logging components
Expand Down Expand Up @@ -1117,9 +1122,9 @@ func (c *ClusterUpConfig) checkProxySettings() string {
if len(dockerHTTPProxy) > 0 || len(dockerHTTPSProxy) > 0 {
dockerNoProxyList := strings.Split(dockerNoProxy, ",")
dockerNoProxySet := sets.NewString(dockerNoProxyList...)
if !dockerNoProxySet.Has(openshift.RegistryServiceClusterIP) {
if !dockerNoProxySet.Has(registry.RegistryServiceClusterIP) {
warnings = append(warnings, fmt.Sprintf("A proxy is configured for Docker, however %[1]s is not included in its NO_PROXY list.\n"+
" %[1]s needs to be included in the Docker daemon's NO_PROXY environment variable so pushes to the local OpenShift registry can succeed.", openshift.RegistryServiceClusterIP))
" %[1]s needs to be included in the Docker daemon's NO_PROXY environment variable so pushes to the local OpenShift registry can succeed.", registry.RegistryServiceClusterIP))
}
}

Expand Down Expand Up @@ -1358,7 +1363,7 @@ func (c *ClusterUpConfig) ShouldInitializeData() bool {
return true
}

if _, err = kclient.Core().Services(openshift.DefaultNamespace).Get(openshift.RegistryServiceName, metav1.GetOptions{}); err != nil {
if _, err = kclient.Core().Services(openshift.DefaultNamespace).Get(registry.SvcDockerRegistry, metav1.GetOptions{}); err != nil {
return true
}

Expand Down

0 comments on commit be6dc5e

Please sign in to comment.