Merge pull request #19891 from ramr/mtls

Router support for mutual tls authentication

images/router/haproxy/conf/haproxy-config.template

@@ -225,6 +225,9 @@ frontend fe_sni
     {{- if isTrue (env "ROUTER_STRICT_SNI") }} strict-sni {{ end }}
     {{- ""}} crt {{firstMatch ".+" .DefaultCertificate "/var/lib/haproxy/conf/default_pub_keys.pem"}}
     {{- ""}} crt-list /var/lib/haproxy/conf/cert_config.map accept-proxy
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CA") }} ca-file {{.}} {{ end }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CRL") }} crl-file {{.}} {{ end }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH") }} verify {{.}} {{ end }}
     {{- if isTrue (env "ROUTER_ENABLE_HTTP2") }} alpn h2,http/1.1{{ end }}
   mode http
 
@@ -235,6 +238,37 @@ frontend fe_sni
   # before matching, or any requests containing uppercase characters will never match.
   http-request set-header Host %[req.hdr(Host),lower]
 
+  {{ if ne (env "ROUTER_MUTUAL_TLS_AUTH" "none") "none" }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_FILTER") }}
+  # If a mutual TLS auth subject filter environment variable is set, we deny
+  # requests if the DN field in the client certificate doesn't match that value.
+  # Please note that this match is a regular expression match.
+  # Example: For DN set to: /CN=header.test/ST=CA/C=US/O=Security/OU=OpenShift3,
+  #          A. ROUTER_MUTUAL_TLS_AUTH_FILTER="header.test"   OR
+  #             ROUTER_MUTUAL_TLS_AUTH_FILTER="head"          OR
+  #             ROUTER_MUTUAL_TLS_AUTH_FILTER="^/CN=header.test/ST=CA/C=US/O=Security/OU=OpenShift3$" /* exact match example */
+  #             the filter would match the DN field (substring or exact match)
+  #             and the request will be passed on to the backend.
+  #          B. ROUTER_MUTUAL_TLS_AUTH_FILTER="legacy-web-client", the request
+  #             will be rejected.
+  acl cert_cn_matches ssl_c_s_dn -m reg {{.}}
+  http-request deny unless cert_cn_matches
+    {{- end }}
+
+  # Add X-SSL* headers to pass client certificate information to the backend.
+  http-request set-header X-SSL                  %[ssl_fc]
+  http-request set-header X-SSL-Client-Verify    %[ssl_c_verify]
+  http-request set-header X-SSL-Client-Serial    %{+Q}[ssl_c_serial,hex]
+  http-request set-header X-SSL-Client-Version   %{+Q}[ssl_c_version]
+  http-request set-header X-SSL-Client-SHA1      %{+Q}[ssl_c_sha1,hex]
+  http-request set-header X-SSL-Client-DN        %{+Q}[ssl_c_s_dn]
+  http-request set-header X-SSL-Client-CN        %{+Q}[ssl_c_s_dn(cn)]
+  http-request set-header X-SSL-Issuer           %{+Q}[ssl_c_i_dn]
+  http-request set-header X-SSL-Client-NotBefore %{+Q}[ssl_c_notbefore]
+  http-request set-header X-SSL-Client-NotAfter  %{+Q}[ssl_c_notafter]
+  http-request set-header X-SSL-Client-DER       %{+Q}[ssl_c_der,base64]
+  {{- end }}
+
   # map to backend
   # Search from most specific to general path (host case).
   # Note: If no match, haproxy uses the default_backend, no other
@@ -261,6 +295,9 @@ backend be_no_sni
 frontend fe_no_sni
   # terminate ssl on edge
   bind 127.0.0.1:{{env "ROUTER_SERVICE_NO_SNI_PORT" "10443"}} ssl no-sslv3 crt {{firstMatch ".+" .DefaultCertificate "/var/lib/haproxy/conf/default_pub_keys.pem"}} accept-proxy
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CA") }} ca-file {{.}} {{ end }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_CRL") }} crl-file {{.}} {{ end }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH") }} verify {{.}} {{ end }}
   mode http
 
   # Strip off Proxy headers to prevent HTTpoxy (https://httpoxy.org/)
@@ -270,6 +307,29 @@ frontend fe_no_sni
   # before matching, or any requests containing uppercase characters will never match.
   http-request set-header Host %[req.hdr(Host),lower]
 
+  {{ if ne (env "ROUTER_MUTUAL_TLS_AUTH" "none") "none" }}
+    {{- with (env "ROUTER_MUTUAL_TLS_AUTH_FILTER") }}
+  # If a mutual TLS auth subject filter environment variable is set, we deny
+  # requests if the DN field in the client certificate doesn't match that value.
+  # Please note that this match is a regular expression match.
+  # See the config section 'frontend fe_sni' for examples.
+  acl cert_cn_matches ssl_c_s_dn -m reg {{.}}
+  http-request deny unless cert_cn_matches
+    {{- end }}
+
+  # Add X-SSL* headers to pass client certificate information to the backend.
+  http-request set-header X-SSL                  %[ssl_fc]
+  http-request set-header X-SSL-Client-Verify    %[ssl_c_verify]
+  http-request set-header X-SSL-Client-Serial    %{+Q}[ssl_c_serial,hex]
+  http-request set-header X-SSL-Client-Version   %{+Q}[ssl_c_version]
+  http-request set-header X-SSL-Client-SHA1      %{+Q}[ssl_c_sha1,hex]
+  http-request set-header X-SSL-Client-DN        %{+Q}[ssl_c_s_dn]
+  http-request set-header X-SSL-Client-CN        %{+Q}[ssl_c_s_dn(cn)]
+  http-request set-header X-SSL-Issuer           %{+Q}[ssl_c_i_dn]
+  http-request set-header X-SSL-Client-NotBefore %{+Q}[ssl_c_notbefore]
+  http-request set-header X-SSL-Client-NotAfter  %{+Q}[ssl_c_notafter]
+  http-request set-header X-SSL-Client-DER       %{+Q}[ssl_c_der,base64]
+  {{- end }}
 
   # map to backend
   # Search from most specific to general path (host case).

pkg/oc/admin/router/router.go

@@ -18,6 +18,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/kubernetes/pkg/api/legacyscheme"
@@ -84,6 +85,11 @@ var (
 	privkeyName       = "router.pem"
 	privkeyPath       = secretsPath + "/" + privkeyName
 
+	defaultMutualTLSAuth = "none"
+	clientCertConfigDir  = "/etc/pki/tls/client-certs"
+	clientCertConfigCA   = "ca.pem"
+	clientCertConfigCRL  = "crl.pem"
+
 	defaultCertificatePath = path.Join(defaultCertificateDir, "tls.crt")
 )
 
@@ -237,6 +243,23 @@ type RouterConfig struct {
 	Threads int32
 
 	Local bool
+
+	// MutualTLSAuth controls access to the router using a mutually agreed
+	// upon TLS authentication mechanism (example client certificates).
+	// One of: required | optional | none  - the default is none.
+	MutualTLSAuth string
+
+	// MutualTLSAuthCA contains the CA certificates that will be used
+	// to verify a client's certificate.
+	MutualTLSAuthCA string
+
+	// MutualTLSAuthCRL contains the certificate revocation list used to
+	// verify a client's certificate.
+	MutualTLSAuthCRL string
+
+	// MutualTLSAuthFilter contains the value to filter requests based on
+	// a client certificate subject field substring match.
+	MutualTLSAuthFilter string
 }
 
 const (
@@ -271,6 +294,8 @@ func NewCmdRouter(f kcmdutil.Factory, parentName, name string, out, errout io.Wr
 		StatsPort:     defaultStatsPort,
 		HostNetwork:   true,
 		HostPorts:     true,
+
+		MutualTLSAuth: defaultMutualTLSAuth,
 	}
 
 	cmd := &cobra.Command{
@@ -325,15 +350,25 @@ func NewCmdRouter(f kcmdutil.Factory, parentName, name string, out, errout io.Wr
 	cmd.Flags().BoolVar(&cfg.Local, "local", cfg.Local, "If true, do not contact the apiserver")
 	cmd.Flags().Int32Var(&cfg.Threads, "threads", cfg.Threads, "Specifies the number of threads for the haproxy router.")
 
+	cmd.Flags().StringVar(&cfg.MutualTLSAuth, "mutual-tls-auth", cfg.MutualTLSAuth, "Controls access to the router using mutually agreed upon TLS configuration (example client certificates). You can choose one of 'required', 'optional', or 'none'. The default is none.")
+	cmd.Flags().StringVar(&cfg.MutualTLSAuthCA, "mutual-tls-auth-ca", cfg.MutualTLSAuthCA, "Optional path to a file containing one or more CA certificates used for mutual TLS authentication. The CA certificate[s] are used by the router to verify a client's certificate.")
+	cmd.Flags().StringVar(&cfg.MutualTLSAuthCRL, "mutual-tls-auth-crl", cfg.MutualTLSAuthCRL, "Optional path to a file containing the certificate revocation list used for mutual TLS authentication. The certificate revocation list is used by the router to verify a client's certificate.")
+	cmd.Flags().StringVar(&cfg.MutualTLSAuthFilter, "mutual-tls-auth-filter", cfg.MutualTLSAuthFilter, "Optional regular expression to filter the client certificates. If the client certificate subject field does _not_ match this regular expression, requests will be rejected by the router.")
+
 	cfg.Action.BindForOutput(cmd.Flags())
 	cmd.Flags().String("output-version", "", "The preferred API versions of the output objects")
 
 	return cmd
 }
 
+// generateMutualTLSSecretName generates a mutual TLS auth secret name.
+func generateMutualTLSSecretName(prefix string) string {
+	return fmt.Sprintf("%s-mutual-tls-auth", prefix)
+}
+
 // generateSecretsConfig generates any Secret and Volume objects, such
 // as SSH private keys, that are necessary for the router container.
-func generateSecretsConfig(cfg *RouterConfig, namespace string, defaultCert []byte, certName string) ([]*kapi.Secret, []kapi.Volume, []kapi.VolumeMount, error) {
+func generateSecretsConfig(cfg *RouterConfig, namespace, certName string, defaultCert, mtlsAuthCA, mtlsAuthCRL []byte) ([]*kapi.Secret, []kapi.Volume, []kapi.VolumeMount, error) {
 	var secrets []*kapi.Secret
 	var volumes []kapi.Volume
 	var mounts []kapi.VolumeMount
@@ -440,6 +475,42 @@ func generateSecretsConfig(cfg *RouterConfig, namespace string, defaultCert []by
 	}
 	mounts = append(mounts, mount)
 
+	mtlsSecretData := map[string][]byte{}
+	if len(mtlsAuthCA) > 0 {
+		mtlsSecretData[clientCertConfigCA] = mtlsAuthCA
+	}
+	if len(mtlsAuthCRL) > 0 {
+		mtlsSecretData[clientCertConfigCRL] = mtlsAuthCRL
+	}
+
+	if len(mtlsSecretData) > 0 {
+		secretName := generateMutualTLSSecretName(cfg.Name)
+		secret := &kapi.Secret{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: secretName,
+			},
+			Data: mtlsSecretData,
+		}
+		secrets = append(secrets, secret)
+
+		volume := kapi.Volume{
+			Name: "mutual-tls-config",
+			VolumeSource: kapi.VolumeSource{
+				Secret: &kapi.SecretVolumeSource{
+					SecretName: secretName,
+				},
+			},
+		}
+		volumes = append(volumes, volume)
+
+		mount := kapi.VolumeMount{
+			Name:      volume.Name,
+			ReadOnly:  true,
+			MountPath: clientCertConfigDir,
+		}
+		mounts = append(mounts, mount)
+	}
+
 	return secrets, volumes, mounts, nil
 }
 
@@ -608,6 +679,14 @@ func RunCmdRouter(f kcmdutil.Factory, cmd *cobra.Command, out, errout io.Writer,
 		if err != nil {
 			return fmt.Errorf("error getting client: %v", err)
 		}
+
+		if len(cfg.MutualTLSAuthCA) > 0 || len(cfg.MutualTLSAuthCRL) > 0 {
+			secretName := generateMutualTLSSecretName(cfg.Name)
+			if _, err := kClient.Core().Secrets(namespace).Get(secretName, metav1.GetOptions{}); err == nil {
+				return fmt.Errorf("router could not be created: mutual tls secret %q already exists", secretName)
+			}
+		}
+
 		service, err := kClient.Core().Services(namespace).Get(name, metav1.GetOptions{})
 		if err != nil {
 			if !generate {
@@ -660,6 +739,20 @@ func RunCmdRouter(f kcmdutil.Factory, cmd *cobra.Command, out, errout io.Writer,
 		return fmt.Errorf("router could not be created; error reading default certificate file: %v", err)
 	}
 
+	mtlsAuthOptions := []string{"required", "optional", "none"}
+	allowedMutualTLSAuthOptions := sets.NewString(mtlsAuthOptions...)
+	if !allowedMutualTLSAuthOptions.Has(cfg.MutualTLSAuth) {
+		return fmt.Errorf("invalid mutual tls auth option %v, expected one of %v", cfg.MutualTLSAuth, mtlsAuthOptions)
+	}
+	mtlsAuthCA, err := fileutil.LoadData(cfg.MutualTLSAuthCA)
+	if err != nil {
+		return fmt.Errorf("reading ca certificates for mutual tls auth: %v", err)
+	}
+	mtlsAuthCRL, err := fileutil.LoadData(cfg.MutualTLSAuthCRL)
+	if err != nil {
+		return fmt.Errorf("reading certificate revocation list for mutual tls auth: %v", err)
+	}
+
 	if len(cfg.StatsPassword) == 0 {
 		cfg.StatsPassword = generateStatsPassword()
 		if !cfg.Action.ShouldPrint() {
@@ -719,6 +812,20 @@ func RunCmdRouter(f kcmdutil.Factory, cmd *cobra.Command, out, errout io.Writer,
 		env["ROUTER_METRICS_TLS_CERT_FILE"] = "/etc/pki/tls/metrics/tls.crt"
 		env["ROUTER_METRICS_TLS_KEY_FILE"] = "/etc/pki/tls/metrics/tls.key"
 	}
+	mtlsAuth := strings.TrimSpace(cfg.MutualTLSAuth)
+	if len(mtlsAuth) > 0 && mtlsAuth != defaultMutualTLSAuth {
+		env["ROUTER_MUTUAL_TLS_AUTH"] = cfg.MutualTLSAuth
+		if len(mtlsAuthCA) > 0 {
+			env["ROUTER_MUTUAL_TLS_AUTH_CA"] = path.Join(clientCertConfigDir, clientCertConfigCA)
+		}
+		if len(mtlsAuthCRL) > 0 {
+			env["ROUTER_MUTUAL_TLS_AUTH_CRL"] = path.Join(clientCertConfigDir, clientCertConfigCRL)
+		}
+		if len(cfg.MutualTLSAuthFilter) > 0 {
+			env["ROUTER_MUTUAL_TLS_AUTH_FILTER"] = strings.Replace(cfg.MutualTLSAuthFilter, " ", "\\ ", -1)
+		}
+	}
+
 	env.Add(secretEnv)
 	if len(defaultCert) > 0 {
 		if cfg.SecretsAsEnv {
@@ -729,7 +836,7 @@ func RunCmdRouter(f kcmdutil.Factory, cmd *cobra.Command, out, errout io.Writer,
 	}
 	env.Add(app.Environment{"DEFAULT_CERTIFICATE_DIR": defaultCertificateDir})
 	var certName = fmt.Sprintf("%s-certs", cfg.Name)
-	secrets, volumes, routerMounts, err := generateSecretsConfig(cfg, namespace, defaultCert, certName)
+	secrets, volumes, routerMounts, err := generateSecretsConfig(cfg, namespace, certName, defaultCert, mtlsAuthCA, mtlsAuthCRL)
 	if err != nil {
 		return fmt.Errorf("router could not be created: %v", err)
 	}