diff --git a/test/extended/router/router.go b/test/extended/router/router.go new file mode 100644 index 000000000000..2bfa5d1ab694 --- /dev/null +++ b/test/extended/router/router.go @@ -0,0 +1,101 @@ +package images + +import ( + "net/http" + "time" + + g "github.com/onsi/ginkgo" + o "github.com/onsi/gomega" + + kapierrs "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + + routev1 "github.com/openshift/api/route/v1" + routeclientset "github.com/openshift/client-go/route/clientset/versioned" + exutil "github.com/openshift/origin/test/extended/util" + "github.com/openshift/origin/test/extended/util/url" +) + +var _ = g.Describe("[Conformance][Area:Networking][Feature:Router]", func() { + defer g.GinkgoRecover() + var ( + host, ns string + + configPath = exutil.FixturePath("testdata", "ingress.yaml") + oc = exutil.NewCLI("router", exutil.KubeConfigPath()) + ) + + g.BeforeEach(func() { + _, err := oc.AdminAppsClient().Apps().DeploymentConfigs("default").Get("router", metav1.GetOptions{}) + if kapierrs.IsNotFound(err) { + g.Skip("no router installed on the cluster") + return + } + o.Expect(err).NotTo(o.HaveOccurred()) + + // wait for the router endpoints to show up + err = wait.PollImmediate(2*time.Second, 120*time.Second, func() (bool, error) { + epts, err := oc.AdminKubeClient().CoreV1().Endpoints("default").Get("router", metav1.GetOptions{}) + o.Expect(err).NotTo(o.HaveOccurred()) + if len(epts.Subsets) == 0 || len(epts.Subsets[0].Addresses) == 0 { + return false, nil + } + host = epts.Subsets[0].Addresses[0].IP + return true, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + ns = oc.KubeFramework().Namespace.Name + }) + + g.AfterEach(func() { + if g.CurrentGinkgoTestDescription().Failed { + exutil.DumpPodLogsStartingWithInNamespace("router", "default", oc.AsAdmin()) + } + }) + + g.Describe("The HAProxy router", func() { + g.It("should respond with 503 to unrecognized hosts", func() { + t := url.NewTester(oc.AdminKubeClient(), ns) + defer t.Close() + t.Within( + time.Minute, + url.Expect("GET", "https://www.google.com").Through(host).SkipTLSVerification().HasStatusCode(503), + url.Expect("GET", "http://www.google.com").Through(host).HasStatusCode(503), + ) + }) + + g.It("should serve routes that were created from an ingress", func() { + g.By("deploying an ingress rule") + err := oc.Run("create").Args("-f", configPath).Execute() + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("waiting for the ingress rule to be converted to routes") + client := routeclientset.NewForConfigOrDie(oc.AdminConfig()) + var r []routev1.Route + err = wait.Poll(time.Second, time.Minute, func() (bool, error) { + routes, err := client.Route().Routes(ns).List(metav1.ListOptions{}) + if err != nil { + return false, err + } + r = routes.Items + return len(routes.Items) == 4, nil + }) + o.Expect(err).NotTo(o.HaveOccurred()) + + g.By("verifying the router reports the correct behavior") + t := url.NewTester(oc.AdminKubeClient(), ns) + defer t.Close() + t.Within( + 3*time.Minute, + url.Expect("GET", "http://1.ingress-test.com/test").Through(host).HasStatusCode(200), + url.Expect("GET", "http://1.ingress-test.com/other/deep").Through(host).HasStatusCode(200), + url.Expect("GET", "http://1.ingress-test.com/").Through(host).HasStatusCode(503), + url.Expect("GET", "http://2.ingress-test.com/").Through(host).HasStatusCode(200), + url.Expect("GET", "https://3.ingress-test.com/").Through(host).SkipTLSVerification().HasStatusCode(200), + url.Expect("GET", "http://3.ingress-test.com/").Through(host).RedirectsTo("https://3.ingress-test.com/", http.StatusFound), + ) + }) + }) +}) diff --git a/test/extended/testdata/bindata.go b/test/extended/testdata/bindata.go index c09624f8f129..6eae1284aff6 100644 --- a/test/extended/testdata/bindata.go +++ b/test/extended/testdata/bindata.go @@ -141,6 +141,7 @@ // test/extended/testdata/image_ecosystem/perl-hotdeploy/perl.json // test/extended/testdata/imagestream-jenkins-slave-pods.yaml // test/extended/testdata/imagestreamtag-jenkins-slave-pods.yaml +// test/extended/testdata/ingress.yaml // test/extended/testdata/jenkins-plugin/build-job-clone.xml // test/extended/testdata/jenkins-plugin/build-job-slave.xml // test/extended/testdata/jenkins-plugin/build-job.xml @@ -7364,6 +7365,99 @@ func testExtendedTestdataImagestreamtagJenkinsSlavePodsYaml() (*asset, error) { return a, nil } +var _testExtendedTestdataIngressYaml = []byte(`kind: List +apiVersion: v1 +items: +# an ingress that should be captured as three routes +- apiVersion: extensions/v1beta1 + kind: Ingress + metadata: + name: test + spec: + tls: + - hosts: + - 3.ingress-test.com + secretName: ingress-endpoint-secret + rules: + - host: 1.ingress-test.com + http: + paths: + - path: /test + backend: + serviceName: ingress-endpoint + servicePort: 80 + - path: /other + backend: + serviceName: ingress-endpoint + servicePort: 80 + - host: 2.ingress-test.com + http: + paths: + - path: / + backend: + serviceName: ingress-endpoint + servicePort: 80 + - host: 3.ingress-test.com + http: + paths: + - path: / + backend: + serviceName: ingress-endpoint + servicePort: 80 +# an empty secret +- apiVersion: v1 + kind: Secret + metadata: + name: ingress-endpoint-secret + type: kubernetes.io/tls + stringData: + tls.key: "" + tls.crt: "" +# a service to be routed to +- apiVersion: v1 + kind: Service + metadata: + name: ingress-endpoint + spec: + selector: + app: ingress-endpoint + ports: + - port: 80 + targetPort: 8080 +# a pod that serves a response +- apiVersion: v1 + kind: Pod + metadata: + name: ingress-endpoint-1 + labels: + app: ingress-endpoint + spec: + terminationGracePeriodSeconds: 1 + containers: + - name: test + image: openshift/hello-openshift + ports: + - containerPort: 8080 + name: http + - containerPort: 100 + protocol: UDP +`) + +func testExtendedTestdataIngressYamlBytes() ([]byte, error) { + return _testExtendedTestdataIngressYaml, nil +} + +func testExtendedTestdataIngressYaml() (*asset, error) { + bytes, err := testExtendedTestdataIngressYamlBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "test/extended/testdata/ingress.yaml", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info} + return a, nil +} + var _testExtendedTestdataJenkinsPluginBuildJobCloneXml = []byte(` @@ -30170,6 +30264,7 @@ var _bindata = map[string]func() (*asset, error){ "test/extended/testdata/image_ecosystem/perl-hotdeploy/perl.json": testExtendedTestdataImage_ecosystemPerlHotdeployPerlJson, "test/extended/testdata/imagestream-jenkins-slave-pods.yaml": testExtendedTestdataImagestreamJenkinsSlavePodsYaml, "test/extended/testdata/imagestreamtag-jenkins-slave-pods.yaml": testExtendedTestdataImagestreamtagJenkinsSlavePodsYaml, + "test/extended/testdata/ingress.yaml": testExtendedTestdataIngressYaml, "test/extended/testdata/jenkins-plugin/build-job-clone.xml": testExtendedTestdataJenkinsPluginBuildJobCloneXml, "test/extended/testdata/jenkins-plugin/build-job-slave.xml": testExtendedTestdataJenkinsPluginBuildJobSlaveXml, "test/extended/testdata/jenkins-plugin/build-job.xml": testExtendedTestdataJenkinsPluginBuildJobXml, @@ -30641,6 +30736,7 @@ var _bintree = &bintree{nil, map[string]*bintree{ }}, "imagestream-jenkins-slave-pods.yaml": &bintree{testExtendedTestdataImagestreamJenkinsSlavePodsYaml, map[string]*bintree{}}, "imagestreamtag-jenkins-slave-pods.yaml": &bintree{testExtendedTestdataImagestreamtagJenkinsSlavePodsYaml, map[string]*bintree{}}, + "ingress.yaml": &bintree{testExtendedTestdataIngressYaml, map[string]*bintree{}}, "jenkins-plugin": &bintree{nil, map[string]*bintree{ "build-job-clone.xml": &bintree{testExtendedTestdataJenkinsPluginBuildJobCloneXml, map[string]*bintree{}}, "build-job-slave.xml": &bintree{testExtendedTestdataJenkinsPluginBuildJobSlaveXml, map[string]*bintree{}}, diff --git a/test/extended/testdata/ingress.yaml b/test/extended/testdata/ingress.yaml new file mode 100644 index 000000000000..26e5baf59133 --- /dev/null +++ b/test/extended/testdata/ingress.yaml @@ -0,0 +1,76 @@ +kind: List +apiVersion: v1 +items: +# an ingress that should be captured as three routes +- apiVersion: extensions/v1beta1 + kind: Ingress + metadata: + name: test + spec: + tls: + - hosts: + - 3.ingress-test.com + secretName: ingress-endpoint-secret + rules: + - host: 1.ingress-test.com + http: + paths: + - path: /test + backend: + serviceName: ingress-endpoint + servicePort: 80 + - path: /other + backend: + serviceName: ingress-endpoint + servicePort: 80 + - host: 2.ingress-test.com + http: + paths: + - path: / + backend: + serviceName: ingress-endpoint + servicePort: 80 + - host: 3.ingress-test.com + http: + paths: + - path: / + backend: + serviceName: ingress-endpoint + servicePort: 80 +# an empty secret +- apiVersion: v1 + kind: Secret + metadata: + name: ingress-endpoint-secret + type: kubernetes.io/tls + stringData: + tls.key: "" + tls.crt: "" +# a service to be routed to +- apiVersion: v1 + kind: Service + metadata: + name: ingress-endpoint + spec: + selector: + app: ingress-endpoint + ports: + - port: 80 + targetPort: 8080 +# a pod that serves a response +- apiVersion: v1 + kind: Pod + metadata: + name: ingress-endpoint-1 + labels: + app: ingress-endpoint + spec: + terminationGracePeriodSeconds: 1 + containers: + - name: test + image: openshift/hello-openshift + ports: + - containerPort: 8080 + name: http + - containerPort: 100 + protocol: UDP diff --git a/test/extended/util/cli.go b/test/extended/util/cli.go index 0cc482d84aa3..50b8d36378ac 100644 --- a/test/extended/util/cli.go +++ b/test/extended/util/cli.go @@ -20,6 +20,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apiserver/pkg/storage/names" kclientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" clientcmd "k8s.io/client-go/tools/clientcmd" kinternalclientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset" e2e "k8s.io/kubernetes/test/e2e/framework" @@ -386,6 +387,14 @@ func (c *CLI) InternalAdminKubeClient() kinternalclientset.Interface { return kubeClient } +func (c *CLI) AdminConfig() *restclient.Config { + _, clientConfig, err := configapi.GetExternalKubeClient(c.adminConfigPath, nil) + if err != nil { + FatalErr(err) + } + return clientConfig +} + // Namespace returns the name of the namespace used in the current test case. // If the namespace is not set, an empty string is returned. func (c *CLI) Namespace() string { diff --git a/test/extended/util/framework.go b/test/extended/util/framework.go index 267ea35f5a32..61c779c80cf5 100644 --- a/test/extended/util/framework.go +++ b/test/extended/util/framework.go @@ -256,7 +256,7 @@ func DumpPodLogs(pods []kapiv1.Pod, oc *CLI) { } dumpContainer := func(container *kapiv1.Container) { - depOutput, err := oc.AsAdmin().Run("logs").Args("pod/"+pod.Name, "-c", container.Name).Output() + depOutput, err := oc.AsAdmin().Run("logs").WithoutNamespace().Args("pod/"+pod.Name, "-c", container.Name, "-n", pod.Namespace).Output() if err == nil { e2e.Logf("Log for pod %q/%q\n---->\n%s\n<----end of log for %[1]q/%[2]q\n", pod.Name, container.Name, depOutput) } else { diff --git a/test/extended/util/url/url.go b/test/extended/util/url/url.go new file mode 100644 index 000000000000..1aedee6b9a06 --- /dev/null +++ b/test/extended/util/url/url.go @@ -0,0 +1,284 @@ +package url + +import ( + "bufio" + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "strings" + "time" + + o "github.com/onsi/gomega" + + "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + kclientset "k8s.io/client-go/kubernetes" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +type Tester struct { + client kclientset.Interface + namespace string + podName string +} + +func NewTester(client kclientset.Interface, ns string) *Tester { + return &Tester{client: client, namespace: ns} +} + +func (ut *Tester) Close() { + if err := ut.client.Core().Pods(ut.namespace).Delete(ut.podName, metav1.NewDeleteOptions(1)); err != nil { + e2e.Logf("Failed to delete exec pod %s: %v", ut.podName, err) + } + ut.podName = "" +} + +func (ut *Tester) Responses(tests ...*Test) []*Response { + script := testsToScript(tests) + if len(ut.podName) == 0 { + name, err := createExecPod(ut.client, ut.namespace, "execpod") + o.Expect(err).NotTo(o.HaveOccurred()) + ut.podName = name + } + output, err := e2e.RunHostCmd(ut.namespace, ut.podName, script) + o.Expect(err).NotTo(o.HaveOccurred()) + responses, err := parseResponses(output) + o.Expect(err).NotTo(o.HaveOccurred()) + if len(responses) != len(tests) { + o.Expect(fmt.Errorf("number of tests did not match number of responses: %d and %d", len(responses), len(tests))).NotTo(o.HaveOccurred()) + } + return responses +} + +func (ut *Tester) Within(t time.Duration, tests ...*Test) { + var errs []error + failing := tests + err := wait.PollImmediate(time.Second, t, func() (bool, error) { + errs = errs[:0] + responses := ut.Responses(failing...) + var next []*Test + for i, res := range responses { + if err := failing[i].Test(i, res); err != nil { + next = append(next, failing[i]) + errs = append(errs, err) + } + } + e2e.Logf("%d/%d failed out of %d", len(errs), len(failing), len(tests)) + // perform one more loop if we haven't seen all tests pass at the same time + if len(next) == 0 && len(failing) != len(tests) { + failing = tests + return false, nil + } + failing = next + return len(errs) == 0, nil + }) + if len(errs) > 0 { + o.Expect(fmt.Errorf("%d/%d tests failed after %s: %v", len(errs), len(tests), t, errs)) + } + o.Expect(err).ToNot(o.HaveOccurred()) +} + +// createExecPod creates a simple centos:7 pod in a sleep loop used as a +// vessel for kubectl exec commands. +// Returns the name of the created pod. +func createExecPod(clientset kclientset.Interface, ns, name string) (string, error) { + e2e.Logf("Creating new exec pod") + immediate := int64(0) + execPod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Command: []string{"/bin/bash", "-c", "exec sleep 10000"}, + Name: "hostexec", + Image: "centos:7", + ImagePullPolicy: v1.PullIfNotPresent, + }, + }, + HostNetwork: true, + TerminationGracePeriodSeconds: &immediate, + }, + } + client := clientset.CoreV1() + created, err := client.Pods(ns).Create(execPod) + if err != nil { + return "", err + } + err = wait.PollImmediate(e2e.Poll, 5*time.Minute, func() (bool, error) { + retrievedPod, err := client.Pods(execPod.Namespace).Get(created.Name, metav1.GetOptions{}) + if err != nil { + return false, nil + } + return retrievedPod.Status.Phase == v1.PodRunning, nil + }) + if err != nil { + return "", err + } + return created.Name, nil +} + +func testsToScript(tests []*Test) string { + testScripts := []string{ + "set -euo pipefail", + `function json_escape() {`, + ` python -c 'import json,sys; print json.dumps(sys.stdin.read())'`, + `}`, + } + for i, test := range tests { + testScripts = append(testScripts, test.ToShell(i)) + } + script := strings.Join(testScripts, "\n") + return script +} + +func parseResponses(out string) ([]*Response, error) { + var responses []*Response + d := json.NewDecoder(bytes.NewReader([]byte(out))) + for i := 0; ; i++ { + r := &Response{} + if err := d.Decode(r); err != nil { + if err == io.EOF { + return responses, nil + } + return nil, fmt.Errorf("response %d could not be decoded: %v", i, err) + } + + if i != r.Test { + return nil, fmt.Errorf("response %d does not match test body %d", i, r.Test) + } + + // parse the HTTP response + res, err := http.ReadResponse(bufio.NewReader(bytes.NewBufferString(r.Headers)), nil) + if err != nil { + return nil, fmt.Errorf("response %d was unparseable: %v\n%s", i, err, r.Headers) + } + if res.StatusCode != r.CURL.Code { + return nil, fmt.Errorf("response %d returned a different status code than was encoded in the headers:\n%s", i, r.Headers) + } + res.Body = ioutil.NopCloser(bytes.NewBuffer(r.Body)) + r.Response = res + + responses = append(responses, r) + } +} + +type Response struct { + Test int `json:"test"` + ReturnCode int `json:"rc"` + Error string `json:"error"` + + CURL CURL `json:"curl"` + Body []byte `json:"body"` + Headers string `json:"headers"` + + Response *http.Response +} + +type CURL struct { + Code int `json:"code"` +} + +type Test struct { + Name string + Req *http.Request + SkipVerify bool + + Wants []func(*http.Response) error +} + +func Expect(method, url string) *Test { + req, err := http.NewRequest(method, url, nil) + if err != nil { + panic(err) + } + return &Test{ + Req: req, + } +} + +func (ut *Test) Through(addr string) *Test { + ut.Req.Header.Set("Host", ut.Req.URL.Host) + ut.Req.URL.Host = addr + return ut +} + +func (ut *Test) HasStatusCode(codes ...int) *Test { + ut.Wants = append(ut.Wants, func(res *http.Response) error { + for _, code := range codes { + if res.StatusCode == code { + return nil + } + } + return fmt.Errorf("status code %d not in %v", res.StatusCode, codes) + }) + return ut +} + +func (ut *Test) RedirectsTo(url string, codes ...int) *Test { + if len(codes) == 0 { + codes = []int{http.StatusFound, http.StatusPermanentRedirect, http.StatusTemporaryRedirect} + } + ut.HasStatusCode(codes...) + ut.Wants = append(ut.Wants, func(res *http.Response) error { + location := res.Header.Get("Location") + if location != url { + return fmt.Errorf("Location header was %q, not %q", location, url) + } + return nil + }) + return ut +} + +func (ut *Test) SkipTLSVerification() *Test { + ut.SkipVerify = true + return ut +} + +func (ut *Test) Test(i int, res *Response) error { + if len(res.Error) > 0 || res.ReturnCode != 0 { + return fmt.Errorf("test %d was not successful: %d %s", i, res.ReturnCode, res.Error) + } + for _, fn := range ut.Wants { + if err := fn(res.Response); err != nil { + return fmt.Errorf("test %d was not successful: %v", i, err) + } + } + if len(ut.Wants) == 0 { + if res.Response.StatusCode < 200 || res.Response.StatusCode >= 300 { + return fmt.Errorf("test %d did not return a 2xx status code: %d", i, res.Response.StatusCode) + } + } + return nil +} + +func (ut *Test) ToShell(i int) string { + var lines []string + if len(ut.Name) > 0 { + lines = append(lines, fmt.Sprintf("# Test: %s (%d)", ut.Name, i)) + } else { + lines = append(lines, fmt.Sprintf("# Test: %d", i)) + } + var headers []string + for k, values := range ut.Req.Header { + for _, v := range values { + headers = append(headers, fmt.Sprintf("-H %q", k+":"+v)) + } + } + lines = append(lines, `rc=0`) + cmd := fmt.Sprintf(`curl -X %s %s -s -S -o /tmp/body -D /tmp/headers %q`, ut.Req.Method, strings.Join(headers, " "), ut.Req.URL) + cmd += ` -w '{"code":%{http_code}}'` + if ut.SkipVerify { + cmd += ` -k` + } + cmd += " 2>/tmp/error 1>/tmp/output || rc=$?" + lines = append(lines, cmd) + lines = append(lines, fmt.Sprintf(`echo "{\"test\":%d,\"rc\":$(echo $rc),\"curl\":$(cat /tmp/output),\"error\":$(cat /tmp/error | json_escape),\"body\":\"$(cat /tmp/body | base64 -w 0 -)\",\"headers\":$(cat /tmp/headers | json_escape)}"`, i)) + return strings.Join(lines, "\n") +} diff --git a/test/extended/util/url/url_test.go b/test/extended/util/url/url_test.go new file mode 100644 index 000000000000..9f0681438184 --- /dev/null +++ b/test/extended/util/url/url_test.go @@ -0,0 +1,13 @@ +package url + +import ( + "fmt" + "testing" +) + +func TestTestsToScript(t *testing.T) { + tests := []*URLTest{ + MustURLTest("GET", "https://www.google.com"), + } + fmt.Println(testsToScript(tests)) +}