make HTTP query to app

pkg/diagnostics/cluster/app_create.go

@@ -2,6 +2,7 @@ package cluster
 
 import (
 	"fmt"
+	"net/http"
 	"os"
 	"os/signal"
 	"strconv"
@@ -10,9 +11,12 @@ import (
 	"time"
 
 	appstypedclient "github.com/openshift/origin/pkg/apps/generated/internalclientset/typed/apps/internalversion"
+	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	kvalidation "k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/storage/names"
 	kapi "k8s.io/kubernetes/pkg/api"
 	"k8s.io/kubernetes/pkg/apis/authorization"
@@ -39,17 +43,18 @@ type AppCreate struct {
 	SARClient           authorizationtypedclient.SelfSubjectAccessReviewsGetter
 
 	// from parameters specific to this diagnostic
-	nodeSelector  string
-	project       string
-	projectBase   string
-	deleteProject bool
-	appName       string
-	appImage      string
-	appPort       int
-	timeout       int64
-	deleteApp     bool
-	label         map[string]string // for selecting components later
-	labelSelector string            // for selecting components later
+	nodeSelector   string
+	project        string
+	projectBase    string
+	deleteProject  bool
+	appName        string
+	appImage       string
+	appPort        int
+	timeout        int64
+	requestTimeout int64
+	deleteApp      bool
+	label          map[string]string // for selecting components later
+	labelSelector  string            // for selecting components later
 
 	out types.DiagnosticResult
 }
@@ -75,6 +80,9 @@ const (
 	AppCreateTimeoutParam   = "timeout"
 	AppCreateTimeoutDefault = "120"
 
+	AppCreateRequestTimeoutParam   = "requestTimeout"
+	AppCreateRequestTimeoutDefault = "5"
+
 	AppCreateDeleteAppParam = "deleteApp"
 )
 
@@ -88,15 +96,16 @@ func (d *AppCreate) Description() string {
 
 func (d *AppCreate) AvailableParameters() map[string]string {
 	return map[string]string{
-		AppCreateProjectParam:       "Project name to use for creating the app (default: generate from projectBase)",
-		AppCreateProjectBaseParam:   fmt.Sprintf("Base name to create project with random name (default: %s)", AppCreateProjectBaseDefault),
-		AppCreateDeleteProjectParam: "Whether to delete the test project when complete (default: true for random projects, false for named ones)",
-		AppCreateAppNameParam:       fmt.Sprintf("Name for the test application to be created (default: %s)", AppCreateAppNameDefault),
-		AppCreateAppImageParam:      fmt.Sprintf("Image for the test application to be created (default: %s)", getDefaultAppImage()),
-		AppCreateAppPortParam:       fmt.Sprintf("Port at which the test application listens (default: %s)", AppCreateAppPortDefault),
-		AppCreateTimeoutParam:       fmt.Sprintf("Seconds to wait for the app to be running (default: %s)", AppCreateTimeoutDefault),
-		AppCreateNodeSelectorParam:  "Node selector for where the test app should land (default: any node)",
-		AppCreateDeleteAppParam:     "Whether to delete the test app when complete (default: true)",
+		AppCreateProjectParam:        "Project name to use for creating the app (default: generate from projectBase)",
+		AppCreateProjectBaseParam:    fmt.Sprintf("Base name to create project with random name (default: %s)", AppCreateProjectBaseDefault),
+		AppCreateDeleteProjectParam:  "Whether to delete the test project when complete (default: true for random projects, false for named ones)",
+		AppCreateAppNameParam:        fmt.Sprintf("Name for the test application to be created (default: %s)", AppCreateAppNameDefault),
+		AppCreateAppImageParam:       fmt.Sprintf("Image for the test application to be created (default: %s)", getDefaultAppImage()),
+		AppCreateAppPortParam:        fmt.Sprintf("Port at which the test application listens (default: %s)", AppCreateAppPortDefault),
+		AppCreateTimeoutParam:        fmt.Sprintf("Seconds to wait for the app to be running (default: %s)", AppCreateTimeoutDefault),
+		AppCreateRequestTimeoutParam: fmt.Sprintf("Time to wait for an HTTP request to the app (default: %s)", AppCreateRequestTimeoutDefault),
+		AppCreateNodeSelectorParam:   "Node selector for where the test app should land (default: any node)",
+		AppCreateDeleteAppParam:      "Whether to delete the test app when complete (default: true)",
 	}
 }
 
@@ -151,10 +160,19 @@ func (d *AppCreate) SetParameters(params map[string]string) error {
 	}
 	timeoutInt, err := strconv.ParseInt(timeout, 0, 0)
 	if err != nil {
-		return fmt.Errorf("'%s' is not a valid timeout: %v", timeout, err)
+		return fmt.Errorf("'%s' is not a valid %s: %v", timeout, AppCreateTimeoutParam, err)
 	}
 	d.timeout = timeoutInt
 
+	requestTimeout := AppCreateRequestTimeoutDefault
+	if params[AppCreateRequestTimeoutParam] != "" {
+		requestTimeout = params[AppCreateRequestTimeoutParam]
+	}
+	d.requestTimeout, err = strconv.ParseInt(requestTimeout, 0, 0)
+	if err != nil {
+		return fmt.Errorf("'%s' is not a valid %s: %v", requestTimeout, AppCreateRequestTimeoutParam, err)
+	}
+
 	if image := params[AppCreateAppImageParam]; image != "" {
 		d.appImage = image
 	}
@@ -210,7 +228,7 @@ func (d *AppCreate) setupProject() bool {
 		d.project = names.SimpleNameGenerator.GenerateName(d.projectBase)
 	}
 
-	d.out.Debug("DCluAC001", fmt.Sprintf("%s: Using project '%s' for diagnostic.", now(), d.project))
+	d.out.Info("DCluAC001", fmt.Sprintf("%s: Using project '%s' for diagnostic.", now(), d.project))
 	if existing, err := d.KubeClient.Core().Namespaces().Get(d.project, metav1.GetOptions{}); existing != nil && err == nil {
 		d.out.Debug("DCluAC002", fmt.Sprintf("%s: Project '%s' already exists.", now(), d.project))
 		return true
@@ -236,7 +254,7 @@ func (d *AppCreate) setupProject() bool {
 
 func getAppCreateService(appName string, appPort int, label map[string]string) *kapi.Service {
 	return &kapi.Service{
-		ObjectMeta: metav1.ObjectMeta{Name: appName},
+		ObjectMeta: metav1.ObjectMeta{Name: appName, Labels: label},
 		Spec: kapi.ServiceSpec{
 			Type:     kapi.ServiceTypeClusterIP,
 			Selector: label,
@@ -320,50 +338,123 @@ func (d *AppCreate) createApp() bool {
 }
 
 type AppCreateResult struct {
-	beginTimestamp time.Time
-	failureTime    time.Duration
-	runningTime    time.Duration
-	endTime        time.Duration
+	beginTimestamp   time.Time
+	failureTime      time.Duration
+	runningTime      time.Duration
+	serviceReadyTime time.Duration
+	respondedTime    time.Duration
+	endTime          time.Duration
 }
 
 func (d *AppCreate) testApp() AppCreateResult {
 	result := AppCreateResult{beginTimestamp: time.Now()}
-	d.out.Debug("DCluAC017", fmt.Sprintf("%s: Waiting for pod to reach running state.", now()))
+	defer func() { // always record end time
+		result.endTime = time.Since(result.beginTimestamp)
+	}()
+	d.out.Debug("DCluAC017", fmt.Sprintf("%s: Waiting %ds for pod to reach running state.", now(), d.timeout))
 
 	// wait for a pod to become active
 	watcher, err := d.KubeClient.Core().Pods(d.project).Watch(metav1.ListOptions{LabelSelector: d.labelSelector, TimeoutSeconds: &d.timeout})
 	if err != nil {
-		result.failureTime = time.Now().Sub(result.beginTimestamp)
-		d.out.Error("DCluAC011", err, fmt.Sprintf("%s: Waiting for '%s' to deploy a pod failed:\n%v", now(), d.appName, err))
+		result.failureTime = time.Since(result.beginTimestamp)
+		d.out.Error("DCluAC011", err, fmt.Sprintf("%s: Waiting %ds for '%s' to deploy a pod failed:\n%v", now(), d.timeout, d.appName, err))
 		return result
 	}
 	for event := range watcher.ResultChan() {
 		running, err := conditions.PodContainerRunning(d.appName)(event)
 		if err != nil {
-			result.failureTime = time.Now().Sub(result.beginTimestamp)
-			d.out.Error("DCluAC012", err, fmt.Sprintf("%s: Error watching for app pod:\n%v", now(), event))
+			result.failureTime = time.Since(result.beginTimestamp)
+			d.out.Error("DCluAC012", err, fmt.Sprintf("%s: Error watching for app pod:\n%v", now(), err))
 			watcher.Stop()
 			return result
 		}
 		if running {
-			result.runningTime = time.Now().Sub(result.beginTimestamp)
-			d.out.Debug("DCluAC018", fmt.Sprintf("%s: Pod is running after %v", now(), result.runningTime))
+			result.runningTime = time.Since(result.beginTimestamp)
+			d.out.Info("DCluAC018", fmt.Sprintf("%s: App is running after %v", now(), result.runningTime))
 			break
 		}
 	}
 	watcher.Stop()
 	if result.runningTime == 0 {
-		result.failureTime = time.Now().Sub(result.beginTimestamp)
+		result.failureTime = time.Since(result.beginTimestamp)
 		d.out.Error("DCluAC019", nil, fmt.Sprintf("%s: App pod was not in running state before timeout (%d sec)", now(), d.timeout))
+		return result
+	}
+
+	// wait for the service to establish endpoints
+	start := time.Now()
+	timeout := int64(10) // normally instantaneous, timeout is mainly a failsafe
+	d.out.Debug("DCluAC027", fmt.Sprintf("%s: Waiting for service to establish endpoints", now()))
+	watcher, err = d.KubeClient.Core().Endpoints(d.project).Watch(metav1.ListOptions{FieldSelector: "metadata.name=" + d.appName, TimeoutSeconds: &timeout})
+	if err != nil {
+		result.failureTime = time.Since(result.beginTimestamp)
+		d.out.Error("DCluAC020", err, fmt.Sprintf("%s: Failed to establish watch for '%s' service to be ready:\n%v", now(), d.appName, err))
+		return result
+	}
+	for event := range watcher.ResultChan() {
+		ready, err := d.testServiceHasEndpoint(event)
+		if err != nil {
+			result.failureTime = time.Since(result.beginTimestamp)
+			d.out.Error("DCluAC021", err, fmt.Sprintf("%s: Error while watching for service endpoint:\n%v", now(), err))
+			watcher.Stop()
+			return result
+		}
+		if ready {
+			result.serviceReadyTime = time.Since(start)
+			d.out.Debug("DCluAC022", fmt.Sprintf("%s: Service has endpoint after %v", now(), result.serviceReadyTime))
+			break
+		}
+	}
+	watcher.Stop()
+	if result.serviceReadyTime == 0 {
+		result.failureTime = time.Since(result.beginTimestamp)
+		d.out.Error("DCluAC023", nil, fmt.Sprintf("%s: Service did not find endpoint before timeout (%d sec)", now(), timeout))
+		return result
 	}
 
 	// check we can actually get a response from the app
+	service, err := d.KubeClient.Core().Services(d.project).Get(d.appName, metav1.GetOptions{})
+	if err != nil {
+		result.failureTime = time.Since(result.beginTimestamp)
+		d.out.Error("DCluAC025", err, fmt.Sprintf("%s: Error retrieving %s service: %v", now(), d.appName, err))
+		return result
+	}
+	ip := service.Spec.ClusterIP
+
+	start = time.Now()
+	d.out.Debug("DCluAC028", fmt.Sprintf("%s: waiting %ds for an HTTP response from the app at %s:8080", now(), d.requestTimeout, ip))
+	client := &http.Client{Timeout: time.Second * time.Duration(d.requestTimeout)}
+	response, err := client.Get(fmt.Sprintf("http://%s:8080/", ip))
+	if err != nil {
+		result.failureTime = time.Since(result.beginTimestamp)
+		d.out.Error("DCluAC029", err, fmt.Sprintf("%s: Request to app returned an error or timed out in %d sec: %v", now(), d.requestTimeout, err))
+		return result
+	}
+	result.respondedTime = time.Since(start)
+	d.out.Info("DCluAC030", fmt.Sprintf("%s: Completed HTTP request successfully in %v", now(), result.respondedTime))
+	response.Body.Close()
 
-	// record end time
-	result.endTime = time.Now().Sub(result.beginTimestamp)
 	return result
 }
 
+// Returns false until the service has at least one endpoint.
+// Will return an error if the service is deleted or any other error occurs.
+func (d *AppCreate) testServiceHasEndpoint(event watch.Event) (bool, error) {
+	switch event.Type {
+	case watch.Deleted:
+		return false, errors.NewNotFound(schema.GroupResource{Resource: "services"}, "")
+	}
+	switch ep := event.Object.(type) {
+	case *kapi.Endpoints:
+		ss := ep.Subsets
+		if len(ss) == 0 || len(ss[0].Addresses) < 1 {
+			return false, nil
+		}
+		return true, nil
+	}
+	return false, nil
+}
+
 func (d *AppCreate) cleanup() {
 	if d.project == "" {
 		return // something stopped early, nothing to clean up