Merge pull request #16821 from miminar/image-pruning-dereference-istags

Automatic merge from submit-queue.

image-pruning: dereference ImageStreamTags

Create strong references to images for each pod/bc/dc/etc that uses `<host>/<repo>:tag` reference.

Resolves [bz#1498604](https://bugzilla.redhat.com/show_bug.cgi?id=1498604) and https://bugzilla.redhat.com/show_bug.cgi?id=1386917



Images can manually removed using `oc delete`. Image stream tags having references to these images become obsolete - we may delete them.

To be sure that we don't remove reference to image that has just been created (and we don't know about it), make sure to honor `--keep-younger-than`.

pkg/apps/graph/nodes/nodes.go

@@ -3,13 +3,69 @@ package nodes
 import (
 	"github.com/gonum/graph"
 
+	kapisext "k8s.io/kubernetes/pkg/apis/extensions"
+
 	osgraph "github.com/openshift/origin/pkg/api/graph"
 	kubegraph "github.com/openshift/origin/pkg/api/kubegraph/nodes"
-	depoyapi "github.com/openshift/origin/pkg/apps/apis/apps"
+	deployapi "github.com/openshift/origin/pkg/apps/apis/apps"
 )
 
+// EnsureDaemonSetNode adds the provided daemon set to the graph if it does not exist
+func EnsureDaemonSetNode(g osgraph.MutableUniqueGraph, ds *kapisext.DaemonSet) *DaemonSetNode {
+	dsName := DaemonSetNodeName(ds)
+	dsNode := osgraph.EnsureUnique(
+		g,
+		dsName,
+		func(node osgraph.Node) graph.Node {
+			return &DaemonSetNode{Node: node, DaemonSet: ds, IsFound: true}
+		},
+	).(*DaemonSetNode)
+
+	podTemplateSpecNode := kubegraph.EnsurePodTemplateSpecNode(g, &ds.Spec.Template, ds.Namespace, dsName)
+	g.AddEdge(dsNode, podTemplateSpecNode, osgraph.ContainsEdgeKind)
+
+	return dsNode
+}
+
+func FindOrCreateSyntheticDaemonSetNode(g osgraph.MutableUniqueGraph, ds *kapisext.DaemonSet) *DaemonSetNode {
+	return osgraph.EnsureUnique(
+		g,
+		DaemonSetNodeName(ds),
+		func(node osgraph.Node) graph.Node {
+			return &DaemonSetNode{Node: node, DaemonSet: ds, IsFound: false}
+		},
+	).(*DaemonSetNode)
+}
+
+// EnsureDeploymentNode adds the provided upstream deployment to the graph if it does not exist
+func EnsureDeploymentNode(g osgraph.MutableUniqueGraph, deployment *kapisext.Deployment) *DeploymentNode {
+	deploymentName := DeploymentNodeName(deployment)
+	deploymentNode := osgraph.EnsureUnique(
+		g,
+		deploymentName,
+		func(node osgraph.Node) graph.Node {
+			return &DeploymentNode{Node: node, Deployment: deployment, IsFound: true}
+		},
+	).(*DeploymentNode)
+
+	podTemplateSpecNode := kubegraph.EnsurePodTemplateSpecNode(g, &deployment.Spec.Template, deployment.Namespace, deploymentName)
+	g.AddEdge(deploymentNode, podTemplateSpecNode, osgraph.ContainsEdgeKind)
+
+	return deploymentNode
+}
+
+func FindOrCreateSyntheticDeploymentNode(g osgraph.MutableUniqueGraph, deployment *kapisext.Deployment) *DeploymentNode {
+	return osgraph.EnsureUnique(
+		g,
+		DeploymentNodeName(deployment),
+		func(node osgraph.Node) graph.Node {
+			return &DeploymentNode{Node: node, Deployment: deployment, IsFound: false}
+		},
+	).(*DeploymentNode)
+}
+
 // EnsureDeploymentConfigNode adds the provided deployment config to the graph if it does not exist
-func EnsureDeploymentConfigNode(g osgraph.MutableUniqueGraph, dc *depoyapi.DeploymentConfig) *DeploymentConfigNode {
+func EnsureDeploymentConfigNode(g osgraph.MutableUniqueGraph, dc *deployapi.DeploymentConfig) *DeploymentConfigNode {
 	dcName := DeploymentConfigNodeName(dc)
 	dcNode := osgraph.EnsureUnique(
 		g,
@@ -27,7 +83,7 @@ func EnsureDeploymentConfigNode(g osgraph.MutableUniqueGraph, dc *depoyapi.Deplo
 	return dcNode
 }
 
-func FindOrCreateSyntheticDeploymentConfigNode(g osgraph.MutableUniqueGraph, dc *depoyapi.DeploymentConfig) *DeploymentConfigNode {
+func FindOrCreateSyntheticDeploymentConfigNode(g osgraph.MutableUniqueGraph, dc *deployapi.DeploymentConfig) *DeploymentConfigNode {
 	return osgraph.EnsureUnique(
 		g,
 		DeploymentConfigNodeName(dc),
@@ -36,3 +92,30 @@ func FindOrCreateSyntheticDeploymentConfigNode(g osgraph.MutableUniqueGraph, dc
 		},
 	).(*DeploymentConfigNode)
 }
+
+// EnsureReplicaSetNode adds the provided replica set to the graph if it does not exist
+func EnsureReplicaSetNode(g osgraph.MutableUniqueGraph, rs *kapisext.ReplicaSet) *ReplicaSetNode {
+	rsName := ReplicaSetNodeName(rs)
+	rsNode := osgraph.EnsureUnique(
+		g,
+		rsName,
+		func(node osgraph.Node) graph.Node {
+			return &ReplicaSetNode{Node: node, ReplicaSet: rs, IsFound: true}
+		},
+	).(*ReplicaSetNode)
+
+	podTemplateSpecNode := kubegraph.EnsurePodTemplateSpecNode(g, &rs.Spec.Template, rs.Namespace, rsName)
+	g.AddEdge(rsNode, podTemplateSpecNode, osgraph.ContainsEdgeKind)
+
+	return rsNode
+}
+
+func FindOrCreateSyntheticReplicaSetNode(g osgraph.MutableUniqueGraph, rs *kapisext.ReplicaSet) *ReplicaSetNode {
+	return osgraph.EnsureUnique(
+		g,
+		ReplicaSetNodeName(rs),
+		func(node osgraph.Node) graph.Node {
+			return &ReplicaSetNode{Node: node, ReplicaSet: rs, IsFound: false}
+		},
+	).(*ReplicaSetNode)
+}

pkg/apps/graph/nodes/types.go

@@ -3,14 +3,73 @@ package nodes
 import (
 	"reflect"
 
+	kapisext "k8s.io/kubernetes/pkg/apis/extensions"
+
 	osgraph "github.com/openshift/origin/pkg/api/graph"
 	deployapi "github.com/openshift/origin/pkg/apps/apis/apps"
 )
 
 var (
+	DaemonSetNodeKind        = reflect.TypeOf(kapisext.DaemonSet{}).Name()
+	DeploymentNodeKind       = reflect.TypeOf(kapisext.Deployment{}).Name()
 	DeploymentConfigNodeKind = reflect.TypeOf(deployapi.DeploymentConfig{}).Name()
+	ReplicaSetNodeKind       = reflect.TypeOf(kapisext.ReplicaSet{}).Name()
 )
 
+func DaemonSetNodeName(o *kapisext.DaemonSet) osgraph.UniqueName {
+	return osgraph.GetUniqueRuntimeObjectNodeName(DaemonSetNodeKind, o)
+}
+
+type DaemonSetNode struct {
+	osgraph.Node
+	DaemonSet *kapisext.DaemonSet
+
+	IsFound bool
+}
+
+func (n DaemonSetNode) Found() bool {
+	return n.IsFound
+}
+
+func (n DaemonSetNode) Object() interface{} {
+	return n.DaemonSet
+}
+
+func (n DaemonSetNode) String() string {
+	return string(DaemonSetNodeName(n.DaemonSet))
+}
+
+func (*DaemonSetNode) Kind() string {
+	return DaemonSetNodeKind
+}
+
+func DeploymentNodeName(o *kapisext.Deployment) osgraph.UniqueName {
+	return osgraph.GetUniqueRuntimeObjectNodeName(DeploymentNodeKind, o)
+}
+
+type DeploymentNode struct {
+	osgraph.Node
+	Deployment *kapisext.Deployment
+
+	IsFound bool
+}
+
+func (n DeploymentNode) Found() bool {
+	return n.IsFound
+}
+
+func (n DeploymentNode) Object() interface{} {
+	return n.Deployment
+}
+
+func (n DeploymentNode) String() string {
+	return string(DeploymentNodeName(n.Deployment))
+}
+
+func (*DeploymentNode) Kind() string {
+	return DeploymentNodeKind
+}
+
 func DeploymentConfigNodeName(o *deployapi.DeploymentConfig) osgraph.UniqueName {
 	return osgraph.GetUniqueRuntimeObjectNodeName(DeploymentConfigNodeKind, o)
 }
@@ -37,3 +96,30 @@ func (n DeploymentConfigNode) String() string {
 func (*DeploymentConfigNode) Kind() string {
 	return DeploymentConfigNodeKind
 }
+
+func ReplicaSetNodeName(o *kapisext.ReplicaSet) osgraph.UniqueName {
+	return osgraph.GetUniqueRuntimeObjectNodeName(ReplicaSetNodeKind, o)
+}
+
+type ReplicaSetNode struct {
+	osgraph.Node
+	ReplicaSet *kapisext.ReplicaSet
+
+	IsFound bool
+}
+
+func (n ReplicaSetNode) Found() bool {
+	return n.IsFound
+}
+
+func (n ReplicaSetNode) Object() interface{} {
+	return n.ReplicaSet
+}
+
+func (n ReplicaSetNode) String() string {
+	return string(ReplicaSetNodeName(n.ReplicaSet))
+}
+
+func (*ReplicaSetNode) Kind() string {
+	return ReplicaSetNodeKind
+}

pkg/cmd/server/bootstrappolicy/policy.go

@@ -551,6 +551,9 @@ func GetOpenshiftBootstrapClusterRoles() []rbac.ClusterRole {
 				rbac.NewRule("list").Groups(kapiGroup).Resources("limitranges").RuleOrDie(),
 				rbac.NewRule("get", "list").Groups(buildGroup, legacyBuildGroup).Resources("buildconfigs", "builds").RuleOrDie(),
 				rbac.NewRule("get", "list").Groups(deployGroup, legacyDeployGroup).Resources("deploymentconfigs").RuleOrDie(),
+				rbac.NewRule("get", "list").Groups(extensionsGroup).Resources("daemonsets").RuleOrDie(),
+				rbac.NewRule("get", "list").Groups(extensionsGroup).Resources("deployments").RuleOrDie(),
+				rbac.NewRule("get", "list").Groups(extensionsGroup).Resources("replicasets").RuleOrDie(),
 
 				rbac.NewRule("delete").Groups(imageGroup, legacyImageGroup).Resources("images").RuleOrDie(),
 				rbac.NewRule("get", "list").Groups(imageGroup, legacyImageGroup).Resources("images", "imagestreams").RuleOrDie(),

pkg/image/graph/nodes/nodes.go

@@ -36,8 +36,12 @@ func EnsureAllImageStreamTagNodes(g osgraph.MutableUniqueGraph, is *imageapi.Ima
 	return ret
 }
 
-func FindImage(g osgraph.MutableUniqueGraph, imageName string) graph.Node {
-	return g.Find(ImageNodeName(&imageapi.Image{ObjectMeta: metav1.ObjectMeta{Name: imageName}}))
+func FindImage(g osgraph.MutableUniqueGraph, imageName string) *ImageNode {
+	n := g.Find(ImageNodeName(&imageapi.Image{ObjectMeta: metav1.ObjectMeta{Name: imageName}}))
+	if imageNode, ok := n.(*ImageNode); ok {
+		return imageNode
+	}
+	return nil
 }
 
 // EnsureDockerRepositoryNode adds the named Docker repository tag reference to the graph if it does

pkg/image/prune/helper.go

@@ -7,6 +7,8 @@ import (
 	"sort"
 	"strings"
 
+	kapi "k8s.io/kubernetes/pkg/api"
+
 	"github.com/docker/distribution/registry/api/errcode"
 	"github.com/golang/glog"
 
@@ -206,3 +208,60 @@ func TryProtocolsWithRegistryURL(registry string, allowInsecure bool, action fun
 type retryPath struct{ err error }
 
 func (rp *retryPath) Error() string { return rp.err.Error() }
+
+// ErrBadReference denotes an invalid reference to image, imagestreamtag or imagestreamimage stored in a
+// particular object. The object is identified by kind, namespace and name.
+type ErrBadReference struct {
+	kind       string
+	namespace  string
+	name       string
+	targetKind string
+	reference  string
+	reason     string
+}
+
+func newErrBadReferenceToImage(reference string, obj *kapi.ObjectReference, reason string) error {
+	kind := "<UnknownType>"
+	namespace := ""
+	name := "<unknown-name>"
+	if obj != nil {
+		kind = obj.Kind
+		namespace = obj.Namespace
+		name = obj.Name
+	}
+
+	return &ErrBadReference{
+		kind:      kind,
+		namespace: namespace,
+		name:      name,
+		reference: reference,
+		reason:    reason,
+	}
+}
+
+func newErrBadReferenceTo(targetKind, reference string, obj *kapi.ObjectReference, reason string) error {
+	return &ErrBadReference{
+		kind:       obj.Kind,
+		namespace:  obj.Namespace,
+		name:       obj.Name,
+		targetKind: targetKind,
+		reference:  reference,
+		reason:     reason,
+	}
+}
+
+func (e *ErrBadReference) Error() string {
+	return e.String()
+}
+
+func (e *ErrBadReference) String() string {
+	name := e.name
+	if len(e.namespace) > 0 {
+		name = e.namespace + "/" + name
+	}
+	targetKind := "docker image"
+	if len(e.targetKind) > 0 {
+		targetKind = e.targetKind
+	}
+	return fmt.Sprintf("%s[%s]: invalid %s reference %q: %s", e.kind, name, targetKind, e.reference, e.reason)
+}