Skip to content

Commit

Permalink
Add -prune option to dockerregistry
Browse files Browse the repository at this point in the history
Signed-off-by: Oleg Bulatov <[email protected]>
  • Loading branch information
Oleg Bulatov committed Jun 14, 2017
1 parent 322171b commit bb45ac4
Show file tree
Hide file tree
Showing 3 changed files with 161 additions and 3 deletions.
43 changes: 41 additions & 2 deletions pkg/cmd/dockerregistry/dockerregistry.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,13 @@ package dockerregistry
import (
"crypto/tls"
"crypto/x509"
"flag"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"strings"
"time"

log "github.com/Sirupsen/logrus"
Expand All @@ -19,6 +21,8 @@ import (
"github.com/docker/distribution/health"
"github.com/docker/distribution/registry/auth"
"github.com/docker/distribution/registry/handlers"
"github.com/docker/distribution/registry/storage"
"github.com/docker/distribution/registry/storage/driver/factory"
"github.com/docker/distribution/uuid"
"github.com/docker/distribution/version"

Expand All @@ -35,8 +39,6 @@ import (
_ "github.com/docker/distribution/registry/storage/driver/s3-aws"
_ "github.com/docker/distribution/registry/storage/driver/swift"

"strings"

"github.com/openshift/origin/pkg/cmd/server/crypto"
"github.com/openshift/origin/pkg/cmd/util/clientcmd"
"github.com/openshift/origin/pkg/dockerregistry/server"
Expand All @@ -45,8 +47,45 @@ import (
registryconfig "github.com/openshift/origin/pkg/dockerregistry/server/configuration"
)

var prune = flag.Bool("prune", false, "prune blobs from the storage and exit")

// ExecutePruner runs the pruner.
func ExecutePruner(configFile io.Reader) {
log.Infof("prune version=%s", version.Version)

config, _, err := registryconfig.Parse(configFile)
if err != nil {
log.Fatalf("error parsing configuration file: %s", err)
}

ctx := context.Background()
ctx, err = configureLogging(ctx, config)
if err != nil {
log.Fatalf("error configuring logging: %s", err)
}

registryClient := server.NewRegistryClient(clientcmd.NewConfig().BindToFile())

storageDriver, err := factory.Create(config.Storage.Type(), config.Storage.Parameters())
if err != nil {
log.Fatalf("error creating storage driver: %s", err)
}

registry, err := storage.NewRegistry(ctx, storageDriver, storage.EnableDelete)
if err != nil {
log.Fatalf("error creating registry: %s", err)
}

server.Prune(ctx, storageDriver, registry, registryClient)
}

// Execute runs the Docker registry.
func Execute(configFile io.Reader) {
if *prune {
ExecutePruner(configFile)
return
}

dockerConfig, extraConfig, err := registryconfig.Parse(configFile)
if err != nil {
log.Fatalf("error parsing configuration file: %s", err)
Expand Down
3 changes: 2 additions & 1 deletion pkg/dockerregistry/server/errorblobstore.go
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,8 @@ func (f statCrossMountCreateOptions) Apply(v interface{}) error {
if err != nil {
context.GetLogger(f.ctx).Infof("cannot mount blob %s from repository %s: %v - disabling cross-repo mount",
opts.Mount.From.Digest().String(),
opts.Mount.From.Name())
opts.Mount.From.Name(),
err)
opts.Mount.ShouldMount = false
return nil
}
Expand Down
118 changes: 118 additions & 0 deletions pkg/dockerregistry/server/prune.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
package server

import (
"fmt"

"github.com/docker/distribution"
"github.com/docker/distribution/context"
"github.com/docker/distribution/digest"
"github.com/docker/distribution/manifest/schema2"
"github.com/docker/distribution/reference"
"github.com/docker/distribution/registry/storage"
"github.com/docker/distribution/registry/storage/driver"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func Prune(ctx context.Context, storageDriver driver.StorageDriver, registry distribution.Namespace, registryClient RegistryClient) {
logger := context.GetLogger(ctx)

repositoryEnumerator, ok := registry.(distribution.RepositoryEnumerator)
if !ok {
logger.Fatal("unable to convert Namespace to RepositoryEnumerator")
}

oc, _, err := registryClient.Clients()
if err != nil {
logger.Fatalf("error getting clients: %s", err)
}

imageList, err := oc.Images().List(metav1.ListOptions{})
if err != nil {
logger.Fatalf("error listing images: %s", err)
}

inuse := make(map[string]string)
for _, image := range imageList.Items {
// Keep the manifest.
inuse[image.Name] = image.DockerImageReference

// Keep the config for a schema 2 manifest.
if image.DockerImageManifestMediaType == schema2.MediaTypeManifest {
inuse[image.DockerImageMetadata.ID] = image.DockerImageReference
}

// Keep image layers.
for _, layer := range image.DockerImageLayers {
inuse[layer.Name] = image.DockerImageReference
}
}

err = repositoryEnumerator.Enumerate(ctx, func(repoName string) error {
logger.Debugln("Processing repository", repoName)

named, err := reference.WithName(repoName)
if err != nil {
return fmt.Errorf("failed to parse repo name %s: %v", repoName, err)
}

repository, err := registry.Repository(ctx, named)
if err != nil {
return err
}

manifestService, err := repository.Manifests(ctx)
if err != nil {
return err
}

manifestEnumerator, ok := manifestService.(distribution.ManifestEnumerator)
if !ok {
return fmt.Errorf("unable to convert ManifestService into ManifestEnumerator")
}

err = manifestEnumerator.Enumerate(ctx, func(dgst digest.Digest) error {
if imageReference, ok := inuse[string(dgst)]; ok {
logger.Debugf("Keep manifest %s@%s (manifest belongs to image %s)", repoName, dgst, imageReference)
return nil
}

logger.Printf("Deleting manifest: %s@%s", repoName, dgst)
err := manifestService.Delete(ctx, dgst)
if err != nil {
return fmt.Errorf("delete manifest %s: %s", dgst, err)
}

return nil
})
if e, ok := err.(driver.PathNotFoundError); ok {
logger.Warnf("prune manifests in repository %s: %s", repoName, e)
} else if err != nil {
return fmt.Errorf("prune manifests in repository %s: %s", repoName, err)
}

return nil
})
if err != nil {
logger.Fatal(err)
}

logger.Debugln("Processing blobs")
vacuum := storage.NewVacuum(ctx, storageDriver)
err = registry.Blobs().Enumerate(ctx, func(dgst digest.Digest) error {
if imageReference, ok := inuse[string(dgst)]; ok {
logger.Debugf("Keep blob %s (blob belongs to image %s)", dgst, imageReference)
return nil
}

err := vacuum.RemoveBlob(string(dgst))
if err != nil {
return fmt.Errorf("delete blob %s: %s", dgst, err)
}

return nil
})
if err != nil {
logger.Fatal(err)
}
}

0 comments on commit bb45ac4

Please sign in to comment.