From f23b6544860a17acff0d1c7584fe9c6ad2bc98cf Mon Sep 17 00:00:00 2001 From: David Eads Date: Mon, 9 Jul 2018 09:13:35 -0400 Subject: [PATCH] bump image builder --- glide.lock | 23 +-- .../cmd/imagebuilder/imagebuilder.go | 2 +- .../imagebuilder/dockerclient/archive.go | 149 ++++++++++-------- .../imagebuilder/dockerclient/archive_test.go | 33 ++++ .../imagebuilder/dockerclient/client.go | 18 ++- .../dockerclient/conformance_test.go | 83 +++++++++- .../dockerclient/copyinfo_test.go | 47 +++++- .../imagebuilder/dockerclient/directory.go | 87 ++++++++++ .../dockerclient/testdata/copy/Dockerfile | 3 + .../dockerclient/testdata/copy/script | 2 + .../dockerclient/testdata/copydir/Dockerfile | 3 + .../dockerclient/testdata/copydir/dir/file | 0 .../testdata/copyrename/Dockerfile | 3 + .../dockerclient/testdata/copyrename/file1 | 2 + 14 files changed, 362 insertions(+), 93 deletions(-) create mode 100644 vendor/github.com/openshift/imagebuilder/dockerclient/directory.go create mode 100644 vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copy/Dockerfile create mode 100644 vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copy/script create mode 100644 vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copydir/Dockerfile create mode 100644 vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copydir/dir/file create mode 100644 vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copyrename/Dockerfile create mode 100644 vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copyrename/file1 diff --git a/glide.lock b/glide.lock index a5d50fb0aa06..93d4795d4148 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 1511c1c08d724d5b2f6c586490bfa154f305c5b0038fb2a4bf3c0404aabc72a6 -updated: 2018-06-24T03:58:28.501190286-04:00 +hash: f6c313daf71c664707bed1f649639897c0690fd47c45615bed5abf7c4971e086 +updated: 2018-07-09T09:13:12.866398637-04:00 imports: - name: bitbucket.org/ww/goautoneg version: 75cd24fc2f2c2a2088577d12123ddee5f54e0675 @@ -656,7 +656,6 @@ imports: version: e47cf1f2e83a02443d7115c54f838be8ee959644 subpackages: - client - - client/v2 - models - pkg/escape - name: github.com/JeffAshton/win_pdh @@ -770,8 +769,6 @@ imports: version: d59fa0ac68bb5dd932ee8d24eed631cdd519efc3 subpackages: - format - - gstruct - - gstruct/errors - internal/assertion - internal/asyncassertion - internal/oraclematcher @@ -947,7 +944,7 @@ imports: - user/informers/externalversions/user/v1 - user/listers/user/v1 - name: github.com/openshift/imagebuilder - version: 937f2b31b364190f90c4bc51a1ae8b80125b374b + version: bfc0aea02ce95dcfc1a83d452270dbc66933709b subpackages: - dockerclient - imageprogress @@ -1140,9 +1137,6 @@ imports: - pbm/types - property - session - - simulator - - simulator/esx - - simulator/vpx - task - vim25 - vim25/debug @@ -1254,11 +1248,7 @@ imports: - name: golang.org/x/tools version: 2382e3994d48b1d22acc2c86bcad0a2aff028e32 subpackages: - - benchmark/parse - container/intsets - - go/ast/astutil - - go/vcs - - imports - name: google.golang.org/api version: 7f657476956314fee258816aaf81c0ff65cf8bee subpackages: @@ -1270,9 +1260,6 @@ imports: - gensupport - googleapi - googleapi/internal/uritemplates - - logging/v2beta1 - - monitoring/v3 - - pubsub/v1 - tpu/v1alpha1 - name: google.golang.org/appengine version: 12d5545dc1cfa6047a286d5e853841b6471f4c19 @@ -1766,7 +1753,7 @@ imports: - util/testing - util/workqueue - name: k8s.io/code-generator - version: 006eedfdf16ff9bda1d82e13062f5e17be611d23 + version: 895b23721034b22084c5afd7814cb9bd14ee5abd repo: https://github.com/openshift/kubernetes-code-generator.git - name: k8s.io/gengo version: 01a732e01d00cb9a81bb0ca050d3e6d2b947927b @@ -1815,7 +1802,7 @@ imports: - pkg/util/proto - pkg/util/proto/validation - name: k8s.io/kubernetes - version: d500745e6519d922e3dc9735cbafc5f2a67e0e44 + version: a745c520ef9f526306f56ae7bd27e33d5062cb59 repo: https://github.com/openshift/kubernetes.git subpackages: - cmd/controller-manager/app diff --git a/vendor/github.com/openshift/imagebuilder/cmd/imagebuilder/imagebuilder.go b/vendor/github.com/openshift/imagebuilder/cmd/imagebuilder/imagebuilder.go index 62a932c27049..499e466f987b 100644 --- a/vendor/github.com/openshift/imagebuilder/cmd/imagebuilder/imagebuilder.go +++ b/vendor/github.com/openshift/imagebuilder/cmd/imagebuilder/imagebuilder.go @@ -82,7 +82,7 @@ func main() { if glog.V(2) { log.Printf("Builder: "+format, args...) } else { - fmt.Fprintf(options.ErrOut, "--> %s\n", fmt.Sprintf(format, args...)) + fmt.Fprintf(options.Out, "--> %s\n", fmt.Sprintf(format, args...)) } } diff --git a/vendor/github.com/openshift/imagebuilder/dockerclient/archive.go b/vendor/github.com/openshift/imagebuilder/dockerclient/archive.go index ee5ca9a30e51..c8c0f68bc92f 100644 --- a/vendor/github.com/openshift/imagebuilder/dockerclient/archive.go +++ b/vendor/github.com/openshift/imagebuilder/dockerclient/archive.go @@ -38,7 +38,9 @@ func FilterArchive(r io.Reader, w io.Writer, fn TransformFileFunc) error { } var body io.Reader = tr + name := h.Name data, ok, skip, err := fn(h, tr) + glog.V(6).Infof("Transform %s -> %s: data=%t ok=%t skip=%t err=%v", name, h.Name, data != nil, ok, skip, err) if err != nil { return err } @@ -100,7 +102,7 @@ func NewLazyArchive(fn CreateFileFunc) io.ReadCloser { return pr } -func archiveFromURL(src, dst, tempDir string) (io.Reader, io.Closer, error) { +func archiveFromURL(src, dst, tempDir string, check DirectoryCheck) (io.Reader, io.Closer, error) { // get filename from URL u, err := url.Parse(src) if err != nil { @@ -151,7 +153,7 @@ func archiveFromURL(src, dst, tempDir string) (io.Reader, io.Closer, error) { return archive, closers{resp.Body.Close, archive.Close}, nil } -func archiveFromDisk(directory string, src, dst string, allowDownload bool, excludes []string) (io.Reader, io.Closer, error) { +func archiveFromDisk(directory string, src, dst string, allowDownload bool, excludes []string, check DirectoryCheck) (io.Reader, io.Closer, error) { var err error if filepath.IsAbs(src) { src, err = filepath.Rel(filepath.Dir(src), src) @@ -173,13 +175,66 @@ func archiveFromDisk(directory string, src, dst string, allowDownload bool, excl directory = filepath.Dir(directory) } - options := archiveOptionsFor(infos, dst, excludes) + options, err := archiveOptionsFor(infos, dst, excludes, check) + if err != nil { + return nil, nil, err + } glog.V(4).Infof("Tar of %s %#v", directory, options) rc, err := archive.TarWithOptions(directory, options) return rc, rc, err } +func archiveFromFile(file string, src, dst string, excludes []string, check DirectoryCheck) (io.Reader, io.Closer, error) { + var err error + if filepath.IsAbs(src) { + src, err = filepath.Rel(filepath.Dir(src), src) + if err != nil { + return nil, nil, err + } + } + + mapper, _, err := newArchiveMapper(src, dst, excludes, true, check) + if err != nil { + return nil, nil, err + } + + f, err := os.Open(file) + if err != nil { + return nil, nil, err + } + + r, err := transformArchive(f, true, mapper.Filter) + return r, f, err +} + +func archiveFromContainer(in io.Reader, src, dst string, excludes []string, check DirectoryCheck) (io.Reader, string, error) { + mapper, archiveRoot, err := newArchiveMapper(src, dst, excludes, false, check) + if err != nil { + return nil, "", err + } + + r, err := transformArchive(in, false, mapper.Filter) + return r, archiveRoot, err +} + +func transformArchive(r io.Reader, compressed bool, fn TransformFileFunc) (io.Reader, error) { + pr, pw := io.Pipe() + go func() { + if compressed { + in, err := archive.DecompressStream(r) + if err != nil { + pw.CloseWithError(err) + return + } + r = in + } + err := FilterArchive(r, pw, fn) + pw.CloseWithError(err) + }() + return pr, nil +} + // * -> test // a (dir) -> test // a (file) -> test @@ -193,9 +248,15 @@ func archivePathMapper(src, dst string, isDestDir bool) (fn func(name string, is } pattern := filepath.Base(srcPattern) + glog.V(6).Infof("creating mapper for srcPattern=%s pattern=%s dst=%s isDestDir=%t", srcPattern, pattern, dst, isDestDir) + // no wildcards if !containsWildcards(pattern) { return func(name string, isDir bool) (string, bool) { + // when extracting from the working directory, Docker prefaces with ./ + if strings.HasPrefix(name, "."+string(filepath.Separator)) { + name = name[2:] + } if name == srcPattern { if isDir { return "", false @@ -232,7 +293,7 @@ func archivePathMapper(src, dst string, isDestDir bool) (fn func(name string, is } prefix += string(filepath.Separator) - // nested with pattern pattern + // nested with pattern return func(name string, isDir bool) (string, bool) { remainder := strings.TrimPrefix(name, prefix) if remainder == name { @@ -251,56 +312,6 @@ func archivePathMapper(src, dst string, isDestDir bool) (fn func(name string, is } } -func archiveFromFile(file string, src, dst string, excludes []string) (io.Reader, io.Closer, error) { - var err error - if filepath.IsAbs(src) { - src, err = filepath.Rel(filepath.Dir(src), src) - if err != nil { - return nil, nil, err - } - } - - mapper, _, err := newArchiveMapper(src, dst, excludes, true) - if err != nil { - return nil, nil, err - } - - f, err := os.Open(file) - if err != nil { - return nil, nil, err - } - - r, err := transformArchive(f, true, mapper.Filter) - return r, f, err -} - -func archiveFromContainer(in io.Reader, src, dst string, excludes []string) (io.Reader, string, error) { - mapper, archiveRoot, err := newArchiveMapper(src, dst, excludes, false) - if err != nil { - return nil, "", err - } - - r, err := transformArchive(in, false, mapper.Filter) - return r, archiveRoot, err -} - -func transformArchive(r io.Reader, compressed bool, fn TransformFileFunc) (io.Reader, error) { - pr, pw := io.Pipe() - go func() { - if compressed { - in, err := archive.DecompressStream(r) - if err != nil { - pw.CloseWithError(err) - return - } - r = in - } - err := FilterArchive(r, pw, fn) - pw.CloseWithError(err) - }() - return pr, nil -} - type archiveMapper struct { exclude *fileutils.PatternMatcher rename func(name string, isDir bool) (string, bool) @@ -308,7 +319,7 @@ type archiveMapper struct { resetOwners bool } -func newArchiveMapper(src, dst string, excludes []string, resetOwners bool) (*archiveMapper, string, error) { +func newArchiveMapper(src, dst string, excludes []string, resetOwners bool, check DirectoryCheck) (*archiveMapper, string, error) { ex, err := fileutils.NewPatternMatcher(excludes) if err != nil { return nil, "", err @@ -316,6 +327,13 @@ func newArchiveMapper(src, dst string, excludes []string, resetOwners bool) (*ar isDestDir := strings.HasSuffix(dst, "/") || path.Base(dst) == "." dst = path.Clean(dst) + if !isDestDir && check != nil { + isDir, err := check.IsDirectory(dst) + if err != nil { + return nil, "", err + } + isDestDir = isDir + } var prefix string archiveRoot := src @@ -380,19 +398,27 @@ func (m *archiveMapper) Filter(h *tar.Header, r io.Reader) ([]byte, bool, bool, return nil, false, false, nil } -func archiveOptionsFor(infos []CopyInfo, dst string, excludes []string) *archive.TarOptions { +func archiveOptionsFor(infos []CopyInfo, dst string, excludes []string, check DirectoryCheck) (*archive.TarOptions, error) { dst = trimLeadingPath(dst) dstIsDir := strings.HasSuffix(dst, "/") || dst == "." || dst == "/" || strings.HasSuffix(dst, "/.") dst = trimTrailingSlash(dst) dstIsRoot := dst == "." || dst == "/" + if !dstIsDir && check != nil { + isDir, err := check.IsDirectory(dst) + if err != nil { + return nil, fmt.Errorf("unable to check whether %s is a directory: %v", dst, err) + } + dstIsDir = isDir + } + options := &archive.TarOptions{ ChownOpts: &idtools.IDPair{UID: 0, GID: 0}, } pm, err := fileutils.NewPatternMatcher(excludes) if err != nil { - return options + return options, nil } for _, info := range infos { @@ -418,12 +444,9 @@ func archiveOptionsFor(infos []CopyInfo, dst string, excludes []string) *archive case len(infos) > 1: // put each input into the target, which is assumed to be a directory ([Dockerfile, dir] -> [a/Dockerfile, a/dir]) options.RebaseNames[infoPath] = path.Join(dst, path.Base(infoPath)) - case info.FileInfo.IsDir() && dstIsDir: - // mapping a directory to an explicit directory ([dir] -> [a]) - options.RebaseNames[infoPath] = dst case info.FileInfo.IsDir(): - // mapping a directory to an implicit directory ([Dockerfile] -> [dir/Dockerfile]) - options.RebaseNames[infoPath] = path.Join(dst, path.Base(infoPath)) + // mapping a directory to a destination, explicit or not ([dir] -> [a]) + options.RebaseNames[infoPath] = dst case info.FromDir: // this is a file that was part of an explicit directory request, no transformation options.RebaseNames[infoPath] = path.Join(dst, path.Base(infoPath)) @@ -437,7 +460,7 @@ func archiveOptionsFor(infos []CopyInfo, dst string, excludes []string) *archive } options.ExcludePatterns = excludes - return options + return options, nil } func sourceToDestinationName(src, dst string, forceDir bool) string { diff --git a/vendor/github.com/openshift/imagebuilder/dockerclient/archive_test.go b/vendor/github.com/openshift/imagebuilder/dockerclient/archive_test.go index 89961c769b63..978c8564bfd3 100644 --- a/vendor/github.com/openshift/imagebuilder/dockerclient/archive_test.go +++ b/vendor/github.com/openshift/imagebuilder/dockerclient/archive_test.go @@ -13,6 +13,20 @@ import ( "github.com/docker/docker/pkg/archive" ) +type testDirectoryCheck map[string]bool + +func (c testDirectoryCheck) IsDirectory(path string) (bool, error) { + if c == nil { + return false, nil + } + + isDir, ok := c[path] + if !ok { + return false, fmt.Errorf("no path defined for %s", path) + } + return isDir, nil +} + type archiveGenerator struct { Headers []*tar.Header } @@ -81,6 +95,7 @@ func Test_archiveFromFile(t *testing.T) { dst string excludes []string expect []string + check map[string]bool }{ { file: testArchive, @@ -233,6 +248,7 @@ func Test_archiveFromFile(t *testing.T) { testCase.src, testCase.dst, testCase.excludes, + testDirectoryCheck(testCase.check), ) if err != nil { t.Fatal(err) @@ -266,6 +282,7 @@ func Test_archiveFromContainer(t *testing.T) { excludes []string expect []string path string + check map[string]bool }{ { gen: newArchiveGenerator().File("file").Dir("test").File("test/file2"), @@ -394,6 +411,14 @@ func Test_archiveFromContainer(t *testing.T) { path: "/a", expect: nil, }, + { + gen: newArchiveGenerator().File("b"), + src: "/a/b", + dst: "/a", + check: map[string]bool{"/a": true}, + path: "/a", + expect: nil, + }, { gen: newArchiveGenerator().Dir("a/").File("a/b"), src: "/a/b", @@ -403,6 +428,13 @@ func Test_archiveFromContainer(t *testing.T) { "/a", }, }, + { + gen: newArchiveGenerator().Dir("./a").File("./a/b"), + src: "a", + dst: "/a", + path: ".", + expect: []string{"/a/b"}, + }, } for i := range testCases { testCase := testCases[i] @@ -412,6 +444,7 @@ func Test_archiveFromContainer(t *testing.T) { testCase.src, testCase.dst, testCase.excludes, + testDirectoryCheck(testCase.check), ) if err != nil { t.Fatal(err) diff --git a/vendor/github.com/openshift/imagebuilder/dockerclient/client.go b/vendor/github.com/openshift/imagebuilder/dockerclient/client.go index 4340191c51f0..fa8b4a8fa2be 100644 --- a/vendor/github.com/openshift/imagebuilder/dockerclient/client.go +++ b/vendor/github.com/openshift/imagebuilder/dockerclient/client.go @@ -455,10 +455,9 @@ func (e *ClientExecutor) PopulateTransientMounts(opts docker.CreateContainerOpti var copies []imagebuilder.Copy for i, mount := range transientMounts { - source := mount.SourcePath copies = append(copies, imagebuilder.Copy{ FromFS: true, - Src: []string{filepath.Join(e.Directory, source)}, + Src: []string{mount.SourcePath}, Dest: filepath.Join(e.ContainerTransientMount, strconv.Itoa(i)), }) } @@ -800,8 +799,9 @@ func (e *ClientExecutor) archiveFromContainer(from string, src, dst string) (io. e.Deferred = append([]func() error{func() error { return e.removeContainer(containerID) }}, e.Deferred...) } + check := newDirectoryCheck(e.Client, e.Container.ID) pr, pw := io.Pipe() - ar, archiveRoot, err := archiveFromContainer(pr, src, dst, nil) + ar, archiveRoot, err := archiveFromContainer(pr, src, dst, nil, check) if err != nil { pr.Close() return nil, nil, err @@ -819,26 +819,30 @@ func (e *ClientExecutor) archiveFromContainer(from string, src, dst string) (io. // TODO: this does not support decompressing nested archives for ADD (when the source is a compressed file) func (e *ClientExecutor) Archive(fromFS bool, src, dst string, allowDownload bool, excludes []string) (io.Reader, io.Closer, error) { + var check DirectoryCheck + if e.Container != nil { + check = newDirectoryCheck(e.Client, e.Container.ID) + } if isURL(src) { if !allowDownload { return nil, nil, fmt.Errorf("source can't be a URL") } glog.V(5).Infof("Archiving %s -> %s from URL", src, dst) - return archiveFromURL(src, dst, e.TempDir) + return archiveFromURL(src, dst, e.TempDir, check) } // the input is from the filesystem, use the source as the input if fromFS { glog.V(5).Infof("Archiving %s %s -> %s from a filesystem location", src, ".", dst) - return archiveFromDisk(src, ".", dst, allowDownload, excludes) + return archiveFromDisk(src, ".", dst, allowDownload, excludes, check) } // if the context is in archive form, read from it without decompressing if len(e.ContextArchive) > 0 { glog.V(5).Infof("Archiving %s %s -> %s from context archive", e.ContextArchive, src, dst) - return archiveFromFile(e.ContextArchive, src, dst, excludes) + return archiveFromFile(e.ContextArchive, src, dst, excludes, check) } // if the context is a directory, we only allow relative includes glog.V(5).Infof("Archiving %q %q -> %q from disk", e.Directory, src, dst) - return archiveFromDisk(e.Directory, src, dst, allowDownload, excludes) + return archiveFromDisk(e.Directory, src, dst, allowDownload, excludes, check) } // ContainerVolumeTracker manages tracking archives of specific paths inside a container. diff --git a/vendor/github.com/openshift/imagebuilder/dockerclient/conformance_test.go b/vendor/github.com/openshift/imagebuilder/dockerclient/conformance_test.go index 27a99ae9f1a5..911083ada297 100644 --- a/vendor/github.com/openshift/imagebuilder/dockerclient/conformance_test.go +++ b/vendor/github.com/openshift/imagebuilder/dockerclient/conformance_test.go @@ -60,7 +60,6 @@ func TestMount(t *testing.T) { out := &bytes.Buffer{} e.Out, e.ErrOut = out, out - e.Directory = "." e.Tag = filepath.Base(tmpDir) e.TransientMounts = []Mount{ {SourcePath: "testdata/volume/", DestinationPath: "/tmp/test"}, @@ -87,6 +86,72 @@ func TestMount(t *testing.T) { } } +func TestCopyFrom(t *testing.T) { + c, err := docker.NewClientFromEnv() + if err != nil { + t.Fatal(err) + } + + testCases := []struct { + name string + create string + copy string + extra string + expect string + }{ + {name: "copy file to root", create: "touch /a /b", copy: "/a /", expect: "[[ -f /a ]]"}, + {name: "copy file to same file", create: "touch /a", copy: "/a /a", expect: "[[ -f /a ]]"}, + {name: "copy file to workdir", create: "touch /a", extra: "WORKDIR /b", copy: "/a .", expect: "[[ -f /b/a ]]"}, + {name: "copy file to workdir rename", create: "touch /a", extra: "WORKDIR /b", copy: "/a ./b", expect: "[[ -f /b/b ]]"}, + {name: "copy folder contents to higher level", create: "mkdir -p /a/b && touch /a/b/1 /a/b/2", copy: "/a/b/ /b/", expect: "[[ -f /b/1 && -f /b/2 && ! -e /a ]]"}, + {name: "copy wildcard folder contents to higher level", create: "mkdir -p /a/b && touch /a/b/1 /a/b/2", copy: "/a/b/* /b/", expect: "ls -al /b/1 /b/2 /b && ! ls -al /a /b/a /b/b"}, + {name: "copy folder with dot contents to higher level", create: "mkdir -p /a/b && touch /a/b/1 /a/b/2", copy: "/a/b/. /b/", expect: "ls -al /b/1 /b/2 /b && ! ls -al /a /b/a /b/b"}, + {name: "copy root file to different root name", create: "touch /b", copy: "/b /a", expect: "ls -al /a && ! ls -al /b"}, + {name: "copy nested file to different root name", create: "mkdir -p /a && touch /a/b", copy: "/a/b /a", expect: "ls -al /a && ! ls -al /b"}, + {name: "copy file to deeper directory with explicit slash", create: "mkdir -p /a && touch /a/1", copy: "/a/1 /a/b/c/", expect: "ls -al /a/b/c/1 && ! ls -al /a/b/1"}, + {name: "copy file to deeper directory without explicit slash", create: "mkdir -p /a && touch /a/1", copy: "/a/1 /a/b/c", expect: "ls -al /a/b/c && ! ls -al /a/b/1"}, + {name: "copy directory to deeper directory without explicit slash", create: "mkdir -p /a && touch /a/1", copy: "/a /a/b/c", expect: "ls -al /a/b/c/1 && ! ls -al /a/b/1"}, + {name: "copy directory to root without explicit slash", create: "mkdir -p /a && touch /a/1", copy: "a /a", expect: "ls -al /a/1 && ! ls -al /a/a"}, + {name: "copy directory trailing to root without explicit slash", create: "mkdir -p /a && touch /a/1", copy: "a/. /a", expect: "ls -al /a/1 && ! ls -al /a/a"}, + } + for i, testCase := range testCases { + name := fmt.Sprintf("%d", i) + if len(testCase.name) > 0 { + name = testCase.name + } + test := testCase + t.Run(name, func(t *testing.T) { + t.Parallel() + e := NewClientExecutor(c) + defer e.Release() + + out := &bytes.Buffer{} + e.Out, e.ErrOut = out, out + b := imagebuilder.NewBuilder(nil) + dockerfile := fmt.Sprintf(` + FROM busybox AS base + RUN %s + FROM busybox + %s + COPY --from=base %s + RUN %s + `, test.create, test.extra, test.copy, test.expect, + ) + t.Log(dockerfile) + node, err := imagebuilder.ParseDockerfile(strings.NewReader(dockerfile)) + if err != nil { + t.Fatal(err) + } + + stages := imagebuilder.NewStages(node, b) + if _, err := e.Stages(b, stages, ""); err != nil { + t.Log(out.String()) + t.Fatal(err) + } + }) + } +} + func TestShell(t *testing.T) { tmpDir, err := ioutil.TempDir("", "dockerbuild-conformance-") if err != nil { @@ -188,6 +253,18 @@ func TestConformanceInternal(t *testing.T) { Name: "directory", ContextDir: "testdata/dir", }, + { + Name: "copy to dir", + ContextDir: "testdata/copy", + }, + { + Name: "copy dir", + ContextDir: "testdata/copydir", + }, + { + Name: "copy to renamed file", + ContextDir: "testdata/copyrename", + }, { Name: "directory with slash", ContextDir: "testdata/overlapdir", @@ -329,8 +406,8 @@ func TestTransientMount(t *testing.T) { e.AllowPull = true e.Directory = "testdata" e.TransientMounts = []Mount{ - {SourcePath: "dir", DestinationPath: "/mountdir"}, - {SourcePath: "Dockerfile.env", DestinationPath: "/mountfile"}, + {SourcePath: "testdata/dir", DestinationPath: "/mountdir"}, + {SourcePath: "testdata/Dockerfile.env", DestinationPath: "/mountfile"}, } e.Tag = fmt.Sprintf("conformance%d", rand.Int63()) diff --git a/vendor/github.com/openshift/imagebuilder/dockerclient/copyinfo_test.go b/vendor/github.com/openshift/imagebuilder/dockerclient/copyinfo_test.go index 7828e3511850..24854441b945 100644 --- a/vendor/github.com/openshift/imagebuilder/dockerclient/copyinfo_test.go +++ b/vendor/github.com/openshift/imagebuilder/dockerclient/copyinfo_test.go @@ -17,6 +17,7 @@ func TestCalcCopyInfo(t *testing.T) { paths map[string]struct{} excludes []string rebaseNames map[string]string + check map[string]bool }{ { origPath: "subdir/*", @@ -106,6 +107,33 @@ func TestCalcCopyInfo(t *testing.T) { "Dockerfile": "copy/Dockerfile", }, }, + { + origPath: "Dockerfile", + dstPath: "copy", + rootPath: "testdata/singlefile", + allowWildcards: true, + errFn: nilErr, + paths: map[string]struct{}{ + "Dockerfile": {}, + }, + rebaseNames: map[string]string{ + "Dockerfile": "copy", + }, + }, + { + origPath: "Dockerfile", + dstPath: "copy", + check: map[string]bool{"copy": true}, + rootPath: "testdata/singlefile", + allowWildcards: true, + errFn: nilErr, + paths: map[string]struct{}{ + "Dockerfile": {}, + }, + rebaseNames: map[string]string{ + "Dockerfile": "copy/Dockerfile", + }, + }, { origPath: "existing/", dstPath: ".", @@ -185,6 +213,20 @@ func TestCalcCopyInfo(t *testing.T) { "subdir": "test", }, }, + { + origPath: "dir", + dstPath: "/dir", + check: map[string]bool{"dir": false}, + rootPath: "testdata/copydir", + allowWildcards: true, + errFn: nilErr, + paths: map[string]struct{}{ + "dir": {}, + }, + rebaseNames: map[string]string{ + "dir": "dir", + }, + }, } for i, test := range tests { @@ -211,7 +253,10 @@ func TestCalcCopyInfo(t *testing.T) { t.Errorf("did not see paths: %#v", expect) } - options := archiveOptionsFor(infos, test.dstPath, test.excludes) + options, err := archiveOptionsFor(infos, test.dstPath, test.excludes, testDirectoryCheck(test.check)) + if err != nil { + t.Fatal(err) + } if !reflect.DeepEqual(test.rebaseNames, options.RebaseNames) { t.Errorf("rebase names did not match:\n%#v\n%#v", test.rebaseNames, options.RebaseNames) } diff --git a/vendor/github.com/openshift/imagebuilder/dockerclient/directory.go b/vendor/github.com/openshift/imagebuilder/dockerclient/directory.go new file mode 100644 index 000000000000..4e0a9f8bf01a --- /dev/null +++ b/vendor/github.com/openshift/imagebuilder/dockerclient/directory.go @@ -0,0 +1,87 @@ +package dockerclient + +import ( + "archive/tar" + "context" + "io" + "io/ioutil" + + "github.com/golang/glog" + + docker "github.com/fsouza/go-dockerclient" +) + +type DirectoryCheck interface { + IsDirectory(path string) (bool, error) +} + +type directoryCheck struct { + containerID string + client *docker.Client +} + +func newDirectoryCheck(client *docker.Client, containerID string) *directoryCheck { + return &directoryCheck{ + containerID: containerID, + client: client, + } +} + +func (c *directoryCheck) IsDirectory(path string) (bool, error) { + if path == "/" || path == "." || path == "./" { + return true, nil + } + + dir, err := isContainerPathDirectory(c.client, c.containerID, path) + if err != nil { + return false, err + } + + return dir, nil +} + +func isContainerPathDirectory(client *docker.Client, containerID, path string) (bool, error) { + pr, pw := io.Pipe() + defer pw.Close() + ctx, cancel := context.WithCancel(context.TODO()) + go func() { + err := client.DownloadFromContainer(containerID, docker.DownloadFromContainerOptions{ + OutputStream: pw, + Path: path, + Context: ctx, + }) + if err != nil { + if apiErr, ok := err.(*docker.Error); ok && apiErr.Status == 404 { + glog.V(4).Infof("path %s did not exist in container %s: %v", path, containerID, err) + err = nil + } + if err != nil && err != context.Canceled { + glog.V(6).Infof("error while checking directory contents for container %s at path %s: %v", containerID, path, err) + } + } + pw.CloseWithError(err) + }() + + tr := tar.NewReader(pr) + + h, err := tr.Next() + if err != nil { + if err == io.EOF { + err = nil + } + return false, err + } + + glog.V(4).Infof("Retrieved first header from container %s at path %s: %#v", containerID, path, h) + + // take the remainder of the input and discard it + go func() { + cancel() + n, err := io.Copy(ioutil.Discard, pr) + if n > 0 || err != nil { + glog.V(6).Infof("Discarded %d bytes from end of container directory check, and got error: %v", n, err) + } + }() + + return h.FileInfo().IsDir(), nil +} diff --git a/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copy/Dockerfile b/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copy/Dockerfile new file mode 100644 index 000000000000..815de493ba73 --- /dev/null +++ b/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copy/Dockerfile @@ -0,0 +1,3 @@ +FROM centos:7 +COPY script /usr/bin +RUN ls -al /usr/bin/script \ No newline at end of file diff --git a/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copy/script b/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copy/script new file mode 100644 index 000000000000..c3c3f3f53f62 --- /dev/null +++ b/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copy/script @@ -0,0 +1,2 @@ +#!/bin/bash +exit 0 \ No newline at end of file diff --git a/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copydir/Dockerfile b/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copydir/Dockerfile new file mode 100644 index 000000000000..92c53fdf6dad --- /dev/null +++ b/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copydir/Dockerfile @@ -0,0 +1,3 @@ +FROM centos:7 +COPY dir /dir +RUN ls -al /dir/file \ No newline at end of file diff --git a/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copydir/dir/file b/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copydir/dir/file new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copyrename/Dockerfile b/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copyrename/Dockerfile new file mode 100644 index 000000000000..575bf2cd4de4 --- /dev/null +++ b/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copyrename/Dockerfile @@ -0,0 +1,3 @@ +FROM centos:7 +COPY file1 /usr/bin/file2 +RUN ls -al /usr/bin/file2 && ! ls -al /usr/bin/file1 \ No newline at end of file diff --git a/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copyrename/file1 b/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copyrename/file1 new file mode 100644 index 000000000000..c3c3f3f53f62 --- /dev/null +++ b/vendor/github.com/openshift/imagebuilder/dockerclient/testdata/copyrename/file1 @@ -0,0 +1,2 @@ +#!/bin/bash +exit 0 \ No newline at end of file