diff --git a/contrib/completions/bash/oc b/contrib/completions/bash/oc index 067373f4f0c2..e3d83c28115c 100644 --- a/contrib/completions/bash/oc +++ b/contrib/completions/bash/oc @@ -11817,6 +11817,61 @@ _oc_image_append() noun_aliases=() } +_oc_image_extract() +{ + last_command="oc_image_extract" + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--created-at=") + local_nonpersistent_flags+=("--created-at=") + flags+=("--dry-run") + local_nonpersistent_flags+=("--dry-run") + flags+=("--filter-by-os=") + local_nonpersistent_flags+=("--filter-by-os=") + flags+=("--force") + local_nonpersistent_flags+=("--force") + flags+=("--insecure") + local_nonpersistent_flags+=("--insecure") + flags+=("--max-per-registry=") + local_nonpersistent_flags+=("--max-per-registry=") + flags+=("--only-files") + local_nonpersistent_flags+=("--only-files") + flags+=("--as=") + flags+=("--as-group=") + flags+=("--cache-dir=") + flags+=("--certificate-authority=") + flags+=("--client-certificate=") + flags+=("--client-key=") + flags+=("--cluster=") + flags+=("--config=") + flags+=("--context=") + flags+=("--insecure-skip-tls-verify") + flags+=("--loglevel=") + flags+=("--logspec=") + flags+=("--match-server-version") + flags+=("--namespace=") + flags_with_completion+=("--namespace") + flags_completion+=("__oc_get_namespaces") + two_word_flags+=("-n") + flags_with_completion+=("-n") + flags_completion+=("__oc_get_namespaces") + flags+=("--request-timeout=") + flags+=("--server=") + two_word_flags+=("-s") + flags+=("--token=") + flags+=("--user=") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + _oc_image_mirror() { last_command="oc_image_mirror" @@ -11884,6 +11939,7 @@ _oc_image() last_command="oc_image" commands=() commands+=("append") + commands+=("extract") commands+=("mirror") flags=() diff --git a/contrib/completions/zsh/oc b/contrib/completions/zsh/oc index 7ed42a23bb85..9f4d086f5182 100644 --- a/contrib/completions/zsh/oc +++ b/contrib/completions/zsh/oc @@ -11959,6 +11959,61 @@ _oc_image_append() noun_aliases=() } +_oc_image_extract() +{ + last_command="oc_image_extract" + commands=() + + flags=() + two_word_flags=() + local_nonpersistent_flags=() + flags_with_completion=() + flags_completion=() + + flags+=("--created-at=") + local_nonpersistent_flags+=("--created-at=") + flags+=("--dry-run") + local_nonpersistent_flags+=("--dry-run") + flags+=("--filter-by-os=") + local_nonpersistent_flags+=("--filter-by-os=") + flags+=("--force") + local_nonpersistent_flags+=("--force") + flags+=("--insecure") + local_nonpersistent_flags+=("--insecure") + flags+=("--max-per-registry=") + local_nonpersistent_flags+=("--max-per-registry=") + flags+=("--only-files") + local_nonpersistent_flags+=("--only-files") + flags+=("--as=") + flags+=("--as-group=") + flags+=("--cache-dir=") + flags+=("--certificate-authority=") + flags+=("--client-certificate=") + flags+=("--client-key=") + flags+=("--cluster=") + flags+=("--config=") + flags+=("--context=") + flags+=("--insecure-skip-tls-verify") + flags+=("--loglevel=") + flags+=("--logspec=") + flags+=("--match-server-version") + flags+=("--namespace=") + flags_with_completion+=("--namespace") + flags_completion+=("__oc_get_namespaces") + two_word_flags+=("-n") + flags_with_completion+=("-n") + flags_completion+=("__oc_get_namespaces") + flags+=("--request-timeout=") + flags+=("--server=") + two_word_flags+=("-s") + flags+=("--token=") + flags+=("--user=") + + must_have_one_flag=() + must_have_one_noun=() + noun_aliases=() +} + _oc_image_mirror() { last_command="oc_image_mirror" @@ -12026,6 +12081,7 @@ _oc_image() last_command="oc_image" commands=() commands+=("append") + commands+=("extract") commands+=("mirror") flags=() diff --git a/docs/man/man1/.files_generated_oc b/docs/man/man1/.files_generated_oc index cb6f5ea2ab17..1d1e2f4d756d 100644 --- a/docs/man/man1/.files_generated_oc +++ b/docs/man/man1/.files_generated_oc @@ -231,6 +231,7 @@ oc-extract.1 oc-get.1 oc-idle.1 oc-image-append.1 +oc-image-extract.1 oc-image-mirror.1 oc-image.1 oc-import-app.json.1 diff --git a/docs/man/man1/oc-image-extract.1 b/docs/man/man1/oc-image-extract.1 new file mode 100644 index 000000000000..b6fd7a0f9896 --- /dev/null +++ b/docs/man/man1/oc-image-extract.1 @@ -0,0 +1,3 @@ +This file is autogenerated, but we've stopped checking such files into the +repository to reduce the need for rebases. Please run hack/generate-docs.sh to +populate this file. diff --git a/pkg/oc/cli/image/archive/archive.go b/pkg/oc/cli/image/archive/archive.go new file mode 100644 index 000000000000..2f7b8b06997e --- /dev/null +++ b/pkg/oc/cli/image/archive/archive.go @@ -0,0 +1,438 @@ +package archive + +import ( + "archive/tar" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "strings" + "syscall" + + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/idtools" + "github.com/docker/docker/pkg/pools" + "github.com/docker/docker/pkg/system" +) + +type ( + // Compression is the state represents if compressed or not. + Compression int + // WhiteoutFormat is the format of whiteouts unpacked + WhiteoutFormat int + + // TarOptions wraps the tar options. + TarOptions struct { + IncludeFiles []string + ExcludePatterns []string + Compression Compression + NoLchown bool + // REMOVED: use remap instead + //UIDMaps []idtools.IDMap + //GIDMaps []idtools.IDMap + ChownOpts *idtools.IDPair + IncludeSourceDir bool + // WhiteoutFormat is the expected on disk format for whiteout files. + // This format will be converted to the standard format on pack + // and from the standard format on unpack. + WhiteoutFormat WhiteoutFormat + // When unpacking, specifies whether overwriting a directory with a + // non-directory is allowed and vice versa. + NoOverwriteDirNonDir bool + // For each include when creating an archive, the included name will be + // replaced with the matching name from this map. + RebaseNames map[string]string + InUserNS bool + + // ADDED: allow bypassing chown + // If false, no chown will be performed + Chown bool + + AlterHeaders AlterHeader + } +) + +// breakoutError is used to differentiate errors related to breaking out +// When testing archive breakout in the unit tests, this error is expected +// in order for the test to pass. +type breakoutError error + +type tarWhiteoutConverter interface { + ConvertWrite(*tar.Header, string, os.FileInfo) (*tar.Header, error) + ConvertRead(*tar.Header, string) (bool, error) +} + +type AlterHeader interface { + Alter(*tar.Header) (bool, error) +} + +type RemapIDs struct { + mappings *idtools.IDMappings +} + +func (r RemapIDs) Alter(hdr *tar.Header) (bool, error) { + ids, err := r.mappings.ToHost(idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid}) + hdr.Uid, hdr.Gid = ids.UID, ids.GID + return true, err +} + +// ApplyLayer is copied from github.com/docker/docker/pkg/archive +func ApplyLayer(dest string, layer io.Reader, options *TarOptions) (int64, error) { + dest = filepath.Clean(dest) + var err error + layer, err = archive.DecompressStream(layer) + if err != nil { + return 0, err + } + return unpackLayer(dest, layer, options) +} + +// unpackLayer is copied from github.com/docker/docker/pkg/archive +// unpackLayer unpack `layer` to a `dest`. The stream `layer` can be +// compressed or uncompressed. +// Returns the size in bytes of the contents of the layer. +func unpackLayer(dest string, layer io.Reader, options *TarOptions) (size int64, err error) { + tr := tar.NewReader(layer) + trBuf := pools.BufioReader32KPool.Get(tr) + defer pools.BufioReader32KPool.Put(trBuf) + + var dirs []*tar.Header + unpackedPaths := make(map[string]struct{}) + + if options == nil { + options = &TarOptions{Chown: true} + } + if options.ExcludePatterns == nil { + options.ExcludePatterns = []string{} + } + // idMappings := idtools.NewIDMappingsFromMaps(options.UIDMaps, options.GIDMaps) + + aufsTempdir := "" + aufsHardlinks := make(map[string]*tar.Header) + + // Iterate through the files in the archive. + for { + hdr, err := tr.Next() + if err == io.EOF { + // end of tar archive + break + } + if err != nil { + return 0, err + } + + size += hdr.Size + + // Normalize name, for safety and for a simple is-root check + hdr.Name = filepath.Clean(hdr.Name) + + if options.AlterHeaders != nil { + ok, err := options.AlterHeaders.Alter(hdr) + if err != nil { + return 0, err + } + if !ok { + continue + } + } + + // Windows does not support filenames with colons in them. Ignore + // these files. This is not a problem though (although it might + // appear that it is). Let's suppose a client is running docker pull. + // The daemon it points to is Windows. Would it make sense for the + // client to be doing a docker pull Ubuntu for example (which has files + // with colons in the name under /usr/share/man/man3)? No, absolutely + // not as it would really only make sense that they were pulling a + // Windows image. However, for development, it is necessary to be able + // to pull Linux images which are in the repository. + // + // TODO Windows. Once the registry is aware of what images are Windows- + // specific or Linux-specific, this warning should be changed to an error + // to cater for the situation where someone does manage to upload a Linux + // image but have it tagged as Windows inadvertently. + if runtime.GOOS == "windows" { + if strings.Contains(hdr.Name, ":") { + continue + } + } + + // Note as these operations are platform specific, so must the slash be. + if !strings.HasSuffix(hdr.Name, string(os.PathSeparator)) { + // Not the root directory, ensure that the parent directory exists. + // This happened in some tests where an image had a tarfile without any + // parent directories. + parent := filepath.Dir(hdr.Name) + parentPath := filepath.Join(dest, parent) + + if _, err := os.Lstat(parentPath); err != nil && os.IsNotExist(err) { + err = system.MkdirAll(parentPath, 0600, "") + if err != nil { + return 0, err + } + } + } + + // Skip AUFS metadata dirs + if strings.HasPrefix(hdr.Name, archive.WhiteoutMetaPrefix) { + // Regular files inside /.wh..wh.plnk can be used as hardlink targets + // We don't want this directory, but we need the files in them so that + // such hardlinks can be resolved. + if strings.HasPrefix(hdr.Name, archive.WhiteoutLinkDir) && hdr.Typeflag == tar.TypeReg { + basename := filepath.Base(hdr.Name) + aufsHardlinks[basename] = hdr + if aufsTempdir == "" { + if aufsTempdir, err = ioutil.TempDir("", "dockerplnk"); err != nil { + return 0, err + } + defer os.RemoveAll(aufsTempdir) + } + if err := createTarFile(filepath.Join(aufsTempdir, basename), dest, hdr, tr, options.Chown, options.ChownOpts, options.InUserNS); err != nil { + return 0, err + } + } + + if hdr.Name != archive.WhiteoutOpaqueDir { + continue + } + } + + path := filepath.Join(dest, hdr.Name) + rel, err := filepath.Rel(dest, path) + if err != nil { + return 0, err + } + + // Note as these operations are platform specific, so must the slash be. + if strings.HasPrefix(rel, ".."+string(os.PathSeparator)) { + return 0, breakoutError(fmt.Errorf("%q is outside of %q", hdr.Name, dest)) + } + base := filepath.Base(path) + + if strings.HasPrefix(base, archive.WhiteoutPrefix) { + dir := filepath.Dir(path) + if base == archive.WhiteoutOpaqueDir { + _, err := os.Lstat(dir) + if err != nil { + return 0, err + } + err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + if os.IsNotExist(err) { + err = nil // parent was deleted + } + return err + } + if path == dir { + return nil + } + if _, exists := unpackedPaths[path]; !exists { + err := os.RemoveAll(path) + return err + } + return nil + }) + if err != nil { + return 0, err + } + } else { + originalBase := base[len(archive.WhiteoutPrefix):] + originalPath := filepath.Join(dir, originalBase) + if err := os.RemoveAll(originalPath); err != nil { + return 0, err + } + } + } else { + // If path exits we almost always just want to remove and replace it. + // The only exception is when it is a directory *and* the file from + // the layer is also a directory. Then we want to merge them (i.e. + // just apply the metadata from the layer). + if fi, err := os.Lstat(path); err == nil { + if !(fi.IsDir() && hdr.Typeflag == tar.TypeDir) { + if err := os.RemoveAll(path); err != nil { + return 0, err + } + } + } + + trBuf.Reset(tr) + srcData := io.Reader(trBuf) + srcHdr := hdr + + // Hard links into /.wh..wh.plnk don't work, as we don't extract that directory, so + // we manually retarget these into the temporary files we extracted them into + if hdr.Typeflag == tar.TypeLink && strings.HasPrefix(filepath.Clean(hdr.Linkname), archive.WhiteoutLinkDir) { + linkBasename := filepath.Base(hdr.Linkname) + srcHdr = aufsHardlinks[linkBasename] + if srcHdr == nil { + return 0, fmt.Errorf("Invalid aufs hardlink") + } + tmpFile, err := os.Open(filepath.Join(aufsTempdir, linkBasename)) + if err != nil { + return 0, err + } + defer tmpFile.Close() + srcData = tmpFile + } + + // if err := remapIDs(idMappings, srcHdr); err != nil { + // return 0, err + // } + + if err := createTarFile(path, dest, srcHdr, srcData, options.Chown, options.ChownOpts, options.InUserNS); err != nil { + return 0, err + } + + // Directory mtimes must be handled at the end to avoid further + // file creation in them to modify the directory mtime + if hdr.Typeflag == tar.TypeDir { + dirs = append(dirs, hdr) + } + unpackedPaths[path] = struct{}{} + } + } + + for _, hdr := range dirs { + path := filepath.Join(dest, hdr.Name) + if err := system.Chtimes(path, hdr.AccessTime, hdr.ModTime); err != nil { + return 0, err + } + } + + return size, nil +} + +func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, Lchown bool, chownOpts *idtools.IDPair, inUserns bool) error { + // hdr.Mode is in linux format, which we can use for sycalls, + // but for os.Foo() calls we need the mode converted to os.FileMode, + // so use hdrInfo.Mode() (they differ for e.g. setuid bits) + hdrInfo := hdr.FileInfo() + + switch hdr.Typeflag { + case tar.TypeDir: + // Create directory unless it exists as a directory already. + // In that case we just want to merge the two + if fi, err := os.Lstat(path); !(err == nil && fi.IsDir()) { + if err := os.Mkdir(path, hdrInfo.Mode()); err != nil { + return err + } + } + + case tar.TypeReg, tar.TypeRegA: + // Source is regular file. We use system.OpenFileSequential to use sequential + // file access to avoid depleting the standby list on Windows. + // On Linux, this equates to a regular os.OpenFile + file, err := system.OpenFileSequential(path, os.O_CREATE|os.O_WRONLY, hdrInfo.Mode()) + if err != nil { + return err + } + if _, err := io.Copy(file, reader); err != nil { + file.Close() + return err + } + file.Close() + + case tar.TypeBlock, tar.TypeChar: + if inUserns { // cannot create devices in a userns + return nil + } + // Handle this is an OS-specific way + if err := handleTarTypeBlockCharFifo(hdr, path); err != nil { + return err + } + + case tar.TypeFifo: + // Handle this is an OS-specific way + if err := handleTarTypeBlockCharFifo(hdr, path); err != nil { + return err + } + + case tar.TypeLink: + targetPath := filepath.Join(extractDir, hdr.Linkname) + // check for hardlink breakout + if !strings.HasPrefix(targetPath, extractDir) { + return breakoutError(fmt.Errorf("invalid hardlink %q -> %q", targetPath, hdr.Linkname)) + } + if err := os.Link(targetPath, path); err != nil { + return err + } + + case tar.TypeSymlink: + // path -> hdr.Linkname = targetPath + // e.g. /extractDir/path/to/symlink -> ../2/file = /extractDir/path/2/file + targetPath := filepath.Join(filepath.Dir(path), hdr.Linkname) + + // the reason we don't need to check symlinks in the path (with FollowSymlinkInScope) is because + // that symlink would first have to be created, which would be caught earlier, at this very check: + if !strings.HasPrefix(targetPath, extractDir) { + return breakoutError(fmt.Errorf("invalid symlink %q -> %q", path, hdr.Linkname)) + } + if err := os.Symlink(hdr.Linkname, path); err != nil { + return err + } + + case tar.TypeXGlobalHeader: + return nil + + default: + return fmt.Errorf("unhandled tar header type %d", hdr.Typeflag) + } + + // Lchown is not supported on Windows. + if Lchown && runtime.GOOS != "windows" { + if chownOpts == nil { + chownOpts = &idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid} + } + if err := os.Lchown(path, chownOpts.UID, chownOpts.GID); err != nil { + return err + } + } + + var errors []string + for key, value := range hdr.Xattrs { + if err := system.Lsetxattr(path, key, []byte(value), 0); err != nil { + if err == syscall.ENOTSUP { + // We ignore errors here because not all graphdrivers support + // xattrs *cough* old versions of AUFS *cough*. However only + // ENOTSUP should be emitted in that case, otherwise we still + // bail. + errors = append(errors, err.Error()) + continue + } + return err + } + + } + + // There is no LChmod, so ignore mode for symlink. Also, this + // must happen after chown, as that can modify the file mode + if err := handleLChmod(hdr, path, hdrInfo); err != nil { + return err + } + + aTime := hdr.AccessTime + if aTime.Before(hdr.ModTime) { + // Last access time should never be before last modified time. + aTime = hdr.ModTime + } + + // system.Chtimes doesn't support a NOFOLLOW flag atm + if hdr.Typeflag == tar.TypeLink { + if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) { + if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil { + return err + } + } + } else if hdr.Typeflag != tar.TypeSymlink { + if err := system.Chtimes(path, aTime, hdr.ModTime); err != nil { + return err + } + } else { + ts := []syscall.Timespec{timeToTimespec(aTime), timeToTimespec(hdr.ModTime)} + if err := system.LUtimesNano(path, ts); err != nil && err != system.ErrNotSupportedPlatform { + return err + } + } + return nil +} diff --git a/pkg/oc/cli/image/archive/archive_linux.go b/pkg/oc/cli/image/archive/archive_linux.go new file mode 100644 index 000000000000..52351954d645 --- /dev/null +++ b/pkg/oc/cli/image/archive/archive_linux.go @@ -0,0 +1,93 @@ +package archive + +import ( + "archive/tar" + "os" + "path/filepath" + "strings" + + "github.com/docker/docker/pkg/archive" + "github.com/docker/docker/pkg/system" + "golang.org/x/sys/unix" +) + +func getWhiteoutConverter(format archive.WhiteoutFormat) tarWhiteoutConverter { + if format == archive.OverlayWhiteoutFormat { + return overlayWhiteoutConverter{} + } + return nil +} + +type overlayWhiteoutConverter struct{} + +func (overlayWhiteoutConverter) ConvertWrite(hdr *tar.Header, path string, fi os.FileInfo) (wo *tar.Header, err error) { + // convert whiteouts to AUFS format + if fi.Mode()&os.ModeCharDevice != 0 && hdr.Devmajor == 0 && hdr.Devminor == 0 { + // we just rename the file and make it normal + dir, filename := filepath.Split(hdr.Name) + hdr.Name = filepath.Join(dir, archive.WhiteoutPrefix+filename) + hdr.Mode = 0600 + hdr.Typeflag = tar.TypeReg + hdr.Size = 0 + } + + if fi.Mode()&os.ModeDir != 0 { + // convert opaque dirs to AUFS format by writing an empty file with the prefix + opaque, err := system.Lgetxattr(path, "trusted.overlay.opaque") + if err != nil { + return nil, err + } + if len(opaque) == 1 && opaque[0] == 'y' { + if hdr.Xattrs != nil { + delete(hdr.Xattrs, "trusted.overlay.opaque") + } + + // create a header for the whiteout file + // it should inherit some properties from the parent, but be a regular file + wo = &tar.Header{ + Typeflag: tar.TypeReg, + Mode: hdr.Mode & int64(os.ModePerm), + Name: filepath.Join(hdr.Name, archive.WhiteoutOpaqueDir), + Size: 0, + Uid: hdr.Uid, + Uname: hdr.Uname, + Gid: hdr.Gid, + Gname: hdr.Gname, + AccessTime: hdr.AccessTime, + ChangeTime: hdr.ChangeTime, + } + } + } + + return +} + +func (overlayWhiteoutConverter) ConvertRead(hdr *tar.Header, path string) (bool, error) { + base := filepath.Base(path) + dir := filepath.Dir(path) + + // if a directory is marked as opaque by the AUFS special file, we need to translate that to overlay + if base == archive.WhiteoutOpaqueDir { + err := unix.Setxattr(dir, "trusted.overlay.opaque", []byte{'y'}, 0) + // don't write the file itself + return false, err + } + + // if a file was deleted and we are using overlay, we need to create a character device + if strings.HasPrefix(base, archive.WhiteoutPrefix) { + originalBase := base[len(archive.WhiteoutPrefix):] + originalPath := filepath.Join(dir, originalBase) + + if err := unix.Mknod(originalPath, unix.S_IFCHR, 0); err != nil { + return false, err + } + if err := os.Chown(originalPath, hdr.Uid, hdr.Gid); err != nil { + return false, err + } + + // don't write the file itself + return false, nil + } + + return true, nil +} diff --git a/pkg/oc/cli/image/archive/archive_other.go b/pkg/oc/cli/image/archive/archive_other.go new file mode 100644 index 000000000000..9069c03a398a --- /dev/null +++ b/pkg/oc/cli/image/archive/archive_other.go @@ -0,0 +1,9 @@ +// +build !linux + +package archive + +import "github.com/docker/docker/pkg/archive" + +func getWhiteoutConverter(format archive.WhiteoutFormat) tarWhiteoutConverter { + return nil +} diff --git a/pkg/oc/cli/image/archive/archive_unix.go b/pkg/oc/cli/image/archive/archive_unix.go new file mode 100644 index 000000000000..9eb92b306515 --- /dev/null +++ b/pkg/oc/cli/image/archive/archive_unix.go @@ -0,0 +1,78 @@ +// +build !windows + +package archive + +import ( + "archive/tar" + "bufio" + "fmt" + "os" + + "github.com/docker/docker/pkg/system" + "golang.org/x/sys/unix" +) + +// runningInUserNS detects whether we are currently running in a user namespace. +// Copied from github.com/opencontainers/runc/libcontainer/system +func runningInUserNS() bool { + file, err := os.Open("/proc/self/uid_map") + if err != nil { + // This kernel-provided file only exists if user namespaces are supported + return false + } + defer file.Close() + + buf := bufio.NewReader(file) + l, _, err := buf.ReadLine() + if err != nil { + return false + } + + line := string(l) + var a, b, c int64 + fmt.Sscanf(line, "%d %d %d", &a, &b, &c) + /* + * We assume we are in the initial user namespace if we have a full + * range - 4294967295 uids starting at uid 0. + */ + if a == 0 && b == 0 && c == 4294967295 { + return false + } + return true +} + +// handleTarTypeBlockCharFifo is an OS-specific helper function used by +// createTarFile to handle the following types of header: Block; Char; Fifo +func handleTarTypeBlockCharFifo(hdr *tar.Header, path string) error { + if runningInUserNS() { + // cannot create a device if running in user namespace + return nil + } + + mode := uint32(hdr.Mode & 07777) + switch hdr.Typeflag { + case tar.TypeBlock: + mode |= unix.S_IFBLK + case tar.TypeChar: + mode |= unix.S_IFCHR + case tar.TypeFifo: + mode |= unix.S_IFIFO + } + + return system.Mknod(path, mode, int(system.Mkdev(hdr.Devmajor, hdr.Devminor))) +} + +func handleLChmod(hdr *tar.Header, path string, hdrInfo os.FileInfo) error { + if hdr.Typeflag == tar.TypeLink { + if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) { + if err := os.Chmod(path, hdrInfo.Mode()); err != nil { + return err + } + } + } else if hdr.Typeflag != tar.TypeSymlink { + if err := os.Chmod(path, hdrInfo.Mode()); err != nil { + return err + } + } + return nil +} diff --git a/pkg/oc/cli/image/archive/archive_windows.go b/pkg/oc/cli/image/archive/archive_windows.go new file mode 100644 index 000000000000..e9d83376ea5e --- /dev/null +++ b/pkg/oc/cli/image/archive/archive_windows.go @@ -0,0 +1,18 @@ +// +build windows + +package archive + +import ( + "archive/tar" + "os" +) + +// handleTarTypeBlockCharFifo is an OS-specific helper function used by +// createTarFile to handle the following types of header: Block; Char; Fifo +func handleTarTypeBlockCharFifo(hdr *tar.Header, path string) error { + return nil +} + +func handleLChmod(hdr *tar.Header, path string, hdrInfo os.FileInfo) error { + return nil +} diff --git a/pkg/oc/cli/image/archive/time_linux.go b/pkg/oc/cli/image/archive/time_linux.go new file mode 100644 index 000000000000..3448569b1ebb --- /dev/null +++ b/pkg/oc/cli/image/archive/time_linux.go @@ -0,0 +1,16 @@ +package archive + +import ( + "syscall" + "time" +) + +func timeToTimespec(time time.Time) (ts syscall.Timespec) { + if time.IsZero() { + // Return UTIME_OMIT special value + ts.Sec = 0 + ts.Nsec = ((1 << 30) - 2) + return + } + return syscall.NsecToTimespec(time.UnixNano()) +} diff --git a/pkg/oc/cli/image/archive/time_unsupported.go b/pkg/oc/cli/image/archive/time_unsupported.go new file mode 100644 index 000000000000..e85aac054080 --- /dev/null +++ b/pkg/oc/cli/image/archive/time_unsupported.go @@ -0,0 +1,16 @@ +// +build !linux + +package archive + +import ( + "syscall" + "time" +) + +func timeToTimespec(time time.Time) (ts syscall.Timespec) { + nsec := int64(0) + if !time.IsZero() { + nsec = time.UnixNano() + } + return syscall.NsecToTimespec(nsec) +} diff --git a/pkg/oc/cli/image/extract/extract.go b/pkg/oc/cli/image/extract/extract.go new file mode 100644 index 000000000000..43fc25302d48 --- /dev/null +++ b/pkg/oc/cli/image/extract/extract.go @@ -0,0 +1,564 @@ +package extract + +import ( + "archive/tar" + "context" + "fmt" + "io" + "math" + "os" + "os/user" + "path" + "path/filepath" + "regexp" + "strconv" + "strings" + + "github.com/golang/glog" + "github.com/spf13/cobra" + + "github.com/docker/distribution" + dockerarchive "github.com/docker/docker/pkg/archive" + + "k8s.io/client-go/rest" + "k8s.io/kubernetes/pkg/kubectl/cmd/templates" + kcmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util" + "k8s.io/kubernetes/pkg/kubectl/genericclioptions" + + imagereference "github.com/openshift/origin/pkg/image/apis/image/reference" + "github.com/openshift/origin/pkg/image/registryclient" + "github.com/openshift/origin/pkg/image/registryclient/dockercredentials" + "github.com/openshift/origin/pkg/oc/cli/image/archive" + imagemanifest "github.com/openshift/origin/pkg/oc/cli/image/manifest" +) + +var ( + desc = templates.LongDesc(` + Extract the contents of an image to disk + + Download an image or parts of an image to the filesystem. Allows users to access the + contents of images without requiring a container runtime engine running. + + Describe what to download by passing arguments to the command - each argument may + have one, two, or three sections that describe the source image, an optional filter on + the contents of the image, and the location: + + IMAGE[[=FROM]=TO] + + The image is a standard image pull spec. The from section may be either a file, a + directory (ends with a '/'), or a file pattern within a directory. The to section + is a directory to extract to. If to is omitted the current directory is assumed. + + If the specified image supports multiple operating systems, the image that matches the + current operating system will be chosen. Otherwise you must pass --filter-by-os to + select the desired image. + + You may further qualify the image by adding a layer selector to the end of the image + string to only extract specific layers within an image. The supported selectors are: + + [] - select the layer at the provided index (zero-indexed) + [,] - select layers by index, exclusive + [~] - select the layer with the matching prefix or return an error + + Negative indices are counted from the end of the list, e.g. [-1] selects the last + layer. + + Experimental: This command is under active development and may change without notice.`) + + example = templates.Examples(` +# Extract the busybox image into the current directory +%[1]s docker.io/library/busybox:latest + +# Extract the busybox image to a temp directory (must exist) +%[1]s docker.io/library/busybox:latest=/tmp/busybox + +# Extract a single file from the image into the current directory +%[1]s docker.io/library/centos:7=/bin/bash=. + +# Extract all .repo files from the image's /etc/yum.repos.d/ folder. +%[1]s docker.io/library/centos:7=/etc/yum.repos.d/*.repo=. + +# Extract the last layer in the image +%[1]s docker.io/library/centos:7[-1] + +# Extract the first three layers of the image +%[1]s docker.io/library/centos:7[:3] + +# Extract the last three layers of the image +%[1]s docker.io/library/centos:7[-3:] +`) +) + +type Options struct { + Mappings []Mapping + + CreatedAt string + + OnlyFiles bool + + FilterOptions imagemanifest.FilterOptions + + MaxPerRegistry int + + DryRun bool + Insecure bool + Force bool + + genericclioptions.IOStreams +} + +func NewOptions(streams genericclioptions.IOStreams) *Options { + return &Options{ + IOStreams: streams, + MaxPerRegistry: 3, + } +} + +// New creates a new command +func New(name string, streams genericclioptions.IOStreams) *cobra.Command { + o := NewOptions(streams) + + cmd := &cobra.Command{ + Use: "extract", + Short: "Copy files from an image to the filesystem", + Long: desc, + Example: fmt.Sprintf(example, name), + Run: func(c *cobra.Command, args []string) { + kcmdutil.CheckErr(o.Complete(c, args)) + kcmdutil.CheckErr(o.Run()) + }, + } + + flag := cmd.Flags() + o.FilterOptions.Bind(flag) + + flag.BoolVar(&o.DryRun, "dry-run", o.DryRun, "Print the actions that would be taken and exit without writing to the destination.") + flag.BoolVar(&o.Insecure, "insecure", o.Insecure, "Allow push and pull operations to registries to be made over HTTP") + + flag.StringVar(&o.CreatedAt, "created-at", o.CreatedAt, "The creation date for this image, in RFC3339 format or milliseconds from the Unix epoch.") + + flag.BoolVar(&o.OnlyFiles, "only-files", o.OnlyFiles, "Only extract regular files and directories from the image.") + + flag.BoolVar(&o.Force, "force", o.Force, "If set, the command will attempt to upload all layers instead of skipping those that are already uploaded.") + flag.IntVar(&o.MaxPerRegistry, "max-per-registry", o.MaxPerRegistry, "Number of concurrent requests allowed per registry.") + + return cmd +} + +type LayerFilter interface { + Filter(layers []distribution.Descriptor) ([]distribution.Descriptor, error) +} + +type Mapping struct { + Image string + ImageRef imagereference.DockerImageReference + + LayerFilter LayerFilter + + From string + To string +} + +func parseMappings(args []string) ([]Mapping, error) { + layerFilter := regexp.MustCompile(`^(.*)\[([^\]]*)\](.*)$`) + + var mappings []Mapping + for _, arg := range args { + parts := strings.SplitN(arg, "=", 3) + var mapping Mapping + switch len(parts) { + case 1: + mappings = append(mappings, Mapping{From: parts[0], To: "."}) + case 2: + mapping = Mapping{Image: parts[0], To: parts[1]} + case 3: + mapping = Mapping{Image: parts[0], From: parts[1], To: parts[2]} + } + if matches := layerFilter.FindStringSubmatch(mapping.Image); len(matches) > 0 { + if len(matches[1]) == 0 || len(matches[2]) == 0 || len(matches[3]) != 0 { + return nil, fmt.Errorf("layer selectors must be of the form IMAGE[\\d:\\d]") + } + mapping.Image = matches[1] + var err error + mapping.LayerFilter, err = parseLayerFilter(matches[2]) + if err != nil { + return nil, err + } + } + if len(mapping.From) > 1 { + mapping.From = strings.TrimPrefix(mapping.From, "/") + } + if len(mapping.To) > 0 { + fi, err := os.Stat(mapping.To) + if err != nil { + return nil, fmt.Errorf("invalid argument: %s", err) + } + if !fi.IsDir() { + return nil, fmt.Errorf("invalid argument: %s is not a directory", arg) + } + } + src, err := imagereference.Parse(mapping.Image) + if err != nil { + return nil, err + } + if len(src.Tag) == 0 && len(src.ID) == 0 { + return nil, fmt.Errorf("source image must point to an image ID or image tag") + } + mapping.ImageRef = src + mappings = append(mappings, mapping) + } + return mappings, nil +} + +func (o *Options) Complete(cmd *cobra.Command, args []string) error { + if err := o.FilterOptions.Complete(cmd.Flags()); err != nil { + return err + } + + if len(args) == 0 { + return fmt.Errorf("you must specify at least one argument in IMAGE, IMAGE=DST, or IMAGE=SRC=DST form") + } + + var err error + o.Mappings, err = parseMappings(args) + if err != nil { + return err + } + return nil +} + +func (o *Options) Run() error { + preserveOwnership := false + u, err := user.Current() + if err != nil { + fmt.Fprintf(os.Stderr, "warning: Could not load current user information: %v\n", err) + } + if u != nil { + if uid, err := strconv.Atoi(u.Uid); err == nil && uid == 0 { + preserveOwnership = true + } + } + + rt, err := rest.TransportFor(&rest.Config{}) + if err != nil { + return err + } + insecureRT, err := rest.TransportFor(&rest.Config{TLSClientConfig: rest.TLSClientConfig{Insecure: true}}) + if err != nil { + return err + } + creds := dockercredentials.NewLocal() + ctx := context.Background() + fromContext := registryclient.NewContext(rt, insecureRT).WithCredentials(creds) + + for _, mapping := range o.Mappings { + from := mapping.ImageRef + + repo, err := fromContext.Repository(ctx, from.DockerClientDefaults().RegistryURL(), from.RepositoryName(), o.Insecure) + if err != nil { + return err + } + + srcManifest, _, location, err := imagemanifest.FirstManifest(ctx, from, repo, o.FilterOptions.Include) + if err != nil { + return fmt.Errorf("unable to read image %s: %v", from, err) + } + + _, layers, err := imagemanifest.ManifestToImageConfig(ctx, srcManifest, repo.Blobs(ctx), location) + if err != nil { + return fmt.Errorf("unable to parse image %s: %v", from, err) + } + + var alter alterations + if o.OnlyFiles { + alter = append(alter, filesOnly{}) + } + if len(mapping.From) > 0 { + switch { + case strings.HasSuffix(mapping.From, "/"): + alter = append(alter, newCopyFromDirectory(mapping.From)) + default: + name, parent := path.Base(mapping.From), path.Dir(mapping.From) + if name == "." || parent == "." { + return fmt.Errorf("unexpected directory from mapping %s", mapping.From) + } + alter = append(alter, newCopyFromPattern(parent, name)) + } + } + + filteredLayers := layers + if mapping.LayerFilter != nil { + filteredLayers, err = mapping.LayerFilter.Filter(filteredLayers) + if err != nil { + return fmt.Errorf("unable to filter layers for %s: %v", from, err) + } + } + + glog.V(5).Infof("Extracting from layers\n:%#v", filteredLayers) + + for i := range filteredLayers { + layer := &layers[i] + + err := func() error { + fromBlobs := repo.Blobs(ctx) + + // source + r, err := fromBlobs.Open(ctx, layer.Digest) + if err != nil { + return fmt.Errorf("unable to access the source layer %s: %v", layer.Digest, err) + } + defer r.Close() + + options := &archive.TarOptions{ + AlterHeaders: alter, + Chown: preserveOwnership, + } + + if o.DryRun { + return printLayer(os.Stdout, r, mapping.To, options) + } + + glog.V(4).Infof("Extracting layer %s with options %#v", layer.Digest, options) + if _, err := archive.ApplyLayer(mapping.To, r, options); err != nil { + return err + } + return nil + }() + if err != nil { + return err + } + } + } + return nil +} + +func printLayer(w io.Writer, r io.Reader, path string, options *archive.TarOptions) error { + rc, err := dockerarchive.DecompressStream(r) + if err != nil { + return err + } + defer rc.Close() + tr := tar.NewReader(rc) + for { + hdr, err := tr.Next() + if err != nil { + if err == io.EOF { + return nil + } + return err + } + if options.AlterHeaders != nil { + ok, err := options.AlterHeaders.Alter(hdr) + if err != nil { + return err + } + if !ok { + continue + } + } + if len(hdr.Name) == 0 { + continue + } + switch hdr.Typeflag { + case tar.TypeDir: + fmt.Fprintf(w, "Creating directory %s\n", filepath.Join(path, hdr.Name)) + case tar.TypeReg, tar.TypeRegA: + fmt.Fprintf(w, "Creating file %s\n", filepath.Join(path, hdr.Name)) + case tar.TypeLink: + fmt.Fprintf(w, "Link %s to %s\n", hdr.Name, filepath.Join(path, hdr.Linkname)) + case tar.TypeSymlink: + fmt.Fprintf(w, "Symlink %s to %s\n", hdr.Name, filepath.Join(path, hdr.Linkname)) + default: + fmt.Fprintf(w, "Extracting %s with type %0x\n", filepath.Join(path, hdr.Name), hdr.Typeflag) + } + } +} + +type alterations []archive.AlterHeader + +func (a alterations) Alter(hdr *tar.Header) (bool, error) { + for _, item := range a { + ok, err := item.Alter(hdr) + if err != nil { + return false, err + } + if !ok { + return false, nil + } + } + return true, nil +} + +type copyFromDirectory struct { + From string +} + +func newCopyFromDirectory(from string) archive.AlterHeader { + if !strings.HasSuffix(from, "/") { + from = from + "/" + } + return ©FromDirectory{From: from} +} + +func (n *copyFromDirectory) Alter(hdr *tar.Header) (bool, error) { + return changeTarEntryParent(hdr, n.From), nil +} + +type copyFromPattern struct { + Base string + Name string +} + +func newCopyFromPattern(dir, name string) archive.AlterHeader { + if !strings.HasSuffix(dir, "/") { + dir = dir + "/" + } + return ©FromPattern{Base: dir, Name: name} +} + +func (n *copyFromPattern) Alter(hdr *tar.Header) (bool, error) { + if !changeTarEntryParent(hdr, n.Base) { + return false, nil + } + matchName := hdr.Name + if i := strings.Index(matchName, "/"); i != -1 { + matchName = matchName[:i] + } + if ok, err := path.Match(n.Name, matchName); !ok || err != nil { + glog.V(5).Infof("Excluded %s due to filter %s", hdr.Name, n.Name) + return false, err + } + return true, nil +} + +func changeTarEntryParent(hdr *tar.Header, from string) bool { + if !strings.HasPrefix(hdr.Name, from) { + return false + } + if len(hdr.Linkname) > 0 { + if strings.HasPrefix(hdr.Linkname, from) { + hdr.Linkname = strings.TrimPrefix(hdr.Linkname, from) + glog.V(5).Infof("Updated link to %s", hdr.Linkname) + } else { + glog.V(4).Infof("Name %s won't correctly point to %s outside of %s", hdr.Name, hdr.Linkname, from) + } + } + hdr.Name = strings.TrimPrefix(hdr.Name, from) + glog.V(5).Infof("Updated name %s", hdr.Name) + return true +} + +type filesOnly struct { +} + +func (_ filesOnly) Alter(hdr *tar.Header) (bool, error) { + switch hdr.Typeflag { + case tar.TypeReg, tar.TypeRegA, tar.TypeDir: + return true, nil + default: + return false, nil + } +} + +func parseLayerFilter(s string) (LayerFilter, error) { + if strings.HasPrefix(s, "~") { + s = s[1:] + return &prefixLayerSelector{Prefix: s}, nil + } + + if strings.Contains(s, ":") { + l := &indexLayerSelector{From: 0, To: math.MaxInt32} + parts := strings.SplitN(s, ":", 2) + if len(parts[0]) > 0 { + i, err := strconv.Atoi(parts[0]) + if err != nil { + return nil, fmt.Errorf("[from:to] must have valid numbers: %v", err) + } + l.From = int32(i) + } + if len(parts[1]) > 0 { + i, err := strconv.Atoi(parts[1]) + if err != nil { + return nil, fmt.Errorf("[from:to] must have valid numbers: %v", err) + } + l.To = int32(i) + } + if l.To > 0 && l.To < l.From { + return nil, fmt.Errorf("[from:to] to must be larger than from") + } + return l, nil + } + + if i, err := strconv.Atoi(s); err == nil { + l := &positionLayerSelector{At: int32(i)} + return l, nil + } + + return nil, fmt.Errorf("the layer selector [%s] is not valid, must be [from:to], [index], or [~digest]", s) +} + +type prefixLayerSelector struct { + Prefix string +} + +func (s *prefixLayerSelector) Filter(layers []distribution.Descriptor) ([]distribution.Descriptor, error) { + var filtered []distribution.Descriptor + for _, d := range layers { + if strings.HasPrefix(d.Digest.String(), s.Prefix) { + filtered = append(filtered, d) + } + } + if len(filtered) == 0 { + return nil, fmt.Errorf("no layers start with '%s'", s.Prefix) + } + if len(filtered) > 1 { + return nil, fmt.Errorf("multiple layers start with '%s', you must be more specific", s.Prefix) + } + return filtered, nil +} + +type indexLayerSelector struct { + From int32 + To int32 +} + +func (s *indexLayerSelector) Filter(layers []distribution.Descriptor) ([]distribution.Descriptor, error) { + l := int32(len(layers)) + from := s.From + to := s.To + if from < 0 { + from = l + from + } + if to < 0 { + to = l + to + } + if to > l { + to = l + } + if from < 0 || to < 0 || from >= l { + if s.To == math.MaxInt32 { + return nil, fmt.Errorf("tried to select [%d:], but image only has %d layers", s.From, l) + } + return nil, fmt.Errorf("tried to select [%d:%d], but image only has %d layers", s.From, s.To, l) + } + if to < from { + to, from = from, to + } + return layers[from:to], nil +} + +type positionLayerSelector struct { + At int32 +} + +func (s *positionLayerSelector) Filter(layers []distribution.Descriptor) ([]distribution.Descriptor, error) { + l := int32(len(layers)) + at := s.At + if at < 0 { + at = l + s.At + } + if at < 0 || at >= l { + return nil, fmt.Errorf("tried to select layer %d, but image only has %d layers", s.At, l) + } + return []distribution.Descriptor{layers[at]}, nil +} diff --git a/pkg/oc/cli/image/image.go b/pkg/oc/cli/image/image.go index 36deb78ba53c..33088b367dc1 100644 --- a/pkg/oc/cli/image/image.go +++ b/pkg/oc/cli/image/image.go @@ -11,6 +11,7 @@ import ( "github.com/openshift/origin/pkg/cmd/templates" "github.com/openshift/origin/pkg/oc/cli/image/append" + "github.com/openshift/origin/pkg/oc/cli/image/extract" "github.com/openshift/origin/pkg/oc/cli/image/mirror" ) @@ -37,6 +38,7 @@ func NewCmdImage(fullName string, f kcmdutil.Factory, streams genericclioptions. Message: "Advanced commands:", Commands: []*cobra.Command{ append.NewCmdAppendImage(name, streams), + extract.New(name, streams), mirror.NewCmdMirrorImage(name, streams), }, },