move compression to a separate package

TODO: align with containerd pkg/archive/compression, which

- also moved the utilities to Compress/Decompress streams
- changed the Extension to not include ".tar"

Signed-off-by: Sebastiaan van Stijn <github@gone.nl>

Author: thaJeztah

archive.go

@@ -8,7 +8,6 @@ import (
 	"compress/bzip2"
 	"compress/gzip"
 	"context"
-	"encoding/binary"
 	"errors"
 	"fmt"
 	"io"
@@ -27,6 +26,8 @@ import (
 	"github.com/moby/patternmatcher"
 	"github.com/moby/sys/sequential"
 	"github.com/moby/sys/user"
+
+	"github.com/moby/go-archive/compression"
 )
 
 // ImpliedDirectoryMode represents the mode (Unix permissions) applied to directories that are implied by files in a
@@ -43,7 +44,9 @@ const ImpliedDirectoryMode = 0o755
 
 type (
 	// Compression is the state represents if compressed or not.
-	Compression int
+	//
+	// Deprecated: use [compression.Compression].
+	Compression = compression.Compression
 	// WhiteoutFormat is the format of whiteouts unpacked
 	WhiteoutFormat int
 
@@ -56,7 +59,7 @@ type (
 	TarOptions struct {
 		IncludeFiles     []string
 		ExcludePatterns  []string
-		Compression      Compression
+		Compression      compression.Compression
 		NoLchown         bool
 		IDMap            user.IdentityMapping
 		ChownOpts        *ChownOpts
@@ -100,11 +103,11 @@ func NewDefaultArchiver() *Archiver {
 type breakoutError error
 
 const (
-	Uncompressed Compression = 0 // Uncompressed represents the uncompressed.
-	Bzip2        Compression = 1 // Bzip2 is bzip2 compression algorithm.
-	Gzip         Compression = 2 // Gzip is gzip compression algorithm.
-	Xz           Compression = 3 // Xz is xz compression algorithm.
-	Zstd         Compression = 4 // Zstd is zstd compression algorithm.
+	Uncompressed = compression.None  // Deprecated: use [compression.None].
+	Bzip2        = compression.Bzip2 // Deprecated: use [compression.Bzip2].
+	Gzip         = compression.Gzip  // Deprecated: use [compression.Gzip].
+	Xz           = compression.Xz    // Deprecated: use [compression.Xz].
+	Zstd         = compression.Zstd  // Deprecated: use [compression.Zstd].
 )
 
 const (
@@ -130,63 +133,11 @@ func IsArchivePath(path string) bool {
 	return err == nil
 }
 
-const (
-	zstdMagicSkippableStart = 0x184D2A50
-	zstdMagicSkippableMask  = 0xFFFFFFF0
-)
-
-var (
-	bzip2Magic = []byte{0x42, 0x5A, 0x68}
-	gzipMagic  = []byte{0x1F, 0x8B, 0x08}
-	xzMagic    = []byte{0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00}
-	zstdMagic  = []byte{0x28, 0xb5, 0x2f, 0xfd}
-)
-
-type matcher = func([]byte) bool
-
-func magicNumberMatcher(m []byte) matcher {
-	return func(source []byte) bool {
-		return bytes.HasPrefix(source, m)
-	}
-}
-
-// zstdMatcher detects zstd compression algorithm.
-// Zstandard compressed data is made of one or more frames.
-// There are two frame formats defined by Zstandard: Zstandard frames and Skippable frames.
-// See https://datatracker.ietf.org/doc/html/rfc8878#section-3 for more details.
-func zstdMatcher() matcher {
-	return func(source []byte) bool {
-		if bytes.HasPrefix(source, zstdMagic) {
-			// Zstandard frame
-			return true
-		}
-		// skippable frame
-		if len(source) < 8 {
-			return false
-		}
-		// magic number from 0x184D2A50 to 0x184D2A5F.
-		if binary.LittleEndian.Uint32(source[:4])&zstdMagicSkippableMask == zstdMagicSkippableStart {
-			return true
-		}
-		return false
-	}
-}
-
 // DetectCompression detects the compression algorithm of the source.
-func DetectCompression(source []byte) Compression {
-	compressionMap := map[Compression]matcher{
-		Bzip2: magicNumberMatcher(bzip2Magic),
-		Gzip:  magicNumberMatcher(gzipMagic),
-		Xz:    magicNumberMatcher(xzMagic),
-		Zstd:  zstdMatcher(),
-	}
-	for _, compression := range []Compression{Bzip2, Gzip, Xz, Zstd} {
-		fn := compressionMap[compression]
-		if fn(source) {
-			return compression
-		}
-	}
-	return Uncompressed
+//
+// Deprecated: use [compression.Detect].
+func DetectCompression(source []byte) compression.Compression {
+	return compression.Detect(source)
 }
 
 func xzDecompress(ctx context.Context, archive io.Reader) (io.ReadCloser, error) {
@@ -278,12 +229,12 @@ func DecompressStream(archive io.Reader) (io.ReadCloser, error) {
 		return nil, err
 	}
 
-	switch comp := DetectCompression(bs); comp {
-	case Uncompressed:
+	switch comp := compression.Detect(bs); comp {
+	case compression.None:
 		return &readCloserWrapper{
 			Reader: buf,
 		}, nil
-	case Gzip:
+	case compression.Gzip:
 		ctx, cancel := context.WithCancel(context.Background())
 
 		gzReader, err := gzDecompress(ctx, buf)
@@ -298,12 +249,12 @@ func DecompressStream(archive io.Reader) (io.ReadCloser, error) {
 				return gzReader.Close()
 			},
 		}, nil
-	case Bzip2:
+	case compression.Bzip2:
 		bz2Reader := bzip2.NewReader(buf)
 		return &readCloserWrapper{
 			Reader: bz2Reader,
 		}, nil
-	case Xz:
+	case compression.Xz:
 		ctx, cancel := context.WithCancel(context.Background())
 
 		xzReader, err := xzDecompress(ctx, buf)
@@ -319,7 +270,7 @@ func DecompressStream(archive io.Reader) (io.ReadCloser, error) {
 				return xzReader.Close()
 			},
 		}, nil
-	case Zstd:
+	case compression.Zstd:
 		zstdReader, err := zstd.NewReader(buf)
 		if err != nil {
 			return nil, err
@@ -343,13 +294,13 @@ type nopWriteCloser struct {
 func (nopWriteCloser) Close() error { return nil }
 
 // CompressStream compresses the dest with specified compression algorithm.
-func CompressStream(dest io.Writer, comp Compression) (io.WriteCloser, error) {
+func CompressStream(dest io.Writer, comp compression.Compression) (io.WriteCloser, error) {
 	switch comp {
-	case Uncompressed:
+	case compression.None:
 		return nopWriteCloser{dest}, nil
-	case Gzip:
+	case compression.Gzip:
 		return gzip.NewWriter(dest), nil
-	case Bzip2, Xz:
+	case compression.Bzip2, compression.Xz:
 		// archive/bzip2 does not support writing, and there is no xz support at all
 		// However, this is not a problem as docker only currently generates gzipped tars
 		return nil, fmt.Errorf("unsupported compression format: %s", (&comp).Extension())
@@ -446,23 +397,6 @@ func ReplaceFileTarWrapper(inputTarStream io.ReadCloser, mods map[string]TarModi
 	return pipeReader
 }
 
-// Extension returns the extension of a file that uses the specified compression algorithm.
-func (c *Compression) Extension() string {
-	switch *c {
-	case Uncompressed:
-		return "tar"
-	case Bzip2:
-		return "tar.bz2"
-	case Gzip:
-		return "tar.gz"
-	case Xz:
-		return "tar.xz"
-	case Zstd:
-		return "tar.zst"
-	}
-	return ""
-}
-
 // assert that we implement [tar.FileInfoNames].
 var _ tar.FileInfoNames = (*nosysFileInfo)(nil)
 
@@ -894,7 +828,7 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, o
 
 // Tar creates an archive from the directory at `path`, and returns it as a
 // stream of bytes.
-func Tar(path string, comp Compression) (io.ReadCloser, error) {
+func Tar(path string, comp compression.Compression) (io.ReadCloser, error) {
 	return TarWithOptions(path, &TarOptions{Compression: comp})
 }
 
@@ -1322,7 +1256,7 @@ func untarHandler(tarArchive io.Reader, dest string, options *TarOptions, decomp
 // TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other.
 // If either Tar or Untar fails, TarUntar aborts and returns the error.
 func (archiver *Archiver) TarUntar(src, dst string) error {
-	archive, err := Tar(src, Uncompressed)
+	archive, err := Tar(src, compression.None)
 	if err != nil {
 		return err
 	}

archive_test.go

@@ -22,6 +22,8 @@ import (
 	"gotest.tools/v3/assert"
 	is "gotest.tools/v3/assert/cmp"
 	"gotest.tools/v3/skip"
+
+	"github.com/moby/go-archive/compression"
 )
 
 var defaultArchiver = NewDefaultArchiver()
@@ -170,7 +172,7 @@ func TestCompressStreamXzUnsupported(t *testing.T) {
 	}
 	defer dest.Close()
 
-	_, err = CompressStream(dest, Xz)
+	_, err = CompressStream(dest, compression.Xz)
 	if err == nil {
 		t.Fatalf("Should fail as xz is unsupported for compression format.")
 	}
@@ -183,7 +185,7 @@ func TestCompressStreamBzip2Unsupported(t *testing.T) {
 	}
 	defer dest.Close()
 
-	_, err = CompressStream(dest, Bzip2)
+	_, err = CompressStream(dest, compression.Bzip2)
 	if err == nil {
 		t.Fatalf("Should fail as bzip2 is unsupported for compression format.")
 	}
@@ -202,25 +204,6 @@ func TestCompressStreamInvalid(t *testing.T) {
 	}
 }
 
-func TestExtension(t *testing.T) {
-	tests := []struct {
-		compression Compression
-		extension   string
-	}{
-		{compression: -1, extension: ""},
-		{compression: Uncompressed, extension: "tar"},
-		{compression: Bzip2, extension: "tar.bz2"},
-		{compression: Gzip, extension: "tar.gz"},
-		{compression: Xz, extension: "tar.xz"},
-		{compression: Zstd, extension: "tar.zst"},
-	}
-	for _, tc := range tests {
-		if actual := tc.compression.Extension(); actual != tc.extension {
-			t.Errorf("expected %s extension got %s", tc.extension, actual)
-		}
-	}
-}
-
 func TestCmdStreamLargeStderr(t *testing.T) {
 	cmd := exec.Command("sh", "-c", "dd if=/dev/zero bs=1k count=1000 of=/dev/stderr; echo hello")
 	out, err := cmdStream(cmd, nil)
@@ -620,7 +603,7 @@ func tarUntar(t *testing.T, origin string, options *TarOptions) ([]Change, error
 	}
 	wrap := io.MultiReader(bytes.NewReader(buf), archive)
 
-	detectedCompression := DetectCompression(buf)
+	detectedCompression := compression.Detect(buf)
 	expected := options.Compression
 	if detectedCompression.Extension() != expected.Extension() {
 		return nil, fmt.Errorf("wrong compression detected; expected: %s, got: %s", expected.Extension(), detectedCompression.Extension())
@@ -637,34 +620,6 @@ func tarUntar(t *testing.T, origin string, options *TarOptions) ([]Change, error
 	return ChangesDirs(origin, tmp)
 }
 
-func TestDetectCompressionZstd(t *testing.T) {
-	// test zstd compression without skippable frames.
-	compressedData := []byte{
-		0x28, 0xb5, 0x2f, 0xfd, // magic number of Zstandard frame: 0xFD2FB528
-		0x04, 0x00, 0x31, 0x00, 0x00, // frame header
-		0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, // data block "docker"
-		0x16, 0x0e, 0x21, 0xc3, // content checksum
-	}
-	compression := DetectCompression(compressedData)
-	if compression != Zstd {
-		t.Fatal("Unexpected compression")
-	}
-	// test zstd compression with skippable frames.
-	hex := []byte{
-		0x50, 0x2a, 0x4d, 0x18, // magic number of skippable frame: 0x184D2A50 to 0x184D2A5F
-		0x04, 0x00, 0x00, 0x00, // frame size
-		0x5d, 0x00, 0x00, 0x00, // user data
-		0x28, 0xb5, 0x2f, 0xfd, // magic number of Zstandard frame: 0xFD2FB528
-		0x04, 0x00, 0x31, 0x00, 0x00, // frame header
-		0x64, 0x6f, 0x63, 0x6b, 0x65, 0x72, // data block "docker"
-		0x16, 0x0e, 0x21, 0xc3, // content checksum
-	}
-	compression = DetectCompression(hex)
-	if compression != Zstd {
-		t.Fatal("Unexpected compression")
-	}
-}
-
 func TestTarUntar(t *testing.T) {
 	origin := t.TempDir()
 	if err := os.WriteFile(filepath.Join(origin, "1"), []byte("hello world"), 0o700); err != nil {
@@ -677,9 +632,9 @@ func TestTarUntar(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	for _, c := range []Compression{
-		Uncompressed,
-		Gzip,
+	for _, c := range []compression.Compression{
+		compression.None,
+		compression.Gzip,
 	} {
 		changes, err := tarUntar(t, origin, &TarOptions{
 			Compression:     c,

archive_unix_test.go

@@ -20,6 +20,8 @@ import (
 	"gotest.tools/v3/assert"
 	is "gotest.tools/v3/assert/cmp"
 	"gotest.tools/v3/skip"
+
+	"github.com/moby/go-archive/compression"
 )
 
 func TestCanonicalTarName(t *testing.T) {
@@ -82,7 +84,7 @@ func TestTarWithHardLink(t *testing.T) {
 	defer os.RemoveAll(dest)
 
 	// we'll do this in two steps to separate failure
-	fh, err := Tar(origin, Uncompressed)
+	fh, err := Tar(origin, compression.None)
 	assert.NilError(t, err)
 
 	// ensure we can read the whole thing with no error, before writing back out
@@ -211,7 +213,7 @@ func TestTarWithBlockCharFifo(t *testing.T) {
 	defer os.RemoveAll(dest)
 
 	// we'll do this in two steps to separate failure
-	fh, err := Tar(origin, Uncompressed)
+	fh, err := Tar(origin, compression.None)
 	assert.NilError(t, err)
 
 	// ensure we can read the whole thing with no error, before writing back out
@@ -254,7 +256,7 @@ func TestTarUntarWithXattr(t *testing.T) {
 	out, err := exec.Command("setcap", "cap_block_suspend+ep", filepath.Join(origin, "2")).CombinedOutput()
 	assert.NilError(t, err, string(out))
 
-	tarball, err := Tar(origin, Uncompressed)
+	tarball, err := Tar(origin, compression.None)
 	assert.NilError(t, err)
 	defer tarball.Close()
 	rdr := tar.NewReader(tarball)
@@ -275,9 +277,9 @@ func TestTarUntarWithXattr(t *testing.T) {
 		}
 	}
 
-	for _, c := range []Compression{
-		Uncompressed,
-		Gzip,
+	for _, c := range []compression.Compression{
+		compression.None,
+		compression.Gzip,
 	} {
 		changes, err := tarUntar(t, origin, &TarOptions{
 			Compression:     c,