diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..8d77e584 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + open-pull-requests-limit: 10 + directory: "/" + schedule: + interval: "daily" + labels: + - "dependencies" + - "bot" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cc45ad04..0af4d51d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,16 +4,20 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + DESTDIR: ./bin + on: + workflow_dispatch: schedule: - cron: '0 8 */6 * *' # every 6 days push: branches: - earthly-main - gh_test_ci + # No branch filter, so PRs stacked on other PR branches + # (not just PRs into earthly-main) get CI too. pull_request: - branches: - - earthly-main jobs: validate: @@ -27,13 +31,22 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - + name: Login to Docker Hub + # Authenticated pulls avoid Docker Hub's anonymous rate limit (429). + # Secrets are unavailable to fork PRs, so skip the login there. + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Run - uses: docker/bake-action@v3 + uses: docker/bake-action@4a9a8d494466d37134e2bfca2d3a8de8fb2681ad # v5.13.0 with: targets: ${{ matrix.target }} @@ -47,13 +60,22 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - + name: Login to Docker Hub + # Authenticated pulls avoid Docker Hub's anonymous rate limit (429). + # Secrets are unavailable to fork PRs, so skip the login there. + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Run - uses: docker/bake-action@v3 + uses: docker/bake-action@4a9a8d494466d37134e2bfca2d3a8de8fb2681ad # v5.13.0 with: targets: ${{ matrix.target }} @@ -61,66 +83,134 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go_version: - - "1.19" + go: + - "1.21" - "1.20" + mode: + - "root" + - "noroot" env: - GO_VERSION: ${{ matrix.go_version }} + GO_VERSION: ${{ matrix.go }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - + name: Login to Docker Hub + # Authenticated pulls avoid Docker Hub's anonymous rate limit (429). + # Secrets are unavailable to fork PRs, so skip the login there. + if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + uses: docker/login-action@c94ce9fb468520275223c153574b00df6fe4bcc9 # v3.7.0 + with: + username: ${{ vars.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - name: Test - uses: docker/bake-action@v3 + uses: docker/bake-action@4a9a8d494466d37134e2bfca2d3a8de8fb2681ad # v5.13.0 with: - targets: test + targets: test-${{ matrix.mode }} + - + name: Upload coverage + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + with: + directory: ${{ env.DESTDIR }}/coverage + flags: unit,${{ matrix.mode }},go-${{ matrix.go }} + token: ${{ secrets.CODECOV_TOKEN }} - test-macos: - runs-on: macos-latest + test-os: + runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: + os: + - macos-latest + - windows-latest go: - - "1.19" + - "1.21" - "1.20" + exclude: + # Go 1.21 test binaries abort on current macOS runner images + # ("dyld: missing LC_UUID load command"); Go 1.20 still covers macOS. + - os: macos-latest + go: "1.21" steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Set up Go - uses: actions/setup-go@v4 + uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 with: go-version: ${{ matrix.go }} - name: Test + shell: bash run: | - go test ./... + go test -coverprofile="coverage.txt" -covermode="atomic" ./... + go tool cover -func="coverage.txt" + - + name: Upload coverage + if: always() + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + with: + file: ./coverage.txt + env_vars: RUNNER_OS + flags: unit,go-${{ matrix.go }} + token: ${{ secrets.CODECOV_TOKEN }} test-freebsd-amd64: - runs-on: macos-latest - timeout-minutes: 60 + # macos-13 (and Vagrant/VirtualBox on it) is retired; run FreeBSD in a VM + # on Linux the way upstream tonistiigi/fsutil does. + runs-on: ubuntu-latest env: - VAGRANT_VAGRANTFILE: hack/Vagrantfile.freebsd13 + GO_VERSION: "1.21" steps: + - + name: Prepare + shell: bash + run: | + goVersion=$(curl --fail --silent --show-error "https://go.dev/dl/?mode=json&include=all" | jq -er --arg version "go${GO_VERSION}." '[.[] | select(.stable and (.version | startswith($version)))][0].version') + echo "GO_VERSION=${goVersion#go}" >> "$GITHUB_ENV" - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - - name: Cache Vagrant boxes - uses: actions/cache@v3 + name: Set up FreeBSD + uses: vmactions/freebsd-vm@c46abacb49f09938ca4e1702d15d836285d694cc # v1.5.9 with: - path: ~/.vagrant.d/boxes - key: ${{ runner.os }}-vagrant-${{ hashFiles('hack/Vagrantfile.freebsd13') }} - restore-keys: | - ${{ runner.os }}-vagrant- - - - name: Set up vagrant - run: vagrant up + release: "14.5" + envs: GO_VERSION + custom-shell-name: bsd + prepare: | + set -ex + mkdir -p /var/tmp + chmod 1777 /var/tmp + archive="go${GO_VERSION}.freebsd-amd64.tar.gz" + fetch -o "/var/tmp/$archive" "https://go.dev/dl/$archive" + tar -C /var/tmp -xzf "/var/tmp/$archive" + pw useradd -n fsutil -m -s /bin/sh - name: Test - run: vagrant ssh -- "cd /vagrant; go test -buildvcs=false ./..." + run: | + bsd /dev/stdin <<'EOF' + set -ex + chown -R fsutil "$GITHUB_WORKSPACE" + su - fsutil -c ' + set -ex + cd "$1" + export TMPDIR=/var/tmp + export PATH="/var/tmp/go/bin:$PATH" + go test -buildvcs=false -coverprofile=coverage.txt -covermode=atomic ./... + ' sh "$GITHUB_WORKSPACE" + EOF + - + name: Upload coverage + if: always() + uses: codecov/codecov-action@b9fd7d16f6d7d1b5d2bec1a2887e65ceed900238 # v4.6.0 + with: + file: ./coverage.txt + flags: unit,freebsd + token: ${{ secrets.CODECOV_TOKEN }} diff --git a/.gitignore b/.gitignore index 51b9602c..e5c343e2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,5 @@ # if you want to ignore files created by your editor/tools, consider using a # global .gitignore or .git/info/exclude see https://help.github.com/articles/ignoring-files -.* -!.github -!.gitignore -!.travis.yml -*.prof +bin/ # support running go modules in vendor mode for local development vendor/ diff --git a/.golangci.yml b/.golangci.yml index 2a8b86ae..7300a0e5 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -14,3 +14,17 @@ linters: - staticcheck - typecheck disable-all: true + +linters-settings: + depguard: + rules: + main: + deny: + # The io/ioutil package has been deprecated. + # https://go.dev/doc/go1.16#ioutil + - pkg: "io/ioutil" + desc: The io/ioutil package has been deprecated. + +# show all +max-issues-per-linter: 0 +max-same-issues: 0 diff --git a/Dockerfile b/Dockerfile index 9584648d..21c1f39f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,9 @@ -#syntax=docker/dockerfile:1 -ARG GO_VERSION=1.20 +# syntax=docker/dockerfile:1 -FROM --platform=$BUILDPLATFORM tonistiigi/xx:1.1.0 AS xx +ARG GO_VERSION=1.21 +ARG XX_VERSION=1.4.0 + +FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS base RUN apk add --no-cache git @@ -18,13 +20,19 @@ FROM base AS test ARG TESTFLAGS RUN --mount=target=. --mount=target=/go/pkg/mod,type=cache \ --mount=target=/root/.cache,type=cache \ - CGO_ENABLED=0 xx-go test -test.v ${TESTFLAGS} ./... + CGO_ENABLED=0 xx-go test -v -coverprofile=/tmp/coverage.txt -covermode=atomic ${TESTFLAGS} ./... FROM base AS test-noroot RUN mkdir /go/pkg && chmod 0777 /go/pkg USER 1000:1000 RUN --mount=target=. \ --mount=target=/tmp/.cache,type=cache \ - CGO_ENABLED=0 GOCACHE=/tmp/gocache xx-go test -test.v ./... + CGO_ENABLED=0 GOCACHE=/tmp/gocache xx-go test -v -coverprofile=/tmp/coverage.txt -covermode=atomic ./... + +FROM scratch AS test-coverage +COPY --from=test /tmp/coverage.txt /coverage-root.txt + +FROM scratch AS test-noroot-coverage +COPY --from=test-noroot /tmp/coverage.txt /coverage-noroot.txt FROM build diff --git a/bench/go.mod b/bench/go.mod index 6ca11879..805f3a38 100644 --- a/bench/go.mod +++ b/bench/go.mod @@ -1,6 +1,6 @@ module github.com/tonistiigi/fsutil/bench -go 1.19 +go 1.20 require ( github.com/containerd/continuity v0.4.1 diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..f5f9e02c --- /dev/null +++ b/codecov.yml @@ -0,0 +1,12 @@ +comment: false + +coverage: + status: + project: # settings affecting project coverage + default: + target: auto # auto % coverage target + threshold: 1% # allow for 1% reduction of coverage without failing + patch: off + +github_checks: + annotations: false diff --git a/copy/copy.go b/copy/copy.go index 558c553f..f8cc0a43 100644 --- a/copy/copy.go +++ b/copy/copy.go @@ -88,7 +88,7 @@ func Copy(ctx context.Context, srcRoot, src, dstRoot, dst string, opts ...Opt) e return err } - c, err := newCopier(dstRoot, ci.Chown, ci.Utime, ci.Mode, ci.XAttrErrorHandler, ci.IncludePatterns, ci.ExcludePatterns, ci.ChangeFunc) + c, err := newCopier(dstRoot, ci.Chown, ci.Utime, ci.Mode, ci.XAttrErrorHandler, ci.IncludePatterns, ci.ExcludePatterns, ci.AlwaysReplaceExistingDestPaths, ci.ChangeFunc) if err != nil { return err } @@ -172,7 +172,11 @@ type CopyInfo struct { IncludePatterns []string // Exclude files/dir matching any of these patterns (even if they match an include pattern) ExcludePatterns []string - ChangeFunc fsutil.ChangeFunc + // If true, any source path that overwrite existing destination paths will always replace + // the existing destination path, even if they are of different types (e.g. a directory will + // replace any existing symlink or file) + AlwaysReplaceExistingDestPaths bool + ChangeFunc fsutil.ChangeFunc } type Opt func(*CopyInfo) @@ -227,16 +231,17 @@ func WithChangeNotifier(fn fsutil.ChangeFunc) Opt { } type copier struct { - chown Chowner - utime *time.Time - mode *int - inodes map[uint64]string - xattrErrorHandler XAttrErrorHandler - includePatternMatcher *patternmatcher.PatternMatcher - excludePatternMatcher *patternmatcher.PatternMatcher - parentDirs []parentDir - changefn fsutil.ChangeFunc - root string + chown Chowner + utime *time.Time + mode *int + inodes map[uint64]string + xattrErrorHandler XAttrErrorHandler + includePatternMatcher *patternmatcher.PatternMatcher + excludePatternMatcher *patternmatcher.PatternMatcher + parentDirs []parentDir + changefn fsutil.ChangeFunc + root string + alwaysReplaceExistingDestPaths bool } type parentDir struct { @@ -245,7 +250,7 @@ type parentDir struct { copied bool } -func newCopier(root string, chown Chowner, tm *time.Time, mode *int, xeh XAttrErrorHandler, includePatterns, excludePatterns []string, changeFunc fsutil.ChangeFunc) (*copier, error) { +func newCopier(root string, chown Chowner, tm *time.Time, mode *int, xeh XAttrErrorHandler, includePatterns, excludePatterns []string, alwaysReplaceExistingDestPaths bool, changeFunc fsutil.ChangeFunc) (*copier, error) { if xeh == nil { xeh = func(dst, src, key string, err error) error { return err @@ -271,15 +276,16 @@ func newCopier(root string, chown Chowner, tm *time.Time, mode *int, xeh XAttrEr } return &copier{ - root: root, - inodes: map[uint64]string{}, - chown: chown, - utime: tm, - xattrErrorHandler: xeh, - mode: mode, - includePatternMatcher: includePatternMatcher, - excludePatternMatcher: excludePatternMatcher, - changefn: changeFunc, + root: root, + inodes: map[uint64]string{}, + chown: chown, + utime: tm, + xattrErrorHandler: xeh, + mode: mode, + includePatternMatcher: includePatternMatcher, + excludePatternMatcher: excludePatternMatcher, + changefn: changeFunc, + alwaysReplaceExistingDestPaths: alwaysReplaceExistingDestPaths, }, nil } @@ -324,6 +330,10 @@ func (c *copier) copy(ctx context.Context, src, srcComponents, target string, ov } if include { + if err := c.removeTargetIfNeeded(src, target, fi, targetFi); err != nil { + return err + } + if err := c.createParentDirs(src, srcComponents, target, overwriteTargetMetadata); err != nil { return err } @@ -440,6 +450,21 @@ func (c *copier) exclude(path string, fi os.FileInfo, parentExcludeMatchInfo pat return m, matchInfo, nil } +func (c *copier) removeTargetIfNeeded(src, target string, srcFi, targetFi os.FileInfo) error { + if !c.alwaysReplaceExistingDestPaths { + return nil + } + if targetFi == nil { + // already doesn't exist + return nil + } + if srcFi.IsDir() && targetFi.IsDir() { + // directories are merged, not replaced + return nil + } + return os.RemoveAll(target) +} + // Delayed creation of parent directories when a file or dir matches an include // pattern. func (c *copier) createParentDirs(src, srcComponents, target string, overwriteTargetMetadata bool) error { diff --git a/copy/copy_darwin.go b/copy/copy_darwin.go index bc93b21c..0cdc00a8 100644 --- a/copy/copy_darwin.go +++ b/copy/copy_darwin.go @@ -13,7 +13,7 @@ import ( func copyFile(source, target string) error { if err := unix.Clonefileat(unix.AT_FDCWD, source, unix.AT_FDCWD, target, unix.CLONE_NOFOLLOW); err != nil { - if err != unix.EINVAL { + if err != unix.EINVAL && err != unix.EXDEV { return err } } else { diff --git a/copy/copy_linux.go b/copy/copy_linux.go index 971cb5c5..6d9b490c 100644 --- a/copy/copy_linux.go +++ b/copy/copy_linux.go @@ -30,7 +30,16 @@ func (c *copier) copyFileInfo(fi os.FileInfo, src, name string) error { m := fi.Mode() if c.mode != nil { - m = (m & ^os.FileMode(0777)) | os.FileMode(*c.mode&0777) + m = os.FileMode(*c.mode).Perm() + if *c.mode&syscall.S_ISGID != 0 { + m |= os.ModeSetgid + } + if *c.mode&syscall.S_ISUID != 0 { + m |= os.ModeSetuid + } + if *c.mode&syscall.S_ISVTX != 0 { + m |= os.ModeSticky + } } if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink { if err := os.Chmod(name, m); err != nil { diff --git a/copy/copy_test.go b/copy/copy_test.go index e77d54e7..425ade68 100644 --- a/copy/copy_test.go +++ b/copy/copy_test.go @@ -359,6 +359,60 @@ func TestCopySymlinks(t *testing.T) { require.Equal(t, "foo.txt", link) } +func TestCopyWithAlwaysReplaceExistingDestPaths(t *testing.T) { + destDir := t.TempDir() + require.NoError(t, fstest.Apply( + fstest.CreateDir("root", 0755), + fstest.CreateDir("root/overwritedir", 0755), + fstest.CreateFile("root/overwritedir/subfile", nil, 0755), + fstest.CreateFile("root/overwritefile", nil, 0755), + fstest.Symlink("dir", "root/overwritesymlink"), + fstest.CreateDir("root/dir", 0755), + fstest.CreateFile("root/dir/dirfile1", nil, 0755), + fstest.CreateDir("root/dir/overwritesubdir", 0755), + fstest.CreateFile("root/dir/overwritesubfile", nil, 0755), + fstest.Symlink("dirfile1", "root/dir/overwritesymlink"), + ).Apply(destDir)) + + srcDir := t.TempDir() + require.NoError(t, fstest.Apply( + fstest.CreateDir("root", 0755), + fstest.CreateFile("root/overwritedir", nil, 0755), + fstest.CreateDir("root/overwritefile", 0755), + fstest.CreateFile("root/overwritefile/foo", nil, 0755), + fstest.CreateDir("root/overwritesymlink", 0755), + fstest.CreateDir("root/dir", 0755), + fstest.CreateFile("root/dir/dirfile2", nil, 0755), + fstest.CreateFile("root/dir/overwritesubdir", nil, 0755), + fstest.CreateDir("root/dir/overwritesubfile", 0755), + fstest.CreateDir("root/dir/overwritesymlink", 0755), + ).Apply(srcDir)) + + expectedDir := t.TempDir() + require.NoError(t, fstest.Apply( + fstest.CreateDir("root", 0755), + fstest.CreateFile("root/overwritedir", nil, 0755), + fstest.CreateDir("root/overwritefile", 0755), + fstest.CreateFile("root/overwritefile/foo", nil, 0755), + fstest.CreateDir("root/overwritesymlink", 0755), + fstest.CreateDir("root/dir", 0755), + fstest.CreateFile("root/dir/dirfile1", nil, 0755), + fstest.CreateFile("root/dir/dirfile2", nil, 0755), + fstest.CreateFile("root/dir/overwritesubdir", nil, 0755), + fstest.CreateDir("root/dir/overwritesubfile", 0755), + fstest.CreateDir("root/dir/overwritesymlink", 0755), + ).Apply(expectedDir)) + + err := Copy(context.TODO(), srcDir, "root", destDir, "root", WithCopyInfo(CopyInfo{ + AlwaysReplaceExistingDestPaths: true, + CopyDirContents: true, + })) + require.NoError(t, err) + + err = fstest.CheckDirectoryEqual(destDir, expectedDir) + require.NoError(t, err) +} + func testCopy(t *testing.T, apply fstest.Applier, exp string) error { t1 := t.TempDir() t2 := t.TempDir() diff --git a/copy/copy_unix.go b/copy/copy_unix.go index 945e96c5..4a7d0c86 100644 --- a/copy/copy_unix.go +++ b/copy/copy_unix.go @@ -31,7 +31,16 @@ func (c *copier) copyFileInfo(fi os.FileInfo, src, name string) error { m := fi.Mode() if c.mode != nil { - m = (m & ^os.FileMode(0777)) | os.FileMode(*c.mode&0777) + m = os.FileMode(*c.mode).Perm() + if *c.mode&syscall.S_ISGID != 0 { + m |= os.ModeSetgid + } + if *c.mode&syscall.S_ISUID != 0 { + m |= os.ModeSetuid + } + if *c.mode&syscall.S_ISVTX != 0 { + m |= os.ModeSticky + } } if (fi.Mode() & os.ModeSymlink) != os.ModeSymlink { if err := os.Chmod(name, m); err != nil { diff --git a/copy/copy_unix_test.go b/copy/copy_unix_test.go index df710d4c..405f757c 100644 --- a/copy/copy_unix_test.go +++ b/copy/copy_unix_test.go @@ -7,6 +7,7 @@ import ( "context" "os" "path/filepath" + "syscall" "testing" "github.com/stretchr/testify/assert" @@ -56,3 +57,39 @@ func TestCopyDevicesAndFifo(t *testing.T) { assert.NotEqual(t, os.ModeSocket, fi.Mode()&os.ModeSocket) // socket copied as stub assert.Equal(t, os.FileMode(0555), fi.Mode()&0777) } + +func TestCopySetuid(t *testing.T) { + requiresRoot(t) + + t1 := t.TempDir() + + err := mknod(filepath.Join(t1, "char"), unix.S_IFCHR|0444, int(unix.Mkdev(1, 9))) + require.NoError(t, err) + + t2 := t.TempDir() + + err = Copy(context.TODO(), t1, ".", t2, ".") + require.NoError(t, err) + + fi, err := os.Lstat(filepath.Join(t2, "char")) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0444), fi.Mode().Perm()) + assert.Equal(t, os.FileMode(0), fi.Mode()&os.ModeSetuid) + assert.Equal(t, os.FileMode(0), fi.Mode()&os.ModeSetgid) + assert.Equal(t, os.FileMode(0), fi.Mode()&os.ModeSticky) + + t3 := t.TempDir() + + p := 0444 | syscall.S_ISUID + err = Copy(context.TODO(), t1, ".", t3, ".", WithCopyInfo(CopyInfo{ + Mode: &p, + })) + require.NoError(t, err) + + fi, err = os.Lstat(filepath.Join(t3, "char")) + require.NoError(t, err) + assert.Equal(t, os.FileMode(0444), fi.Mode().Perm()) + assert.Equal(t, os.ModeSetuid, fi.Mode()&os.ModeSetuid) + assert.Equal(t, os.FileMode(0), fi.Mode()&os.ModeSetgid) + assert.Equal(t, os.FileMode(0), fi.Mode()&os.ModeSticky) +} diff --git a/diff_containerd.go b/diff_containerd.go index d8619abf..86d64602 100644 --- a/diff_containerd.go +++ b/diff_containerd.go @@ -5,6 +5,7 @@ import ( "context" "io" "os" + "path/filepath" "strings" "github.com/tonistiigi/fsutil/types" @@ -127,7 +128,7 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn, fil f1 = nil continue } else if rmdir == "" && f1.stat.IsDir() { - rmdir = f1.path + string(os.PathSeparator) + rmdir = f1.path + string(filepath.Separator) } else if rmdir != "" { rmdir = "" } @@ -138,7 +139,7 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b walkerFn, fil return err } if f1.stat.IsDir() && !f2copy.stat.IsDir() { - rmdir = f1.path + string(os.PathSeparator) + rmdir = f1.path + string(filepath.Separator) } else if rmdir != "" { rmdir = "" } diff --git a/diskwriter.go b/diskwriter.go index 10b60851..a62c6b0c 100644 --- a/diskwriter.go +++ b/diskwriter.go @@ -37,6 +37,7 @@ type DiskWriter struct { ctx context.Context cancel func() eg *errgroup.Group + egCtx context.Context filter FilterFunc dirModTimes map[string]int64 } @@ -50,13 +51,14 @@ func NewDiskWriter(ctx context.Context, dest string, opt DiskWriterOpt) (*DiskWr } ctx, cancel := context.WithCancel(ctx) - eg, ctx := errgroup.WithContext(ctx) + eg, egCtx := errgroup.WithContext(ctx) return &DiskWriter{ opt: opt, dest: dest, eg: eg, ctx: ctx, + egCtx: egCtx, cancel: cancel, filter: opt.Filter, dirModTimes: map[string]int64{}, @@ -98,7 +100,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er } }() - destPath := filepath.Join(dw.dest, filepath.FromSlash(p)) + destPath := filepath.Join(dw.dest, p) if kind == ChangeKindDelete { if dw.filter != nil { @@ -183,12 +185,12 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er } default: isRegularFile = true - file, err := os.OpenFile(newPath, os.O_CREATE|os.O_WRONLY, fi.Mode()) //todo: windows + file, err := os.OpenFile(newPath, os.O_CREATE|os.O_WRONLY, fi.Mode()) if err != nil { return errors.Wrapf(err, "failed to create %s", newPath) } if dw.opt.SyncDataCb != nil { - if err := dw.processChange(ChangeKindAdd, p, fi, file); err != nil { + if err := dw.processChange(dw.ctx, ChangeKindAdd, p, fi, file); err != nil { file.Close() return err } @@ -219,7 +221,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er dw.requestAsyncFileData(p, destPath, fi, &statCopy) } } else { - return dw.processChange(kind, p, fi, nil) + return dw.processChange(dw.ctx, kind, p, fi, nil) } return nil @@ -228,7 +230,7 @@ func (dw *DiskWriter) HandleChange(kind ChangeKind, p string, fi os.FileInfo, er func (dw *DiskWriter) requestAsyncFileData(p, dest string, fi os.FileInfo, st *types.Stat) { // todo: limit worker threads dw.eg.Go(func() error { - if err := dw.processChange(ChangeKindAdd, p, fi, &lazyFileWriter{ + if err := dw.processChange(dw.egCtx, ChangeKindAdd, p, fi, &lazyFileWriter{ dest: dest, }); err != nil { return err @@ -237,7 +239,7 @@ func (dw *DiskWriter) requestAsyncFileData(p, dest string, fi os.FileInfo, st *t }) } -func (dw *DiskWriter) processChange(kind ChangeKind, p string, fi os.FileInfo, w io.WriteCloser) error { +func (dw *DiskWriter) processChange(ctx context.Context, kind ChangeKind, p string, fi os.FileInfo, w io.WriteCloser) error { origw := w var hw *hashedWriter if dw.opt.NotifyCb != nil { @@ -252,7 +254,7 @@ func (dw *DiskWriter) processChange(kind ChangeKind, p string, fi os.FileInfo, w if fn == nil && dw.opt.AsyncDataCb != nil { fn = dw.opt.AsyncDataCb } - if err := fn(dw.ctx, p, w); err != nil { + if err := fn(ctx, p, w); err != nil { return err } } else { @@ -313,7 +315,7 @@ type lazyFileWriter struct { func (lfw *lazyFileWriter) Write(dt []byte) (int, error) { if lfw.f == nil { - file, err := os.OpenFile(lfw.dest, os.O_WRONLY, 0) //todo: windows + file, err := os.OpenFile(lfw.dest, os.O_WRONLY, 0) if os.IsPermission(err) { // retry after chmod fi, er := os.Stat(lfw.dest) diff --git a/diskwriter_test.go b/diskwriter_test.go index 4b6edb3b..ed8e0e7d 100644 --- a/diskwriter_test.go +++ b/diskwriter_test.go @@ -7,15 +7,12 @@ import ( "os" "path/filepath" "sync" - "syscall" "testing" "time" "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "golang.org/x/sys/unix" ) // requiresRoot skips tests that require root @@ -54,12 +51,12 @@ func TestWriterSimple(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `dir bar + assert.Equal(t, `dir bar file bar/foo symlink:../foo bar/foo2 file foo file foo2 >foo -`) +`, b.String()) } @@ -91,9 +88,9 @@ func TestWriterFileToDir(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `dir foo + assert.Equal(t, `dir foo file foo/bar -`) +`, b.String()) } func TestWriterDirToFile(t *testing.T) { @@ -124,8 +121,8 @@ func TestWriterDirToFile(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `file foo -`) + assert.Equal(t, `file foo +`, b.String()) } func TestWalkerWriterSimple(t *testing.T) { @@ -153,117 +150,16 @@ func TestWalkerWriterSimple(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo symlink:../foo bar/foo2 file foo file foo2 -`) +`), b.String()) dt, err := os.ReadFile(filepath.Join(dest, "foo")) assert.NoError(t, err) assert.Equal(t, []byte("mydata"), dt) - -} - -func TestWalkerWriterAsync(t *testing.T) { - d, err := tmpDir(changeStream([]string{ - "ADD foo dir", - "ADD foo/foo1 file data1", - "ADD foo/foo2 file data2", - "ADD foo/foo3 file data3", - "ADD foo/foo4 file >foo/foo3", - "ADD foo5 file data5", - })) - assert.NoError(t, err) - defer os.RemoveAll(d) - - dest := t.TempDir() - - dw, err := NewDiskWriter(context.TODO(), dest, DiskWriterOpt{ - AsyncDataCb: newWriteToFunc(d, 300*time.Millisecond), - }) - assert.NoError(t, err) - - st := time.Now() - - err = Walk(context.Background(), d, nil, readAsAdd(dw.HandleChange)) - assert.NoError(t, err) - - err = dw.Wait(context.TODO()) - assert.NoError(t, err) - - dt, err := os.ReadFile(filepath.Join(dest, "foo/foo3")) - assert.NoError(t, err) - assert.Equal(t, "data3", string(dt)) - - dt, err = os.ReadFile(filepath.Join(dest, "foo/foo4")) - assert.NoError(t, err) - assert.Equal(t, "data3", string(dt)) - - fi1, err := os.Lstat(filepath.Join(dest, "foo/foo3")) - assert.NoError(t, err) - fi2, err := os.Lstat(filepath.Join(dest, "foo/foo4")) - assert.NoError(t, err) - stat1, ok1 := fi1.Sys().(*syscall.Stat_t) - stat2, ok2 := fi2.Sys().(*syscall.Stat_t) - if ok1 && ok2 { - assert.Equal(t, stat1.Ino, stat2.Ino) - } - - dt, err = os.ReadFile(filepath.Join(dest, "foo5")) - assert.NoError(t, err) - assert.Equal(t, "data5", string(dt)) - - duration := time.Since(st) - assert.True(t, duration < 500*time.Millisecond) -} - -func TestWalkerWriterDevices(t *testing.T) { - requiresRoot(t) - - d, err := tmpDir(changeStream([]string{ - "ADD foo dir", - "ADD foo/foo1 file data1", - })) - require.NoError(t, err) - - err = unix.Mknod(filepath.Join(d, "foo/block"), syscall.S_IFBLK|0600, mkdev(2, 3)) - require.NoError(t, err) - - err = unix.Mknod(filepath.Join(d, "foo/char"), syscall.S_IFCHR|0400, mkdev(1, 9)) - require.NoError(t, err) - - dest := t.TempDir() - - dw, err := NewDiskWriter(context.TODO(), dest, DiskWriterOpt{ - SyncDataCb: newWriteToFunc(d, 0), - }) - assert.NoError(t, err) - - err = Walk(context.Background(), d, nil, readAsAdd(dw.HandleChange)) - assert.NoError(t, err) - - err = dw.Wait(context.TODO()) - assert.NoError(t, err) - - fi, err := os.Lstat(filepath.Join(dest, "foo/char")) - require.NoError(t, err) - - stat, ok := fi.Sys().(*syscall.Stat_t) - require.True(t, ok) - - assert.Equal(t, uint64(1), stat.Rdev>>8) - assert.Equal(t, uint64(9), stat.Rdev&0xff) - - fi, err = os.Lstat(filepath.Join(dest, "foo/block")) - require.NoError(t, err) - - stat, ok = fi.Sys().(*syscall.Stat_t) - require.True(t, ok) - - assert.Equal(t, uint64(2), stat.Rdev>>8) - assert.Equal(t, uint64(3), stat.Rdev&0xff) } func readAsAdd(f HandleChangeFn) filepath.WalkFunc { diff --git a/diskwriter_unix_test.go b/diskwriter_unix_test.go new file mode 100644 index 00000000..7cd6eb55 --- /dev/null +++ b/diskwriter_unix_test.go @@ -0,0 +1,117 @@ +//go:build !windows +// +build !windows + +package fsutil + +import ( + "context" + "os" + "path/filepath" + "syscall" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/sys/unix" +) + +func TestWalkerWriterAsync(t *testing.T) { + d, err := tmpDir(changeStream([]string{ + "ADD foo dir", + "ADD foo/foo1 file data1", + "ADD foo/foo2 file data2", + "ADD foo/foo3 file data3", + "ADD foo/foo4 file >foo/foo3", + "ADD foo5 file data5", + })) + assert.NoError(t, err) + defer os.RemoveAll(d) + + dest := t.TempDir() + + dw, err := NewDiskWriter(context.TODO(), dest, DiskWriterOpt{ + AsyncDataCb: newWriteToFunc(d, 300*time.Millisecond), + }) + assert.NoError(t, err) + + st := time.Now() + + err = Walk(context.Background(), d, nil, readAsAdd(dw.HandleChange)) + assert.NoError(t, err) + + err = dw.Wait(context.TODO()) + assert.NoError(t, err) + + dt, err := os.ReadFile(filepath.Join(dest, "foo/foo3")) + assert.NoError(t, err) + assert.Equal(t, "data3", string(dt)) + + dt, err = os.ReadFile(filepath.Join(dest, "foo/foo4")) + assert.NoError(t, err) + assert.Equal(t, "data3", string(dt)) + + fi1, err := os.Lstat(filepath.Join(dest, "foo/foo3")) + assert.NoError(t, err) + fi2, err := os.Lstat(filepath.Join(dest, "foo/foo4")) + assert.NoError(t, err) + stat1, ok1 := fi1.Sys().(*syscall.Stat_t) + stat2, ok2 := fi2.Sys().(*syscall.Stat_t) + if ok1 && ok2 { + assert.Equal(t, stat1.Ino, stat2.Ino) + } + + dt, err = os.ReadFile(filepath.Join(dest, "foo5")) + assert.NoError(t, err) + assert.Equal(t, "data5", string(dt)) + + duration := time.Since(st) + assert.True(t, duration < 500*time.Millisecond) +} + +func TestWalkerWriterDevices(t *testing.T) { + requiresRoot(t) + + d, err := tmpDir(changeStream([]string{ + "ADD foo dir", + "ADD foo/foo1 file data1", + })) + require.NoError(t, err) + + err = unix.Mknod(filepath.Join(d, "foo/block"), syscall.S_IFBLK|0600, mkdev(2, 3)) + require.NoError(t, err) + + err = unix.Mknod(filepath.Join(d, "foo/char"), syscall.S_IFCHR|0400, mkdev(1, 9)) + require.NoError(t, err) + + dest := t.TempDir() + + dw, err := NewDiskWriter(context.TODO(), dest, DiskWriterOpt{ + SyncDataCb: newWriteToFunc(d, 0), + }) + assert.NoError(t, err) + + err = Walk(context.Background(), d, nil, readAsAdd(dw.HandleChange)) + assert.NoError(t, err) + + err = dw.Wait(context.TODO()) + assert.NoError(t, err) + + fi, err := os.Lstat(filepath.Join(dest, "foo/char")) + require.NoError(t, err) + + stat, ok := fi.Sys().(*syscall.Stat_t) + require.True(t, ok) + + assert.Equal(t, uint64(1), stat.Rdev>>8) + assert.Equal(t, uint64(9), stat.Rdev&0xff) + + fi, err = os.Lstat(filepath.Join(dest, "foo/block")) + require.NoError(t, err) + + stat, ok = fi.Sys().(*syscall.Stat_t) + require.True(t, ok) + + assert.Equal(t, uint64(2), stat.Rdev>>8) + assert.Equal(t, uint64(3), stat.Rdev&0xff) +} diff --git a/docker-bake.hcl b/docker-bake.hcl index 6ba3c867..48127d4f 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -1,5 +1,9 @@ variable "GO_VERSION" { - default = "1.20" + default = null +} + +variable "DESTDIR" { + default = "./bin" } group "default" { @@ -18,12 +22,14 @@ group "test" { target "test-root" { inherits = ["build"] - target = "test" + target = "test-coverage" + output = ["${DESTDIR}/coverage"] } target "test-noroot" { inherits = ["build"] - target = "test-noroot" + target = "test-noroot-coverage" + output = ["${DESTDIR}/coverage"] } target "lint" { diff --git a/filter.go b/filter.go index 1925146c..48398009 100644 --- a/filter.go +++ b/filter.go @@ -68,9 +68,9 @@ type filterFS struct { // NewFilterFS creates a new FS that filters the given FS using the given // FilterOpt. - +// // The returned FS will not contain any paths that do not match the provided -// include and exclude patterns, or that are are exlcluded using the mapping +// include and exclude patterns, or that are are excluded using the mapping // function. // // The FS is assumed to be a snapshot of the filesystem at the time of the @@ -98,7 +98,7 @@ func NewFilterFS(fs FS, opt *FilterOpt) (FS, error) { } patternChars := "*[]?^" - if os.PathSeparator != '\\' { + if filepath.Separator != '\\' { patternChars += `\` } @@ -179,6 +179,7 @@ func (fs *filterFS) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc includeMatchInfo patternmatcher.MatchInfo excludeMatchInfo patternmatcher.MatchInfo calledFn bool + skipFn bool } // used only for include/exclude handling @@ -339,6 +340,9 @@ func (fs *filterFS) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc } } for i, parentDir := range parentDirs { + if parentDir.skipFn { + return filepath.SkipDir + } if parentDir.calledFn { continue } @@ -358,15 +362,21 @@ func (fs *filterFS) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc } if fs.mapFn != nil { result := fs.mapFn(parentStat.Path, parentStat) - if result == MapResultSkipDir || result == MapResultExclude { + if result == MapResultExclude { continue + } else if result == MapResultSkipDir { + parentDirs[i].skipFn = true + return filepath.SkipDir } } - if err := fn(parentStat.Path, &DirEntryInfo{Stat: parentStat}, nil); err != nil { + parentDirs[i].calledFn = true + if err := fn(parentStat.Path, &DirEntryInfo{Stat: parentStat}, nil); err == filepath.SkipDir { + parentDirs[i].skipFn = true + return filepath.SkipDir + } else if err != nil { return err } - parentDirs[i].calledFn = true } if err := fn(stat.Path, &DirEntryInfo{Stat: stat}, nil); err != nil { return err diff --git a/filter_test.go b/filter_test.go index 5d25ffdd..406425da 100644 --- a/filter_test.go +++ b/filter_test.go @@ -30,10 +30,9 @@ func TestWalkerSimple(t *testing.T) { err = Walk(context.Background(), d, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `file foo + assert.Equal(t, `file foo file foo2 -`) - +`, b.String()) } func TestInvalidExcludePatterns(t *testing.T) { @@ -62,9 +61,9 @@ func TestWalkerInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -72,9 +71,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -82,9 +81,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -92,9 +91,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -111,7 +110,7 @@ file bar/foo assert.NoError(t, err) assert.Equal(t, `file foo2 -`, string(b.Bytes())) +`, b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -119,9 +118,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -129,9 +128,9 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -139,9 +138,33 @@ file bar/foo }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo -`, string(b.Bytes())) +`), b.String()) +} + +func TestWalkerIncludeReturnSkipDir(t *testing.T) { + d, err := tmpDir(changeStream([]string{ + "ADD foo dir", + "ADD foo/x dir", + "ADD foo/y dir", + "ADD foo/x/a.txt file", + "ADD foo/y/b.txt file", + })) + assert.NoError(t, err) + defer os.RemoveAll(d) + + found := []string{} + + err = Walk(context.Background(), d, &FilterOpt{ + IncludePatterns: []string{"**/*.txt"}, + }, func(path string, info gofs.FileInfo, err error) error { + found = append(found, path) + return filepath.SkipDir + }) + assert.NoError(t, err) + + assert.Equal(t, []string{"foo"}, found) } func TestWalkerExclude(t *testing.T) { @@ -159,24 +182,40 @@ func TestWalkerExclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `file bar + assert.Equal(t, filepath.FromSlash(`file bar dir foo file foo/bar2 -`, string(b.Bytes())) +`), b.String()) } func TestWalkerFollowLinks(t *testing.T) { - d, err := tmpDir(changeStream([]string{ - "ADD bar file", - "ADD foo dir", - "ADD foo/l1 symlink /baz/one", - "ADD foo/l2 symlink /baz/two", - "ADD baz dir", - "ADD baz/one file", - "ADD baz/two symlink ../bax", - "ADD bax file", - "ADD bay file", // not included - })) + var d string + var err error + if runtime.GOOS == "windows" { + d, err = tmpDir(changeStream([]string{ + "ADD bar file", + "ADD foo dir", + "ADD foo/l1 symlink C:/baz/one", + "ADD foo/l2 symlink C:/baz/two", + "ADD baz dir", + "ADD baz/one file", + "ADD baz/two symlink ../bax", + "ADD bax file", + "ADD bay file", // not included + })) + } else { + d, err = tmpDir(changeStream([]string{ + "ADD bar file", + "ADD foo dir", + "ADD foo/l1 symlink /baz/one", + "ADD foo/l2 symlink /baz/two", + "ADD baz dir", + "ADD baz/one file", + "ADD baz/two symlink ../bax", + "ADD bax file", + "ADD bay file", // not included + })) + } assert.NoError(t, err) defer os.RemoveAll(d) b := &bytes.Buffer{} @@ -185,7 +224,18 @@ func TestWalkerFollowLinks(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `file bar + if runtime.GOOS == "windows" { + assert.Equal(t, filepath.FromSlash(`file bar +file bax +dir baz +file baz/one +symlink:../bax baz/two +dir foo +symlink:C:/baz/one foo/l1 +symlink:C:/baz/two foo/l2 +`), b.String()) + } else { + assert.Equal(t, filepath.FromSlash(`file bar file bax dir baz file baz/one @@ -193,7 +243,8 @@ symlink:../bax baz/two dir foo symlink:/baz/one foo/l1 symlink:/baz/two foo/l2 -`, string(b.Bytes())) +`), b.String()) + } } func TestWalkerFollowLinksToRoot(t *testing.T) { @@ -212,12 +263,12 @@ func TestWalkerFollowLinksToRoot(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `file bar + assert.Equal(t, filepath.FromSlash(`file bar file bax dir bay file bay/baz symlink:. foo -`, string(b.Bytes())) +`), b.String()) } func TestWalkerMap(t *testing.T) { @@ -241,10 +292,10 @@ func TestWalkerMap(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir _foo + assert.Equal(t, filepath.FromSlash(`dir _foo file _foo/bar2 file _foo2 -`, string(b.Bytes())) +`), b.String()) } func TestWalkerMapSkipDir(t *testing.T) { @@ -275,10 +326,37 @@ func TestWalkerMapSkipDir(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, `dir includeDir + assert.Equal(t, filepath.FromSlash(`dir includeDir file includeDir/a.txt -`, string(b.Bytes())) - assert.Equal(t, []string{"excludeDir", "includeDir", "includeDir/a.txt"}, walked) +`), b.String()) + assert.Equal(t, []string{"excludeDir", "includeDir", filepath.FromSlash("includeDir/a.txt")}, walked) +} + +func TestWalkerMapSkipDirWithPattern(t *testing.T) { + d, err := tmpDir(changeStream([]string{ + "ADD x dir", + "ADD x/a.txt file", + "ADD y dir", + "ADD y/b.txt file", + })) + assert.NoError(t, err) + defer os.RemoveAll(d) + + b := &bytes.Buffer{} + err = Walk(context.Background(), d, &FilterOpt{ + IncludePatterns: []string{"**/*.txt"}, + Map: func(_ string, s *types.Stat) MapResult { + if filepath.Base(s.Path) == "x" { + return MapResultSkipDir + } + return MapResultKeep + }, + }, bufWalk(b)) + assert.NoError(t, err) + + assert.Equal(t, filepath.FromSlash(`dir y +file y/b.txt +`), b.String()) } func TestWalkerPermissionDenied(t *testing.T) { @@ -313,7 +391,7 @@ func TestWalkerPermissionDenied(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) assert.Equal(t, `dir foo -`, string(b.Bytes())) +`, b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -321,7 +399,7 @@ func TestWalkerPermissionDenied(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) assert.Equal(t, `dir foo -`, string(b.Bytes())) +`, b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -337,7 +415,7 @@ func TestWalkerPermissionDenied(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) assert.Equal(t, `dir foo -`, string(b.Bytes())) +`, b.String()) } func bufWalk(buf *bytes.Buffer) filepath.WalkFunc { @@ -582,7 +660,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -596,7 +674,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { dir foo/bar file foo/bar/bee file foo2 - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -604,7 +682,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -615,7 +693,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { dir foo dir foo/bar file foo/bar/bee - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -623,14 +701,14 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar file a/b/bar/foo dir bar file bar/foo - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -638,7 +716,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -651,7 +729,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { dir foo dir foo/bar file foo/bar/bee - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -659,7 +737,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -667,7 +745,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { file a/b/bar/fop dir bar file bar/foo - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -675,7 +753,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ``, string(b.Bytes())) + trimEqual(t, ``, b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -683,7 +761,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -695,7 +773,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { dir foo/bar file foo/bar/bee file foo2 - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -703,7 +781,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar @@ -711,7 +789,7 @@ func TestWalkerDoublestarInclude(t *testing.T) { file a/b/bar/fop dir bar file bar/foo - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -719,14 +797,14 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/bar file a/b/bar/foo dir bar file bar/foo - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -734,11 +812,11 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir foo dir foo/bar file foo/bar/bee - `, string(b.Bytes())) + `), b.String()) b.Reset() err = Walk(context.Background(), d, &FilterOpt{ @@ -746,12 +824,12 @@ func TestWalkerDoublestarInclude(t *testing.T) { }, bufWalk(b)) assert.NoError(t, err) - trimEqual(t, ` + trimEqual(t, filepath.FromSlash(` dir a dir a/b dir a/b/baz dir baz - `, string(b.Bytes())) + `), b.String()) } func TestFSWalk(t *testing.T) { @@ -768,23 +846,23 @@ func TestFSWalk(t *testing.T) { b := &bytes.Buffer{} err = f.Walk(context.Background(), "", bufWalkDir(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo2 file foo -`) +`), b.String()) b = &bytes.Buffer{} err = f.Walk(context.Background(), "foo", bufWalkDir(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `file foo -`) + assert.Equal(t, `file foo +`, b.String()) b = &bytes.Buffer{} err = f.Walk(context.Background(), "bar", bufWalkDir(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `dir bar + assert.Equal(t, filepath.FromSlash(`dir bar file bar/foo2 -`) +`), b.String()) } func TestFSWalkNested(t *testing.T) { @@ -804,9 +882,9 @@ func TestFSWalkNested(t *testing.T) { b := &bytes.Buffer{} err = f2.Walk(context.Background(), "", bufWalkDir(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `dir foo + assert.Equal(t, filepath.FromSlash(`dir foo file foo/bar -`) +`), b.String()) f2, err = NewFilterFS(f, &FilterOpt{ ExcludePatterns: []string{"!foo/bar"}, @@ -819,7 +897,7 @@ file foo/bar b = &bytes.Buffer{} err = f2.Walk(context.Background(), "", bufWalkDir(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), ``) + assert.Equal(t, ``, b.String()) f2, err = NewFilterFS(f, &FilterOpt{ ExcludePatterns: []string{"foo"}, @@ -832,7 +910,7 @@ file foo/bar b = &bytes.Buffer{} err = f2.Walk(context.Background(), "", bufWalkDir(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), ``) + assert.Equal(t, ``, b.String()) } func TestFilteredOpen(t *testing.T) { @@ -853,8 +931,8 @@ func TestFilteredOpen(t *testing.T) { b := &bytes.Buffer{} err = f.Walk(context.Background(), "", bufWalkDir(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `file foo -`) + assert.Equal(t, `file foo +`, b.String()) r, err := f.Open("foo") assert.NoError(t, err) diff --git a/followlinks.go b/followlinks.go index 3a0bd2bb..559291d7 100644 --- a/followlinks.go +++ b/followlinks.go @@ -137,8 +137,8 @@ func (r *symlinkResolver) readSymlink(p string, allowWildcard bool) ([]string, e func statFile(fs FS, root string) (os.DirEntry, error) { var out os.DirEntry - root = filepath.Clean(root) - if root == "/" || root == "." { + root = filepath.FromSlash(filepath.Clean(root)) + if root == string(filepath.Separator) || root == "." { return nil, nil } @@ -168,8 +168,8 @@ func statFile(fs FS, root string) (os.DirEntry, error) { func readDir(fs FS, root string) ([]os.DirEntry, error) { var out []os.DirEntry - root = filepath.Clean(root) - if root == "/" || root == "." { + root = filepath.FromSlash(filepath.Clean(root)) + if root == string(filepath.Separator) || root == "." { root = "." out = make([]gofs.DirEntry, 0) } diff --git a/fs_test.go b/fs_test.go index 54e7653b..fc60c1f7 100644 --- a/fs_test.go +++ b/fs_test.go @@ -4,6 +4,7 @@ import ( "context" gofs "io/fs" "os" + "path/filepath" "testing" "github.com/containerd/continuity/fs/fstest" @@ -32,7 +33,7 @@ func TestWalk(t *testing.T) { }) require.NoError(t, err) - require.Equal(t, paths, []string{"dir", "dir/foo"}) + require.Equal(t, []string{"dir", filepath.FromSlash("dir/foo")}, paths) require.Len(t, files, 2) require.Equal(t, "dir", files[0].Name()) require.Equal(t, "foo", files[1].Name()) @@ -49,7 +50,7 @@ func TestWalk(t *testing.T) { require.Equal(t, len("contents"), int(fis[1].Size())) require.Equal(t, "dir", fis[0].(*StatInfo).Path) - require.Equal(t, "dir/foo", fis[1].(*StatInfo).Path) + require.Equal(t, filepath.FromSlash("dir/foo"), fis[1].(*StatInfo).Path) } func TestWalkDir(t *testing.T) { @@ -98,7 +99,7 @@ func TestWalkDir(t *testing.T) { }) require.NoError(t, err) - require.Equal(t, paths, []string{"1", "1/dir", "1/dir/foo", "2", "2/dir", "2/dir/bar"}) + require.Equal(t, []string{"1", filepath.FromSlash("1/dir"), filepath.FromSlash("1/dir/foo"), "2", filepath.FromSlash("2/dir"), filepath.FromSlash("2/dir/bar")}, paths) require.Equal(t, "1", files[0].Name()) require.Equal(t, "dir", files[1].Name()) require.Equal(t, "foo", files[2].Name()) diff --git a/go.mod b/go.mod index 3dceea97..93977f05 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/tonistiigi/fsutil -go 1.19 +go 1.20 require ( github.com/Microsoft/go-winio v0.5.2 diff --git a/hack/Vagrantfile.freebsd13 b/hack/Vagrantfile.freebsd13 index 0014347e..9cd7e062 100644 --- a/hack/Vagrantfile.freebsd13 +++ b/hack/Vagrantfile.freebsd13 @@ -1,31 +1,20 @@ -# This code is taken from the Vagrantfile from libjail-rs -# https://github.com/fubarnetes/libjail-rs/blob/727353bd6565c5e7a9be2664258d0197a1c8bb35/Vagrantfile -# licensed under BSD-3 Clause License: -# BSD 3-Clause License - -# Copyright (c) 2018, Fabian Freyer All rights reserved. - -# Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - -# * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. - -# * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - -# * Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. - -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# -*- mode: ruby -*- +# vi: set ft=ruby : Vagrant.configure("2") do |config| - # Stable version - # config.vm.define "fbsd_13_2" do |fbsd_13_2| fbsd_13_2.vm.box = "freebsd/FreeBSD-13.2-STABLE" end - config.vm.synced_folder ".", "/vagrant", type: "rsync", rsync__auto: true + config.vm.boot_timeout = 900 + config.vm.synced_folder ".", "/vagrant", type: "rsync" + config.ssh.keep_alive = true - config.vm.provision "shell", inline: <<-SHELL - pkg bootstrap - pkg install -y go - SHELL + config.vm.provision "init", type: "shell", run: "once" do |sh| + sh.inline = <<~SHELL + pkg bootstrap + pkg install -y go121 + ln -s /usr/local/bin/go121 /usr/local/bin/go + SHELL + end end diff --git a/hack/dockerfiles/gomod.Dockerfile b/hack/dockerfiles/gomod.Dockerfile index a7c5101c..d9f1d6d9 100644 --- a/hack/dockerfiles/gomod.Dockerfile +++ b/hack/dockerfiles/gomod.Dockerfile @@ -1,5 +1,6 @@ # syntax=docker/dockerfile:1 -ARG GO_VERSION=1.20 + +ARG GO_VERSION=1.21 FROM golang:${GO_VERSION}-alpine AS gomod RUN apk add --no-cache git diff --git a/hack/dockerfiles/lint.Dockerfile b/hack/dockerfiles/lint.Dockerfile index 2561db30..d6d173dc 100644 --- a/hack/dockerfiles/lint.Dockerfile +++ b/hack/dockerfiles/lint.Dockerfile @@ -1,9 +1,15 @@ # syntax=docker/dockerfile:1 -ARG GO_VERSION=1.20 + +ARG GO_VERSION=1.21 +ARG GOLANGCI_LINT_VERSION=1.54.2 FROM golang:${GO_VERSION}-alpine RUN apk add --no-cache git gcc musl-dev -RUN wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v1.53.3 +ENV GOFLAGS="-buildvcs=false" +ARG GOLANGCI_LINT_VERSION +RUN wget -O- -nv https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s v${GOLANGCI_LINT_VERSION} WORKDIR /go/src/github.com/tonistiigi/fsutil -RUN --mount=target=/go/src/github.com/tonistiigi/fsutil --mount=target=/root/.cache,type=cache --mount=target=/go/pkg/mod,type=cache \ - golangci-lint run +RUN --mount=target=. \ + --mount=target=/root/.cache,type=cache \ + --mount=target=/go/pkg/mod,type=cache \ + golangci-lint run diff --git a/hardlinks.go b/hardlinks.go index ef8bbfb5..d9bf2fc1 100644 --- a/hardlinks.go +++ b/hardlinks.go @@ -1,6 +1,9 @@ package fsutil import ( + "context" + "io" + gofs "io/fs" "os" "syscall" @@ -46,3 +49,68 @@ func (v *Hardlinks) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err return nil } + +// WithHardlinkReset returns a FS that fixes hardlinks for FS that has been filtered +// so that original hardlink sources might be missing +func WithHardlinkReset(fs FS) FS { + return &hardlinkFilter{fs: fs} +} + +type hardlinkFilter struct { + fs FS +} + +var _ FS = &hardlinkFilter{} + +func (r *hardlinkFilter) Walk(ctx context.Context, target string, fn gofs.WalkDirFunc) error { + seenFiles := make(map[string]string) + return r.fs.Walk(ctx, target, func(path string, entry gofs.DirEntry, err error) error { + if err != nil { + return err + } + + fi, err := entry.Info() + if err != nil { + return err + } + + if fi.IsDir() || fi.Mode()&os.ModeSymlink != 0 { + return fn(path, entry, nil) + } + + stat, ok := fi.Sys().(*types.Stat) + if !ok { + return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"}) + } + + if stat.Linkname != "" { + if v, ok := seenFiles[stat.Linkname]; !ok { + seenFiles[stat.Linkname] = stat.Path + stat.Linkname = "" + entry = &dirEntryWithStat{DirEntry: entry, stat: stat} + } else { + if v != stat.Path { + stat.Linkname = v + entry = &dirEntryWithStat{DirEntry: entry, stat: stat} + } + } + } + + seenFiles[path] = stat.Path + + return fn(path, entry, nil) + }) +} + +func (r *hardlinkFilter) Open(p string) (io.ReadCloser, error) { + return r.fs.Open(p) +} + +type dirEntryWithStat struct { + gofs.DirEntry + stat *types.Stat +} + +func (d *dirEntryWithStat) Info() (gofs.FileInfo, error) { + return &StatInfo{d.stat}, nil +} diff --git a/readme.md b/readme.md index 5ce685b7..c4171688 100644 --- a/readme.md +++ b/readme.md @@ -1,3 +1,8 @@ +[![PkgGoDev](https://img.shields.io/badge/go.dev-docs-007d9c?style=flat-square&logo=go&logoColor=white)](https://pkg.go.dev/github.com/tonistiigi/fsutil) +[![CI Status](https://img.shields.io/github/actions/workflow/status/tonistiigi/fsutil/ci.yml?label=ci&logo=github&style=flat-square)](https://github.com/tonistiigi/fsutil/actions?query=workflow%3Aci) +[![Go Report Card](https://goreportcard.com/badge/github.com/tonistiigi/fsutil?style=flat-square)](https://goreportcard.com/report/github.com/tonistiigi/fsutil) +[![Codecov](https://img.shields.io/codecov/c/github/tonistiigi/fsutil?logo=codecov&style=flat-square)](https://codecov.io/gh/tonistiigi/fsutil) + Incremental file directory sync tools in golang. ``` diff --git a/receive.go b/receive.go index 9924ce07..afb118bb 100644 --- a/receive.go +++ b/receive.go @@ -1,10 +1,42 @@ +// send.go and receive.go describe the fsutil file-transfer protocol, which +// allows transferring file trees across a network connection. +// +// The protocol operates as follows: +// - The client (the receiver) connects to the server (the sender). +// - The sender walks the target tree lexicographically and sends a series of +// STAT packets that describe each file (an empty stat indicates EOF). +// - The receiver sends a REQ packet for each file it requires the contents for, +// using the ID for the file (determined as its index in the STAT sequence). +// - The sender sends a DATA packet with byte arrays for the contents of the +// file, associated with an ID (an empty array indicates EOF). +// - Once the receiver has received all files it wants, it sends a FIN packet, +// and the file transfer is complete. +// If an error is encountered on either side, an ERR packet is sent containing +// a human-readable error. +// +// All paths transferred over the protocol are normalized to unix-style paths, +// regardless of which platforms are present on either side. These path +// conversions are performed right before sending a STAT packet (for the +// sender) or right after receiving the corresponding STAT packet (for the +// receiver); this abstraction doesn't leak into the rest of fsutil, which +// operates on native platform-specific paths. +// +// Note that in the case of cross-platform file transfers, the transfer is +// best-effort. Some filenames that are valid on a unix sender would not be +// valid on a windows receiver, so these paths are rejected as they are +// received. Additionally, file metadata, like user/group owners and xattrs do +// not have an exact correspondence on windows, and so would be discarded by +// a windows receiver. + package fsutil import ( "context" "io" "os" + "path/filepath" "sync" + "syscall" "github.com/pkg/errors" "github.com/tonistiigi/fsutil/types" @@ -192,13 +224,24 @@ func (r *receiver) run(ctx context.Context) error { if r.verboseProgressCb != nil { r.verboseProgressCb(p.Stat.Path, StatusStat, p.Size()) } + + // normalize unix wire-specific paths to platform-specific paths + path := filepath.FromSlash(p.Stat.Path) + if filepath.ToSlash(path) != p.Stat.Path { + // e.g. a linux path foo/bar\baz cannot be represented on windows + return errors.WithStack(&os.PathError{Path: p.Stat.Path, Err: syscall.EINVAL, Op: "unrepresentable path"}) + } + p.Stat.Path = path + p.Stat.Linkname = filepath.FromSlash(p.Stat.Linkname) + if fileCanRequestData(os.FileMode(p.Stat.Mode)) { r.mu.Lock() r.files[p.Stat.Path] = i r.mu.Unlock() } i++ - cp := ¤tPath{path: p.Stat.Path, stat: p.Stat} + + cp := ¤tPath{path: path, stat: p.Stat} if err := r.orderValidator.HandleChange(ChangeKindAdd, cp.path, &StatInfo{cp.stat}, nil); err != nil { return err } diff --git a/receive_test.go b/receive_test.go index 3d6cf464..4ffff200 100644 --- a/receive_test.go +++ b/receive_test.go @@ -9,6 +9,7 @@ import ( gofs "io/fs" "os" "path/filepath" + "runtime" "sync" "testing" "time" @@ -117,7 +118,7 @@ func TestCopyDirectoryTimestamps(t *testing.T) { eg.Go(func() error { defer s1.(*fakeConnProto).closeSend() - return Send(ctx, s1, fs, nil) + return Send(ctx, s1, fs, nil, nil) }) eg.Go(func() error { return Receive(ctx, s2, dest, ReceiveOpt{}) @@ -144,10 +145,10 @@ func TestCopySwitchDirToFile(t *testing.T) { dest, err := tmpDir(changeStream([]string{ "ADD foo dir", - "ADD foo/bar dile data2", + "ADD foo/bar file data2", })) assert.NoError(t, err) - defer os.RemoveAll(d) + defer os.RemoveAll(dest) copy := func(src, dest string) (*changes, error) { ts := newNotificationBuffer() @@ -180,8 +181,13 @@ func TestCopySwitchDirToFile(t *testing.T) { NotifyHashed: chs.HandleChange, ContentHasher: simpleSHA256Hasher, Filter: func(_ string, s *types.Stat) bool { - s.Uid = uint32(os.Getuid()) - s.Gid = uint32(os.Getgid()) + if runtime.GOOS != "windows" { + // On Windows, Getuid() and Getgid() always return -1 + // See: https://pkg.go.dev/os#Getgid + // See: https://pkg.go.dev/os#Geteuid + s.Uid = uint32(os.Getuid()) + s.Gid = uint32(os.Getgid()) + } return true }, }) @@ -206,8 +212,78 @@ func TestCopySwitchDirToFile(t *testing.T) { err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `file foo -`) + assert.Equal(t, `file foo +`, b.String()) +} + +func TestHardlinkFilter(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("hardlink stat linknames are not populated on windows") + } + + d, err := tmpDir(changeStream([]string{ + "ADD bar file data1", + "ADD foo file >bar", + "ADD foo2 file >bar", + })) + assert.NoError(t, err) + defer os.RemoveAll(d) + + assert.NoError(t, err) + defer os.RemoveAll(d) + fs, err := NewFS(d) + assert.NoError(t, err) + fs, err = NewFilterFS(fs, &FilterOpt{}) + assert.NoError(t, err) + fs, err = NewFilterFS(fs, &FilterOpt{ + IncludePatterns: []string{"foo*"}, + Map: func(_ string, s *types.Stat) MapResult { + s.Uid = 0 + s.Gid = 0 + return MapResultKeep + }, + }) + assert.NoError(t, err) + + dest := t.TempDir() + + eg, ctx := errgroup.WithContext(context.Background()) + s1, s2 := sockPairProto(ctx) + + eg.Go(func() error { + defer s1.(*fakeConnProto).closeSend() + return Send(ctx, s1, fs, nil, nil) + }) + eg.Go(func() error { + return Receive(ctx, s2, dest, ReceiveOpt{ + Filter: func(p string, s *types.Stat) bool { + if p == "foo2" { + require.Equal(t, "foo", s.Linkname) + } + if runtime.GOOS != "windows" { + // On Windows, Getuid() and Getgid() always return -1 + // See: https://pkg.go.dev/os#Getgid + // See: https://pkg.go.dev/os#Geteuid + s.Uid = uint32(os.Getuid()) + s.Gid = uint32(os.Getgid()) + } + return true + }, + }) + }) + assert.NoError(t, eg.Wait()) + + dt, err := os.ReadFile(filepath.Join(dest, "foo")) + assert.NoError(t, err) + assert.Equal(t, "data1", string(dt)) + + st1, err := os.Stat(filepath.Join(dest, "foo")) + assert.NoError(t, err) + + st2, err := os.Stat(filepath.Join(dest, "foo2")) + assert.NoError(t, err) + + assert.True(t, os.SameFile(st1, st2)) } func TestCopySimple(t *testing.T) { @@ -244,17 +320,27 @@ func TestCopySimple(t *testing.T) { tm := time.Now().Truncate(time.Hour) + var processCbWasCalled bool + progressCb := func(size int, last bool) { + processCbWasCalled = true + } + eg.Go(func() error { defer s1.(*fakeConnProto).closeSend() - return Send(ctx, s1, fs, nil, nil) + return Send(ctx, s1, fs, progressCb, nil) }) eg.Go(func() error { return Receive(ctx, s2, dest, ReceiveOpt{ NotifyHashed: chs.HandleChange, ContentHasher: simpleSHA256Hasher, Filter: func(p string, s *types.Stat) bool { - s.Uid = uint32(os.Getuid()) - s.Gid = uint32(os.Getgid()) + if runtime.GOOS != "windows" { + // On Windows, Getuid() and Getgid() always return -1 + // See: https://pkg.go.dev/os#Getgid + // See: https://pkg.go.dev/os#Geteuid + s.Uid = uint32(os.Getuid()) + s.Gid = uint32(os.Getgid()) + } s.ModTime = tm.UnixNano() return true }, @@ -262,12 +348,24 @@ func TestCopySimple(t *testing.T) { }) assert.NoError(t, eg.Wait()) + assert.True(t, processCbWasCalled) b := &bytes.Buffer{} err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `file foo + if runtime.GOOS == "windows" { + assert.Equal(t, `file foo +file foo2 +dir zzz +file zzz\aa +dir zzz\bb +dir zzz\bb\cc +symlink:..\..\ zzz\bb\cc\dd +file zzz.aa +`, b.String()) + } else { + assert.Equal(t, `file foo file foo2 dir zzz file zzz/aa @@ -275,7 +373,8 @@ dir zzz/bb dir zzz/bb/cc symlink:../../ zzz/bb/cc/dd file zzz.aa -`) +`, b.String()) + } dt, err := os.ReadFile(filepath.Join(dest, "zzz/aa")) assert.NoError(t, err) @@ -286,24 +385,36 @@ file zzz.aa assert.Equal(t, "dat2", string(dt)) fi, err := os.Stat(filepath.Join(dest, "foo2")) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, tm, fi.ModTime()) - h, ok := ts.Hash("zzz/aa") + h, ok := ts.Hash(filepath.FromSlash("zzz/aa")) assert.True(t, ok) - assert.Equal(t, digest.Digest("sha256:99b6ef96ee0572b5b3a4eb28f00b715d820bfd73836e59cc1565e241f4d1bb2f"), h) + if runtime.GOOS == "windows" { + assert.Equal(t, digest.Digest("sha256:5da6b6a222dca8d9260384da15378d4389f7e16943e812e08c39759b8514b456"), h) + } else { + assert.Equal(t, digest.Digest("sha256:99b6ef96ee0572b5b3a4eb28f00b715d820bfd73836e59cc1565e241f4d1bb2f"), h) + } h, ok = ts.Hash("foo2") assert.True(t, ok) - assert.Equal(t, digest.Digest("sha256:dd2529f7749ba45ea55de3b2e10086d6494cc45a94e57650c2882a6a14b4ff32"), h) + if runtime.GOOS == "windows" { + assert.Equal(t, digest.Digest("sha256:cd83620e5308f6ddb9953a82b2c7450832eac78521dbf067d2882318cabc1311"), h) + } else { + assert.Equal(t, digest.Digest("sha256:dd2529f7749ba45ea55de3b2e10086d6494cc45a94e57650c2882a6a14b4ff32"), h) + } - h, ok = ts.Hash("zzz/bb/cc/dd") + h, ok = ts.Hash(filepath.FromSlash("zzz/bb/cc/dd")) assert.True(t, ok) - assert.Equal(t, digest.Digest("sha256:eca07e8f2d09bd574ea2496312e6de1685ef15b8e6a49a534ed9e722bcac8adc"), h) + if runtime.GOOS == "windows" { + assert.Equal(t, digest.Digest("sha256:47dc68d117ae85dc688103d6ba2cee54caabbbcf606e54ca62fda6a3d9deae19"), h) + } else { + assert.Equal(t, digest.Digest("sha256:eca07e8f2d09bd574ea2496312e6de1685ef15b8e6a49a534ed9e722bcac8adc"), h) + } - k, ok := chs.c["zzz/aa"] - assert.Equal(t, ok, true) - assert.Equal(t, k, ChangeKindAdd) + k, ok := chs.c[filepath.FromSlash("zzz/aa")] + assert.Equal(t, true, ok) + assert.Equal(t, ChangeKindAdd, k) err = os.WriteFile(filepath.Join(d, "zzz/bb/cc/foo"), []byte("data5"), 0600) assert.NoError(t, err) @@ -325,8 +436,13 @@ file zzz.aa NotifyHashed: chs.HandleChange, ContentHasher: simpleSHA256Hasher, Filter: func(_ string, s *types.Stat) bool { - s.Uid = uint32(os.Getuid()) - s.Gid = uint32(os.Getgid()) + if runtime.GOOS != "windows" { + // On Windows, Getuid() and Getgid() always return -1 + // See: https://pkg.go.dev/os#Getgid + // See: https://pkg.go.dev/os#Geteuid + s.Uid = uint32(os.Getuid()) + s.Gid = uint32(os.Getgid()) + } s.ModTime = tm.UnixNano() return true }, @@ -339,7 +455,18 @@ file zzz.aa err = Walk(context.Background(), dest, nil, bufWalk(b)) assert.NoError(t, err) - assert.Equal(t, string(b.Bytes()), `file foo + if runtime.GOOS == "windows" { + assert.Equal(t, `file foo +dir zzz +file zzz\aa +dir zzz\bb +dir zzz\bb\cc +symlink:..\..\ zzz\bb\cc\dd +file zzz\bb\cc\foo +file zzz.aa +`, b.String()) + } else { + assert.Equal(t, `file foo dir zzz file zzz/aa dir zzz/bb @@ -347,36 +474,45 @@ dir zzz/bb/cc symlink:../../ zzz/bb/cc/dd file zzz/bb/cc/foo file zzz.aa -`) +`, b.String()) + } dt, err = os.ReadFile(filepath.Join(dest, "zzz/bb/cc/foo")) assert.NoError(t, err) assert.Equal(t, "data5", string(dt)) - h, ok = ts.Hash("zzz/bb/cc/dd") + h, ok = ts.Hash(filepath.FromSlash("zzz/bb/cc/dd")) assert.True(t, ok) - assert.Equal(t, digest.Digest("sha256:eca07e8f2d09bd574ea2496312e6de1685ef15b8e6a49a534ed9e722bcac8adc"), h) + if runtime.GOOS == "windows" { + assert.Equal(t, digest.Digest("sha256:47dc68d117ae85dc688103d6ba2cee54caabbbcf606e54ca62fda6a3d9deae19"), h) + } else { + assert.Equal(t, digest.Digest("sha256:eca07e8f2d09bd574ea2496312e6de1685ef15b8e6a49a534ed9e722bcac8adc"), h) + } - h, ok = ts.Hash("zzz/bb/cc/foo") + h, ok = ts.Hash(filepath.FromSlash("zzz/bb/cc/foo")) assert.True(t, ok) - assert.Equal(t, digest.Digest("sha256:cd14a931fc2e123ded338093f2864b173eecdee578bba6ec24d0724272326c3a"), h) + if runtime.GOOS == "windows" { + assert.Equal(t, digest.Digest("sha256:9184a7db8d056ee43838613279db9a7ab02272e50d5e20d253393521bb34aa46"), h) + } else { + assert.Equal(t, digest.Digest("sha256:cd14a931fc2e123ded338093f2864b173eecdee578bba6ec24d0724272326c3a"), h) + } _, ok = ts.Hash("foo2") assert.False(t, ok) k, ok = chs.c["foo2"] - assert.Equal(t, ok, true) - assert.Equal(t, k, ChangeKindDelete) + assert.Equal(t, true, ok) + assert.Equal(t, ChangeKindDelete, k) - k, ok = chs.c["zzz/bb/cc/foo"] - assert.Equal(t, ok, true) - assert.Equal(t, k, ChangeKindAdd) + k, ok = chs.c[filepath.FromSlash("zzz/bb/cc/foo")] + assert.Equal(t, true, ok) + assert.Equal(t, ChangeKindAdd, k) - _, ok = chs.c["zzz/aa"] - assert.Equal(t, ok, false) + _, ok = chs.c[filepath.FromSlash("zzz/aa")] + assert.Equal(t, false, ok) _, ok = chs.c["zzz.aa"] - assert.Equal(t, ok, false) + assert.Equal(t, false, ok) } func sockPairProto(ctx context.Context) (Stream, Stream) { @@ -498,8 +634,15 @@ func simpleSHA256Hasher(s *types.Stat) (hash.Hash, error) { // Unlike Linux, on FreeBSD's stat() call returns -1 in st_rdev for regular files ss.Devminor = 0 ss.Devmajor = 0 + if runtime.GOOS == "windows" { + // On Windows, Getuid() and Getgid() always return -1 + // See: https://pkg.go.dev/os#Getgid + // See: https://pkg.go.dev/os#Geteuid + ss.Uid = 0 + ss.Gid = 0 + } - if os.FileMode(ss.Mode)&os.ModeSymlink != 0 { + if os.FileMode(ss.Mode)&os.ModeSymlink != 0 && runtime.GOOS != "windows" { ss.Mode = ss.Mode | 0777 } diff --git a/send.go b/send.go index 7fe01a07..8e694221 100644 --- a/send.go +++ b/send.go @@ -4,6 +4,7 @@ import ( "context" "io" "os" + "path/filepath" "sync" "syscall" @@ -28,7 +29,7 @@ type Stream interface { func Send(ctx context.Context, conn Stream, fs FS, progressCb func(int, bool), verboseProgressCb VerboseProgressCB) error { s := &sender{ conn: &syncStream{Stream: conn}, - fs: fs, + fs: WithHardlinkReset(fs), files: make(map[uint32]string), progressCb: progressCb, verboseProgressCb: verboseProgressCb, @@ -50,6 +51,7 @@ type sender struct { progressCb func(int, bool) verboseProgressCb VerboseProgressCB progressCurrent int + progressCurrentMu sync.Mutex sendpipeline chan *sendHandle } @@ -116,6 +118,9 @@ func (s *sender) updateProgress(size int, last bool) { defer s.mu.Unlock() s.progressCurrent += size if s.progressCb != nil { + s.progressCurrentMu.Lock() + defer s.progressCurrentMu.Unlock() + s.progressCurrent += size s.progressCb(s.progressCurrent, last) } } @@ -164,7 +169,8 @@ func (s *sender) walk(ctx context.Context) error { if !ok { return errors.WithStack(&os.PathError{Path: path, Err: syscall.EBADMSG, Op: "fileinfo without stat info"}) } - + stat.Path = filepath.ToSlash(stat.Path) + stat.Linkname = filepath.ToSlash(stat.Linkname) p := &types.Packet{ Type: types.PACKET_STAT, Stat: stat, diff --git a/stat.go b/stat.go index 2ab8da11..44441cb6 100644 --- a/stat.go +++ b/stat.go @@ -19,7 +19,7 @@ func mkstat(path, relpath string, fi os.FileInfo, inodemap map[uint64]string) (* relpath = filepath.ToSlash(relpath) stat := &types.Stat{ - Path: relpath, + Path: filepath.FromSlash(relpath), Mode: uint32(fi.Mode()), ModTime: fi.ModTime().UnixNano(), } diff --git a/validator.go b/validator.go index 9bd7d94d..8b0dd91e 100644 --- a/validator.go +++ b/validator.go @@ -2,8 +2,7 @@ package fsutil import ( "os" - "path" - "runtime" + "path/filepath" "sort" "strings" "syscall" @@ -28,21 +27,18 @@ func (v *Validator) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err if v.parentDirs == nil { v.parentDirs = make([]parent, 1, 10) } - if runtime.GOOS == "windows" { - p = strings.Replace(p, "\\", "", -1) - } - if p != path.Clean(p) { + if p != filepath.Clean(p) { return errors.WithStack(&os.PathError{Path: p, Err: syscall.EINVAL, Op: "unclean path"}) } - if path.IsAbs(p) { + if filepath.IsAbs(p) { return errors.WithStack(&os.PathError{Path: p, Err: syscall.EINVAL, Op: "absolute path"}) } - dir := path.Dir(p) - base := path.Base(p) + dir := filepath.Dir(p) + base := filepath.Base(p) if dir == "." { dir = "" } - if dir == ".." || strings.HasPrefix(p, "../") { + if dir == ".." || strings.HasPrefix(p, filepath.FromSlash("../")) { return errors.WithStack(&os.PathError{Path: p, Err: syscall.EINVAL, Op: "escape check"}) } @@ -56,12 +52,12 @@ func (v *Validator) HandleChange(kind ChangeKind, p string, fi os.FileInfo, err } if dir != v.parentDirs[len(v.parentDirs)-1].dir || v.parentDirs[i].last >= base { - return errors.Errorf("changes out of order: %q %q", p, path.Join(v.parentDirs[i].dir, v.parentDirs[i].last)) + return errors.Errorf("changes out of order: %q %q", p, filepath.Join(v.parentDirs[i].dir, v.parentDirs[i].last)) } v.parentDirs[i].last = base if kind != ChangeKindDelete && fi.IsDir() { v.parentDirs = append(v.parentDirs, parent{ - dir: path.Join(dir, base), + dir: filepath.Join(dir, base), last: "", }) } @@ -76,7 +72,7 @@ func ComparePath(p1, p2 string) int { switch { case p1[i] == p2[i]: continue - case p2[i] != '/' && p1[i] < p2[i] || p1[i] == '/': + case p2[i] != filepath.Separator && p1[i] < p2[i] || p1[i] == filepath.Separator: return -1 default: return 1 diff --git a/validator_test.go b/validator_test.go index aabb0732..b3acf688 100644 --- a/validator_test.go +++ b/validator_test.go @@ -3,6 +3,7 @@ package fsutil import ( "fmt" "os" + "path/filepath" "strings" "testing" @@ -210,7 +211,7 @@ func parseChange(str string) *change { default: panic(errStr) } - c.path = f[1] + c.path = filepath.FromSlash(f[1]) st := &types.Stat{} switch f[2] { case "file":