From 7e7e9a81d1111406d8b114f46ab255c634a07339 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Tue, 3 Feb 2026 17:11:37 -0800 Subject: [PATCH 01/30] initial revert to return to old fs logic --- config/config.go | 37 -- go.mod | 1 + go.sum | 2 + internal/ufs/LICENSE | 21 - internal/ufs/README.md | 21 - internal/ufs/doc.go | 12 - internal/ufs/error.go | 183 ------ internal/ufs/file.go | 180 ------ internal/ufs/file_posix.go | 49 -- internal/ufs/filesystem.go | 168 ----- internal/ufs/fs_quota.go | 169 ----- internal/ufs/fs_unix.go | 895 --------------------------- internal/ufs/fs_unix_test.go | 768 ----------------------- internal/ufs/go.LICENSE | 27 - internal/ufs/mkdir_unix.go | 63 -- internal/ufs/path_unix.go | 80 --- internal/ufs/quota_writer.go | 115 ---- internal/ufs/removeall_unix.go | 261 -------- internal/ufs/stat_unix.go | 67 -- internal/ufs/walk.go | 124 ---- internal/ufs/walk_unix.go | 314 ---------- parser/helpers.go | 18 +- parser/parser.go | 80 ++- router/downloader/downloader.go | 13 +- router/router_download.go | 18 +- router/router_server.go | 10 +- router/router_server_files.go | 16 +- router/router_transfer.go | 1 - server/backup.go | 15 +- server/backup/backup.go | 3 +- server/backup/backup_local.go | 6 +- server/backup/backup_s3.go | 6 +- server/config_parser.go | 14 +- server/filesystem/archive.go | 158 ++--- server/filesystem/archive_test.go | 35 +- server/filesystem/compress.go | 232 +++---- server/filesystem/compress_test.go | 7 +- server/filesystem/disk_space.go | 131 ++-- server/filesystem/errors.go | 9 +- server/filesystem/filesystem.go | 655 +++++++++++--------- server/filesystem/filesystem_test.go | 151 ++--- server/filesystem/path.go | 105 +++- server/filesystem/path_test.go | 143 +++-- server/filesystem/stat.go | 49 +- server/filesystem/stat_darwin.go | 13 + server/filesystem/stat_linux.go | 20 +- server/filesystem/stat_windows.go | 12 + server/install.go | 17 +- server/manager.go | 5 +- server/transfer/archive.go | 4 +- sftp/handler.go | 23 +- 51 files changed, 1147 insertions(+), 4379 deletions(-) delete mode 100644 internal/ufs/LICENSE delete mode 100644 internal/ufs/README.md delete mode 100644 internal/ufs/doc.go delete mode 100644 internal/ufs/error.go delete mode 100644 internal/ufs/file.go delete mode 100644 internal/ufs/file_posix.go delete mode 100644 internal/ufs/filesystem.go delete mode 100644 internal/ufs/fs_quota.go delete mode 100644 internal/ufs/fs_unix.go delete mode 100644 internal/ufs/fs_unix_test.go delete mode 100644 internal/ufs/go.LICENSE delete mode 100644 internal/ufs/mkdir_unix.go delete mode 100644 internal/ufs/path_unix.go delete mode 100644 internal/ufs/quota_writer.go delete mode 100644 internal/ufs/removeall_unix.go delete mode 100644 internal/ufs/stat_unix.go delete mode 100644 internal/ufs/walk.go delete mode 100644 internal/ufs/walk_unix.go create mode 100644 server/filesystem/stat_darwin.go create mode 100644 server/filesystem/stat_windows.go diff --git a/config/config.go b/config/config.go index a83936d88..c88186a0d 100644 --- a/config/config.go +++ b/config/config.go @@ -13,7 +13,6 @@ import ( "regexp" "strings" "sync" - "sync/atomic" "text/template" "time" @@ -22,7 +21,6 @@ import ( "github.com/apex/log" "github.com/creasty/defaults" "github.com/gbrlsnchs/jwt/v3" - "golang.org/x/sys/unix" "gopkg.in/yaml.v2" "github.com/pterodactyl/wings/system" @@ -251,8 +249,6 @@ type SystemConfiguration struct { Backups Backups `yaml:"backups"` Transfers Transfers `yaml:"transfers"` - - OpenatMode string `default:"auto" yaml:"openat_mode"` } type CrashDetection struct { @@ -798,39 +794,6 @@ func getSystemName() (string, error) { return release["ID"], nil } -var ( - openat2 atomic.Bool - openat2Set atomic.Bool -) - -func UseOpenat2() bool { - if openat2Set.Load() { - return openat2.Load() - } - defer openat2Set.Store(true) - - c := Get() - openatMode := c.System.OpenatMode - switch openatMode { - case "openat2": - openat2.Store(true) - return true - case "openat": - openat2.Store(false) - return false - default: - fd, err := unix.Openat2(unix.AT_FDCWD, "/", &unix.OpenHow{}) - if err != nil { - log.WithError(err).Warn("error occurred while checking for openat2 support, falling back to openat") - openat2.Store(false) - return false - } - _ = unix.Close(fd) - openat2.Store(true) - return true - } -} - // Expand expands an input string by calling [os.ExpandEnv] to expand all // environment variables, then checks if the value is prefixed with `file://` // to support reading the value from a file. diff --git a/go.mod b/go.mod index 6ed6d60f4..af2e23de2 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/iancoleman/strcase v0.3.0 github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0 github.com/juju/ratelimit v1.0.2 + github.com/karrick/godirwalk v1.17.0 github.com/klauspost/compress v1.18.0 github.com/klauspost/pgzip v1.2.6 github.com/magiconair/properties v1.8.9 diff --git a/go.sum b/go.sum index 9e2465a94..198c38245 100644 --- a/go.sum +++ b/go.sum @@ -216,6 +216,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= +github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= +github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/internal/ufs/LICENSE b/internal/ufs/LICENSE deleted file mode 100644 index 287f516f9..000000000 --- a/internal/ufs/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2024 Matthew Penner - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/internal/ufs/README.md b/internal/ufs/README.md deleted file mode 100644 index c9a844104..000000000 --- a/internal/ufs/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# Filesystem - -Coming Soon™ - -> TODO - -## Licensing - -Most code in this package is licensed under `MIT` with some exceptions. - -The following files are licensed under `BSD-3-Clause` due to them being copied -verbatim or derived from [Go](https://go.dev)'s source code. - -- [`file_posix.go`](./file_posix.go) -- [`mkdir_unix.go`](./mkdir_unix.go) -- [`path_unix.go`](./path_unix.go) -- [`removeall_unix.go`](./removeall_unix.go) -- [`stat_unix.go`](./stat_unix.go) -- [`walk.go`](./walk.go) - -These changes are not associated with nor endorsed by The Go Authors. diff --git a/internal/ufs/doc.go b/internal/ufs/doc.go deleted file mode 100644 index 85ad991b3..000000000 --- a/internal/ufs/doc.go +++ /dev/null @@ -1,12 +0,0 @@ -// SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: Copyright (c) 2024 Matthew Penner - -// Package ufs provides an abstraction layer for performing I/O on filesystems. -// This package is designed to be used in-place of standard `os` package I/O -// calls, and is not designed to be used as a generic filesystem abstraction -// like the `io/fs` package. -// -// The primary use-case of this package was to provide a "chroot-like" `os` -// wrapper, so we can safely sandbox I/O operations within a directory and -// use untrusted arbitrary paths. -package ufs diff --git a/internal/ufs/error.go b/internal/ufs/error.go deleted file mode 100644 index 842d404fa..000000000 --- a/internal/ufs/error.go +++ /dev/null @@ -1,183 +0,0 @@ -// SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: Copyright (c) 2024 Matthew Penner - -package ufs - -import ( - "errors" - iofs "io/fs" - "os" - "syscall" - - "golang.org/x/sys/unix" -) - -var ( - // ErrIsDirectory is an error for when an operation that operates only on - // files is given a path to a directory. - ErrIsDirectory = errors.New("is a directory") - // ErrNotDirectory is an error for when an operation that operates only on - // directories is given a path to a file. - ErrNotDirectory = errors.New("not a directory") - // ErrBadPathResolution is an error for when a sand-boxed filesystem - // resolves a given path to a forbidden location. - ErrBadPathResolution = errors.New("bad path resolution") - // ErrNotRegular is an error for when an operation that operates only on - // regular files is passed something other than a regular file. - ErrNotRegular = errors.New("not a regular file") - - // ErrClosed is an error for when an entry was accessed after being closed. - ErrClosed = iofs.ErrClosed - // ErrInvalid is an error for when an invalid argument was used. - ErrInvalid = iofs.ErrInvalid - // ErrExist is an error for when an entry already exists. - ErrExist = iofs.ErrExist - // ErrNotExist is an error for when an entry does not exist. - ErrNotExist = iofs.ErrNotExist - // ErrPermission is an error for when the required permissions to perform an - // operation are missing. - ErrPermission = iofs.ErrPermission -) - -// LinkError records an error during a link or symlink or rename -// system call and the paths that caused it. -type LinkError = os.LinkError - -// PathError records an error and the operation and file path that caused it. -type PathError = iofs.PathError - -// SyscallError records an error from a specific system call. -type SyscallError = os.SyscallError - -// NewSyscallError returns, as an error, a new [*os.SyscallError] with the -// given system call name and error details. As a convenience, if err is nil, -// [NewSyscallError] returns nil. -func NewSyscallError(syscall string, err error) error { - return os.NewSyscallError(syscall, err) -} - -// convertErrorType converts errors into our custom errors to ensure consistent -// error values. -func convertErrorType(err error) error { - if err == nil { - return nil - } - - var pErr *PathError - if errors.As(err, &pErr) { - if errno, ok := pErr.Err.(syscall.Errno); ok { - return errnoToPathError(errno, pErr.Op, pErr.Path) - } - return pErr - } - - // If the error wasn't already a path error and is a errno, wrap it with - // details that we can use to know there is something wrong with our - // error wrapping somewhere. - var errno syscall.Errno - if errors.As(err, &errno) { - return &PathError{ - Op: "!(UNKNOWN)", - Path: "!(UNKNOWN)", - Err: err, - } - } - - return err -} - -// ensurePathError ensures that err is a PathError. The op and path arguments -// are only used of the error isn't already a PathError. -func ensurePathError(err error, op, path string) error { - if err == nil { - return nil - } - - // Check if the error is already a PathError. - var pErr *PathError - if errors.As(err, &pErr) { - // If underlying error is a errno, convert it. - // - // DO NOT USE `errors.As` or whatever here, the error will either be - // an errno, or it will be wrapped already. - if errno, ok := pErr.Err.(syscall.Errno); ok { - return errnoToPathError(errno, pErr.Op, pErr.Path) - } - // Return the PathError as-is without modification. - return pErr - } - - // If the error is directly an errno, convert it to a PathError. - var errno syscall.Errno - if errors.As(err, &errno) { - return errnoToPathError(errno, op, path) - } - - // Otherwise just wrap it as a PathError without any additional changes. - return &PathError{ - Op: op, - Path: path, - Err: err, - } -} - -// errnoToPathError converts an errno into a proper path error. -func errnoToPathError(err syscall.Errno, op, path string) error { - switch err { - // File exists - case unix.EEXIST: - return &PathError{ - Op: op, - Path: path, - Err: ErrExist, - } - // Is a directory - case unix.EISDIR: - return &PathError{ - Op: op, - Path: path, - Err: ErrIsDirectory, - } - // Not a directory - case unix.ENOTDIR: - return &PathError{ - Op: op, - Path: path, - Err: ErrNotDirectory, - } - // No such file or directory - case unix.ENOENT: - return &PathError{ - Op: op, - Path: path, - Err: ErrNotExist, - } - // Operation not permitted - case unix.EPERM: - return &PathError{ - Op: op, - Path: path, - Err: ErrPermission, - } - // Invalid cross-device link - case unix.EXDEV: - return &PathError{ - Op: op, - Path: path, - Err: ErrBadPathResolution, - } - // Too many levels of symbolic links - case unix.ELOOP: - return &PathError{ - Op: op, - Path: path, - Err: ErrBadPathResolution, - } - default: - return &PathError{ - Op: op, - Path: path, - Err: err, - } - } -} diff --git a/internal/ufs/file.go b/internal/ufs/file.go deleted file mode 100644 index 9902c8b64..000000000 --- a/internal/ufs/file.go +++ /dev/null @@ -1,180 +0,0 @@ -// SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: Copyright (c) 2024 Matthew Penner - -package ufs - -import ( - "io" - iofs "io/fs" - - "golang.org/x/sys/unix" -) - -// DirEntry is an entry read from a directory. -type DirEntry = iofs.DirEntry - -// File describes readable and/or writable file from a Filesystem. -type File interface { - // Name returns the base name of the file. - Name() string - - // Stat returns the FileInfo structure describing the file. - // If there is an error, it will be of type *PathError. - Stat() (FileInfo, error) - - // ReadDir reads the contents of the directory associated with the file f - // and returns a slice of DirEntry values in directory order. - // Subsequent calls on the same file will yield later DirEntry records in the directory. - // - // If n > 0, ReadDir returns at most n DirEntry records. - // In this case, if ReadDir returns an empty slice, it will return an error explaining why. - // At the end of a directory, the error is io.EOF. - // - // If n <= 0, ReadDir returns all the DirEntry records remaining in the directory. - // When it succeeds, it returns a nil error (not io.EOF). - ReadDir(n int) ([]DirEntry, error) - - // Readdirnames reads the contents of the directory associated with file - // and returns a slice of up to n names of files in the directory, - // in directory order. Subsequent calls on the same file will yield - // further names. - // - // If n > 0, Readdirnames returns at most n names. In this case, if - // Readdirnames returns an empty slice, it will return a non-nil error - // explaining why. At the end of a directory, the error is io.EOF. - // - // If n <= 0, Readdirnames returns all the names from the directory in - // a single slice. In this case, if Readdirnames succeeds (reads all - // the way to the end of the directory), it returns the slice and a - // nil error. If it encounters an error before the end of the - // directory, Readdirnames returns the names read until that point and - // a non-nil error. - Readdirnames(n int) (names []string, err error) - - // Fd returns the integer Unix file descriptor referencing the open file. - // If f is closed, the file descriptor becomes invalid. - // If f is garbage collected, a finalizer may close the file descriptor, - // making it invalid; see runtime.SetFinalizer for more information on when - // a finalizer might be run. On Unix systems this will cause the SetDeadline - // methods to stop working. - // Because file descriptors can be reused, the returned file descriptor may - // only be closed through the Close method of f, or by its finalizer during - // garbage collection. Otherwise, during garbage collection the finalizer - // may close an unrelated file descriptor with the same (reused) number. - // - // As an alternative, see the f.SyscallConn method. - Fd() uintptr - - // Truncate changes the size of the file. - // It does not change the I/O offset. - // If there is an error, it will be of type *PathError. - Truncate(size int64) error - - io.Closer - - io.Reader - io.ReaderAt - io.ReaderFrom - - io.Writer - io.WriterAt - - io.Seeker -} - -// FileInfo describes a file and is returned by Stat and Lstat. -type FileInfo = iofs.FileInfo - -// FileMode represents a file's mode and permission bits. -// The bits have the same definition on all systems, so that -// information about files can be moved from one system -// to another portably. Not all bits apply to all systems. -// The only required bit is ModeDir for directories. -type FileMode = iofs.FileMode - -// The defined file mode bits are the most significant bits of the FileMode. -// The nine least-significant bits are the standard Unix rwxrwxrwx permissions. -// The values of these bits should be considered part of the public API and -// may be used in wire protocols or disk representations: they must not be -// changed, although new bits might be added. -const ( - // ModeDir represents a directory. - // d: is a directory - ModeDir = iofs.ModeDir - // ModeAppend represents an append-only file. - // a: append-only - ModeAppend = iofs.ModeAppend - // ModeExclusive represents an exclusive file. - // l: exclusive use - ModeExclusive = iofs.ModeExclusive - // ModeTemporary . - // T: temporary file; Plan 9 only. - ModeTemporary = iofs.ModeTemporary - // ModeSymlink . - // L: symbolic link. - ModeSymlink = iofs.ModeSymlink - // ModeDevice . - // D: device file. - ModeDevice = iofs.ModeDevice - // ModeNamedPipe . - // p: named pipe (FIFO) - ModeNamedPipe = iofs.ModeNamedPipe - // ModeSocket . - // S: Unix domain socket. - ModeSocket = iofs.ModeSocket - // ModeSetuid . - // u: setuid - ModeSetuid = iofs.ModeSetuid - // ModeSetgid . - // g: setgid - ModeSetgid = iofs.ModeSetgid - // ModeCharDevice . - // c: Unix character device, when ModeDevice is set - ModeCharDevice = iofs.ModeCharDevice - // ModeSticky . - // t: sticky - ModeSticky = iofs.ModeSticky - // ModeIrregular . - // ?: non-regular file; nothing else is known about this file. - ModeIrregular = iofs.ModeIrregular - - // ModeType . - ModeType = iofs.ModeType - - // ModePerm . - // Unix permission bits, 0o777. - ModePerm = iofs.ModePerm -) - -// Re-using the same names as Go's official `unix` and `os` package do. -const ( - // O_RDONLY opens the file read-only. - O_RDONLY = unix.O_RDONLY - // O_WRONLY opens the file write-only. - O_WRONLY = unix.O_WRONLY - // O_RDWR opens the file read-write. - O_RDWR = unix.O_RDWR - // O_APPEND appends data to the file when writing. - O_APPEND = unix.O_APPEND - // O_CREATE creates a new file if it doesn't exist. - O_CREATE = unix.O_CREAT - // O_EXCL is used with O_CREATE, file must not exist. - O_EXCL = unix.O_EXCL - // O_SYNC open for synchronous I/O. - O_SYNC = unix.O_SYNC - // O_TRUNC truncates regular writable file when opened. - O_TRUNC = unix.O_TRUNC - // O_DIRECTORY opens a directory only. If the entry is not a directory an - // error will be returned. - O_DIRECTORY = unix.O_DIRECTORY - // O_NOFOLLOW opens the exact path given without following symlinks. - O_NOFOLLOW = unix.O_NOFOLLOW - O_CLOEXEC = unix.O_CLOEXEC - O_LARGEFILE = unix.O_LARGEFILE -) - -const ( - AT_SYMLINK_NOFOLLOW = unix.AT_SYMLINK_NOFOLLOW - AT_REMOVEDIR = unix.AT_REMOVEDIR - AT_EMPTY_PATH = unix.AT_EMPTY_PATH -) diff --git a/internal/ufs/file_posix.go b/internal/ufs/file_posix.go deleted file mode 100644 index 7ce0634d6..000000000 --- a/internal/ufs/file_posix.go +++ /dev/null @@ -1,49 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -// Code in this file was copied from `go/src/os/file_posix.go`. - -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the `go.LICENSE` file. - -//go:build unix || (js && wasm) || wasip1 || windows - -package ufs - -import ( - "golang.org/x/sys/unix" -) - -// ignoringEINTR makes a function call and repeats it if it returns an -// EINTR error. This appears to be required even though we install all -// signal handlers with SA_RESTART: see https://go.dev/issue/22838, -// https://go.dev/issue/38033, https://go.dev/issue/38836, -// https://go.dev/issue/40846. Also, https://go.dev/issue/20400 and -// https://go.dev/issue/36644 are issues in which a signal handler is -// installed without setting SA_RESTART. None of these are the common case, -// but there are enough of them that it seems that we can't avoid -// an EINTR loop. -func ignoringEINTR(fn func() error) error { - for { - err := fn() - if err != unix.EINTR { - return err - } - } -} - -// syscallMode returns the syscall-specific mode bits from Go's portable mode bits. -func syscallMode(i FileMode) (o FileMode) { - o |= i.Perm() - if i&ModeSetuid != 0 { - o |= unix.S_ISUID - } - if i&ModeSetgid != 0 { - o |= unix.S_ISGID - } - if i&ModeSticky != 0 { - o |= unix.S_ISVTX - } - // No mapping for Go's ModeTemporary (plan9 only). - return -} diff --git a/internal/ufs/filesystem.go b/internal/ufs/filesystem.go deleted file mode 100644 index 3fa168244..000000000 --- a/internal/ufs/filesystem.go +++ /dev/null @@ -1,168 +0,0 @@ -// SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: Copyright (c) 2024 Matthew Penner - -package ufs - -import ( - "time" -) - -// Filesystem represents a filesystem capable of performing I/O operations. -type Filesystem interface { - // Chmod changes the mode of the named file to mode. - // - // If the file is a symbolic link, it changes the mode of the link's target. - // If there is an error, it will be of type *PathError. - // - // A different subset of the mode bits are used, depending on the - // operating system. - // - // On Unix, the mode's permission bits, ModeSetuid, ModeSetgid, and - // ModeSticky are used. - // - // On Windows, only the 0200 bit (owner writable) of mode is used; it - // controls whether the file's read-only attribute is set or cleared. - // The other bits are currently unused. For compatibility with Go 1.12 - // and earlier, use a non-zero mode. Use mode 0400 for a read-only - // file and 0600 for a readable+writable file. - // - // On Plan 9, the mode's permission bits, ModeAppend, ModeExclusive, - // and ModeTemporary are used. - Chmod(name string, mode FileMode) error - - // Chown changes the numeric uid and gid of the named file. - // - // If the file is a symbolic link, it changes the uid and gid of the link's target. - // A uid or gid of -1 means to not change that value. - // If there is an error, it will be of type *PathError. - // - // On Windows or Plan 9, Chown always returns the syscall.EWINDOWS or - // EPLAN9 error, wrapped in *PathError. - Chown(name string, uid, gid int) error - - // Lchown changes the numeric uid and gid of the named file. - // - // If the file is a symbolic link, it changes the uid and gid of the link itself. - // If there is an error, it will be of type *PathError. - // - // On Windows, it always returns the syscall.EWINDOWS error, wrapped - // in *PathError. - Lchown(name string, uid, gid int) error - - // Chtimes changes the access and modification times of the named - // file, similar to the Unix utime() or utimes() functions. - // - // The underlying filesystem may truncate or round the values to a - // less precise time unit. - // - // If there is an error, it will be of type *PathError. - Chtimes(name string, atime, mtime time.Time) error - - // Create creates or truncates the named file. If the file already exists, - // it is truncated. - // - // If the file does not exist, it is created with mode 0666 - // (before umask). If successful, methods on the returned File can - // be used for I/O; the associated file descriptor has mode O_RDWR. - // If there is an error, it will be of type *PathError. - Create(name string) (File, error) - - // Mkdir creates a new directory with the specified name and permission - // bits (before umask). - // - // If there is an error, it will be of type *PathError. - Mkdir(name string, perm FileMode) error - - // MkdirAll creates a directory named path, along with any necessary - // parents, and returns nil, or else returns an error. - // - // The permission bits perm (before umask) are used for all - // directories that MkdirAll creates. - // If path is already a directory, MkdirAll does nothing - // and returns nil. - MkdirAll(path string, perm FileMode) error - - // Open opens the named file for reading. - // - // If successful, methods on the returned file can be used for reading; the - // associated file descriptor has mode O_RDONLY. - // - // If there is an error, it will be of type *PathError. - Open(name string) (File, error) - - // OpenFile is the generalized open call; most users will use Open - // or Create instead. It opens the named file with specified flag - // (O_RDONLY etc.). - // - // If the file does not exist, and the O_CREATE flag - // is passed, it is created with mode perm (before umask). If successful, - // methods on the returned File can be used for I/O. - // - // If there is an error, it will be of type *PathError. - OpenFile(name string, flag int, perm FileMode) (File, error) - - // ReadDir reads the named directory, - // - // returning all its directory entries sorted by filename. - // If an error occurs reading the directory, ReadDir returns the entries it - // was able to read before the error, along with the error. - ReadDir(name string) ([]DirEntry, error) - - // Remove removes the named file or (empty) directory. - // - // If there is an error, it will be of type *PathError. - Remove(name string) error - - // RemoveAll removes path and any children it contains. - // - // It removes everything it can but returns the first error - // it encounters. If the path does not exist, RemoveAll - // returns nil (no error). - // - // If there is an error, it will be of type *PathError. - RemoveAll(path string) error - - // Rename renames (moves) oldpath to newpath. - // - // If newpath already exists and is not a directory, Rename replaces it. - // OS-specific restrictions may apply when oldpath and newpath are in different directories. - // Even within the same directory, on non-Unix platforms Rename is not an atomic operation. - // - // If there is an error, it will be of type *LinkError. - Rename(oldname, newname string) error - - // Stat returns a FileInfo describing the named file. - // - // If there is an error, it will be of type *PathError. - Stat(name string) (FileInfo, error) - - // Lstat returns a FileInfo describing the named file. - // - // If the file is a symbolic link, the returned FileInfo - // describes the symbolic link. Lstat makes no attempt to follow the link. - // - // If there is an error, it will be of type *PathError. - Lstat(name string) (FileInfo, error) - - // Symlink creates newname as a symbolic link to oldname. - // - // On Windows, a symlink to a non-existent oldname creates a file symlink; - // if oldname is later created as a directory the symlink will not work. - // - // If there is an error, it will be of type *LinkError. - Symlink(oldname, newname string) error - - // WalkDir walks the file tree rooted at root, calling fn for each file or - // directory in the tree, including root. - // - // All errors that arise visiting files and directories are filtered by fn: - // see the [WalkDirFunc] documentation for details. - // - // The files are walked in lexical order, which makes the output deterministic - // but requires WalkDir to read an entire directory into memory before proceeding - // to walk that directory. - // - // WalkDir does not follow symbolic links found in directories, - // but if root itself is a symbolic link, its target will be walked. - WalkDir(root string, fn WalkDirFunc) error -} diff --git a/internal/ufs/fs_quota.go b/internal/ufs/fs_quota.go deleted file mode 100644 index cc89cbd0d..000000000 --- a/internal/ufs/fs_quota.go +++ /dev/null @@ -1,169 +0,0 @@ -// SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: Copyright (c) 2024 Matthew Penner - -package ufs - -import ( - "sync/atomic" -) - -// Quota is a wrapper around [*UnixFS] that provides the ability to limit the -// disk usage of the filesystem. -// -// NOTE: this is not a full complete quota filesystem, it provides utilities for -// tracking and checking the usage of the filesystem. The only operation that is -// automatically accounted against the quota are file deletions. -type Quota struct { - // fs is the underlying filesystem that runs the actual I/O operations. - *UnixFS - - // limit is the size limit of the filesystem. - // - // limit is atomic to allow the limit to be safely changed after the - // filesystem was created. - // - // A limit of `-1` disables any write operation from being performed. - // A limit of `0` disables any limit checking. - limit atomic.Int64 - - // usage is the current usage of the filesystem. - // - // If usage is set to `-1`, it hasn't been calculated yet. - usage atomic.Int64 -} - -// NewQuota creates a new Quota filesystem using an existing UnixFS and a limit. -func NewQuota(fs *UnixFS, limit int64) *Quota { - qfs := Quota{UnixFS: fs} - qfs.limit.Store(limit) - return &qfs -} - -// Close closes the filesystem. -func (fs *Quota) Close() (err error) { - err = fs.UnixFS.Close() - return -} - -// Limit returns the limit of the filesystem. -func (fs *Quota) Limit() int64 { - return fs.limit.Load() -} - -// SetLimit returns the limit of the filesystem. -func (fs *Quota) SetLimit(newLimit int64) int64 { - return fs.limit.Swap(newLimit) -} - -// Usage returns the current usage of the filesystem. -func (fs *Quota) Usage() int64 { - return fs.usage.Load() -} - -// SetUsage updates the total usage of the filesystem. -func (fs *Quota) SetUsage(newUsage int64) int64 { - return fs.usage.Swap(newUsage) -} - -// Add adds `i` to the tracked usage total. -func (fs *Quota) Add(i int64) int64 { - usage := fs.Usage() - - // If adding `i` to the usage will put us below 0, cap it. (`i` can be negative) - if usage+i < 0 { - fs.usage.Store(0) - return 0 - } - return fs.usage.Add(i) -} - -// CanFit checks if the given size can fit in the filesystem without exceeding -// the limit of the filesystem. -func (fs *Quota) CanFit(size int64) bool { - // Get the size limit of the filesystem. - limit := fs.Limit() - switch limit { - case -1: - // A limit of -1 means no write operations are allowed. - return false - case 0: - // A limit of 0 means unlimited. - return true - } - - // Any other limit is a value we need to check. - usage := fs.Usage() - if usage == -1 { - // We don't know what the current usage is yet. - return true - } - - // If the current usage + the requested size are under the limit of the - // filesystem, allow it. - if usage+size <= limit { - return true - } - - // Welp, the size would exceed the limit of the filesystem, deny it. - return false -} - -// Remove removes the named file or (empty) directory. -// -// If there is an error, it will be of type [*PathError]. -func (fs *Quota) Remove(name string) error { - // For information on why this interface is used here, check its - // documentation. - s, err := fs.RemoveStat(name) - if err != nil { - return err - } - - // Don't reduce the quota's usage as `name` is not a regular file. - if !s.Mode().IsRegular() { - return nil - } - - // Remove the size of the deleted file from the quota usage. - fs.Add(-s.Size()) - return nil -} - -// RemoveAll removes path and any children it contains. -// -// It removes everything it can but returns the first error -// it encounters. If the path does not exist, RemoveAll -// returns nil (no error). -// -// If there is an error, it will be of type [*PathError]. -func (fs *Quota) RemoveAll(name string) error { - name, err := fs.unsafePath(name) - if err != nil { - return err - } - // While removeAll internally checks this, I want to make sure we check it - // and return the proper error so our tests can ensure that this will never - // be a possibility. - if name == "." { - return &PathError{ - Op: "removeall", - Path: name, - Err: ErrBadPathResolution, - } - } - return fs.removeAll(name) -} - -func (fs *Quota) removeAll(path string) error { - return removeAll(fs, path) -} - -func (fs *Quota) unlinkat(dirfd int, name string, flags int) error { - if flags == 0 { - s, err := fs.Lstatat(dirfd, name) - if err == nil && s.Mode().IsRegular() { - fs.Add(-s.Size()) - } - } - return fs.UnixFS.unlinkat(dirfd, name, flags) -} diff --git a/internal/ufs/fs_unix.go b/internal/ufs/fs_unix.go deleted file mode 100644 index dff36c9b8..000000000 --- a/internal/ufs/fs_unix.go +++ /dev/null @@ -1,895 +0,0 @@ -// SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: Copyright (c) 2024 Matthew Penner - -//go:build unix - -package ufs - -import ( - "errors" - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - "time" - - "golang.org/x/sys/unix" -) - -// UnixFS is a filesystem that uses the unix package to make io calls. -// -// This is used for proper sand-boxing and full control over the exact syscalls -// being performed. -type UnixFS struct { - // basePath is the base path for file operations to take place in. - basePath string - - // useOpenat2 controls whether the `openat2` syscall is used instead of the - // older `openat` syscall. - useOpenat2 bool -} - -// NewUnixFS creates a new sandboxed unix filesystem. BasePath is used as the -// sandbox path, operations on BasePath itself are not allowed, but any -// operations on its descendants are. Symlinks pointing outside BasePath are -// checked and prevented from enabling an escape in a non-raceable manor. -func NewUnixFS(basePath string, useOpenat2 bool) (*UnixFS, error) { - basePath = strings.TrimSuffix(basePath, "/") - fs := &UnixFS{ - basePath: basePath, - useOpenat2: useOpenat2, - } - return fs, nil -} - -// BasePath returns the base path of the UnixFS sandbox, file operations -// pointing outside this path are prohibited and will be blocked by all -// operations implemented by UnixFS. -func (fs *UnixFS) BasePath() string { - return fs.basePath -} - -// Close releases the file descriptor used to sandbox operations within the -// base path of the filesystem. -func (fs *UnixFS) Close() error { - return nil -} - -// Chmod changes the mode of the named file to mode. -// -// If the file is a symbolic link, it changes the mode of the link's target. -// If there is an error, it will be of type *PathError. -// -// A different subset of the mode bits are used, depending on the -// operating system. -// -// On Unix, the mode's permission bits, ModeSetuid, ModeSetgid, and -// ModeSticky are used. -// -// On Windows, only the 0200 bit (owner writable) of mode is used; it -// controls whether the file's read-only attribute is set or cleared. -// The other bits are currently unused. For compatibility with Go 1.12 -// and earlier, use a non-zero mode. Use mode 0400 for a read-only -// file and 0600 for a readable+writable file. -// -// On Plan 9, the mode's permission bits, ModeAppend, ModeExclusive, -// and ModeTemporary are used. -func (fs *UnixFS) Chmod(name string, mode FileMode) error { - dirfd, name, closeFd, err := fs.safePath(name) - defer closeFd() - if err != nil { - return err - } - return fs.fchmodat("chmod", dirfd, name, mode) -} - -// Chmodat is like Chmod but it takes a dirfd and name instead of a full path. -func (fs *UnixFS) Chmodat(dirfd int, name string, mode FileMode) error { - return fs.fchmodat("chmodat", dirfd, name, mode) -} - -func (fs *UnixFS) fchmodat(op string, dirfd int, name string, mode FileMode) error { - return ensurePathError(unix.Fchmodat(dirfd, name, uint32(mode), 0), op, name) -} - -// Chown changes the numeric uid and gid of the named file. -// -// If the file is a symbolic link, it changes the uid and gid of the link's target. -// A uid or gid of -1 means to not change that value. -// If there is an error, it will be of type *PathError. -// -// On Windows or Plan 9, Chown always returns the syscall.EWINDOWS or -// EPLAN9 error, wrapped in *PathError. -func (fs *UnixFS) Chown(name string, uid, gid int) error { - return ensurePathError(fs.fchown(name, uid, gid, 0), "chown", name) -} - -// Lchown changes the numeric uid and gid of the named file. -// -// If the file is a symbolic link, it changes the uid and gid of the link itself. -// If there is an error, it will be of type *PathError. -// -// On Windows, it always returns the syscall.EWINDOWS error, wrapped -// in *PathError. -func (fs *UnixFS) Lchown(name string, uid, gid int) error { - // With AT_SYMLINK_NOFOLLOW, Fchownat acts like Lchown but allows us to - // pass a dirfd. - return ensurePathError(fs.fchown(name, uid, gid, AT_SYMLINK_NOFOLLOW), "lchown", name) -} - -// fchown is a re-usable Fchownat syscall used by Chown and Lchown. -func (fs *UnixFS) fchown(name string, uid, gid, flags int) error { - dirfd, name, closeFd, err := fs.safePath(name) - defer closeFd() - if err != nil { - return err - } - return unix.Fchownat(dirfd, name, uid, gid, flags) -} - -// Chownat is like Chown but allows passing an existing directory file -// descriptor rather than needing to resolve one. -func (fs *UnixFS) Chownat(dirfd int, name string, uid, gid int) error { - return ensurePathError(unix.Fchownat(dirfd, name, uid, gid, 0), "chownat", name) -} - -// Lchownat is like Lchown but allows passing an existing directory file -// descriptor rather than needing to resolve one. -func (fs *UnixFS) Lchownat(dirfd int, name string, uid, gid int) error { - return ensurePathError(unix.Fchownat(dirfd, name, uid, gid, AT_SYMLINK_NOFOLLOW), "lchownat", name) -} - -// Chtimes changes the access and modification times of the named -// file, similar to the Unix utime() or utimes() functions. -// -// The underlying filesystem may truncate or round the values to a -// less precise time unit. -// -// If there is an error, it will be of type *PathError. -func (fs *UnixFS) Chtimes(name string, atime, mtime time.Time) error { - dirfd, name, closeFd, err := fs.safePath(name) - defer closeFd() - if err != nil { - return err - } - return fs.Chtimesat(dirfd, name, atime, mtime) -} - -// Chtimesat is like Chtimes but allows passing an existing directory file -// descriptor rather than needing to resolve one. -func (fs *UnixFS) Chtimesat(dirfd int, name string, atime, mtime time.Time) error { - var utimes [2]unix.Timespec - set := func(i int, t time.Time) { - if t.IsZero() { - utimes[i] = unix.Timespec{Sec: unix.UTIME_OMIT, Nsec: unix.UTIME_OMIT} - } else { - utimes[i] = unix.NsecToTimespec(t.UnixNano()) - } - } - set(0, atime) - set(1, mtime) - - // This does support `AT_SYMLINK_NOFOLLOW` as well if needed. - return ensurePathError(unix.UtimesNanoAt(dirfd, name, utimes[0:], 0), "chtimes", name) -} - -// Create creates or truncates the named file. If the file already exists, -// it is truncated. -// -// If the file does not exist, it is created with mode 0666 -// (before umask). If successful, methods on the returned File can -// be used for I/O; the associated file descriptor has mode O_RDWR. -// If there is an error, it will be of type *PathError. -func (fs *UnixFS) Create(name string) (File, error) { - return fs.OpenFile(name, O_CREATE|O_WRONLY|O_TRUNC, 0o644) -} - -// Mkdir creates a new directory with the specified name and permission -// bits (before umask). -// -// If there is an error, it will be of type *PathError. -func (fs *UnixFS) Mkdir(name string, mode FileMode) error { - dirfd, name, closeFd, err := fs.safePath(name) - defer closeFd() - if err != nil { - return err - } - return fs.mkdirat("mkdir", dirfd, name, mode) -} - -func (fs *UnixFS) Mkdirat(dirfd int, name string, mode FileMode) error { - return fs.mkdirat("mkdirat", dirfd, name, mode) -} - -func (fs *UnixFS) mkdirat(op string, dirfd int, name string, mode FileMode) error { - return ensurePathError(unix.Mkdirat(dirfd, name, uint32(mode)), op, name) -} - -// MkdirAll creates a directory named path, along with any necessary -// parents, and returns nil, or else returns an error. -// -// The permission bits perm (before umask) are used for all -// directories that MkdirAll creates. -// If path is already a directory, MkdirAll does nothing -// and returns nil. -func (fs *UnixFS) MkdirAll(name string, mode FileMode) error { - // Ensure name is somewhat clean before continuing. - name, err := fs.unsafePath(name) - if err != nil { - return err - } - return fs.mkdirAll(name, mode) -} - -// Open opens the named file for reading. -// -// If successful, methods on the returned file can be used for reading; the -// associated file descriptor has mode O_RDONLY. -// -// If there is an error, it will be of type *PathError. -func (fs *UnixFS) Open(name string) (File, error) { - return fs.OpenFile(name, O_RDONLY, 0) -} - -// OpenFile is the generalized open call; most users will use Open -// or Create instead. It opens the named file with specified flag -// (O_RDONLY etc.). -// -// If the file does not exist, and the O_CREATE flag -// is passed, it is created with mode perm (before umask). If successful, -// methods on the returned File can be used for I/O. -// -// If there is an error, it will be of type *PathError. -func (fs *UnixFS) OpenFile(name string, flag int, mode FileMode) (File, error) { - fd, err := fs.openFile(name, flag, mode) - if err != nil { - return nil, err - } - // Do not close `fd` here, it is passed to a file that needs the fd, the - // caller of this function is responsible for calling Close() on the File - // to release the file descriptor. - return os.NewFile(uintptr(fd), name), nil -} - -func (fs *UnixFS) openFile(name string, flag int, mode FileMode) (int, error) { - dirfd, name, closeFd, err := fs.safePath(name) - defer closeFd() - if err != nil { - return 0, err - } - return fs.openat(dirfd, name, flag, mode) -} - -func (fs *UnixFS) OpenFileat(dirfd int, name string, flag int, mode FileMode) (File, error) { - fd, err := fs.openat(dirfd, name, flag, mode) - if err != nil { - return nil, err - } - // Do not close `fd` here, it is passed to a file that needs the fd, the - // caller of this function is responsible for calling Close() on the File - // to release the file descriptor. - return os.NewFile(uintptr(fd), name), nil -} - -// ReadDir reads the named directory, -// -// returning all its directory entries sorted by filename. -// If an error occurs reading the directory, ReadDir returns the entries it -// was able to read before the error, along with the error. -func (fs *UnixFS) ReadDir(path string) ([]DirEntry, error) { - dirfd, name, closeFd, err := fs.safePath(path) - defer closeFd() - if err != nil { - return nil, err - } - fd, err := fs.openat(dirfd, name, O_DIRECTORY|O_RDONLY, 0) - if err != nil { - return nil, err - } - defer func() { - _ = unix.Close(fd) - }() - return fs.readDir(fd, name, ".", nil) -} - -// RemoveStat is a combination of Stat and Remove, it is used to more -// efficiently remove a file when the caller needs to stat it before -// removing it. -// -// This optimized function exists for our QuotaFS abstraction, which needs -// to track writes to a filesystem. When removing a file, the QuotaFS needs -// to know if the entry is a file and if so, how large it is. Because we -// need to Stat a file in order to get its mode and size, we will already -// know if the entry needs to be removed by using Unlink or Rmdir. The -// standard `Remove` method just tries both Unlink and Rmdir (in that order) -// as it ends up usually being faster and more efficient than calling Stat + -// the proper operation in the first place. -func (fs *UnixFS) RemoveStat(name string) (FileInfo, error) { - dirfd, name, closeFd, err := fs.safePath(name) - defer closeFd() - if err != nil { - return nil, err - } - - // Lstat name, we use Lstat as Unlink doesn't care about symlinks. - s, err := fs.Lstatat(dirfd, name) - if err != nil { - return nil, err - } - - if s.IsDir() { - err = fs.unlinkat(dirfd, name, AT_REMOVEDIR) // Rmdir - } else { - err = fs.unlinkat(dirfd, name, 0) - } - if err != nil { - return s, ensurePathError(err, "rename", name) - } - return s, nil -} - -// Remove removes the named file or (empty) directory. -// -// If there is an error, it will be of type *PathError. -func (fs *UnixFS) Remove(name string) error { - dirfd, name, closeFd, err := fs.safePath(name) - defer closeFd() - if err != nil { - return err - } - - // Prevent trying to Remove the base directory. - if name == "." { - return &PathError{ - Op: "remove", - Path: name, - Err: ErrBadPathResolution, - } - } - - // System call interface forces us to know - // whether name is a file or directory. - // Try both: it is cheaper on average than - // doing a Stat plus the right one. - err = fs.unlinkat(dirfd, name, 0) - if err == nil { - return nil - } - err1 := fs.unlinkat(dirfd, name, AT_REMOVEDIR) // Rmdir - if err1 == nil { - return nil - } - - // Both failed: figure out which error to return. - // OS X and Linux differ on whether unlink(dir) - // returns EISDIR, so can't use that. However, - // both agree that rmdir(file) returns ENOTDIR, - // so we can use that to decide which error is real. - // Rmdir might also return ENOTDIR if given a bad - // file path, like /etc/passwd/foo, but in that case, - // both errors will be ENOTDIR, so it's okay to - // use the error from unlink. - if err1 != unix.ENOTDIR { - err = err1 - } - return ensurePathError(err, "remove", name) -} - -// RemoveAll removes path and any children it contains. -// -// It removes everything it can but returns the first error -// it encounters. If the path does not exist, RemoveAll -// returns nil (no error). -// -// If there is an error, it will be of type *PathError. -func (fs *UnixFS) RemoveAll(name string) error { - name, err := fs.unsafePath(name) - if err != nil { - return err - } - - // While removeAll internally checks this, I want to make sure we check it - // and return the proper error so our tests can ensure that this will never - // be a possibility. - if name == "." { - return &PathError{ - Op: "removeall", - Path: name, - Err: ErrBadPathResolution, - } - } - - return fs.removeAll(name) -} - -// RemoveContents recursively removes the contents of name. -// -// It removes everything it can but returns the first error -// it encounters. If the path does not exist, RemoveContents -// returns nil (no error). -// -// If there is an error, it will be of type [*PathError]. -func (fs *UnixFS) RemoveContents(name string) error { - name, err := fs.unsafePath(name) - if err != nil { - return err - } - - // Unlike RemoveAll, we don't remove `name` itself, only it's contents. - // So there is no need to check for a name of `.` here. - - return fs.removeContents(name) -} - -func (fs *UnixFS) unlinkat(dirfd int, name string, flags int) error { - return ignoringEINTR(func() error { - return unix.Unlinkat(dirfd, name, flags) - }) -} - -// Rename renames (moves) oldpath to newpath. -// -// If newpath already exists and is not a directory, Rename replaces it. -// OS-specific restrictions may apply when oldpath and newpath are in different directories. -// Even within the same directory, on non-Unix platforms Rename is not an atomic operation. -// -// If there is an error, it will be of type *LinkError. -func (fs *UnixFS) Rename(oldpath, newpath string) error { - // Simple case: both paths are the same. - if oldpath == newpath { - return nil - } - - olddirfd, oldname, closeFd, err := fs.safePath(oldpath) - defer closeFd() - if err != nil { - return err - } - // Ensure that we are not trying to rename the base directory itself. - // While unix.Renameat ends up throwing a "device or resource busy" error, - // that doesn't mean we are protecting the system properly. - if oldname == "." { - return &PathError{ - Op: "rename", - Path: oldname, - Err: ErrBadPathResolution, - } - } - // Stat the old target to return proper errors. - if _, err := fs.Lstatat(olddirfd, oldname); err != nil { - return err - } - - newdirfd, newname, closeFd2, err := fs.safePath(newpath) - if err != nil { - closeFd2() - if !errors.Is(err, ErrNotExist) { - return err - } - var pathErr *PathError - if !errors.As(err, &pathErr) { - return err - } - if err := fs.MkdirAll(pathErr.Path, 0o755); err != nil { - return err - } - newdirfd, newname, closeFd2, err = fs.safePath(newpath) - defer closeFd2() - if err != nil { - return err - } - } else { - defer closeFd2() - } - - // Ensure that we are not trying to rename the base directory itself. - // While unix.Renameat ends up throwing a "device or resource busy" error, - // that doesn't mean we are protecting the system properly. - if newname == "." { - return &PathError{ - Op: "rename", - Path: newname, - Err: ErrBadPathResolution, - } - } - // Stat the new target to return proper errors. - _, err = fs.Lstatat(newdirfd, newname) - switch { - case err == nil: - return &PathError{ - Op: "rename", - Path: newname, - Err: ErrExist, - } - case !errors.Is(err, ErrNotExist): - return err - } - if err := unix.Renameat(olddirfd, oldname, newdirfd, newname); err != nil { - return &LinkError{Op: "rename", Old: oldpath, New: newpath, Err: err} - } - return nil -} - -// Stat returns a FileInfo describing the named file. -// -// If there is an error, it will be of type *PathError. -func (fs *UnixFS) Stat(name string) (FileInfo, error) { - return fs._fstat("stat", name, 0) -} - -// Statat is like Stat but allows passing an existing directory file -// descriptor rather than needing to resolve one. -func (fs *UnixFS) Statat(dirfd int, name string) (FileInfo, error) { - return fs._fstatat("statat", dirfd, name, 0) -} - -// Lstat returns a FileInfo describing the named file. -// -// If the file is a symbolic link, the returned FileInfo -// describes the symbolic link. Lstat makes no attempt to follow the link. -// -// If there is an error, it will be of type *PathError. -func (fs *UnixFS) Lstat(name string) (FileInfo, error) { - return fs._fstat("lstat", name, AT_SYMLINK_NOFOLLOW) -} - -// Lstatat is like Lstat but allows passing an existing directory file -// descriptor rather than needing to resolve one. -func (fs *UnixFS) Lstatat(dirfd int, name string) (FileInfo, error) { - return fs._fstatat("lstatat", dirfd, name, AT_SYMLINK_NOFOLLOW) -} - -func (fs *UnixFS) fstat(name string, flags int) (FileInfo, error) { - return fs._fstat("fstat", name, flags) -} - -func (fs *UnixFS) _fstat(op string, name string, flags int) (FileInfo, error) { - dirfd, name, closeFd, err := fs.safePath(name) - defer closeFd() - if err != nil { - return nil, err - } - return fs._fstatat(op, dirfd, name, flags) -} - -func (fs *UnixFS) fstatat(dirfd int, name string, flags int) (FileInfo, error) { - return fs._fstatat("fstatat", dirfd, name, flags) -} - -func (fs *UnixFS) _fstatat(op string, dirfd int, name string, flags int) (FileInfo, error) { - var s fileStat - if err := ignoringEINTR(func() error { - return unix.Fstatat(dirfd, name, &s.sys, flags) - }); err != nil { - return nil, ensurePathError(err, op, name) - } - fillFileStatFromSys(&s, name) - return &s, nil -} - -// Symlink creates newname as a symbolic link to oldname. -// -// On Windows, a symlink to a non-existent oldname creates a file symlink; -// if oldname is later created as a directory the symlink will not work. -// -// If there is an error, it will be of type *LinkError. -func (fs *UnixFS) Symlink(oldpath, newpath string) error { - dirfd, newpath, closeFd, err := fs.safePath(newpath) - defer closeFd() - if err != nil { - return err - } - if err := ignoringEINTR(func() error { - // We aren't concerned with oldpath here as a symlink can point anywhere - // it wants. - return unix.Symlinkat(oldpath, dirfd, newpath) - }); err != nil { - return &LinkError{Op: "symlink", Old: oldpath, New: newpath, Err: err} - } - return nil -} - -// Touch will attempt to open a file for reading and/or writing. If the file -// does not exist it will be created, and any missing parent directories will -// also be created. The opened file may be truncated, only if `flag` has -// O_TRUNC set. -func (fs *UnixFS) Touch(path string, flag int, mode FileMode) (File, error) { - if flag&O_CREATE == 0 { - flag |= O_CREATE - } - dirfd, name, closeFd, err, _ := fs.TouchPath(path) - defer closeFd() - if err != nil { - return nil, err - } - return fs.OpenFileat(dirfd, name, flag, mode) -} - -// TouchPath is like SafePath except that it will create any missing directories -// in the path. Unlike SafePath, TouchPath returns an additional boolean which -// indicates whether the parent directories already existed, this is intended to -// be used as a way to know if the final destination could already exist. -func (fs *UnixFS) TouchPath(path string) (int, string, func(), error, bool) { - dirfd, name, closeFd, err := fs.safePath(path) - switch { - case err == nil: - return dirfd, name, closeFd, nil, true - case !errors.Is(err, ErrNotExist): - return dirfd, name, closeFd, err, false - } - - var pathErr *PathError - if !errors.As(err, &pathErr) { - return dirfd, name, closeFd, err, false - } - if err := fs.MkdirAll(pathErr.Path, 0o755); err != nil { - return dirfd, name, closeFd, err, false - } - - // Close the previous file descriptor since we are going to be opening - // a new one. - closeFd() - - // Run safe path again now that the parent directories have been created. - dirfd, name, closeFd, err = fs.safePath(path) - return dirfd, name, closeFd, err, false -} - -// WalkDir walks the file tree rooted at root, calling fn for each file or -// directory in the tree, including root. -// -// All errors that arise visiting files and directories are filtered by fn: -// see the [WalkDirFunc] documentation for details. -// -// The files are walked in lexical order, which makes the output deterministic -// but requires WalkDir to read an entire directory into memory before proceeding -// to walk that directory. -// -// WalkDir does not follow symbolic links found in directories, -// but if root itself is a symbolic link, its target will be walked. -func (fs *UnixFS) WalkDir(root string, fn WalkDirFunc) error { - return WalkDir(fs, root, fn) -} - -// openat is a wrapper around both unix.Openat and unix.Openat2. If the UnixFS -// was configured to enable openat2 support, unix.Openat2 will be used instead -// of unix.Openat due to having better security properties for our use-case. -func (fs *UnixFS) openat(dirfd int, name string, flag int, mode FileMode) (int, error) { - if flag&O_NOFOLLOW == 0 { - flag |= O_NOFOLLOW - } - - var fd int - for { - var err error - if fs.useOpenat2 { - fd, err = fs._openat2(dirfd, name, uint64(flag), uint64(syscallMode(mode))) - } else { - fd, err = fs._openat(dirfd, name, flag, uint32(syscallMode(mode))) - } - if err == nil { - break - } - // We have to check EINTR here, per issues https://go.dev/issue/11180 and https://go.dev/issue/39237. - if err == unix.EINTR { - continue - } - return 0, err - } - - // If we are using openat2, we don't need the additional security checks. - if fs.useOpenat2 { - return fd, nil - } - - // If we are not using openat2, do additional path checking. This assumes - // that openat2 is using `RESOLVE_BENEATH` to avoid the same security - // issue. - var finalPath string - finalPath, err := filepath.EvalSymlinks(filepath.Join("/proc/self/fd/", strconv.Itoa(fd))) - if err != nil { - if !errors.Is(err, ErrNotExist) { - return fd, fmt.Errorf("failed to evaluate symlink: %w", convertErrorType(err)) - } - - // The target of one of the symlinks (EvalSymlinks is recursive) - // does not exist. So get the path that does not exist and use - // that for further validation instead. - var pErr *PathError - if !errors.As(err, &pErr) { - return fd, fmt.Errorf("failed to evaluate symlink: %w", convertErrorType(err)) - } - - // Update the final path to whatever directory or path didn't exist while - // recursing any symlinks. - finalPath = pErr.Path - // Ensure the error is wrapped correctly. - err = convertErrorType(err) - } - - // Check if the path is within our root. - if !fs.unsafeIsPathInsideOfBase(finalPath) { - op := "openat" - if fs.useOpenat2 { - op = "openat2" - } - return fd, &PathError{ - Op: op, - Path: name, - Err: ErrBadPathResolution, - } - } - - // Return the file descriptor and any potential error. - return fd, err -} - -// _openat is a wrapper around unix.Openat. This method should never be directly -// called, use `openat` instead. -func (fs *UnixFS) _openat(dirfd int, name string, flag int, mode uint32) (int, error) { - // Ensure the O_CLOEXEC flag is set. - // Go sets this in the os package, but since we are directly using unix - // we need to set it ourselves. - if flag&O_CLOEXEC == 0 { - flag |= O_CLOEXEC - } - // O_LARGEFILE is set by Openat for us automatically. - fd, err := unix.Openat(dirfd, name, flag, mode) - switch { - case err == nil: - return fd, nil - case err == unix.EINTR: - return fd, err - case err == unix.EAGAIN: - return fd, err - default: - return fd, ensurePathError(err, "openat", name) - } -} - -// _openat2 is a wonderful syscall that supersedes the `openat` syscall. It has -// improved validation and security characteristics that weren't available or -// considered when `openat` was originally implemented. As such, it is only -// present in Kernel 5.6 and above. -// -// This method should never be directly called, use `openat` instead. -func (fs *UnixFS) _openat2(dirfd int, name string, flag, mode uint64) (int, error) { - // Ensure the O_CLOEXEC flag is set. - // Go sets this when using the os package, but since we are directly using - // the unix package we need to set it ourselves. - if flag&O_CLOEXEC == 0 { - flag |= O_CLOEXEC - } - // Ensure the O_LARGEFILE flag is set. - // Go sets this for unix.Open, unix.Openat, but not unix.Openat2. - if flag&O_LARGEFILE == 0 { - flag |= O_LARGEFILE - } - fd, err := unix.Openat2(dirfd, name, &unix.OpenHow{ - Flags: flag, - Mode: mode, - // This is the bread and butter of preventing a symlink escape, without - // this option, we have to handle path validation fully on our own. - // - // This is why using Openat2 over Openat is preferred if available. - Resolve: unix.RESOLVE_BENEATH, - }) - switch { - case err == nil: - return fd, nil - case err == unix.EINTR: - return fd, err - case err == unix.EAGAIN: - return fd, err - default: - return fd, ensurePathError(err, "openat2", name) - } -} - -func (fs *UnixFS) SafePath(path string) (int, string, func(), error) { - return fs.safePath(path) -} - -func (fs *UnixFS) safePath(path string) (dirfd int, file string, closeFd func(), err error) { - // Default closeFd to a NO-OP. - closeFd = func() {} - - // Use unsafePath to clean the path and strip BasePath if path is absolute. - var name string - name, err = fs.unsafePath(path) - if err != nil { - return - } - - // Open the base path. We use this as the sandbox root for any further - // operations. - var fsDirfd int - fsDirfd, err = fs._openat(AT_EMPTY_PATH, fs.basePath, O_DIRECTORY|O_RDONLY, 0) - if err != nil { - return - } - - // Split the parent from the last element in the path, this gives us the - // "file name" and the full path to its parent. - var dir string - dir, file = filepath.Split(name) - // If dir is empty then name is not nested. - if dir == "" { - dirfd = fsDirfd - closeFd = func() { _ = unix.Close(dirfd) } - - // Return dirfd, name, an empty closeFd func, and no error - return - } - - // Dir will usually contain a trailing slash as filepath.Split doesn't - // trim slashes. - dir = strings.TrimSuffix(dir, "/") - dirfd, err = fs.openat(fsDirfd, dir, O_DIRECTORY|O_RDONLY, 0) - if err != nil { - // An error occurred while opening the directory, but we already opened - // the filesystem root, so we still need to ensure it gets closed. - closeFd = func() { _ = unix.Close(fsDirfd) } - } else { - // Set closeFd to close the newly opened directory file descriptor. - closeFd = func() { - _ = unix.Close(dirfd) - _ = unix.Close(fsDirfd) - } - } - - // Return dirfd, name, the closeFd func, and err - return -} - -// unsafePath strips and joins the given path with the filesystem's base path, -// cleaning the result. The cleaned path is then checked if it starts with the -// filesystem's base path to obvious any obvious path traversal escapes. The -// fully resolved path (if symlinks are followed) may not be within the -// filesystem's base path, additional checks are required to safely use paths -// returned by this function. -func (fs *UnixFS) unsafePath(path string) (string, error) { - // Calling filepath.Clean on the path will resolve it to it's absolute path, - // removing any path traversal arguments (such as ..), leaving us with an - // absolute path we can then use. - // - // This will also trim the filesystem's base path from the given path and - // join the base path back on to ensure the path starts with the base path - // without appending it twice. - r := filepath.Clean(filepath.Join(fs.basePath, strings.TrimPrefix(path, fs.basePath))) - - if fs.unsafeIsPathInsideOfBase(r) { - // This is kinda ironic isn't it. - // We do this as we are operating with dirfds and `*at` syscalls which - // behave differently if given an absolute path. - // - // First trim the BasePath, then trim any leading slashes. - r = strings.TrimPrefix(strings.TrimPrefix(r, fs.basePath), "/") - // If the path is empty then return "." as the path is pointing to the - // root. - if r == "" { - return ".", nil - } - return r, nil - } - - return "", &PathError{ - Op: "safePath", - Path: path, - Err: ErrBadPathResolution, - } -} - -// unsafeIsPathInsideOfBase checks if the given path is inside the filesystem's -// base path. -// -// NOTE: this method doesn't clean the given path or attempt to join the -// filesystem's base path. This is purely a basic prefix check against the -// given path. -func (fs *UnixFS) unsafeIsPathInsideOfBase(path string) bool { - return strings.HasPrefix( - strings.TrimSuffix(path, "/")+"/", - fs.basePath+"/", - ) -} diff --git a/internal/ufs/fs_unix_test.go b/internal/ufs/fs_unix_test.go deleted file mode 100644 index e64bb823b..000000000 --- a/internal/ufs/fs_unix_test.go +++ /dev/null @@ -1,768 +0,0 @@ -// SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: Copyright (c) 2024 Matthew Penner - -//go:build unix - -package ufs_test - -import ( - "errors" - "os" - "path/filepath" - "reflect" - "slices" - "strconv" - "testing" - - "github.com/pterodactyl/wings/internal/ufs" -) - -type testUnixFS struct { - *ufs.UnixFS - - TmpDir string - Root string -} - -func (fs *testUnixFS) Cleanup() { - _ = fs.Close() - _ = os.RemoveAll(fs.TmpDir) -} - -func newTestUnixFS() (*testUnixFS, error) { - tmpDir, err := os.MkdirTemp(os.TempDir(), "ufs") - if err != nil { - return nil, err - } - root := filepath.Join(tmpDir, "root") - if err := os.Mkdir(root, 0o755); err != nil { - return nil, err - } - // fmt.Println(tmpDir) - fs, err := ufs.NewUnixFS(root, true) - if err != nil { - return nil, err - } - tfs := &testUnixFS{ - UnixFS: fs, - TmpDir: tmpDir, - Root: root, - } - return tfs, nil -} - -func TestUnixFS(t *testing.T) { - t.Parallel() - - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - // Test creating a file within the root. - _, _, closeFd, err := fs.SafePath("/") - closeFd() - if err != nil { - t.Error(err) - return - } - - f, err := fs.Touch("directory/file", ufs.O_RDWR, 0o644) - if err != nil { - t.Error(err) - return - } - _ = f.Close() - - // Test creating a file within the root. - f, err = fs.Create("test") - if err != nil { - t.Error(err) - return - } - _ = f.Close() - - // Test stating a file within the root. - if _, err := fs.Stat("test"); err != nil { - t.Error(err) - return - } - - // Test creating a directory within the root. - if err := fs.Mkdir("ima_directory", 0o755); err != nil { - t.Error(err) - return - } - - // Test creating a nested directory within the root. - if err := fs.Mkdir("ima_directory/ima_nother_directory", 0o755); err != nil { - t.Error(err) - return - } - - // Test creating a file inside a directory within the root. - f, err = fs.Create("ima_directory/ima_file") - if err != nil { - t.Error(err) - return - } - _ = f.Close() - - // Test listing directory entries. - if _, err := fs.ReadDir("ima_directory"); err != nil { - t.Error(err) - return - } - - // Test symlink pointing outside the root. - if err := os.Symlink(fs.TmpDir, filepath.Join(fs.Root, "ima_bad_link")); err != nil { - t.Error(err) - return - } - f, err = fs.Create("ima_bad_link/ima_bad_file") - if err == nil { - _ = f.Close() - t.Error("expected an error") - return - } - if err := fs.Mkdir("ima_bad_link/ima_bad_directory", 0o755); err == nil { - t.Error("expected an error") - return - } - - // Test symlink pointing outside the root inside a parent directory. - if err := fs.Symlink(fs.TmpDir, filepath.Join(fs.Root, "ima_directory/ima_bad_link")); err != nil { - t.Error(err) - return - } - if err := fs.Mkdir("ima_directory/ima_bad_link/ima_bad_directory", 0o755); err == nil { - t.Error("expected an error") - return - } - - // Test symlink pointing outside the root with a child directory. - if err := os.Mkdir(filepath.Join(fs.TmpDir, "ima_directory"), 0o755); err != nil { - t.Error(err) - return - } - f, err = fs.Create("ima_bad_link/ima_directory/ima_bad_file") - if err == nil { - _ = f.Close() - t.Error("expected an error") - return - } - if err := fs.Mkdir("ima_bad_link/ima_directory/ima_bad_directory", 0o755); err == nil { - t.Error("expected an error") - return - } - - if _, err := fs.ReadDir("ima_bad_link/ima_directory"); err == nil { - t.Error("expected an error") - return - } - - // Create multiple nested directories. - if err := fs.MkdirAll("ima_directory/ima_directory/ima_directory/ima_directory", 0o755); err != nil { - t.Error(err) - return - } - if _, err := fs.ReadDir("ima_directory/ima_directory"); err != nil { - t.Error(err) - return - } - - // Test creating a directory under a symlink with a pre-existing directory. - if err := fs.MkdirAll("ima_bad_link/ima_directory/ima_bad_directory/ima_bad_directory", 0o755); err == nil { - t.Error("expected an error") - return - } - - // Test deletion - if err := fs.Remove("test"); err != nil { - t.Error(err) - return - } - if err := fs.Remove("ima_bad_link"); err != nil { - t.Error(err) - return - } - - // Test recursive deletion - if err := fs.RemoveAll("ima_directory"); err != nil { - t.Error(err) - return - } - - // Test recursive deletion underneath a bad symlink - if err := fs.Mkdir("ima_directory", 0o755); err != nil { - t.Error(err) - return - } - if err := fs.Symlink(fs.TmpDir, filepath.Join(fs.Root, "ima_directory/ima_bad_link")); err != nil { - t.Error(err) - return - } - if err := fs.RemoveAll("ima_directory/ima_bad_link/ima_bad_file"); err == nil { - t.Error("expected an error") - return - } - - // This should delete the symlink itself. - if err := fs.RemoveAll("ima_directory/ima_bad_link"); err != nil { - t.Error(err) - return - } - - //for i := 0; i < 5; i++ { - // dirName := "dir" + strconv.Itoa(i) - // if err := fs.Mkdir(dirName, 0o755); err != nil { - // t.Error(err) - // return - // } - // for j := 0; j < 5; j++ { - // f, err := fs.Create(filepath.Join(dirName, "file"+strconv.Itoa(j))) - // if err != nil { - // t.Error(err) - // return - // } - // _ = f.Close() - // } - //} - // - //if err := fs.WalkDir2("", func(fd int, path string, info filesystem.DirEntry, err error) error { - // if err != nil { - // return err - // } - // fmt.Println(path) - // return nil - //}); err != nil { - // t.Error(err) - // return - //} -} - -func TestUnixFS_Chmod(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - // TODO: implement -} - -func TestUnixFS_Chown(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - // TODO: implement -} - -func TestUnixFS_Lchown(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - // TODO: implement -} - -func TestUnixFS_Chtimes(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - // TODO: implement -} - -func TestUnixFS_Create(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - // TODO: implement -} - -func TestUnixFS_Mkdir(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - // TODO: implement -} - -func TestUnixFS_MkdirAll(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - if err := fs.MkdirAll("/a/bunch/of/directories", 0o755); err != nil { - t.Error(err) - return - } - - // TODO: stat sanity check -} - -func TestUnixFS_Open(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - // TODO: implement -} - -func TestUnixFS_OpenFile(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - // TODO: implement -} - -func TestUnixFS_ReadDir(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - // TODO: implement -} - -func TestUnixFS_Remove(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - t.Run("base directory", func(t *testing.T) { - // Try to remove the base directory. - if err := fs.Remove(""); !errors.Is(err, ufs.ErrBadPathResolution) { - t.Errorf("expected an a bad path resolution error, but got: %v", err) - return - } - }) - - t.Run("path traversal", func(t *testing.T) { - // Try to remove the base directory. - if err := fs.RemoveAll("../root"); !errors.Is(err, ufs.ErrBadPathResolution) { - t.Errorf("expected an a bad path resolution error, but got: %v", err) - return - } - }) -} - -func TestUnixFS_RemoveAll(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - t.Run("base directory", func(t *testing.T) { - // Try to remove the base directory. - if err := fs.RemoveAll(""); !errors.Is(err, ufs.ErrBadPathResolution) { - t.Errorf("expected an a bad path resolution error, but got: %v", err) - return - } - }) - - t.Run("path traversal", func(t *testing.T) { - // Try to remove the base directory. - if err := fs.RemoveAll("../root"); !errors.Is(err, ufs.ErrBadPathResolution) { - t.Errorf("expected an a bad path resolution error, but got: %v", err) - return - } - }) -} - -func TestUnixFS_Rename(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - t.Run("rename base directory", func(t *testing.T) { - // Try to rename the base directory. - if err := fs.Rename("", "yeet"); !errors.Is(err, ufs.ErrBadPathResolution) { - t.Errorf("expected an a bad path resolution error, but got: %v", err) - return - } - }) - - t.Run("rename over base directory", func(t *testing.T) { - // Create a directory that we are going to try and move over top of the - // existing base directory. - if err := fs.Mkdir("overwrite_dir", 0o755); err != nil { - t.Error(err) - return - } - - // Try to rename over the base directory. - if err := fs.Rename("overwrite_dir", ""); !errors.Is(err, ufs.ErrBadPathResolution) { - t.Errorf("expected an a bad path resolution error, but got: %v", err) - return - } - }) - - t.Run("directory rename", func(t *testing.T) { - // Create a directory to rename to something else. - if err := fs.Mkdir("test_directory", 0o755); err != nil { - t.Error(err) - return - } - - // Try to rename "test_directory" to "directory". - if err := fs.Rename("test_directory", "directory"); err != nil { - t.Errorf("expected no error, but got: %v", err) - return - } - - // Sanity check - if _, err := os.Lstat(filepath.Join(fs.Root, "directory")); err != nil { - t.Errorf("Lstat errored when performing sanity check: %v", err) - return - } - }) - - t.Run("file rename", func(t *testing.T) { - // Create a directory to rename to something else. - f, err := fs.Create("test_file") - if err != nil { - t.Error(err) - return - } - _ = f.Close() - - // Try to rename "test_file" to "file". - if err := fs.Rename("test_file", "file"); err != nil { - t.Errorf("expected no error, but got: %v", err) - return - } - - // Sanity check - if _, err := os.Lstat(filepath.Join(fs.Root, "file")); err != nil { - t.Errorf("Lstat errored when performing sanity check: %v", err) - return - } - }) -} - -func TestUnixFS_Stat(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - // TODO: implement -} - -func TestUnixFS_Lstat(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - // TODO: implement -} - -func TestUnixFS_Symlink(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - // TODO: implement -} - -func TestUnixFS_Touch(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - t.Run("base directory", func(t *testing.T) { - path := "i_touched_a_file" - f, err := fs.Touch(path, ufs.O_RDWR, 0o644) - if err != nil { - t.Error(err) - return - } - _ = f.Close() - - // Sanity check - if _, err := os.Lstat(filepath.Join(fs.Root, path)); err != nil { - t.Errorf("Lstat errored when performing sanity check: %v", err) - return - } - }) - - t.Run("existing parent directory", func(t *testing.T) { - dir := "some_parent_directory" - if err := fs.Mkdir(dir, 0o755); err != nil { - t.Errorf("error creating parent directory: %v", err) - return - } - path := filepath.Join(dir, "i_touched_a_file") - f, err := fs.Touch(path, ufs.O_RDWR, 0o644) - if err != nil { - t.Errorf("error touching file: %v", err) - return - } - _ = f.Close() - - // Sanity check - if _, err := os.Lstat(filepath.Join(fs.Root, path)); err != nil { - t.Errorf("Lstat errored when performing sanity check: %v", err) - return - } - }) - - t.Run("non-existent parent directory", func(t *testing.T) { - path := "some_other_directory/i_touched_a_file" - f, err := fs.Touch(path, ufs.O_RDWR, 0o644) - if err != nil { - t.Errorf("error touching file: %v", err) - return - } - _ = f.Close() - - // Sanity check - if _, err := os.Lstat(filepath.Join(fs.Root, path)); err != nil { - t.Errorf("Lstat errored when performing sanity check: %v", err) - return - } - }) - - t.Run("non-existent parent directories", func(t *testing.T) { - path := "some_other_directory/some_directory/i_touched_a_file" - f, err := fs.Touch(path, ufs.O_RDWR, 0o644) - if err != nil { - t.Errorf("error touching file: %v", err) - return - } - _ = f.Close() - - // Sanity check - if _, err := os.Lstat(filepath.Join(fs.Root, path)); err != nil { - t.Errorf("Lstat errored when performing sanity check: %v", err) - return - } - }) -} - -func TestUnixFS_WalkDir(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - //for i := 0; i < 5; i++ { - // dirName := "dir" + strconv.Itoa(i) - // if err := fs.Mkdir(dirName, 0o755); err != nil { - // t.Error(err) - // return - // } - // for j := 0; j < 5; j++ { - // f, err := fs.Create(filepath.Join(dirName, "file"+strconv.Itoa(j))) - // if err != nil { - // t.Error(err) - // return - // } - // _ = f.Close() - // } - //} - // - //if err := fs.WalkDir(".", func(path string, info ufs.DirEntry, err error) error { - // if err != nil { - // return err - // } - // t.Log(path) - // return nil - //}); err != nil { - // t.Error(err) - // return - //} -} - -func TestUnixFS_WalkDirat(t *testing.T) { - t.Parallel() - fs, err := newTestUnixFS() - if err != nil { - t.Fatal(err) - return - } - defer fs.Cleanup() - - for i := 0; i < 2; i++ { - dirName := "base" + strconv.Itoa(i) - if err := fs.Mkdir(dirName, 0o755); err != nil { - t.Error(err) - return - } - for j := 0; j < 1; j++ { - f, err := fs.Create(filepath.Join(dirName, "file"+strconv.Itoa(j))) - if err != nil { - t.Error(err) - return - } - _ = f.Close() - if err := fs.Mkdir(filepath.Join(dirName, "dir"+strconv.Itoa(j)), 0o755); err != nil { - t.Error(err) - return - } - f, err = fs.Create(filepath.Join(dirName, "dir"+strconv.Itoa(j), "file"+strconv.Itoa(j))) - if err != nil { - t.Error(err) - return - } - _ = f.Close() - } - } - - t.Run("walk starting at the filesystem root", func(t *testing.T) { - pathsTraversed, err := fs.testWalkDirAt("") - if err != nil { - t.Error(err) - return - } - expect := []Path{ - {Name: ".", Relative: "."}, - {Name: "base0", Relative: "base0"}, - {Name: "dir0", Relative: "base0/dir0"}, - {Name: "file0", Relative: "base0/dir0/file0"}, - {Name: "file0", Relative: "base0/file0"}, - {Name: "base1", Relative: "base1"}, - {Name: "dir0", Relative: "base1/dir0"}, - {Name: "file0", Relative: "base1/dir0/file0"}, - {Name: "file0", Relative: "base1/file0"}, - } - if !reflect.DeepEqual(pathsTraversed, expect) { - t.Log(pathsTraversed) - t.Log(expect) - t.Error("walk doesn't match") - return - } - }) - - t.Run("walk starting in a directory", func(t *testing.T) { - pathsTraversed, err := fs.testWalkDirAt("base0") - if err != nil { - t.Error(err) - return - } - expect := []Path{ - // TODO: what should relative actually be here? - // The behaviour differs from walking the directory root vs a sub - // directory. When walking from the root, dirfd is the directory we - // are walking from and both name and relative are `.`. However, - // when walking from a subdirectory, fd is the parent of the - // subdirectory, and name is the subdirectory. - {Name: "base0", Relative: "."}, - {Name: "dir0", Relative: "dir0"}, - {Name: "file0", Relative: "dir0/file0"}, - {Name: "file0", Relative: "file0"}, - } - if !reflect.DeepEqual(pathsTraversed, expect) { - t.Log(pathsTraversed) - t.Log(expect) - t.Error("walk doesn't match") - return - } - }) -} - -type Path struct { - Name string - Relative string -} - -func (fs *testUnixFS) testWalkDirAt(path string) ([]Path, error) { - dirfd, name, closeFd, err := fs.SafePath(path) - defer closeFd() - if err != nil { - return nil, err - } - var pathsTraversed []Path - if err := fs.WalkDirat(dirfd, name, func(_ int, name, relative string, _ ufs.DirEntry, err error) error { - if err != nil { - return err - } - pathsTraversed = append(pathsTraversed, Path{Name: name, Relative: relative}) - return nil - }); err != nil { - return nil, err - } - slices.SortStableFunc(pathsTraversed, func(a, b Path) int { - if a.Relative > b.Relative { - return 1 - } - if a.Relative < b.Relative { - return -1 - } - return 0 - }) - return pathsTraversed, nil -} diff --git a/internal/ufs/go.LICENSE b/internal/ufs/go.LICENSE deleted file mode 100644 index 6a66aea5e..000000000 --- a/internal/ufs/go.LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/internal/ufs/mkdir_unix.go b/internal/ufs/mkdir_unix.go deleted file mode 100644 index eb2942b3a..000000000 --- a/internal/ufs/mkdir_unix.go +++ /dev/null @@ -1,63 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -// Code in this file was derived from `go/src/os/path.go`. - -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the `go.LICENSE` file. - -//go:build unix - -package ufs - -// mkdirAll is a recursive Mkdir implementation that properly handles symlinks. -func (fs *UnixFS) mkdirAll(name string, mode FileMode) error { - // Fast path: if we can tell whether path is a directory or file, stop with success or error. - dir, err := fs.Lstat(name) - if err == nil { - if dir.Mode()&ModeSymlink != 0 { - // If the final path is a symlink, resolve its target and use that - // to check instead. - dir, err = fs.Stat(name) - if err != nil { - return err - } - } - if dir.IsDir() { - return nil - } - return &PathError{Op: "mkdir", Path: name, Err: ErrNotDirectory} - } - - // Slow path: make sure parent exists and then call Mkdir for path. - i := len(name) - for i > 0 && name[i-1] == '/' { // Skip trailing path separator. - i-- - } - - j := i - for j > 0 && name[j-1] != '/' { // Scan backward over element. - j-- - } - - if j > 1 { - // Create parent. - err = fs.mkdirAll(name[:j-1], mode) - if err != nil { - return err - } - } - - // Parent now exists; invoke Mkdir and use its result. - err = fs.Mkdir(name, mode) - if err != nil { - // Handle arguments like "foo/." by - // double-checking that directory doesn't exist. - dir, err1 := fs.Lstat(name) - if err1 == nil && dir.IsDir() { - return nil - } - return err - } - return nil -} diff --git a/internal/ufs/path_unix.go b/internal/ufs/path_unix.go deleted file mode 100644 index f82f09a7e..000000000 --- a/internal/ufs/path_unix.go +++ /dev/null @@ -1,80 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -// Code in this file was copied from `go/src/os/path.go` -// and `go/src/os/path_unix.go`. - -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the `go.LICENSE` file. - -//go:build unix - -package ufs - -import ( - "os" -) - -// basename removes trailing slashes and the leading directory name from path name. -func basename(name string) string { - i := len(name) - 1 - // Remove trailing slashes - for ; i > 0 && name[i] == '/'; i-- { - name = name[:i] - } - // Remove leading directory name - for i--; i >= 0; i-- { - if name[i] == '/' { - name = name[i+1:] - break - } - } - return name -} - -// endsWithDot reports whether the final component of path is ".". -func endsWithDot(path string) bool { - if path == "." { - return true - } - if len(path) >= 2 && path[len(path)-1] == '.' && os.IsPathSeparator(path[len(path)-2]) { - return true - } - return false -} - -// splitPath returns the base name and parent directory. -func splitPath(path string) (string, string) { - // if no better parent is found, the path is relative from "here" - dirname := "." - - // Remove all but one leading slash. - for len(path) > 1 && path[0] == '/' && path[1] == '/' { - path = path[1:] - } - - i := len(path) - 1 - - // Remove trailing slashes. - for ; i > 0 && path[i] == '/'; i-- { - path = path[:i] - } - - // if no slashes in path, base is path - basename := path - - // Remove leading directory path - for i--; i >= 0; i-- { - if path[i] == '/' { - if i == 0 { - dirname = path[:1] - } else { - dirname = path[:i] - } - basename = path[i+1:] - break - } - } - - return dirname, basename -} diff --git a/internal/ufs/quota_writer.go b/internal/ufs/quota_writer.go deleted file mode 100644 index 9ba15989d..000000000 --- a/internal/ufs/quota_writer.go +++ /dev/null @@ -1,115 +0,0 @@ -// SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: Copyright (c) 2024 Matthew Penner - -package ufs - -import ( - "io" - "sync/atomic" -) - -// CountedWriter is a writer that counts the amount of data written to the -// underlying writer. -type CountedWriter struct { - File - - counter atomic.Int64 - err error -} - -// NewCountedWriter returns a new countedWriter that counts the amount of bytes -// written to the underlying writer. -func NewCountedWriter(f File) *CountedWriter { - return &CountedWriter{File: f} -} - -// BytesWritten returns the amount of bytes that have been written to the -// underlying writer. -func (w *CountedWriter) BytesWritten() int64 { - return w.counter.Load() -} - -// Error returns the error from the writer if any. If the error is an EOF, nil -// will be returned. -func (w *CountedWriter) Error() error { - if w.err == io.EOF { - return nil - } - return w.err -} - -// Write writes bytes to the underlying writer while tracking the total amount -// of bytes written. -func (w *CountedWriter) Write(p []byte) (int, error) { - if w.err != nil { - return 0, io.EOF - } - - // Write is a very simple operation for us to handle. - n, err := w.File.Write(p) - w.counter.Add(int64(n)) - w.err = err - - // TODO: is this how we actually want to handle errors with this? - if err == io.EOF { - return n, io.EOF - } - return n, nil -} - -func (w *CountedWriter) ReadFrom(r io.Reader) (n int64, err error) { - cr := NewCountedReader(r) - n, err = w.File.ReadFrom(cr) - w.counter.Add(n) - return -} - -// CountedReader is a reader that counts the amount of data read from the -// underlying reader. -type CountedReader struct { - reader io.Reader - - counter atomic.Int64 - err error -} - -var _ io.Reader = (*CountedReader)(nil) - -// NewCountedReader returns a new countedReader that counts the amount of bytes -// read from the underlying reader. -func NewCountedReader(r io.Reader) *CountedReader { - return &CountedReader{reader: r} -} - -// BytesRead returns the amount of bytes that have been read from the underlying -// reader. -func (r *CountedReader) BytesRead() int64 { - return r.counter.Load() -} - -// Error returns the error from the reader if any. If the error is an EOF, nil -// will be returned. -func (r *CountedReader) Error() error { - if r.err == io.EOF { - return nil - } - return r.err -} - -// Read reads bytes from the underlying reader while tracking the total amount -// of bytes read. -func (r *CountedReader) Read(p []byte) (int, error) { - if r.err != nil { - return 0, io.EOF - } - - n, err := r.reader.Read(p) - r.counter.Add(int64(n)) - r.err = err - - // TODO: is this how we actually want to handle errors with this? - if err == io.EOF { - return n, io.EOF - } - return n, nil -} diff --git a/internal/ufs/removeall_unix.go b/internal/ufs/removeall_unix.go deleted file mode 100644 index d756021af..000000000 --- a/internal/ufs/removeall_unix.go +++ /dev/null @@ -1,261 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -// Code in this file was derived from `go/src/os/removeall_at.go`. - -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the `go.LICENSE` file. - -//go:build unix - -package ufs - -import ( - "errors" - "io" - "os" - - "golang.org/x/sys/unix" -) - -type unixFS interface { - Open(name string) (File, error) - Remove(name string) error - unlinkat(dirfd int, path string, flags int) error -} - -func (fs *UnixFS) removeAll(path string) error { - return removeAll(fs, path) -} - -func removeAll(fs unixFS, path string) error { - if path == "" { - // fail silently to retain compatibility with previous behavior - // of RemoveAll. See issue https://go.dev/issue/28830. - return nil - } - - // The rmdir system call does not permit removing ".", - // so we don't permit it either. - if endsWithDot(path) { - return &PathError{Op: "removeall", Path: path, Err: unix.EINVAL} - } - - // Simple case: if Remove works, we're done. - err := fs.Remove(path) - if err == nil || errors.Is(err, ErrNotExist) { - return nil - } - - // RemoveAll recurses by deleting the path base from - // its parent directory - parentDir, base := splitPath(path) - - parent, err := fs.Open(parentDir) - if err != nil { - if !errors.Is(err, ErrNotExist) { - return err - } - // If parent does not exist, base cannot exist. Fail silently - return nil - } - defer parent.Close() - - if err := removeAllFrom(fs, parent, base); err != nil { - if pathErr, ok := err.(*PathError); ok { - pathErr.Path = parentDir + string(os.PathSeparator) + pathErr.Path - err = convertErrorType(pathErr) - } else { - err = ensurePathError(err, "removeallfrom", base) - } - return err - } - return nil -} - -func (fs *UnixFS) removeContents(path string) error { - return removeContents(fs, path) -} - -func removeContents(fs unixFS, path string) error { - if path == "" { - // fail silently to retain compatibility with previous behavior - // of RemoveAll. See issue https://go.dev/issue/28830. - return nil - } - - // RemoveAll recurses by deleting the path base from - // its parent directory - parentDir, base := splitPath(path) - - parent, err := fs.Open(parentDir) - if err != nil { - if !errors.Is(err, ErrNotExist) { - return err - } - // If parent does not exist, base cannot exist. Fail silently - return nil - } - defer parent.Close() - - if err := removeContentsFrom(fs, parent, base); err != nil { - if pathErr, ok := err.(*PathError); ok { - pathErr.Path = parentDir + string(os.PathSeparator) + pathErr.Path - err = convertErrorType(pathErr) - } else { - err = ensurePathError(err, "removecontentsfrom", base) - } - return err - } - return nil -} - -// removeContentsFrom recursively removes all descendants of parent without -// removing parent itself. Parent must be a directory. -func removeContentsFrom(fs unixFS, parent File, base string) error { - parentFd := int(parent.Fd()) - - var recurseErr error - for { - const reqSize = 1024 - var respSize int - - // Open the directory to recurse into - file, err := openFdAt(parentFd, base) - if err != nil { - if errors.Is(err, ErrNotExist) { - return nil - } - recurseErr = &PathError{Op: "openfdat", Path: base, Err: err} - break - } - - for { - numErr := 0 - - names, readErr := file.Readdirnames(reqSize) - // Errors other than EOF should stop us from continuing. - if readErr != nil && readErr != io.EOF { - _ = file.Close() - if errors.Is(readErr, ErrNotExist) { - return nil - } - return &PathError{Op: "readdirnames", Path: base, Err: readErr} - } - - respSize = len(names) - for _, name := range names { - err := removeAllFrom(fs, file, name) - if err != nil { - if pathErr, ok := err.(*PathError); ok { - pathErr.Path = base + string(os.PathSeparator) + pathErr.Path - } - numErr++ - if recurseErr == nil { - recurseErr = err - } - } - } - - // If we can delete any entry, break to start new iteration. - // Otherwise, we discard current names, get next entries and try deleting them. - if numErr != reqSize { - break - } - } - - // Removing files from the directory may have caused - // the OS to reshuffle it. Simply calling Readdirnames - // again may skip some entries. The only reliable way - // to avoid this is to close and re-open the - // directory. See issue https://go.dev/issue/20841. - _ = file.Close() - - // Finish when the end of the directory is reached - if respSize < reqSize { - break - } - } - - return nil -} - -func removeAllFrom(fs unixFS, parent File, base string) error { - parentFd := int(parent.Fd()) - - // Simple case: if Unlink (aka remove) works, we're done. - err := fs.unlinkat(parentFd, base, 0) - if err == nil || errors.Is(err, ErrNotExist) { - return nil - } - - // EISDIR means that we have a directory, and we need to - // remove its contents. - // EPERM or EACCES means that we don't have write permission on - // the parent directory, but this entry might still be a directory - // whose contents need to be removed. - // Otherwise, just return the error. - if err != unix.EISDIR && err != unix.EPERM && err != unix.EACCES { - return &PathError{Op: "unlinkat", Path: base, Err: err} - } - - // Is this a directory we need to recurse into? - var statInfo unix.Stat_t - statErr := ignoringEINTR(func() error { - return unix.Fstatat(parentFd, base, &statInfo, AT_SYMLINK_NOFOLLOW) - }) - if statErr != nil { - if errors.Is(statErr, ErrNotExist) { - return nil - } - return &PathError{Op: "fstatat", Path: base, Err: statErr} - } - if statInfo.Mode&unix.S_IFMT != unix.S_IFDIR { - // Not a directory; return the error from the unix.Unlinkat. - return &PathError{Op: "unlinkat", Path: base, Err: err} - } - - // Remove all contents will remove the contents of the directory. - // - // It was split out of this function to allow the deletion of the - // contents of a directory, without deleting the directory itself. - recurseErr := removeContentsFrom(fs, parent, base) - - // Remove the directory itself. - unlinkErr := fs.unlinkat(parentFd, base, AT_REMOVEDIR) - if unlinkErr == nil || errors.Is(unlinkErr, ErrNotExist) { - return nil - } - - if recurseErr != nil { - return recurseErr - } - - return ensurePathError(err, "unlinkat", base) -} - -// openFdAt opens path relative to the directory in fd. -// Other than that this should act like openFileNolog. -// This acts like openFileNolog rather than OpenFile because -// we are going to (try to) remove the file. -// The contents of this file are not relevant for test caching. -func openFdAt(dirfd int, name string) (File, error) { - var fd int - for { - var err error - fd, err = unix.Openat(dirfd, name, O_RDONLY|O_CLOEXEC|O_NOFOLLOW, 0) - if err == nil { - break - } - - // See comment in openFileNolog. - if err == unix.EINTR { - continue - } - - return nil, err - } - // This is stupid, os.NewFile immediately casts `fd` to an `int`, but wants - // it to be passed as a `uintptr`. - return os.NewFile(uintptr(fd), name), nil -} diff --git a/internal/ufs/stat_unix.go b/internal/ufs/stat_unix.go deleted file mode 100644 index 3339fbbbd..000000000 --- a/internal/ufs/stat_unix.go +++ /dev/null @@ -1,67 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -// Code in this file was copied from `go/src/os/stat_linux.go` -// and `go/src/os/types_unix.go`. - -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the `go.LICENSE` file. - -//go:build unix - -package ufs - -import ( - "time" - - "golang.org/x/sys/unix" -) - -type fileStat struct { - name string - size int64 - mode FileMode - modTime time.Time - sys unix.Stat_t -} - -var _ FileInfo = (*fileStat)(nil) - -func (fs *fileStat) Size() int64 { return fs.size } -func (fs *fileStat) Mode() FileMode { return fs.mode } -func (fs *fileStat) ModTime() time.Time { return fs.modTime } -func (fs *fileStat) Sys() any { return &fs.sys } -func (fs *fileStat) Name() string { return fs.name } -func (fs *fileStat) IsDir() bool { return fs.Mode().IsDir() } - -func fillFileStatFromSys(fs *fileStat, name string) { - fs.name = basename(name) - fs.size = fs.sys.Size - fs.modTime = time.Unix(fs.sys.Mtim.Unix()) - fs.mode = FileMode(fs.sys.Mode & 0o777) - switch fs.sys.Mode & unix.S_IFMT { - case unix.S_IFBLK: - fs.mode |= ModeDevice - case unix.S_IFCHR: - fs.mode |= ModeDevice | ModeCharDevice - case unix.S_IFDIR: - fs.mode |= ModeDir - case unix.S_IFIFO: - fs.mode |= ModeNamedPipe - case unix.S_IFLNK: - fs.mode |= ModeSymlink - case unix.S_IFREG: - // nothing to do - case unix.S_IFSOCK: - fs.mode |= ModeSocket - } - if fs.sys.Mode&unix.S_ISGID != 0 { - fs.mode |= ModeSetgid - } - if fs.sys.Mode&unix.S_ISUID != 0 { - fs.mode |= ModeSetuid - } - if fs.sys.Mode&unix.S_ISVTX != 0 { - fs.mode |= ModeSticky - } -} diff --git a/internal/ufs/walk.go b/internal/ufs/walk.go deleted file mode 100644 index 93cd30635..000000000 --- a/internal/ufs/walk.go +++ /dev/null @@ -1,124 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause - -// Code in this file was derived from `go/src/io/fs/walk.go`. - -// Copyright 2020 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the `go.LICENSE` file. - -package ufs - -import ( - iofs "io/fs" - "path" -) - -// SkipDir is used as a return value from [WalkDirFunc] to indicate that -// the directory named in the call is to be skipped. It is not returned -// as an error by any function. -var SkipDir = iofs.SkipDir - -// SkipAll is used as a return value from [WalkDirFunc] to indicate that -// all remaining files and directories are to be skipped. It is not returned -// as an error by any function. -var SkipAll = iofs.SkipAll - -// WalkDirFunc is the type of the function called by [WalkDir] to visit -// each file or directory. -// -// The path argument contains the argument to [WalkDir] as a prefix. -// That is, if WalkDir is called with root argument "dir" and finds a file -// named "a" in that directory, the walk function will be called with -// argument "dir/a". -// -// The d argument is the [DirEntry] for the named path. -// -// The error result returned by the function controls how [WalkDir] -// continues. If the function returns the special value [SkipDir], WalkDir -// skips the current directory (path if d.IsDir() is true, otherwise -// path's parent directory). If the function returns the special value -// [SkipAll], WalkDir skips all remaining files and directories. Otherwise, -// if the function returns a non-nil error, WalkDir stops entirely and -// returns that error. -// -// The err argument reports an error related to path, signaling that -// [WalkDir] will not walk into that directory. The function can decide how -// to handle that error; as described earlier, returning the error will -// cause WalkDir to stop walking the entire tree. -// -// [WalkDir] calls the function with a non-nil err argument in two cases. -// -// First, if the initial [Stat] on the root directory fails, WalkDir -// calls the function with path set to root, d set to nil, and err set to -// the error from [fs.Stat]. -// -// Second, if a directory's ReadDir method (see [ReadDirFile]) fails, WalkDir calls the -// function with path set to the directory's path, d set to an -// [DirEntry] describing the directory, and err set to the error from -// ReadDir. In this second case, the function is called twice with the -// path of the directory: the first call is before the directory read is -// attempted and has err set to nil, giving the function a chance to -// return [SkipDir] or [SkipAll] and avoid the ReadDir entirely. The second call -// is after a failed ReadDir and reports the error from ReadDir. -// (If ReadDir succeeds, there is no second call.) -type WalkDirFunc func(path string, d DirEntry, err error) error - -// WalkDir walks the file tree rooted at root, calling fn for each file or -// directory in the tree, including root. -// -// All errors that arise visiting files and directories are filtered by fn: -// see the [WalkDirFunc] documentation for details. -// -// The files are walked in lexical order, which makes the output deterministic -// but requires WalkDir to read an entire directory into memory before proceeding -// to walk that directory. -// -// WalkDir does not follow symbolic links found in directories, -// but if root itself is a symbolic link, its target will be walked. -func WalkDir(fs Filesystem, root string, fn WalkDirFunc) error { - info, err := fs.Stat(root) - if err != nil { - err = fn(root, nil, err) - } else { - err = walkDir(fs, root, iofs.FileInfoToDirEntry(info), fn) - } - if err == SkipDir || err == SkipAll { - return nil - } - return err -} - -// walkDir recursively descends path, calling walkDirFn. -func walkDir(fs Filesystem, name string, d DirEntry, walkDirFn WalkDirFunc) error { - if err := walkDirFn(name, d, nil); err != nil || !d.IsDir() { - if err == SkipDir && d.IsDir() { - // Successfully skipped directory. - err = nil - } - return err - } - - dirs, err := fs.ReadDir(name) - if err != nil { - // Second call, to report ReadDir error. - err = walkDirFn(name, d, err) - if err != nil { - if err == SkipDir && d.IsDir() { - err = nil - } - return err - } - } - - for _, d1 := range dirs { - name1 := path.Join(name, d1.Name()) - if err := walkDir(fs, name1, d1, walkDirFn); err != nil { - if err == SkipDir { - break - } - return err - } - } - - return nil -} diff --git a/internal/ufs/walk_unix.go b/internal/ufs/walk_unix.go deleted file mode 100644 index 065afc229..000000000 --- a/internal/ufs/walk_unix.go +++ /dev/null @@ -1,314 +0,0 @@ -// SPDX-License-Identifier: BSD-2-Clause - -// Some code in this file was derived from https://github.com/karrick/godirwalk. - -//go:build unix - -package ufs - -import ( - "bytes" - "fmt" - iofs "io/fs" - "os" - "path" - "reflect" - "unsafe" - - "golang.org/x/sys/unix" -) - -type WalkDiratFunc func(dirfd int, name, relative string, d DirEntry, err error) error - -func (fs *UnixFS) WalkDirat(dirfd int, name string, fn WalkDiratFunc) error { - info, err := fs.Lstatat(dirfd, name) - if err != nil { - err = fn(dirfd, name, ".", nil, err) - } else { - b := newScratchBuffer() - err = fs.walkDir(b, dirfd, name, ".", iofs.FileInfoToDirEntry(info), fn) - } - if err == SkipDir || err == SkipAll { - return nil - } - return err -} - -func (fs *UnixFS) walkDir(b []byte, parentfd int, name, relative string, d DirEntry, walkDirFn WalkDiratFunc) error { - if err := walkDirFn(parentfd, name, relative, d, nil); err != nil || !d.IsDir() { - if err == SkipDir && d.IsDir() { - // Successfully skipped directory. - err = nil - } - return err - } - - dirfd, err := fs.openat(parentfd, name, O_DIRECTORY|O_RDONLY, 0) - if dirfd != 0 { - defer unix.Close(dirfd) - } - if err != nil { - return err - } - - dirs, err := fs.readDir(dirfd, name, relative, b) - if err != nil { - // Second call, to report ReadDir error. - err = walkDirFn(dirfd, name, relative, d, err) - if err != nil { - if err == SkipDir && d.IsDir() { - err = nil - } - return err - } - } - - for _, d1 := range dirs { - name := d1.Name() - // This fancy logic ensures that if we start walking from a subdirectory - // that we don't make the path relative to the root of the filesystem. - // - // For example, if we walk from the root of a filesystem, relative would - // be "." and path.Join would end up just returning name. But if relative - // was a subdirectory, relative could be "dir" and path.Join would make - // it "dir/child" even though we are walking starting at dir. - var rel string - if relative == "." { - rel = name - } else { - rel = path.Join(relative, name) - } - if err := fs.walkDir(b, dirfd, name, rel, d1, walkDirFn); err != nil { - if err == SkipDir { - break - } - return err - } - } - - return nil -} - -// ReadDirMap . -// TODO: document -func ReadDirMap[T any](fs *UnixFS, path string, fn func(DirEntry) (T, error)) ([]T, error) { - dirfd, name, closeFd, err := fs.safePath(path) - defer closeFd() - if err != nil { - return nil, err - } - fd, err := fs.openat(dirfd, name, O_DIRECTORY|O_RDONLY, 0) - if err != nil { - return nil, err - } - defer unix.Close(fd) - - entries, err := fs.readDir(fd, ".", path, nil) - if err != nil { - return nil, err - } - - out := make([]T, len(entries)) - for i, e := range entries { - idx := i - e := e - v, err := fn(e) - if err != nil { - return nil, err - } - out[idx] = v - } - - return out, nil -} - -// nameOffset is a compile time constant -const nameOffset = int(unsafe.Offsetof(unix.Dirent{}.Name)) - -func nameFromDirent(de *unix.Dirent) (name []byte) { - // Because this GOOS' syscall.Dirent does not provide a field that specifies - // the name length, this function must first calculate the max possible name - // length, and then search for the NULL byte. - ml := int(de.Reclen) - nameOffset - - // Convert syscall.Dirent.Name, which is array of int8, to []byte, by - // overwriting Cap, Len, and Data slice header fields to the max possible - // name length computed above, and finding the terminating NULL byte. - // - // TODO: is there an alternative to the deprecated SliceHeader? - // SliceHeader was mainly deprecated due to it being misused for avoiding - // allocations when converting a byte slice to a string, ref; - // https://go.dev/issue/53003 - sh := (*reflect.SliceHeader)(unsafe.Pointer(&name)) - sh.Cap = ml - sh.Len = ml - sh.Data = uintptr(unsafe.Pointer(&de.Name[0])) - - if index := bytes.IndexByte(name, 0); index >= 0 { - // Found NULL byte; set slice's cap and len accordingly. - sh.Cap = index - sh.Len = index - return - } - - // NOTE: This branch is not expected, but included for defensive - // programming, and provides a hard stop on the name based on the structure - // field array size. - sh.Cap = len(de.Name) - sh.Len = sh.Cap - return -} - -// modeTypeFromDirent converts a syscall defined constant, which is in purview -// of OS, to a constant defined by Go, assumed by this project to be stable. -// -// When the syscall constant is not recognized, this function falls back to a -// Stat on the file system. -func (fs *UnixFS) modeTypeFromDirent(de *unix.Dirent, fd int, name string) (FileMode, error) { - switch de.Type { - case unix.DT_REG: - return 0, nil - case unix.DT_DIR: - return ModeDir, nil - case unix.DT_LNK: - return ModeSymlink, nil - case unix.DT_CHR: - return ModeDevice | ModeCharDevice, nil - case unix.DT_BLK: - return ModeDevice, nil - case unix.DT_FIFO: - return ModeNamedPipe, nil - case unix.DT_SOCK: - return ModeSocket, nil - default: - // If syscall returned unknown type (e.g., DT_UNKNOWN, DT_WHT), then - // resolve actual mode by reading file information. - return fs.modeType(fd, name) - } -} - -// modeType returns the mode type of the file system entry identified by -// osPathname by calling os.LStat function, to intentionally not follow symbolic -// links. -// -// Even though os.LStat provides all file mode bits, we want to ensure same -// values returned to caller regardless of whether we obtained file mode bits -// from syscall or stat call. Therefore, mask out the additional file mode bits -// that are provided by stat but not by the syscall, so users can rely on their -// values. -func (fs *UnixFS) modeType(dirfd int, name string) (FileMode, error) { - fi, err := fs.Lstatat(dirfd, name) - if err != nil { - return 0, fmt.Errorf("ufs: error finding mode type for %s during readDir: %w", name, err) - } - return fi.Mode() & ModeType, nil -} - -var minimumScratchBufferSize = os.Getpagesize() - -func newScratchBuffer() []byte { - return make([]byte, minimumScratchBufferSize) -} - -func (fs *UnixFS) readDir(fd int, name, relative string, b []byte) ([]DirEntry, error) { - scratchBuffer := b - if scratchBuffer == nil || len(scratchBuffer) < minimumScratchBufferSize { - scratchBuffer = newScratchBuffer() - } - - var entries []DirEntry - var workBuffer []byte - - var sde unix.Dirent - for { - if len(workBuffer) == 0 { - n, err := unix.Getdents(fd, scratchBuffer) - if err != nil { - if err == unix.EINTR { - continue - } - return nil, ensurePathError(err, "getdents", name) - } - if n <= 0 { - // end of directory: normal exit - return entries, nil - } - workBuffer = scratchBuffer[:n] // trim work buffer to number of bytes read - } - - // "Go is like C, except that you just put `unsafe` all over the place". - copy((*[unsafe.Sizeof(unix.Dirent{})]byte)(unsafe.Pointer(&sde))[:], workBuffer) - workBuffer = workBuffer[sde.Reclen:] // advance buffer for next iteration through loop - - if sde.Ino == 0 { - continue // inode set to 0 indicates an entry that was marked as deleted - } - - nameSlice := nameFromDirent(&sde) - nameLength := len(nameSlice) - - if nameLength == 0 || (nameSlice[0] == '.' && (nameLength == 1 || (nameLength == 2 && nameSlice[1] == '.'))) { - continue - } - - childName := string(nameSlice) - mt, err := fs.modeTypeFromDirent(&sde, fd, childName) - if err != nil { - return nil, err - } - var rel string - if relative == "." { - rel = name - } else { - rel = path.Join(relative, childName) - } - entries = append(entries, &dirent{dirfd: fd, name: childName, path: rel, modeType: mt, fs: fs}) - } -} - -// dirent stores the name and file system mode type of discovered file system -// entries. -type dirent struct { - dirfd int - name string - path string - modeType FileMode - - fs *UnixFS -} - -func (de dirent) Name() string { - return de.name -} - -func (de dirent) IsDir() bool { - return de.modeType&ModeDir != 0 -} - -func (de dirent) Type() FileMode { - return de.modeType -} - -func (de dirent) Info() (FileInfo, error) { - if de.fs == nil { - return nil, nil - } - return de.fs.Lstatat(de.dirfd, de.name) - // return de.fs.Lstat(de.path) -} - -func (de dirent) Open() (File, error) { - if de.fs == nil { - return nil, nil - } - return de.fs.OpenFileat(de.dirfd, de.name, O_RDONLY, 0) - // return de.fs.OpenFile(de.path, O_RDONLY, 0) -} - -// reset releases memory held by entry err and name, and resets mode type to 0. -func (de *dirent) reset() { - de.name = "" - de.path = "" - de.modeType = 0 - de.dirfd = 0 -} diff --git a/parser/helpers.go b/parser/helpers.go index be09c686f..a8e8ec789 100644 --- a/parser/helpers.go +++ b/parser/helpers.go @@ -2,6 +2,8 @@ package parser import ( "bytes" + "io" + "os" "regexp" "strconv" "strings" @@ -27,14 +29,24 @@ var configMatchRegex = regexp.MustCompile(`{{\s?config\.([\w.-]+)\s?}}`) // matching: // // -// -// -// +// // // // noinspection RegExpRedundantEscape var xmlValueMatchRegex = regexp.MustCompile(`^\[([\w]+)='(.*)'\]$`) +// Gets the []byte representation of a configuration file to be passed through to other +// handler functions. If the file does not currently exist, it will be created. +func readFileBytes(path string) ([]byte, error) { + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644) + if err != nil { + return nil, err + } + defer file.Close() + + return io.ReadAll(file) +} + // Gets the value of a key based on the value type defined. func (cfr *ConfigurationFileReplacement) getKeyValue(value string) interface{} { if cfr.ReplaceWith.Type() == jsonparser.Boolean { diff --git a/parser/parser.go b/parser/parser.go index e7c98b3b2..12d1ed9d0 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -5,6 +5,8 @@ import ( "bytes" "encoding/json" "io" + "os" + "path/filepath" "strconv" "strings" @@ -18,7 +20,6 @@ import ( "gopkg.in/yaml.v3" "github.com/pterodactyl/wings/config" - "github.com/pterodactyl/wings/internal/ufs" ) // The file parsing options that are available for a server configuration file. @@ -188,12 +189,11 @@ func (cfr *ConfigurationFileReplacement) UnmarshalJSON(data []byte) error { return nil } -// Parse parses a given configuration file and updates all the values within -// as defined in the API response from the Panel. -func (f *ConfigurationFile) Parse(file ufs.File) error { - // log.WithField("path", path).WithField("parser", f.Parser.String()).Debug("parsing server configuration file") +// Parses a given configuration file and updates all of the values within as defined +// in the API response from the Panel. +func (f *ConfigurationFile) Parse(path string, internal bool) error { + log.WithField("path", path).WithField("parser", f.Parser.String()).Debug("parsing server configuration file") - // What the fuck is going on here? if mb, err := json.Marshal(config.Get()); err != nil { return err } else { @@ -204,24 +204,56 @@ func (f *ConfigurationFile) Parse(file ufs.File) error { switch f.Parser { case Properties: - err = f.parsePropertiesFile(file) + err = f.parsePropertiesFile(path) + break case File: - err = f.parseTextFile(file) + err = f.parseTextFile(path) + break case Yaml, "yml": - err = f.parseYamlFile(file) + err = f.parseYamlFile(path) + break case Json: - err = f.parseJsonFile(file) + err = f.parseJsonFile(path) + break case Ini: - err = f.parseIniFile(file) + err = f.parseIniFile(path) + break case Xml: - err = f.parseXmlFile(file) + err = f.parseXmlFile(path) + break } + + if errors.Is(err, os.ErrNotExist) { + // File doesn't exist, we tried creating it, and same error is returned? Pretty + // sure this pathway is impossible, but if not, abort here. + if internal { + return nil + } + + b := strings.TrimSuffix(path, filepath.Base(path)) + if err := os.MkdirAll(b, 0o755); err != nil { + return errors.WithMessage(err, "failed to create base directory for missing configuration file") + } else { + if _, err := os.Create(path); err != nil { + return errors.WithMessage(err, "failed to create missing configuration file") + } + } + + return f.Parse(path, true) + } + return err } // Parses an xml file. -func (f *ConfigurationFile) parseXmlFile(file ufs.File) error { +func (f *ConfigurationFile) parseXmlFile(path string) error { doc := etree.NewDocument() + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644) + if err != nil { + return err + } + defer file.Close() + if _, err := doc.ReadFrom(file); err != nil { return err } @@ -299,9 +331,17 @@ func (f *ConfigurationFile) parseXmlFile(file ufs.File) error { } // Parses an ini file. -func (f *ConfigurationFile) parseIniFile(file ufs.File) error { - // Wrap the file in a NopCloser so the ini package doesn't close the file. - cfg, err := ini.Load(io.NopCloser(file)) +func (f *ConfigurationFile) parseIniFile(path string) error { + // Ini package can't handle a non-existent file, so handle that automatically here + // by creating it if not exists. Then, immediately close the file since we will use + // other methods to write the new contents. + file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644) + if err != nil { + return err + } + file.Close() + + cfg, err := ini.Load(path) if err != nil { return err } @@ -380,8 +420,8 @@ func (f *ConfigurationFile) parseIniFile(file ufs.File) error { // Parses a json file updating any matching key/value pairs. If a match is not found, the // value is set regardless in the file. See the commentary in parseYamlFile for more details // about what is happening during this process. -func (f *ConfigurationFile) parseJsonFile(file ufs.File) error { - b, err := io.ReadAll(file) +func (f *ConfigurationFile) parseJsonFile(path string) error { + b, err := readFileBytes(path) if err != nil { return err } @@ -407,8 +447,8 @@ func (f *ConfigurationFile) parseJsonFile(file ufs.File) error { // Parses a yaml file and updates any matching key/value pairs before persisting // it back to the disk. -func (f *ConfigurationFile) parseYamlFile(file ufs.File) error { - b, err := io.ReadAll(file) +func (f *ConfigurationFile) parseYamlFile(path string) error { + b, err := readFileBytes(path) if err != nil { return err } diff --git a/router/downloader/downloader.go b/router/downloader/downloader.go index 232c93483..650222213 100644 --- a/router/downloader/downloader.go +++ b/router/downloader/downloader.go @@ -199,8 +199,13 @@ func (dl *Download) Execute() error { return errors.New("downloader: got bad response status from endpoint: " + res.Status) } - if res.ContentLength < 1 { - return errors.New("downloader: request is missing ContentLength") + // If there is a Content-Length header on this request go ahead and check that we can + // even write the whole file before beginning this process. If there is no header present + // we'll just have to give it a spin and see how it goes. + if res.ContentLength > 0 { + if err := dl.server.Filesystem().HasSpaceFor(res.ContentLength); err != nil { + return errors.WrapIf(err, "downloader: failed to write file: not enough space") + } } if dl.req.UseHeader { @@ -227,10 +232,8 @@ func (dl *Download) Execute() error { p := dl.Path() dl.server.Log().WithField("path", p).Debug("writing remote file to disk") - // Write the file while tracking the progress, Write will check that the - // size of the file won't exceed the disk limit. r := io.TeeReader(res.Body, dl.counter(res.ContentLength)) - if err := dl.server.Filesystem().Write(p, r, res.ContentLength, 0o644); err != nil { + if err := dl.server.Filesystem().Writefile(p, r); err != nil { return errors.WrapIf(err, "downloader: failed to write file to server directory") } return nil diff --git a/router/router_download.go b/router/router_download.go index 8ebcaa557..59fec5194 100644 --- a/router/router_download.go +++ b/router/router_download.go @@ -56,8 +56,6 @@ func getDownloadBackup(c *gin.Context) { return } - // The use of `os` here is safe as backups are not stored within server - // accessible directories. f, err := os.Open(b.Path()) if err != nil { middleware.CaptureAndAbort(c, err) @@ -89,19 +87,27 @@ func getDownloadFile(c *gin.Context) { return } - f, st, err := s.Filesystem().File(token.FilePath) + p, _ := s.Filesystem().SafePath(token.FilePath) + st, err := os.Stat(p) + // If there is an error or we're somehow trying to download a directory, just + // respond with the appropriate error. if err != nil { middleware.CaptureAndAbort(c, err) return - } - defer f.Close() - if st.IsDir() { + } else if st.IsDir() { c.AbortWithStatusJSON(http.StatusNotFound, gin.H{ "error": "The requested resource was not found on this server.", }) return } + f, err := os.Open(p) + if err != nil { + middleware.CaptureAndAbort(c, err) + return + } + defer f.Close() + c.Header("Content-Length", strconv.Itoa(int(st.Size()))) c.Header("Content-Disposition", "attachment; filename="+strconv.Quote(st.Name())) c.Header("Content-Type", "application/octet-stream") diff --git a/router/router_server.go b/router/router_server.go index ec1f772bd..f4090f060 100644 --- a/router/router_server.go +++ b/router/router_server.go @@ -227,19 +227,19 @@ func deleteServer(c *gin.Context) { // // In addition, servers with large amounts of files can take some time to finish deleting, // so we don't want to block the HTTP call while waiting on this. - go func(s *server.Server) { - fs := s.Filesystem() - p := fs.Path() - _ = fs.UnixFS().Close() + go func(p string) { if err := os.RemoveAll(p); err != nil { log.WithFields(log.Fields{"path": p, "error": err}).Warn("failed to remove server files during deletion process") } - }(s) + }(s.Filesystem().Path()) middleware.ExtractManager(c).Remove(func(server *server.Server) bool { return server.ID() == s.ID() }) + // Deallocate the reference to this server. + s = nil + c.Status(http.StatusNoContent) } diff --git a/router/router_server_files.go b/router/router_server_files.go index 09ad8cd1f..323d23bc3 100644 --- a/router/router_server_files.go +++ b/router/router_server_files.go @@ -30,7 +30,7 @@ import ( // getServerFileContents returns the contents of a file on the server. func getServerFileContents(c *gin.Context) { s := middleware.ExtractServer(c) - p := strings.TrimLeft(c.Query("file"), "/") + p := "/" + strings.TrimLeft(c.Query("file"), "/") f, st, err := s.Filesystem().File(p) if err != nil { middleware.CaptureAndAbort(c, err) @@ -129,6 +129,7 @@ func putServerRenameFiles(c *gin.Context) { } if err := fs.Rename(pf, pt); err != nil { // Return nil if the error is an is not exists. + // NOTE: os.IsNotExist() does not work if the error is wrapped. if errors.Is(err, os.ErrNotExist) { s.Log().WithField("error", err). WithField("from_path", pf). @@ -247,7 +248,7 @@ func postServerWriteFile(c *gin.Context) { return } - if err := s.Filesystem().Write(f, c.Request.Body, c.Request.ContentLength, 0o644); err != nil { + if err := s.Filesystem().Writefile(f, c.Request.Body); err != nil { if filesystem.IsErrorCode(err, filesystem.ErrCodeIsDirectory) { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ "error": "Cannot write file, name conflicts with an existing directory by the same name.", @@ -597,9 +598,15 @@ func postServerUploadFiles(c *gin.Context) { } for _, header := range headers { + p, err := s.Filesystem().SafePath(filepath.Join(directory, header.Filename)) + if err != nil { + middleware.CaptureAndAbort(c, err) + return + } + // We run this in a different method so I can use defer without any of // the consequences caused by calling it in a loop. - if err := handleFileUpload(filepath.Join(directory, header.Filename), s, header); err != nil { + if err := handleFileUpload(p, s, header); err != nil { middleware.CaptureAndAbort(c, err) return } else { @@ -621,8 +628,7 @@ func handleFileUpload(p string, s *server.Server, header *multipart.FileHeader) if err := s.Filesystem().IsIgnored(p); err != nil { return err } - - if err := s.Filesystem().Write(p, file, header.Size, 0o644); err != nil { + if err := s.Filesystem().Writefile(p, file); err != nil { return err } return nil diff --git a/router/router_transfer.go b/router/router_transfer.go index 1b062b054..1e78a39a4 100644 --- a/router/router_transfer.go +++ b/router/router_transfer.go @@ -106,7 +106,6 @@ func postTransfers(c *gin.Context) { if !successful && err != nil { // Delete all extracted files. go func(trnsfr *transfer.Transfer) { - _ = trnsfr.Server.Filesystem().UnixFS().Close() if err := os.RemoveAll(trnsfr.Server.Filesystem().Path()); err != nil && !os.IsNotExist(err) { trnsfr.Log().WithError(err).Warn("failed to delete local server files") } diff --git a/server/backup.go b/server/backup.go index 1568290d5..892a356f8 100644 --- a/server/backup.go +++ b/server/backup.go @@ -67,7 +67,7 @@ func (s *Server) Backup(b backup.BackupInterface) error { } } - ad, err := b.Generate(s.Context(), s.Filesystem(), ignored) + ad, err := b.Generate(s.Context(), s.Filesystem().Path(), ignored) if err != nil { if err := s.notifyPanelOfBackup(b.Identifier(), &backup.ArchiveDetails{}, false); err != nil { s.Log().WithFields(log.Fields{ @@ -154,14 +154,17 @@ func (s *Server) RestoreBackup(b backup.BackupInterface, reader io.ReadCloser) ( err = b.Restore(s.Context(), reader, func(file string, info fs.FileInfo, r io.ReadCloser) error { defer r.Close() s.Events().Publish(DaemonMessageEvent, "(restoring): "+file) - // TODO: since this will be called a lot, it may be worth adding an optimized - // Write with Chtimes method to the UnixFS that is able to re-use the - // same dirfd and file name. - if err := s.Filesystem().Write(file, r, info.Size(), info.Mode()); err != nil { + + if err := s.Filesystem().Writefile(file, r); err != nil { + return err + } + if err := s.Filesystem().Chmod(file, info.Mode()); err != nil { return err } + atime := info.ModTime() - return s.Filesystem().Chtimes(file, atime, atime) + mtime := atime + return s.Filesystem().Chtimes(file, atime, mtime) }) return errors.WithStackIf(err) diff --git a/server/backup/backup.go b/server/backup/backup.go index 01e73d0dd..5e452ee6f 100644 --- a/server/backup/backup.go +++ b/server/backup/backup.go @@ -16,7 +16,6 @@ import ( "github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/remote" - "github.com/pterodactyl/wings/server/filesystem" ) var format = archives.CompressedArchive{ @@ -48,7 +47,7 @@ type BackupInterface interface { WithLogContext(map[string]interface{}) // Generate creates a backup in whatever the configured source for the // specific implementation is. - Generate(context.Context, *filesystem.Filesystem, string) (*ArchiveDetails, error) + Generate(context.Context, string, string) (*ArchiveDetails, error) // Ignored returns the ignored files for this backup instance. Ignored() string // Checksum returns a SHA1 checksum for the generated backup. diff --git a/server/backup/backup_local.go b/server/backup/backup_local.go index 2351416f8..ca7f9749b 100644 --- a/server/backup/backup_local.go +++ b/server/backup/backup_local.go @@ -59,10 +59,10 @@ func (b *LocalBackup) WithLogContext(c map[string]interface{}) { // Generate generates a backup of the selected files and pushes it to the // defined location for this instance. -func (b *LocalBackup) Generate(ctx context.Context, fsys *filesystem.Filesystem, ignore string) (*ArchiveDetails, error) { +func (b *LocalBackup) Generate(ctx context.Context, basePath, ignore string) (*ArchiveDetails, error) { a := &filesystem.Archive{ - Filesystem: fsys, - Ignore: ignore, + BasePath: basePath, + Ignore: ignore, } b.log().WithField("path", b.Path()).Info("creating backup for server") diff --git a/server/backup/backup_s3.go b/server/backup/backup_s3.go index e281ca70a..b651b2702 100644 --- a/server/backup/backup_s3.go +++ b/server/backup/backup_s3.go @@ -48,12 +48,12 @@ func (s *S3Backup) WithLogContext(c map[string]interface{}) { // Generate creates a new backup on the disk, moves it into the S3 bucket via // the provided presigned URL, and then deletes the backup from the disk. -func (s *S3Backup) Generate(ctx context.Context, fsys *filesystem.Filesystem, ignore string) (*ArchiveDetails, error) { +func (s *S3Backup) Generate(ctx context.Context, basePath, ignore string) (*ArchiveDetails, error) { defer s.Remove() a := &filesystem.Archive{ - Filesystem: fsys, - Ignore: ignore, + BasePath: basePath, + Ignore: ignore, } s.log().WithField("path", s.Path()).Info("creating backup for server") diff --git a/server/config_parser.go b/server/config_parser.go index f7f230232..4c51724bd 100644 --- a/server/config_parser.go +++ b/server/config_parser.go @@ -4,11 +4,9 @@ import ( "runtime" "github.com/gammazero/workerpool" - - "github.com/pterodactyl/wings/internal/ufs" ) -// UpdateConfigurationFiles updates all the defined configuration files for +// UpdateConfigurationFiles updates all of the defined configuration files for // a server automatically to ensure that they always use the specified values. func (s *Server) UpdateConfigurationFiles() { pool := workerpool.New(runtime.NumCPU()) @@ -20,18 +18,18 @@ func (s *Server) UpdateConfigurationFiles() { f := cf pool.Submit(func() { - file, err := s.Filesystem().UnixFS().Touch(f.FileName, ufs.O_RDWR|ufs.O_CREATE, 0o644) + p, err := s.Filesystem().SafePath(f.FileName) if err != nil { - s.Log().WithField("file_name", f.FileName).WithField("error", err).Error("failed to open file for configuration") + s.Log().WithField("error", err).Error("failed to generate safe path for configuration file") + return } - defer file.Close() - if err := f.Parse(file); err != nil { + if err := f.Parse(p, false); err != nil { s.Log().WithField("error", err).Error("failed to parse and update server configuration file") } - s.Log().WithField("file_name", f.FileName).Debug("finished processing server configuration file") + s.Log().WithField("path", f.FileName).Debug("finished processing server configuration file") }) } diff --git a/server/filesystem/archive.go b/server/filesystem/archive.go index 16ae7f9ed..c3bd65f30 100644 --- a/server/filesystem/archive.go +++ b/server/filesystem/archive.go @@ -3,6 +3,7 @@ package filesystem import ( "archive/tar" "context" + "fmt" "io" "io/fs" "os" @@ -13,12 +14,12 @@ import ( "emperror.dev/errors" "github.com/apex/log" "github.com/juju/ratelimit" + "github.com/karrick/godirwalk" "github.com/klauspost/pgzip" ignore "github.com/sabhiram/go-gitignore" "github.com/pterodactyl/wings/config" "github.com/pterodactyl/wings/internal/progress" - "github.com/pterodactyl/wings/internal/ufs" ) const memory = 4 * 1024 @@ -56,25 +57,22 @@ func (p *TarProgress) Write(v []byte) (int, error) { } type Archive struct { - // Filesystem to create the archive with. - Filesystem *Filesystem + // BasePath is the absolute path to create the archive from where Files and Ignore are + // relative to. + BasePath string // Ignore is a gitignore string (most likely read from a file) of files to ignore // from the archive. Ignore string - // BaseDirectory . - BaseDirectory string - - // Files specifies the files to archive, this takes priority over the Ignore - // option, if unspecified, all files in the BaseDirectory will be archived - // unless Ignore is set. + // Files specifies the files to archive, this takes priority over the Ignore option, if + // unspecified, all files in the BasePath will be archived unless Ignore is set. + // + // All items in Files must be absolute within BasePath. Files []string // Progress wraps the writer of the archive to pass through the progress tracker. Progress *progress.Progress - - w *TarProgress } // Create creates an archive at dst with all the files defined in the @@ -105,28 +103,14 @@ func (a *Archive) Create(ctx context.Context, dst string) error { return a.Stream(ctx, writer) } -type walkFunc func(dirfd int, name, relative string, d ufs.DirEntry) error - -// Stream streams the creation of the archive to the given writer. +// Stream . func (a *Archive) Stream(ctx context.Context, w io.Writer) error { - if a.Filesystem == nil { - return errors.New("filesystem: archive.Filesystem is unset") - } - - // The base directory may come with a prefixed `/`, strip it to prevent - // problems. - a.BaseDirectory = strings.TrimPrefix(a.BaseDirectory, "/") - - if filesLen := len(a.Files); filesLen > 0 { - files := make([]string, filesLen) - for i, f := range a.Files { - if !strings.HasPrefix(f, a.Filesystem.Path()) { - files[i] = f - continue - } - files[i] = strings.TrimPrefix(strings.TrimPrefix(f, a.Filesystem.Path()), "/") + for _, f := range a.Files { + if strings.HasPrefix(f, a.BasePath) { + continue } - a.Files = files + + return fmt.Errorf("archive: all entries in Files must be absolute and within BasePath: %s\n", f) } // Choose which compression level to use based on the compression_level configuration option @@ -136,6 +120,8 @@ func (a *Archive) Stream(ctx context.Context, w io.Writer) error { compressionLevel = pgzip.NoCompression case "best_compression": compressionLevel = pgzip.BestCompression + case "best_speed": + fallthrough default: compressionLevel = pgzip.BestSpeed } @@ -149,124 +135,107 @@ func (a *Archive) Stream(ctx context.Context, w io.Writer) error { tw := tar.NewWriter(gw) defer tw.Close() - a.w = NewTarProgress(tw, a.Progress) + pw := NewTarProgress(tw, a.Progress) - fs := a.Filesystem.unixFS + // Configure godirwalk. + options := &godirwalk.Options{ + FollowSymbolicLinks: false, + Unsorted: true, + } // If we're specifically looking for only certain files, or have requested // that certain files be ignored we'll update the callback function to reflect // that request. - var callback walkFunc + var callback godirwalk.WalkFunc if len(a.Files) == 0 && len(a.Ignore) > 0 { i := ignore.CompileIgnoreLines(strings.Split(a.Ignore, "\n")...) - callback = a.callback(func(_ int, _, relative string, _ ufs.DirEntry) error { - if i.MatchesPath(relative) { - return SkipThis + + callback = a.callback(pw, func(_ string, rp string) error { + if i.MatchesPath(rp) { + return godirwalk.SkipThis } + return nil }) } else if len(a.Files) > 0 { - callback = a.withFilesCallback() + callback = a.withFilesCallback(pw) } else { - callback = a.callback() - } - - // Open the base directory we were provided. - dirfd, name, closeFd, err := fs.SafePath(a.BaseDirectory) - defer closeFd() - if err != nil { - return err + callback = a.callback(pw) } - // Recursively walk the base directory. - return fs.WalkDirat(dirfd, name, func(dirfd int, name, relative string, d ufs.DirEntry, err error) error { - if err != nil { - return err - } + // Set the callback function, wrapped with support for context cancellation. + options.Callback = func(path string, de *godirwalk.Dirent) error { select { case <-ctx.Done(): return ctx.Err() default: - return callback(dirfd, name, relative, d) + return callback(path, de) } - }) + } + + // Recursively walk the path we are archiving. + return godirwalk.Walk(a.BasePath, options) } // Callback function used to determine if a given file should be included in the archive // being generated. -func (a *Archive) callback(opts ...walkFunc) walkFunc { - // Get the base directory we need to strip when walking. - // - // This is important as when we are walking, the last part of the base directory - // is present on all the paths we walk. - var base string - if a.BaseDirectory != "" { - base = filepath.Base(a.BaseDirectory) + "/" - } - return func(dirfd int, name, relative string, d ufs.DirEntry) error { +func (a *Archive) callback(tw *TarProgress, opts ...func(path string, relative string) error) func(path string, de *godirwalk.Dirent) error { + return func(path string, de *godirwalk.Dirent) error { // Skip directories because we are walking them recursively. - if d.IsDir() { + if de.IsDir() { return nil } - // If base isn't empty, strip it from the relative path. This fixes an - // issue when creating an archive starting from a nested directory. - // - // See https://github.com/pterodactyl/panel/issues/5030 for more details. - if base != "" { - relative = strings.TrimPrefix(relative, base) - } + relative := filepath.ToSlash(strings.TrimPrefix(path, a.BasePath+string(filepath.Separator))) // Call the additional options passed to this callback function. If any of them return // a non-nil error we will exit immediately. for _, opt := range opts { - if err := opt(dirfd, name, relative, d); err != nil { - if err == SkipThis { - return nil - } + if err := opt(path, relative); err != nil { return err } } // Add the file to the archive, if it is nested in a directory, // the directory will be automatically "created" in the archive. - return a.addToArchive(dirfd, name, relative, d) + return a.addToArchive(path, relative, tw) } } -var SkipThis = errors.New("skip this") - // Pushes only files defined in the Files key to the final archive. -func (a *Archive) withFilesCallback() walkFunc { - return a.callback(func(_ int, _, relative string, _ ufs.DirEntry) error { +func (a *Archive) withFilesCallback(tw *TarProgress) func(path string, de *godirwalk.Dirent) error { + return a.callback(tw, func(p string, rp string) error { for _, f := range a.Files { // Allow exact file matches, otherwise check if file is within a parent directory. // // The slashes are added in the prefix checks to prevent partial name matches from being // included in the archive. - if f != relative && !strings.HasPrefix(strings.TrimSuffix(relative, "/")+"/", strings.TrimSuffix(f, "/")+"/") { + if f != p && !strings.HasPrefix(strings.TrimSuffix(p, "/")+"/", strings.TrimSuffix(f, "/")+"/") { continue } // Once we have a match return a nil value here so that the loop stops and the // call to this function will correctly include the file in the archive. If there // are no matches we'll never make it to this line, and the final error returned - // will be the ufs.SkipDir error. + // will be the godirwalk.SkipThis error. return nil } - return SkipThis + return godirwalk.SkipThis }) } // Adds a given file path to the final archive being created. -func (a *Archive) addToArchive(dirfd int, name, relative string, entry ufs.DirEntry) error { - s, err := entry.Info() +func (a *Archive) addToArchive(p string, rp string, w *TarProgress) error { + // Lstat the file, this will give us the same information as Stat except that it will not + // follow a symlink to its target automatically. This is important to avoid including + // files that exist outside the server root unintentionally in the backup. + s, err := os.Lstat(p) if err != nil { - if errors.Is(err, ufs.ErrNotExist) { + if os.IsNotExist(err) { return nil } - return errors.WrapIff(err, "failed executing os.Lstat on '%s'", name) + return errors.WrapIff(err, "failed executing os.Lstat on '%s'", rp) } // Skip socket files as they are unsupported by archive/tar. @@ -286,7 +255,7 @@ func (a *Archive) addToArchive(dirfd int, name, relative string, entry ufs.DirEn if err != nil { // Ignore the not exist errors specifically, since there is nothing important about that. if !os.IsNotExist(err) { - log.WithField("name", name).WithField("readlink_err", err.Error()).Warn("failed reading symlink for target path; skipping...") + log.WithField("path", rp).WithField("readlink_err", err.Error()).Warn("failed reading symlink for target path; skipping...") } return nil } @@ -295,17 +264,17 @@ func (a *Archive) addToArchive(dirfd int, name, relative string, entry ufs.DirEn // Get the tar FileInfoHeader in order to add the file to the archive. header, err := tar.FileInfoHeader(s, filepath.ToSlash(target)) if err != nil { - return errors.WrapIff(err, "failed to get tar#FileInfoHeader for '%s'", name) + return errors.WrapIff(err, "failed to get tar#FileInfoHeader for '%s'", rp) } // Fix the header name if the file is not a symlink. if s.Mode()&fs.ModeSymlink == 0 { - header.Name = relative + header.Name = rp } // Write the tar FileInfoHeader to the archive. - if err := a.w.WriteHeader(header); err != nil { - return errors.WrapIff(err, "failed to write tar#FileInfoHeader for '%s'", name) + if err := w.WriteHeader(header); err != nil { + return errors.WrapIff(err, "failed to write tar#FileInfoHeader for '%s'", rp) } // If the size of the file is less than 1 (most likely for symlinks), skip writing the file. @@ -327,7 +296,7 @@ func (a *Archive) addToArchive(dirfd int, name, relative string, entry ufs.DirEn } // Open the file. - f, err := a.Filesystem.unixFS.OpenFileat(dirfd, name, ufs.O_RDONLY, 0) + f, err := os.Open(p) if err != nil { if os.IsNotExist(err) { return nil @@ -337,8 +306,9 @@ func (a *Archive) addToArchive(dirfd int, name, relative string, entry ufs.DirEn defer f.Close() // Copy the file's contents to the archive using our buffer. - if _, err := io.CopyBuffer(a.w, io.LimitReader(f, header.Size), buf); err != nil { + if _, err := io.CopyBuffer(w, io.LimitReader(f, header.Size), buf); err != nil { return errors.WrapIff(err, "failed to copy '%s' to archive", header.Name) } + return nil } diff --git a/server/filesystem/archive_test.go b/server/filesystem/archive_test.go index 26e3fe964..8f4fe6e1e 100644 --- a/server/filesystem/archive_test.go +++ b/server/filesystem/archive_test.go @@ -20,34 +20,43 @@ func TestArchive_Stream(t *testing.T) { g.Describe("Archive", func() { g.AfterEach(func() { // Reset the filesystem after each run. - _ = fs.TruncateRootDirectory() + rfs.reset() + }) + + g.It("throws an error when passed invalid file paths", func() { + a := &Archive{ + BasePath: fs.Path(), + Files: []string{ + // To use the archiver properly, this needs to be filepath.Join(BasePath, "yeet") + // However, this test tests that we actually validate that behavior. + "yeet", + }, + } + + g.Assert(a.Create(context.Background(), "")).IsNotNil() }) g.It("creates archive with intended files", func() { g.Assert(fs.CreateDirectory("test", "/")).IsNil() g.Assert(fs.CreateDirectory("test2", "/")).IsNil() - r := strings.NewReader("hello, world!\n") - err := fs.Write("test/file.txt", r, r.Size(), 0o644) + err := fs.Writefile("test/file.txt", strings.NewReader("hello, world!\n")) g.Assert(err).IsNil() - r = strings.NewReader("hello, world!\n") - err = fs.Write("test2/file.txt", r, r.Size(), 0o644) + err = fs.Writefile("test2/file.txt", strings.NewReader("hello, world!\n")) g.Assert(err).IsNil() - r = strings.NewReader("hello, world!\n") - err = fs.Write("test_file.txt", r, r.Size(), 0o644) + err = fs.Writefile("test_file.txt", strings.NewReader("hello, world!\n")) g.Assert(err).IsNil() - r = strings.NewReader("hello, world!\n") - err = fs.Write("test_file.txt.old", r, r.Size(), 0o644) + err = fs.Writefile("test_file.txt.old", strings.NewReader("hello, world!\n")) g.Assert(err).IsNil() a := &Archive{ - Filesystem: fs, + BasePath: fs.Path(), Files: []string{ - "test", - "test_file.txt", + filepath.Join(fs.Path(), "test"), + filepath.Join(fs.Path(), "test_file.txt"), }, } @@ -110,7 +119,7 @@ func getFiles(f iofs.ReadDirFS, name string) ([]string, error) { if files == nil { return nil, nil } - + v = append(v, files...) continue } diff --git a/server/filesystem/compress.go b/server/filesystem/compress.go index f2775cb31..ed2810d67 100644 --- a/server/filesystem/compress.go +++ b/server/filesystem/compress.go @@ -5,6 +5,7 @@ import ( "fmt" "io" iofs "io/fs" + "os" "path" "path/filepath" "strings" @@ -12,11 +13,7 @@ import ( "time" "emperror.dev/errors" - "github.com/klauspost/compress/zip" - "github.com/mholt/archives" - - "github.com/pterodactyl/wings/internal/ufs" - "github.com/pterodactyl/wings/server/filesystem/archiverext" + "github.com/mholt/archiver/v4" ) // CompressFiles compresses all the files matching the given paths in the @@ -28,70 +25,46 @@ import ( // All paths are relative to the dir that is passed in as the first argument, // and the compressed file will be placed at that location named // `archive-{date}.tar.gz`. -func (fs *Filesystem) CompressFiles(dir string, paths []string) (ufs.FileInfo, error) { - a := &Archive{Filesystem: fs, BaseDirectory: dir, Files: paths} - d := path.Join( - dir, - fmt.Sprintf("archive-%s.tar.gz", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", "")), - ) - f, err := fs.unixFS.OpenFile(d, ufs.O_WRONLY|ufs.O_CREATE, 0o644) +func (fs *Filesystem) CompressFiles(dir string, paths []string) (os.FileInfo, error) { + cleanedRootDir, err := fs.SafePath(dir) if err != nil { return nil, err } - defer f.Close() - cw := ufs.NewCountedWriter(f) - if err := a.Stream(context.Background(), cw); err != nil { - return nil, err - } - if !fs.unixFS.CanFit(cw.BytesWritten()) { - _ = fs.unixFS.Remove(d) - return nil, newFilesystemError(ErrCodeDiskSpace, nil) + + // Take all the paths passed in and merge them together with the root directory we've gotten. + for i, p := range paths { + paths[i] = filepath.Join(cleanedRootDir, p) } - fs.unixFS.Add(cw.BytesWritten()) - return f.Stat() -} -func (fs *Filesystem) archiverFileSystem(ctx context.Context, p string) (iofs.FS, error) { - f, err := fs.unixFS.Open(p) + cleaned, err := fs.ParallelSafePath(paths) if err != nil { return nil, err } - // Do not use defer to close `f`, it will likely be used later. - format, _, err := archives.Identify(ctx, filepath.Base(p), f) - if err != nil && !errors.Is(err, archives.NoMatch) { - _ = f.Close() - return nil, err - } + a := &Archive{BasePath: cleanedRootDir, Files: cleaned} + d := path.Join( + cleanedRootDir, + fmt.Sprintf("archive-%s.tar.gz", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", "")), + ) - // Reset the file reader. - if _, err := f.Seek(0, io.SeekStart); err != nil { - _ = f.Close() + if err := a.Create(context.Background(), d); err != nil { return nil, err } - info, err := f.Stat() + f, err := os.Stat(d) if err != nil { - _ = f.Close() + _ = os.Remove(d) return nil, err } - if format != nil { - switch ff := format.(type) { - case archives.Zip: - // zip.Reader is more performant than ArchiveFS, because zip.Reader caches content information - // and zip.Reader can open several content files concurrently because of io.ReaderAt requirement - // while ArchiveFS can't. - // zip.Reader doesn't suffer from issue #330 and #310 according to local test (but they should be fixed anyway) - return zip.NewReader(f, info.Size()) - case archives.Extraction: - return &archives.ArchiveFS{Stream: io.NewSectionReader(f, 0, info.Size()), Format: ff, Context: ctx}, nil - case archives.Compression: - return archiverext.FileFS{File: f, Compression: ff}, nil - } + if err := fs.HasSpaceFor(f.Size()); err != nil { + _ = os.Remove(d) + return nil, err } - _ = f.Close() - return nil, archives.NoMatch + + fs.addDisk(f.Size()) + + return f, nil } // SpaceAvailableForDecompression looks through a given archive and determines @@ -103,15 +76,24 @@ func (fs *Filesystem) SpaceAvailableForDecompression(ctx context.Context, dir st return nil } - fsys, err := fs.archiverFileSystem(ctx, filepath.Join(dir, file)) + source, err := fs.SafePath(filepath.Join(dir, file)) if err != nil { - if errors.Is(err, archives.NoMatch) { + return err + } + + // Get the cached size in a parallel process so that if it is not cached we are not + // waiting an unnecessary amount of time on this call. + dirSize, err := fs.DiskUsage(false) + + fsys, err := archiver.FileSystem(ctx, source) + if err != nil { + if errors.Is(err, archiver.ErrNoMatch) { return newFilesystemError(ErrCodeUnknownArchive, err) } return err } - var size atomic.Int64 + var size int64 return iofs.WalkDir(fsys, ".", func(path string, d iofs.DirEntry, err error) error { if err != nil { return err @@ -126,7 +108,7 @@ func (fs *Filesystem) SpaceAvailableForDecompression(ctx context.Context, dir st if err != nil { return err } - if !fs.unixFS.CanFit(size.Add(info.Size())) { + if atomic.AddInt64(&size, info.Size())+dirSize > fs.MaxDisk() { return newFilesystemError(ErrCodeDiskSpace, nil) } return nil @@ -140,23 +122,38 @@ func (fs *Filesystem) SpaceAvailableForDecompression(ctx context.Context, dir st // zip-slip attack being attempted by validating that the final path is within // the server data directory. func (fs *Filesystem) DecompressFile(ctx context.Context, dir string, file string) error { - f, err := fs.unixFS.Open(filepath.Join(dir, file)) + source, err := fs.SafePath(filepath.Join(dir, file)) + if err != nil { + return err + } + return fs.DecompressFileUnsafe(ctx, dir, source) +} + +// DecompressFileUnsafe will decompress any file on the local disk without checking +// if it is owned by the server. The file will be SAFELY decompressed and extracted +// into the server's directory. +func (fs *Filesystem) DecompressFileUnsafe(ctx context.Context, dir string, file string) error { + // Ensure that the archive actually exists on the system. + if _, err := os.Stat(file); err != nil { + return errors.WithStack(err) + } + + f, err := os.Open(file) if err != nil { return err } defer f.Close() // Identify the type of archive we are dealing with. - format, input, err := archives.Identify(ctx, filepath.Base(file), f) + format, input, err := archiver.Identify(filepath.Base(file), f) if err != nil { - if errors.Is(err, archives.NoMatch) { + if errors.Is(err, archiver.ErrNoMatch) { return newFilesystemError(ErrCodeUnknownArchive, err) } return err } return fs.extractStream(ctx, extractStreamOptions{ - FileName: file, Directory: dir, Format: format, Reader: input, @@ -165,13 +162,14 @@ func (fs *Filesystem) DecompressFile(ctx context.Context, dir string, file strin // ExtractStreamUnsafe . func (fs *Filesystem) ExtractStreamUnsafe(ctx context.Context, dir string, r io.Reader) error { - format, input, err := archives.Identify(ctx, "archive.tar.gz", r) + format, input, err := archiver.Identify("archive.tar.gz", r) if err != nil { - if errors.Is(err, archives.NoMatch) { + if errors.Is(err, archiver.ErrNoMatch) { return newFilesystemError(ErrCodeUnknownArchive, err) } return err } + return fs.extractStream(ctx, extractStreamOptions{ Directory: dir, Format: format, @@ -185,99 +183,41 @@ type extractStreamOptions struct { // File name of the archive. FileName string // Format of the archive. - Format archives.Format + Format archiver.Format // Reader for the archive. Reader io.Reader } func (fs *Filesystem) extractStream(ctx context.Context, opts extractStreamOptions) error { - // See if it's a compressed archive, such as TAR or a ZIP - ex, ok := opts.Format.(archives.Extractor) - if !ok { - // If not, check if it's a single-file compression, such as - // .log.gz, .sql.gz, and so on - de, ok := opts.Format.(archives.Decompressor) - if !ok { - return nil - } - - // Strip the compression suffix - p := filepath.Join(opts.Directory, strings.TrimSuffix(opts.FileName, opts.Format.Extension())) - - // Make sure it's not ignored - if err := fs.IsIgnored(p); err != nil { - return nil - } - - reader, err := de.OpenReader(opts.Reader) - if err != nil { - return err - } - defer reader.Close() - - // Open the file for creation/writing - f, err := fs.unixFS.OpenFile(p, ufs.O_WRONLY|ufs.O_CREATE, 0o644) - if err != nil { - return err - } - defer f.Close() - - // Read in 4 KB chunks - buf := make([]byte, 4096) - for { - n, err := reader.Read(buf) - if n > 0 { - - // Check quota before writing the chunk - if quotaErr := fs.HasSpaceFor(int64(n)); quotaErr != nil { - return quotaErr - } - - // Write the chunk - if _, writeErr := f.Write(buf[:n]); writeErr != nil { - return writeErr - } - - // Add to quota - fs.addDisk(int64(n)) + // Decompress and extract archive + if ex, ok := opts.Format.(archiver.Extractor); ok { + return ex.Extract(ctx, opts.Reader, nil, func(ctx context.Context, f archiver.File) error { + if f.IsDir() { + return nil } - + p := filepath.Join(opts.Directory, f.NameInArchive) + // If it is ignored, just don't do anything with the file and skip over it. + if err := fs.IsIgnored(p); err != nil { + return nil + } + r, err := f.Open() if err != nil { - // EOF are expected - if err == io.EOF { - break - } - - // Return any other return err } - } - - return nil - } - - // Decompress and extract archive - return ex.Extract(ctx, opts.Reader, func(ctx context.Context, f archives.FileInfo) error { - if f.IsDir() { - return nil - } - p := filepath.Join(opts.Directory, f.NameInArchive) - // If it is ignored, just don't do anything with the file and skip over it. - if err := fs.IsIgnored(p); err != nil { + defer r.Close() + if err := fs.Writefile(p, r); err != nil { + return wrapError(err, opts.FileName) + } + // Update the file permissions to the one set in the archive. + if err := fs.Chmod(p, f.Mode()); err != nil { + return wrapError(err, opts.FileName) + } + // Update the file modification time to the one set in the archive. + if err := fs.Chtimes(p, f.ModTime(), f.ModTime()); err != nil { + return wrapError(err, opts.FileName) + } return nil - } - r, err := f.Open() - if err != nil { - return err - } - defer r.Close() - if err := fs.Write(p, r, f.Size(), f.Mode()); err != nil { - return wrapError(err, opts.FileName) - } - // Update the file modification time to the one set in the archive. - if err := fs.Chtimes(p, f.ModTime(), f.ModTime()); err != nil { - return wrapError(err, opts.FileName) - } - return nil - }) + }) + } + return nil } diff --git a/server/filesystem/compress_test.go b/server/filesystem/compress_test.go index 80cf70800..d287424ea 100644 --- a/server/filesystem/compress_test.go +++ b/server/filesystem/compress_test.go @@ -3,18 +3,17 @@ package filesystem import ( "context" "os" + "sync/atomic" "testing" . "github.com/franela/goblin" ) // Given an archive named test.{ext}, with the following file structure: -// // test/ // |──inside/ // |────finside.txt // |──outside.txt -// // this test will ensure that it's being decompressed as expected func TestFilesystem_DecompressFile(t *testing.T) { g := Goblin(t) @@ -48,7 +47,9 @@ func TestFilesystem_DecompressFile(t *testing.T) { } g.AfterEach(func() { - _ = fs.TruncateRootDirectory() + rfs.reset() + atomic.StoreInt64(&fs.diskUsed, 0) + atomic.StoreInt64(&fs.diskLimit, 0) }) }) } diff --git a/server/filesystem/disk_space.go b/server/filesystem/disk_space.go index f8760f3d5..95f82b8d6 100644 --- a/server/filesystem/disk_space.go +++ b/server/filesystem/disk_space.go @@ -1,29 +1,26 @@ package filesystem import ( - "golang.org/x/sys/unix" - "slices" "sync" "sync/atomic" + "syscall" "time" "emperror.dev/errors" "github.com/apex/log" - - "github.com/pterodactyl/wings/internal/ufs" + "github.com/karrick/godirwalk" ) type SpaceCheckingOpts struct { AllowStaleResponse bool } -// TODO: can this be replaced with some sort of atomic? Like atomic.Pointer? type usageLookupTime struct { sync.RWMutex value time.Time } -// Set sets the last time that a disk space lookup was performed. +// Update the last time that a disk space lookup was performed. func (ult *usageLookupTime) Set(t time.Time) { ult.Lock() ult.value = t @@ -38,15 +35,14 @@ func (ult *usageLookupTime) Get() time.Time { return ult.value } -// MaxDisk returns the maximum amount of disk space that this Filesystem -// instance is allowed to use. +// Returns the maximum amount of disk space that this Filesystem instance is allowed to use. func (fs *Filesystem) MaxDisk() int64 { - return fs.unixFS.Limit() + return atomic.LoadInt64(&fs.diskLimit) } -// SetDiskLimit sets the disk space limit for this Filesystem instance. +// Sets the disk space limit for this Filesystem instance. func (fs *Filesystem) SetDiskLimit(i int64) { - fs.unixFS.SetLimit(i) + atomic.SwapInt64(&fs.diskLimit, i) } // The same concept as HasSpaceAvailable however this will return an error if there is @@ -69,7 +65,7 @@ func (fs *Filesystem) HasSpaceErr(allowStaleValue bool) error { func (fs *Filesystem) HasSpaceAvailable(allowStaleValue bool) bool { size, err := fs.DiskUsage(allowStaleValue) if err != nil { - log.WithField("root", fs.Path()).WithField("error", err).Warn("failed to determine root fs directory size") + log.WithField("root", fs.root).WithField("error", err).Warn("failed to determine root fs directory size") } // If space is -1 or 0 just return true, means they're allowed unlimited. @@ -88,7 +84,7 @@ func (fs *Filesystem) HasSpaceAvailable(allowStaleValue bool) bool { // function for critical logical checks. It should only be used in areas where the actual disk usage // does not need to be perfect, e.g. API responses for server resource usage. func (fs *Filesystem) CachedUsage() int64 { - return fs.unixFS.Usage() + return atomic.LoadInt64(&fs.diskUsed) } // Internal helper function to allow other parts of the codebase to check the total used disk space @@ -118,14 +114,14 @@ func (fs *Filesystem) DiskUsage(allowStaleValue bool) (int64, error) { // currently performing a lookup, just do the disk usage calculation in the background. go func(fs *Filesystem) { if _, err := fs.updateCachedDiskUsage(); err != nil { - log.WithField("root", fs.Path()).WithField("error", err).Warn("failed to update fs disk usage from within routine") + log.WithField("root", fs.root).WithField("error", err).Warn("failed to update fs disk usage from within routine") } }(fs) } } // Return the currently cached value back to the calling function. - return fs.unixFS.Usage(), nil + return atomic.LoadInt64(&fs.diskUsed), nil } // Updates the currently used disk space for a server. @@ -153,56 +149,76 @@ func (fs *Filesystem) updateCachedDiskUsage() (int64, error) { // error encountered. fs.lastLookupTime.Set(time.Now()) - fs.unixFS.SetUsage(size) + atomic.StoreInt64(&fs.diskUsed, size) return size, err } -// DirectorySize calculates the size of a directory and its descendants. -func (fs *Filesystem) DirectorySize(root string) (int64, error) { - dirfd, name, closeFd, err := fs.unixFS.SafePath(root) - defer closeFd() +// Determines the directory size of a given location by running parallel tasks to iterate +// through all of the folders. Returns the size in bytes. This can be a fairly taxing operation +// on locations with tons of files, so it is recommended that you cache the output. +func (fs *Filesystem) DirectorySize(dir string) (int64, error) { + d, err := fs.SafePath(dir) if err != nil { return 0, err } + var size int64 + var st syscall.Stat_t var hardLinks []uint64 - var size atomic.Int64 - err = fs.unixFS.WalkDirat(dirfd, name, func(dirfd int, name, _ string, d ufs.DirEntry, err error) error { - if err != nil { - return errors.Wrap(err, "walkdirat err") - } - - // Only calculate the size of regular files. - if !d.Type().IsRegular() { - return nil - } + err = godirwalk.Walk(d, &godirwalk.Options{ + Unsorted: true, + Callback: func(p string, e *godirwalk.Dirent) error { + // If this is a symlink then resolve the final destination of it before trying to continue walking + // over its contents. If it resolves outside the server data directory just skip everything else for + // it. Otherwise, allow it to continue. + if e.IsSymlink() { + if _, err := fs.SafePath(p); err != nil { + if IsErrorCode(err, ErrCodePathResolution) { + return godirwalk.SkipThis + } + + return err + } + } - info, err := fs.unixFS.Lstatat(dirfd, name) - if err != nil { - return errors.Wrap(err, "lstatat err") - } + if !e.IsDir() { + _ = syscall.Lstat(p, &st) + atomic.AddInt64(&size, st.Size) + } - var sysFileInfo = info.Sys().(*unix.Stat_t) - if sysFileInfo.Nlink > 1 { - // Hard links have the same inode number - if slices.Contains(hardLinks, sysFileInfo.Ino) { - // Don't add hard links size twice - return nil - } else { - hardLinks = append(hardLinks, sysFileInfo.Ino) + var sysFileInfo = info.Sys().(*unix.Stat_t) + if sysFileInfo.Nlink > 1 { + // Hard links have the same inode number + if slices.Contains(hardLinks, sysFileInfo.Ino) { + // Don't add hard links size twice + return nil + } else { + hardLinks = append(hardLinks, sysFileInfo.Ino) + } } - } - size.Add(info.Size()) - return nil + size.Add(info.Size()) + return nil + }, }) - return size.Load(), errors.WrapIf(err, "server/filesystem: directorysize: failed to walk directory") + + return size, errors.WrapIf(err, "server/filesystem: directorysize: failed to walk directory") } +// Helper function to determine if a server has space available for a file of a given size. +// If space is available, no error will be returned, otherwise an ErrNotEnoughSpace error +// will be raised. func (fs *Filesystem) HasSpaceFor(size int64) error { - if !fs.unixFS.CanFit(size) { + if fs.MaxDisk() == 0 { + return nil + } + s, err := fs.DiskUsage(true) + if err != nil { + return err + } + if (s + size) > fs.MaxDisk() { return newFilesystemError(ErrCodeDiskSpace, nil) } return nil @@ -210,5 +226,24 @@ func (fs *Filesystem) HasSpaceFor(size int64) error { // Updates the disk usage for the Filesystem instance. func (fs *Filesystem) addDisk(i int64) int64 { - return fs.unixFS.Add(i) + size := atomic.LoadInt64(&fs.diskUsed) + + // Sorry go gods. This is ugly but the best approach I can come up with for right + // now without completely re-evaluating the logic we use for determining disk space. + // + // Normally I would just be using the atomic load right below, but I'm not sure about + // the scenarios where it is 0 because nothing has run that would trigger a disk size + // calculation? + // + // Perhaps that isn't even a concern for the sake of this? + if !fs.isTest { + size, _ = fs.DiskUsage(true) + } + + // If we're dropping below 0 somehow just cap it to 0. + if (size + i) < 0 { + return atomic.SwapInt64(&fs.diskUsed, 0) + } + + return atomic.AddInt64(&fs.diskUsed, i) } diff --git a/server/filesystem/errors.go b/server/filesystem/errors.go index b977fe6b2..afae74adc 100644 --- a/server/filesystem/errors.go +++ b/server/filesystem/errors.go @@ -2,12 +2,11 @@ package filesystem import ( "fmt" + "os" "path/filepath" "emperror.dev/errors" "github.com/apex/log" - - "github.com/pterodactyl/wings/internal/ufs" ) type ErrorCode string @@ -87,15 +86,15 @@ func (e *Error) Unwrap() error { // Generates an error logger instance with some basic information. func (fs *Filesystem) error(err error) *log.Entry { - return log.WithField("subsystem", "filesystem").WithField("root", fs.Path()).WithField("error", err) + return log.WithField("subsystem", "filesystem").WithField("root", fs.root).WithField("error", err) } // Handle errors encountered when walking through directories. // // If there is a path resolution error just skip the item entirely. Only return this for a // directory, otherwise return nil. Returning this error for a file will stop the walking -// for the remainder of the directory. This is assuming an FileInfo struct was even returned. -func (fs *Filesystem) handleWalkerError(err error, f ufs.FileInfo) error { +// for the remainder of the directory. This is assuming an os.FileInfo struct was even returned. +func (fs *Filesystem) handleWalkerError(err error, f os.FileInfo) error { if !IsErrorCode(err, ErrCodePathResolution) { return err } diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index 42c56f2de..ccc2daa4f 100644 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -1,11 +1,13 @@ package filesystem import ( - "fmt" + "bufio" "io" + "io/ioutil" "os" + "path" "path/filepath" - "slices" + "sort" "strconv" "strings" "sync" @@ -13,208 +15,220 @@ import ( "time" "emperror.dev/errors" - "github.com/apex/log" "github.com/gabriel-vasile/mimetype" + "github.com/karrick/godirwalk" ignore "github.com/sabhiram/go-gitignore" "github.com/pterodactyl/wings/config" - "github.com/pterodactyl/wings/internal/ufs" + "github.com/pterodactyl/wings/system" ) type Filesystem struct { - unixFS *ufs.Quota - mu sync.RWMutex lastLookupTime *usageLookupTime - lookupInProgress atomic.Bool + lookupInProgress *system.AtomicBool + diskUsed int64 diskCheckInterval time.Duration denylist *ignore.GitIgnore + // The maximum amount of disk space (in bytes) that this Filesystem instance can use. + diskLimit int64 + + // The root data directory path for this Filesystem instance. + root string + isTest bool } // New creates a new Filesystem instance for a given server. -func New(root string, size int64, denylist []string) (*Filesystem, error) { - if err := os.MkdirAll(root, 0o755); err != nil { - return nil, err - } - unixFS, err := ufs.NewUnixFS(root, config.UseOpenat2()) - if err != nil { - return nil, err - } - quota := ufs.NewQuota(unixFS, size) - +func New(root string, size int64, denylist []string) *Filesystem { return &Filesystem{ - unixFS: quota, - + root: root, + diskLimit: size, diskCheckInterval: time.Duration(config.Get().System.DiskCheckInterval), lastLookupTime: &usageLookupTime{}, + lookupInProgress: system.NewAtomicBool(false), denylist: ignore.CompileIgnoreLines(denylist...), - }, nil + } } // Path returns the root path for the Filesystem instance. func (fs *Filesystem) Path() string { - return fs.unixFS.BasePath() -} - -// ReadDir reads directory entries. -func (fs *Filesystem) ReadDir(path string) ([]ufs.DirEntry, error) { - return fs.unixFS.ReadDir(path) -} - -// ReadDirStat is like ReadDir except that it returns FileInfo for each entry -// instead of just a DirEntry. -func (fs *Filesystem) ReadDirStat(path string) ([]ufs.FileInfo, error) { - return ufs.ReadDirMap(fs.unixFS.UnixFS, path, func(e ufs.DirEntry) (ufs.FileInfo, error) { - return e.Info() - }) + return fs.root } // File returns a reader for a file instance as well as the stat information. -func (fs *Filesystem) File(p string) (ufs.File, Stat, error) { - f, err := fs.unixFS.Open(p) +func (fs *Filesystem) File(p string) (*os.File, Stat, error) { + cleaned, err := fs.SafePath(p) if err != nil { - return nil, Stat{}, err + return nil, Stat{}, errors.WithStackIf(err) } - st, err := statFromFile(f) + st, err := fs.Stat(cleaned) if err != nil { - _ = f.Close() - return nil, Stat{}, err + if errors.Is(err, os.ErrNotExist) { + return nil, Stat{}, newFilesystemError(ErrNotExist, err) + } + return nil, Stat{}, errors.WithStackIf(err) + } + if st.IsDir() { + return nil, Stat{}, newFilesystemError(ErrCodeIsDirectory, nil) + } + f, err := os.Open(cleaned) + if err != nil { + return nil, Stat{}, errors.WithStackIf(err) } return f, st, nil } -func (fs *Filesystem) UnixFS() *ufs.UnixFS { - return fs.unixFS.UnixFS -} - // Touch acts by creating the given file and path on the disk if it is not present // already. If it is present, the file is opened using the defaults which will truncate // the contents. The opened file is then returned to the caller. -func (fs *Filesystem) Touch(p string, flag int) (ufs.File, error) { - return fs.unixFS.Touch(p, flag, 0o644) +func (fs *Filesystem) Touch(p string, flag int) (*os.File, error) { + cleaned, err := fs.SafePath(p) + if err != nil { + return nil, err + } + f, err := os.OpenFile(cleaned, flag, 0o644) + if err == nil { + return f, nil + } + if f != nil { + _ = f.Close() + } + // If the error is not because it doesn't exist then we just need to bail at this point. + if !errors.Is(err, os.ErrNotExist) { + return nil, errors.Wrap(err, "server/filesystem: touch: failed to open file handle") + } + // Only create and chown the directory if it doesn't exist. + if _, err := os.Stat(filepath.Dir(cleaned)); errors.Is(err, os.ErrNotExist) { + // Create the path leading up to the file we're trying to create, setting the final perms + // on it as we go. + if err := os.MkdirAll(filepath.Dir(cleaned), 0o755); err != nil { + return nil, errors.Wrap(err, "server/filesystem: touch: failed to create directory tree") + } + if err := fs.Chown(filepath.Dir(cleaned)); err != nil { + return nil, err + } + } + o := &fileOpener{} + // Try to open the file now that we have created the pathing necessary for it, and then + // Chown that file so that the permissions don't mess with things. + f, err = o.open(cleaned, flag, 0o644) + if err != nil { + return nil, errors.Wrap(err, "server/filesystem: touch: failed to open file with wait") + } + _ = fs.Chown(cleaned) + return f, nil } // Writefile writes a file to the system. If the file does not already exist one // will be created. This will also properly recalculate the disk space used by // the server when writing new files or modifying existing ones. -// -// DEPRECATED: use `Write` instead. func (fs *Filesystem) Writefile(p string, r io.Reader) error { - var currentSize int64 - st, err := fs.unixFS.Stat(p) - if err != nil && !errors.Is(err, ufs.ErrNotExist) { - return errors.Wrap(err, "server/filesystem: writefile: failed to stat file") - } else if err == nil { - if st.IsDir() { - // TODO: resolved - return errors.WithStack(&Error{code: ErrCodeIsDirectory, resolved: ""}) - } - currentSize = st.Size() - } - - // Touch the file and return the handle to it at this point. This will - // create or truncate the file, and create any necessary parent directories - // if they are missing. - file, err := fs.unixFS.Touch(p, ufs.O_RDWR|ufs.O_TRUNC, 0o644) + cleaned, err := fs.SafePath(p) if err != nil { - return fmt.Errorf("error touching file: %w", err) - } - defer file.Close() - - // Do not use CopyBuffer here, it is wasteful as the file implements - // io.ReaderFrom, which causes it to not use the buffer anyways. - n, err := io.Copy(file, r) - - // Adjust the disk usage to account for the old size and the new size of the file. - fs.unixFS.Add(n - currentSize) - - if err := fs.chownFile(p); err != nil { - return fmt.Errorf("error chowning file: %w", err) + return err } - // Return the error from io.Copy. - return err -} -func (fs *Filesystem) Write(p string, r io.Reader, newSize int64, mode ufs.FileMode) error { var currentSize int64 - st, err := fs.unixFS.Stat(p) - if err != nil && !errors.Is(err, ufs.ErrNotExist) { + // If the file does not exist on the system already go ahead and create the pathway + // to it and an empty file. We'll then write to it later on after this completes. + stat, err := os.Stat(cleaned) + if err != nil && !os.IsNotExist(err) { return errors.Wrap(err, "server/filesystem: writefile: failed to stat file") } else if err == nil { - if st.IsDir() { - // TODO: resolved - return errors.WithStack(&Error{code: ErrCodeIsDirectory, resolved: ""}) + if stat.IsDir() { + return errors.WithStack(&Error{code: ErrCodeIsDirectory, resolved: cleaned}) } - currentSize = st.Size() + currentSize = stat.Size() } + br := bufio.NewReader(r) // Check that the new size we're writing to the disk can fit. If there is currently // a file we'll subtract that current file size from the size of the buffer to determine // the amount of new data we're writing (or amount we're removing if smaller). - if err := fs.HasSpaceFor(newSize - currentSize); err != nil { + if err := fs.HasSpaceFor(int64(br.Size()) - currentSize); err != nil { return err } - // Touch the file and return the handle to it at this point. This will - // create or truncate the file, and create any necessary parent directories - // if they are missing. - file, err := fs.unixFS.Touch(p, ufs.O_RDWR|ufs.O_TRUNC, mode) + // Touch the file and return the handle to it at this point. This will create the file, + // any necessary directories, and set the proper owner of the file. + file, err := fs.Touch(cleaned, os.O_RDWR|os.O_CREATE|os.O_TRUNC) if err != nil { return err } defer file.Close() - if newSize == 0 { - // Subtract the previous size of the file if the new size is 0. - fs.unixFS.Add(-currentSize) - } else { - // Do not use CopyBuffer here, it is wasteful as the file implements - // io.ReaderFrom, which causes it to not use the buffer anyways. - var n int64 - n, err = io.Copy(file, io.LimitReader(r, newSize)) + buf := make([]byte, 1024*4) + sz, err := io.CopyBuffer(file, r, buf) - // Adjust the disk usage to account for the old size and the new size of the file. - fs.unixFS.Add(n - currentSize) - } + // Adjust the disk usage to account for the old size and the new size of the file. + fs.addDisk(sz - currentSize) - if err := fs.chownFile(p); err != nil { - return err - } - // Return any remaining error. - return err + return fs.unsafeChown(cleaned) } -// CreateDirectory creates a new directory (name) at a specified path (p) for -// the server. +// Creates a new directory (name) at a specified path (p) for the server. func (fs *Filesystem) CreateDirectory(name string, p string) error { - return fs.unixFS.MkdirAll(filepath.Join(p, name), 0o755) + cleaned, err := fs.SafePath(path.Join(p, name)) + if err != nil { + return err + } + return os.MkdirAll(cleaned, 0o755) } -func (fs *Filesystem) Rename(oldpath, newpath string) error { - return fs.unixFS.Rename(oldpath, newpath) -} +// Rename moves (or renames) a file or directory. +func (fs *Filesystem) Rename(from string, to string) error { + cleanedFrom, err := fs.SafePath(from) + if err != nil { + return errors.WithStack(err) + } -func (fs *Filesystem) Symlink(oldpath, newpath string) error { - return fs.unixFS.Symlink(oldpath, newpath) -} + cleanedTo, err := fs.SafePath(to) + if err != nil { + return errors.WithStack(err) + } -func (fs *Filesystem) chownFile(name string) error { - if fs.isTest { - return nil + // If the target file or directory already exists the rename function will fail, so just + // bail out now. + if _, err := os.Stat(cleanedTo); err == nil { + return os.ErrExist } - uid := config.Get().System.User.Uid - gid := config.Get().System.User.Gid - return fs.unixFS.Lchown(name, uid, gid) + if cleanedTo == fs.Path() { + return errors.New("attempting to rename into an invalid directory space") + } + + d := strings.TrimSuffix(cleanedTo, path.Base(cleanedTo)) + // Ensure that the directory we're moving into exists correctly on the system. Only do this if + // we're not at the root directory level. + if d != fs.Path() { + if mkerr := os.MkdirAll(d, 0o755); mkerr != nil { + return errors.WithMessage(mkerr, "failed to create directory structure for file rename") + } + } + + if err := os.Rename(cleanedFrom, cleanedTo); err != nil { + return errors.WithStack(err) + } + return nil } -// Chown recursively iterates over a file or directory and sets the permissions on all of the +// Recursively iterates over a file or directory and sets the permissions on all of the // underlying files. Iterate over all of the files and directories. If it is a file just // go ahead and perform the chown operation. Otherwise dig deeper into the directory until // we've run out of directories to dig into. -func (fs *Filesystem) Chown(p string) error { +func (fs *Filesystem) Chown(path string) error { + cleaned, err := fs.SafePath(path) + if err != nil { + return err + } + return fs.unsafeChown(cleaned) +} + +// unsafeChown chowns the given path, without checking if the path is safe. This should only be used +// when the path has already been checked. +func (fs *Filesystem) unsafeChown(path string) error { if fs.isTest { return nil } @@ -222,44 +236,54 @@ func (fs *Filesystem) Chown(p string) error { uid := config.Get().System.User.Uid gid := config.Get().System.User.Gid - dirfd, name, closeFd, err := fs.unixFS.SafePath(p) - defer closeFd() - if err != nil { - return err - } - // Start by just chowning the initial path that we received. - if err := fs.unixFS.Lchownat(dirfd, name, uid, gid); err != nil { + if err := os.Chown(path, uid, gid); err != nil { return errors.Wrap(err, "server/filesystem: chown: failed to chown path") } // If this is not a directory we can now return from the function, there is nothing // left that we need to do. - if st, err := fs.unixFS.Lstatat(dirfd, name); err != nil || !st.IsDir() { + if st, err := os.Stat(path); err != nil || !st.IsDir() { return nil } - // This walker is probably some of the most efficient code in Wings. It has - // an internally re-used buffer for listing directory entries and doesn't - // need to check if every individual path it touches is safe as the code - // doesn't traverse symlinks, is immune to symlink timing attacks, and - // gives us a dirfd and file name to make a direct syscall with. - if err := fs.unixFS.WalkDirat(dirfd, name, func(dirfd int, name, _ string, info ufs.DirEntry, err error) error { - if err != nil { - return err - } - if err := fs.unixFS.Lchownat(dirfd, name, uid, gid); err != nil { - return err - } + // If this was a directory, begin walking over its contents recursively and ensure that all + // of the subfiles and directories get their permissions updated as well. + err := godirwalk.Walk(path, &godirwalk.Options{ + Unsorted: true, + Callback: func(p string, e *godirwalk.Dirent) error { + // Do not attempt to chown a symlink. Go's os.Chown function will affect the symlink + // so if it points to a location outside the data directory the user would be able to + // (un)intentionally modify that files permissions. + if e.IsSymlink() { + if e.IsDir() { + return godirwalk.SkipThis + } + + return nil + } + + return os.Chown(p, uid, gid) + }, + }) + return errors.Wrap(err, "server/filesystem: chown: failed to chown during walk function") +} + +func (fs *Filesystem) Chmod(path string, mode os.FileMode) error { + cleaned, err := fs.SafePath(path) + if err != nil { + return err + } + + if fs.isTest { return nil - }); err != nil { - return fmt.Errorf("server/filesystem: chown: failed to chown during walk function: %w", err) } - return nil -} -func (fs *Filesystem) Chmod(path string, mode ufs.FileMode) error { - return fs.unixFS.Chmod(path, mode) + if err := os.Chmod(cleaned, mode); err != nil { + return err + } + + return nil } // Begin looping up to 50 times to try and create a unique copy file name. This will take @@ -270,7 +294,7 @@ func (fs *Filesystem) Chmod(path string, mode ufs.FileMode) error { // Could probably make this more efficient by checking if there are any files matching the copy // pattern, and trying to find the highest number and then incrementing it by one rather than // looping endlessly. -func (fs *Filesystem) findCopySuffix(dirfd int, name, extension string) (string, error) { +func (fs *Filesystem) findCopySuffix(dir string, name string, extension string) (string, error) { var i int suffix := " copy" @@ -282,10 +306,11 @@ func (fs *Filesystem) findCopySuffix(dirfd int, name, extension string) (string, n := name + suffix + extension // If we stat the file and it does not exist that means we're good to create the copy. If it // does exist, we'll just continue to the next loop and try again. - if _, err := fs.unixFS.Lstatat(dirfd, n); err != nil { - if !errors.Is(err, ufs.ErrNotExist) { + if _, err := fs.Stat(path.Join(dir, n)); err != nil { + if !errors.Is(err, os.ErrNotExist) { return "", err } + break } @@ -297,68 +322,53 @@ func (fs *Filesystem) findCopySuffix(dirfd int, name, extension string) (string, return name + suffix + extension, nil } -// Copy copies a given file to the same location and appends a suffix to the -// file to indicate that it has been copied. +// Copies a given file to the same location and appends a suffix to the file to indicate that +// it has been copied. func (fs *Filesystem) Copy(p string) error { - dirfd, name, closeFd, err := fs.unixFS.SafePath(p) - defer closeFd() + cleaned, err := fs.SafePath(p) if err != nil { return err } - source, err := fs.unixFS.OpenFileat(dirfd, name, ufs.O_RDONLY, 0) - if err != nil { - return err - } - defer source.Close() - info, err := source.Stat() + + s, err := os.Stat(cleaned) if err != nil { return err - } - if info.IsDir() || !info.Mode().IsRegular() { + } else if s.IsDir() || !s.Mode().IsRegular() { // If this is a directory or not a regular file, just throw a not-exist error // since anything calling this function should understand what that means. - return ufs.ErrNotExist + return os.ErrNotExist } - currentSize := info.Size() // Check that copying this file wouldn't put the server over its limit. - if err := fs.HasSpaceFor(currentSize); err != nil { + if err := fs.HasSpaceFor(s.Size()); err != nil { return err } - base := info.Name() + base := filepath.Base(cleaned) + relative := strings.TrimSuffix(strings.TrimPrefix(cleaned, fs.Path()), base) extension := filepath.Ext(base) - baseName := strings.TrimSuffix(base, extension) + name := strings.TrimSuffix(base, extension) // Ensure that ".tar" is also counted as apart of the file extension. // There might be a better way to handle this for other double file extensions, // but this is a good workaround for now. - if strings.HasSuffix(baseName, ".tar") { + if strings.HasSuffix(name, ".tar") { extension = ".tar" + extension - baseName = strings.TrimSuffix(baseName, ".tar") + name = strings.TrimSuffix(name, ".tar") } - newName, err := fs.findCopySuffix(dirfd, baseName, extension) + source, err := os.Open(cleaned) if err != nil { return err } - dst, err := fs.unixFS.OpenFileat(dirfd, newName, ufs.O_WRONLY|ufs.O_CREATE, info.Mode()) + defer source.Close() + + n, err := fs.findCopySuffix(relative, name, extension) if err != nil { return err } - // Do not use CopyBuffer here, it is wasteful as the file implements - // io.ReaderFrom, which causes it to not use the buffer anyways. - n, err := io.Copy(dst, io.LimitReader(source, currentSize)) - fs.unixFS.Add(n) - - if !fs.isTest { - if err := fs.unixFS.Lchownat(dirfd, newName, config.Get().System.User.Uid, config.Get().System.User.Gid); err != nil { - return err - } - } - // Return the error from io.Copy. - return err + return fs.Writefile(path.Join(relative, n), source) } // TruncateRootDirectory removes _all_ files and directories from a server's @@ -370,128 +380,211 @@ func (fs *Filesystem) TruncateRootDirectory() error { if err := os.Mkdir(fs.Path(), 0o755); err != nil { return err } - _ = fs.unixFS.Close() - unixFS, err := ufs.NewUnixFS(fs.Path(), config.UseOpenat2()) - if err != nil { - return err - } - var limit int64 - if fs.isTest { - limit = 0 - } else { - limit = fs.unixFS.Limit() - } - fs.unixFS = ufs.NewQuota(unixFS, limit) + atomic.StoreInt64(&fs.diskUsed, 0) return nil } // Delete removes a file or folder from the system. Prevents the user from // accidentally (or maliciously) removing their root server data directory. func (fs *Filesystem) Delete(p string) error { - return fs.unixFS.RemoveAll(p) -} + // This is one of the few (only?) places in the codebase where we're explicitly not using + // the SafePath functionality when working with user provided input. If we did, you would + // not be able to delete a file that is a symlink pointing to a location outside the data + // directory. + // + // We also want to avoid resolving a symlink that points _within_ the data directory and thus + // deleting the actual source file for the symlink rather than the symlink itself. For these + // purposes just resolve the actual file path using filepath.Join() and confirm that the path + // exists within the data directory. + resolved := fs.unsafeFilePath(p) + if !fs.unsafeIsInDataDirectory(resolved) { + return NewBadPathResolution(p, resolved) + } + + // Block any whoopsies. + if resolved == fs.Path() { + return errors.New("cannot delete root server directory") + } + + st, err := os.Lstat(resolved) + if err != nil { + if !os.IsNotExist(err) { + fs.error(err).Warn("error while attempting to stat file before deletion") + return err + } -//type fileOpener struct { -// fs *Filesystem -// busy uint -//} -// -//// Attempts to open a given file up to "attempts" number of times, using a backoff. If the file -//// cannot be opened because of a "text file busy" error, we will attempt until the number of attempts -//// has been exhaused, at which point we will abort with an error. -//func (fo *fileOpener) open(path string, flags int, perm ufs.FileMode) (ufs.File, error) { -// for { -// f, err := fo.fs.unixFS.OpenFile(path, flags, perm) -// -// // If there is an error because the text file is busy, go ahead and sleep for a few -// // hundred milliseconds and then try again up to three times before just returning the -// // error back to the caller. -// // -// // Based on code from: https://github.com/golang/go/issues/22220#issuecomment-336458122 -// if err != nil && fo.busy < 3 && strings.Contains(err.Error(), "text file busy") { -// time.Sleep(100 * time.Millisecond << fo.busy) -// fo.busy++ -// continue -// } -// -// return f, err -// } -//} + // The following logic is used to handle a case where a user attempts to + // delete a file that does not exist through a directory symlink. + // We don't want to reveal that the file does not exist, so we validate + // the path of the symlink and return a bad path error if it is invalid. + + // The requested file or directory doesn't exist, so at this point we + // need to iterate up the path chain until we hit a directory that + // _does_ exist and can be validated. + parts := strings.Split(filepath.Dir(resolved), "/") + + // Range over all the path parts and form directory paths from the end + // moving up until we have a valid resolution, or we run out of paths to + // try. + for k := range parts { + try := strings.Join(parts[:(len(parts)-k)], "/") + if !fs.unsafeIsInDataDirectory(try) { + break + } -// ListDirectory lists the contents of a given directory and returns stat -// information about each file and folder within it. -func (fs *Filesystem) ListDirectory(p string) ([]Stat, error) { - // Read entries from the path on the filesystem, using the mapped reader, so - // we can map the DirEntry slice into a Stat slice with mimetype information. - out, err := ufs.ReadDirMap(fs.unixFS.UnixFS, p, func(e ufs.DirEntry) (Stat, error) { - info, err := e.Info() - if err != nil { - return Stat{}, err + t, err := filepath.EvalSymlinks(try) + if err == nil { + if !fs.unsafeIsInDataDirectory(t) { + return NewBadPathResolution(p, t) + } + break + } } - var d string - if e.Type().IsDir() { - d = "inode/directory" - } else { - d = "application/octet-stream" - } - var m *mimetype.MIME - if e.Type().IsRegular() { - // TODO: I should probably find a better way to do this. - eO := e.(interface { - Open() (ufs.File, error) - }) - f, err := eO.Open() - if err != nil { - return Stat{}, err - } - m, err = mimetype.DetectReader(f) - if err != nil { - log.Error(err.Error()) + // Always return early if the file does not exist. + return nil + } + + // If the file is not a symlink, we need to check that it is not within a + // symlinked directory that points outside the data directory. + if st.Mode()&os.ModeSymlink == 0 { + ep, err := filepath.EvalSymlinks(resolved) + if err != nil { + if !os.IsNotExist(err) { + return err } - _ = f.Close() + } else if !fs.unsafeIsInDataDirectory(ep) { + return NewBadPathResolution(p, ep) } + } - st := Stat{FileInfo: info, Mimetype: d} - if m != nil { - st.Mimetype = m.String() + if st.IsDir() { + if s, err := fs.DirectorySize(resolved); err == nil { + fs.addDisk(-s) } - return st, nil - }) + } else { + fs.addDisk(-st.Size()) + } + + return os.RemoveAll(resolved) +} + +type fileOpener struct { + busy uint +} + +// Attempts to open a given file up to "attempts" number of times, using a backoff. If the file +// cannot be opened because of a "text file busy" error, we will attempt until the number of attempts +// has been exhaused, at which point we will abort with an error. +func (fo *fileOpener) open(path string, flags int, perm os.FileMode) (*os.File, error) { + for { + f, err := os.OpenFile(path, flags, perm) + + // If there is an error because the text file is busy, go ahead and sleep for a few + // hundred milliseconds and then try again up to three times before just returning the + // error back to the caller. + // + // Based on code from: https://github.com/golang/go/issues/22220#issuecomment-336458122 + if err != nil && fo.busy < 3 && strings.Contains(err.Error(), "text file busy") { + time.Sleep(100 * time.Millisecond << fo.busy) + fo.busy++ + continue + } + + return f, err + } +} + +// ListDirectory lists the contents of a given directory and returns stat +// information about each file and folder within it. +func (fs *Filesystem) ListDirectory(p string) ([]Stat, error) { + cleaned, err := fs.SafePath(p) + if err != nil { + return nil, err + } + + files, err := ioutil.ReadDir(cleaned) if err != nil { return nil, err } - // Sort entries alphabetically. - slices.SortStableFunc(out, func(a, b Stat) int { - switch { - case a.Name() == b.Name(): - return 0 - case a.Name() > b.Name(): - return 1 - default: - return -1 + var wg sync.WaitGroup + + // You must initialize the output of this directory as a non-nil value otherwise + // when it is marshaled into a JSON object you'll just get 'null' back, which will + // break the panel badly. + out := make([]Stat, len(files)) + + // Iterate over all of the files and directories returned and perform an async process + // to get the mime-type for them all. + for i, file := range files { + wg.Add(1) + + go func(idx int, f os.FileInfo) { + defer wg.Done() + + var m *mimetype.MIME + d := "inode/directory" + if !f.IsDir() { + cleanedp := filepath.Join(cleaned, f.Name()) + if f.Mode()&os.ModeSymlink != 0 { + cleanedp, _ = fs.SafePath(filepath.Join(cleaned, f.Name())) + } + + // Don't try to detect the type on a pipe — this will just hang the application and + // you'll never get a response back. + // + // @see https://github.com/pterodactyl/panel/issues/4059 + if cleanedp != "" && f.Mode()&os.ModeNamedPipe == 0 { + m, _ = mimetype.DetectFile(filepath.Join(cleaned, f.Name())) + } else { + // Just pass this for an unknown type because the file could not safely be resolved within + // the server data path. + d = "application/octet-stream" + } + } + + st := Stat{FileInfo: f, Mimetype: d} + if m != nil { + st.Mimetype = m.String() + } + out[idx] = st + }(i, file) + } + + wg.Wait() + + // Sort the output alphabetically to begin with since we've run the output + // through an asynchronous process and the order is gonna be very random. + sort.SliceStable(out, func(i, j int) bool { + if out[i].Name() == out[j].Name() || out[i].Name() > out[j].Name() { + return true } + return false }) - // Sort folders before other file types. - slices.SortStableFunc(out, func(a, b Stat) int { - switch { - case a.IsDir() && b.IsDir(): - return 0 - case a.IsDir(): - return -1 - default: - return 1 - } + // Then, sort it so that directories are listed first in the output. Everything + // will continue to be alphabetized at this point. + sort.SliceStable(out, func(i, j int) bool { + return out[i].IsDir() }) return out, nil } func (fs *Filesystem) Chtimes(path string, atime, mtime time.Time) error { + cleaned, err := fs.SafePath(path) + if err != nil { + return err + } + if fs.isTest { return nil } - return fs.unixFS.Chtimes(path, atime, mtime) + + if err := os.Chtimes(cleaned, atime, mtime); err != nil { + return err + } + + return nil } diff --git a/server/filesystem/filesystem_test.go b/server/filesystem/filesystem_test.go index e5c6e613b..23f43bfe1 100644 --- a/server/filesystem/filesystem_test.go +++ b/server/filesystem/filesystem_test.go @@ -7,13 +7,12 @@ import ( "math/rand" "os" "path/filepath" + "sync/atomic" "testing" "unicode/utf8" . "github.com/franela/goblin" - "github.com/pterodactyl/wings/internal/ufs" - "github.com/pterodactyl/wings/config" ) @@ -29,23 +28,15 @@ func NewFs() (*Filesystem, *rootFs) { tmpDir, err := os.MkdirTemp(os.TempDir(), "pterodactyl") if err != nil { panic(err) - return nil, nil } + // defer os.RemoveAll(tmpDir) rfs := rootFs{root: tmpDir} - p := filepath.Join(tmpDir, "server") - if err := os.Mkdir(p, 0o755); err != nil { - panic(err) - return nil, nil - } + rfs.reset() - fs, _ := New(p, 0, []string{}) + fs := New(filepath.Join(tmpDir, "/server"), 0, []string{}) fs.isTest = true - if err := fs.TruncateRootDirectory(); err != nil { - panic(err) - return nil, nil - } return fs, &rfs } @@ -54,7 +45,7 @@ type rootFs struct { root string } -func getFileContent(file ufs.File) string { +func getFileContent(file *os.File) string { var w bytes.Buffer if _, err := bufio.NewReader(file).WriteTo(&w); err != nil { panic(err) @@ -63,11 +54,11 @@ func getFileContent(file ufs.File) string { } func (rfs *rootFs) CreateServerFile(p string, c []byte) error { - f, err := os.Create(filepath.Join(rfs.root, "server", p)) + f, err := os.Create(filepath.Join(rfs.root, "/server", p)) if err == nil { - _, _ = f.Write(c) - _ = f.Close() + f.Write(c) + f.Close() } return err @@ -78,7 +69,19 @@ func (rfs *rootFs) CreateServerFileFromString(p string, c string) error { } func (rfs *rootFs) StatServerFile(p string) (os.FileInfo, error) { - return os.Stat(filepath.Join(rfs.root, "server", p)) + return os.Stat(filepath.Join(rfs.root, "/server", p)) +} + +func (rfs *rootFs) reset() { + if err := os.RemoveAll(filepath.Join(rfs.root, "/server")); err != nil { + if !os.IsNotExist(err) { + panic(err) + } + } + + if err := os.Mkdir(filepath.Join(rfs.root, "/server"), 0o755); err != nil { + panic(err) + } } func TestFilesystem_Openfile(t *testing.T) { @@ -90,8 +93,7 @@ func TestFilesystem_Openfile(t *testing.T) { _, _, err := fs.File("foo/bar.txt") g.Assert(err).IsNotNil() - // TODO - //g.Assert(IsErrorCode(err, ErrNotExist)).IsTrue() + g.Assert(IsErrorCode(err, ErrNotExist)).IsTrue() }) g.It("returns file stat information", func() { @@ -106,14 +108,14 @@ func TestFilesystem_Openfile(t *testing.T) { }) g.AfterEach(func() { - _ = fs.TruncateRootDirectory() + rfs.reset() }) }) } func TestFilesystem_Writefile(t *testing.T) { g := Goblin(t) - fs, _ := NewFs() + fs, rfs := NewFs() g.Describe("Open and WriteFile", func() { buf := &bytes.Buffer{} @@ -123,22 +125,22 @@ func TestFilesystem_Writefile(t *testing.T) { g.It("can create a new file", func() { r := bytes.NewReader([]byte("test file content")) - g.Assert(fs.CachedUsage()).Equal(int64(0)) + g.Assert(atomic.LoadInt64(&fs.diskUsed)).Equal(int64(0)) - err := fs.Write("test.txt", r, r.Size(), 0o644) + err := fs.Writefile("test.txt", r) g.Assert(err).IsNil() f, _, err := fs.File("test.txt") g.Assert(err).IsNil() defer f.Close() g.Assert(getFileContent(f)).Equal("test file content") - g.Assert(fs.CachedUsage()).Equal(r.Size()) + g.Assert(atomic.LoadInt64(&fs.diskUsed)).Equal(r.Size()) }) g.It("can create a new file inside a nested directory with leading slash", func() { r := bytes.NewReader([]byte("test file content")) - err := fs.Write("/some/nested/test.txt", r, r.Size(), 0o644) + err := fs.Writefile("/some/nested/test.txt", r) g.Assert(err).IsNil() f, _, err := fs.File("/some/nested/test.txt") @@ -150,7 +152,7 @@ func TestFilesystem_Writefile(t *testing.T) { g.It("can create a new file inside a nested directory without a trailing slash", func() { r := bytes.NewReader([]byte("test file content")) - err := fs.Write("some/../foo/bar/test.txt", r, r.Size(), 0o644) + err := fs.Writefile("some/../foo/bar/test.txt", r) g.Assert(err).IsNil() f, _, err := fs.File("foo/bar/test.txt") @@ -162,13 +164,13 @@ func TestFilesystem_Writefile(t *testing.T) { g.It("cannot create a file outside the root directory", func() { r := bytes.NewReader([]byte("test file content")) - err := fs.Write("/some/../foo/../../test.txt", r, r.Size(), 0o644) + err := fs.Writefile("/some/../foo/../../test.txt", r) g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) g.It("cannot write a file that exceeds the disk limits", func() { - fs.SetDiskLimit(1024) + atomic.StoreInt64(&fs.diskLimit, 1024) b := make([]byte, 1025) _, err := rand.Read(b) @@ -176,18 +178,18 @@ func TestFilesystem_Writefile(t *testing.T) { g.Assert(len(b)).Equal(1025) r := bytes.NewReader(b) - err = fs.Write("test.txt", r, int64(len(b)), 0o644) + err = fs.Writefile("test.txt", r) g.Assert(err).IsNotNil() g.Assert(IsErrorCode(err, ErrCodeDiskSpace)).IsTrue() }) g.It("truncates the file when writing new contents", func() { r := bytes.NewReader([]byte("original data")) - err := fs.Write("test.txt", r, r.Size(), 0o644) + err := fs.Writefile("test.txt", r) g.Assert(err).IsNil() r = bytes.NewReader([]byte("new data")) - err = fs.Write("test.txt", r, r.Size(), 0o644) + err = fs.Writefile("test.txt", r) g.Assert(err).IsNil() f, _, err := fs.File("test.txt") @@ -198,7 +200,10 @@ func TestFilesystem_Writefile(t *testing.T) { g.AfterEach(func() { buf.Truncate(0) - _ = fs.TruncateRootDirectory() + rfs.reset() + + atomic.StoreInt64(&fs.diskUsed, 0) + atomic.StoreInt64(&fs.diskLimit, 0) }) }) } @@ -231,17 +236,17 @@ func TestFilesystem_CreateDirectory(t *testing.T) { g.It("should not allow the creation of directories outside the root", func() { err := fs.CreateDirectory("test", "e/../../something") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) g.It("should not increment the disk usage", func() { err := fs.CreateDirectory("test", "/") g.Assert(err).IsNil() - g.Assert(fs.CachedUsage()).Equal(int64(0)) + g.Assert(atomic.LoadInt64(&fs.diskUsed)).Equal(int64(0)) }) g.AfterEach(func() { - _ = fs.TruncateRootDirectory() + rfs.reset() }) }) } @@ -263,25 +268,25 @@ func TestFilesystem_Rename(t *testing.T) { err = fs.Rename("source.txt", "target.txt") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrExist)).IsTrue("err is not ErrExist") + g.Assert(errors.Is(err, os.ErrExist)).IsTrue() }) g.It("returns an error if the final destination is the root directory", func() { err := fs.Rename("source.txt", "/") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + g.Assert(errors.Is(err, os.ErrExist)).IsTrue() }) g.It("returns an error if the source destination is the root directory", func() { - err := fs.Rename("/", "target.txt") + err := fs.Rename("source.txt", "/") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + g.Assert(errors.Is(err, os.ErrExist)).IsTrue() }) g.It("does not allow renaming to a location outside the root", func() { err := fs.Rename("source.txt", "../target.txt") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) g.It("does not allow renaming from a location outside the root", func() { @@ -289,7 +294,7 @@ func TestFilesystem_Rename(t *testing.T) { err = fs.Rename("/../ext-source.txt", "target.txt") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) g.It("allows a file to be renamed", func() { @@ -298,7 +303,7 @@ func TestFilesystem_Rename(t *testing.T) { _, err = rfs.StatServerFile("source.txt") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue("err is not ErrNotExist") + g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue() st, err := rfs.StatServerFile("target.txt") g.Assert(err).IsNil() @@ -315,7 +320,7 @@ func TestFilesystem_Rename(t *testing.T) { _, err = rfs.StatServerFile("source_dir") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue("err is not ErrNotExist") + g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue() st, err := rfs.StatServerFile("target_dir") g.Assert(err).IsNil() @@ -325,7 +330,7 @@ func TestFilesystem_Rename(t *testing.T) { g.It("returns an error if the source does not exist", func() { err := fs.Rename("missing.txt", "target.txt") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue("err is not ErrNotExist") + g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue() }) g.It("creates directories if they are missing", func() { @@ -338,7 +343,7 @@ func TestFilesystem_Rename(t *testing.T) { }) g.AfterEach(func() { - _ = fs.TruncateRootDirectory() + rfs.reset() }) }) } @@ -353,13 +358,13 @@ func TestFilesystem_Copy(t *testing.T) { panic(err) } - fs.unixFS.SetUsage(int64(utf8.RuneCountInString("test content"))) + atomic.StoreInt64(&fs.diskUsed, int64(utf8.RuneCountInString("test content"))) }) g.It("should return an error if the source does not exist", func() { err := fs.Copy("foo.txt") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue("err is not ErrNotExist") + g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue() }) g.It("should return an error if the source is outside the root", func() { @@ -367,11 +372,11 @@ func TestFilesystem_Copy(t *testing.T) { err = fs.Copy("../ext-source.txt") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) g.It("should return an error if the source directory is outside the root", func() { - err := os.MkdirAll(filepath.Join(rfs.root, "nested/in/dir"), 0o755) + err := os.MkdirAll(filepath.Join(rfs.root, "/nested/in/dir"), 0o755) g.Assert(err).IsNil() err = rfs.CreateServerFileFromString("/../nested/in/dir/ext-source.txt", "external content") @@ -379,28 +384,28 @@ func TestFilesystem_Copy(t *testing.T) { err = fs.Copy("../nested/in/dir/ext-source.txt") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() err = fs.Copy("nested/in/../../../nested/in/dir/ext-source.txt") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) g.It("should return an error if the source is a directory", func() { - err := os.Mkdir(filepath.Join(rfs.root, "server/dir"), 0o755) + err := os.Mkdir(filepath.Join(rfs.root, "/server/dir"), 0o755) g.Assert(err).IsNil() err = fs.Copy("dir") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue("err is not ErrNotExist") + g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue() }) g.It("should return an error if there is not space to copy the file", func() { - fs.SetDiskLimit(2) + atomic.StoreInt64(&fs.diskLimit, 2) err := fs.Copy("source.txt") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodeDiskSpace)).IsTrue("err is not ErrCodeDiskSpace") + g.Assert(IsErrorCode(err, ErrCodeDiskSpace)).IsTrue() }) g.It("should create a copy of the file and increment the disk used", func() { @@ -428,7 +433,7 @@ func TestFilesystem_Copy(t *testing.T) { g.Assert(err).IsNil() } - g.Assert(fs.CachedUsage()).Equal(int64(utf8.RuneCountInString("test content")) * 3) + g.Assert(atomic.LoadInt64(&fs.diskUsed)).Equal(int64(utf8.RuneCountInString("test content")) * 3) }) g.It("should create a copy inside of a directory", func() { @@ -449,7 +454,10 @@ func TestFilesystem_Copy(t *testing.T) { }) g.AfterEach(func() { - _ = fs.TruncateRootDirectory() + rfs.reset() + + atomic.StoreInt64(&fs.diskUsed, 0) + atomic.StoreInt64(&fs.diskLimit, 0) }) }) } @@ -464,7 +472,7 @@ func TestFilesystem_Delete(t *testing.T) { panic(err) } - fs.unixFS.SetUsage(int64(utf8.RuneCountInString("test content"))) + atomic.StoreInt64(&fs.diskUsed, int64(utf8.RuneCountInString("test content"))) }) g.It("does not delete files outside the root directory", func() { @@ -472,13 +480,13 @@ func TestFilesystem_Delete(t *testing.T) { err = fs.Delete("../ext-source.txt") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) g.It("does not allow the deletion of the root directory", func() { err := fs.Delete("/") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + g.Assert(err.Error()).Equal("cannot delete root server directory") }) g.It("does not return an error if the target does not exist", func() { @@ -496,9 +504,9 @@ func TestFilesystem_Delete(t *testing.T) { _, err = rfs.StatServerFile("source.txt") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue("err is not ErrNotExist") + g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue() - g.Assert(fs.CachedUsage()).Equal(int64(0)) + g.Assert(atomic.LoadInt64(&fs.diskUsed)).Equal(int64(0)) }) g.It("deletes all items inside a directory if the directory is deleted", func() { @@ -516,16 +524,16 @@ func TestFilesystem_Delete(t *testing.T) { g.Assert(err).IsNil() } - fs.unixFS.SetUsage(int64(utf8.RuneCountInString("test content") * 3)) + atomic.StoreInt64(&fs.diskUsed, int64(utf8.RuneCountInString("test content")*3)) err = fs.Delete("foo") g.Assert(err).IsNil() - g.Assert(fs.unixFS.Usage()).Equal(int64(0)) + g.Assert(atomic.LoadInt64(&fs.diskUsed)).Equal(int64(0)) for _, s := range sources { _, err = rfs.StatServerFile(s) g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue("err is not ErrNotExist") + g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue() } }) @@ -581,7 +589,7 @@ func TestFilesystem_Delete(t *testing.T) { // Delete a file inside the symlinked directory. err = fs.Delete("symlink/source.txt") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() // Ensure the file outside the root directory still exists. _, err = os.Lstat(filepath.Join(rfs.root, "foo/source.txt")) @@ -600,11 +608,14 @@ func TestFilesystem_Delete(t *testing.T) { // Delete a file inside the symlinked directory. err = fs.Delete("symlink/source.txt") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) g.AfterEach(func() { - _ = fs.TruncateRootDirectory() + rfs.reset() + + atomic.StoreInt64(&fs.diskUsed, 0) + atomic.StoreInt64(&fs.diskLimit, 0) }) }) } diff --git a/server/filesystem/path.go b/server/filesystem/path.go index edb1cad6d..3952e5d2e 100644 --- a/server/filesystem/path.go +++ b/server/filesystem/path.go @@ -1,28 +1,71 @@ package filesystem import ( + "context" + iofs "io/fs" + "os" "path/filepath" "strings" + "sync" "emperror.dev/errors" + "golang.org/x/sync/errgroup" ) // Checks if the given file or path is in the server's file denylist. If so, an Error // is returned, otherwise nil is returned. func (fs *Filesystem) IsIgnored(paths ...string) error { for _, p := range paths { - //sp, err := fs.SafePath(p) - //if err != nil { - // return err - //} - // TODO: update logic to use unixFS - if fs.denylist.MatchesPath(p) { - return errors.WithStack(&Error{code: ErrCodeDenylistFile, path: p, resolved: p}) + sp, err := fs.SafePath(p) + if err != nil { + return err + } + if fs.denylist.MatchesPath(sp) { + return errors.WithStack(&Error{code: ErrCodeDenylistFile, path: p, resolved: sp}) } } return nil } +// Normalizes a directory being passed in to ensure the user is not able to escape +// from their data directory. After normalization if the directory is still within their home +// path it is returned. If they managed to "escape" an error will be returned. +// +// This logic is actually copied over from the SFTP server code. Ideally that eventually +// either gets ported into this application, or is able to make use of this package. +func (fs *Filesystem) SafePath(p string) (string, error) { + // Start with a cleaned up path before checking the more complex bits. + r := fs.unsafeFilePath(p) + + // At the same time, evaluate the symlink status and determine where this file or folder + // is truly pointing to. + ep, err := filepath.EvalSymlinks(r) + if err != nil && !os.IsNotExist(err) { + return "", errors.Wrap(err, "server/filesystem: failed to evaluate symlink") + } else if os.IsNotExist(err) { + // The target of one of the symlinks (EvalSymlinks is recursive) does not exist. + // So we get what target path does not exist and check if it's within the data + // directory. If it is, we return the original path, otherwise we return an error. + pErr, ok := err.(*iofs.PathError) + if !ok { + return "", errors.Wrap(err, "server/filesystem: failed to evaluate symlink") + } + ep = pErr.Path + } + + // If the requested directory from EvalSymlinks begins with the server root directory go + // ahead and return it. If not we'll return an error which will block any further action + // on the file. + if fs.unsafeIsInDataDirectory(ep) { + // Returning the original path here instead of the resolved path ensures that + // whatever the user is trying to do will work as expected. If we returned the + // resolved path, the user would be unable to know that it is in fact a symlink. + return r, nil + } + + return "", NewBadPathResolution(p, r) +} + // Generate a path to the file by cleaning it up and appending the root server path to it. This // DOES NOT guarantee that the file resolves within the server data directory. You'll want to use // the fs.unsafeIsInDataDirectory(p) function to confirm. @@ -41,3 +84,51 @@ func (fs *Filesystem) unsafeFilePath(p string) string { func (fs *Filesystem) unsafeIsInDataDirectory(p string) bool { return strings.HasPrefix(strings.TrimSuffix(p, "/")+"/", strings.TrimSuffix(fs.Path(), "/")+"/") } + +// Executes the fs.SafePath function in parallel against an array of paths. If any of the calls +// fails an error will be returned. +func (fs *Filesystem) ParallelSafePath(paths []string) ([]string, error) { + var cleaned []string + + // Simple locker function to avoid racy appends to the array of cleaned paths. + m := new(sync.Mutex) + push := func(c string) { + m.Lock() + cleaned = append(cleaned, c) + m.Unlock() + } + + // Create an error group that we can use to run processes in parallel while retaining + // the ability to cancel the entire process immediately should any of it fail. + g, ctx := errgroup.WithContext(context.Background()) + + // Iterate over all of the paths and generate a cleaned path, if there is an error for any + // of the files, abort the process. + for _, p := range paths { + // Create copy so we can use it within the goroutine correctly. + pi := p + + // Recursively call this function to continue digging through the directory tree within + // a separate goroutine. If the context is canceled abort this process. + g.Go(func() error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + // If the callback returns true, go ahead and keep walking deeper. This allows + // us to programmatically continue deeper into directories, or stop digging + // if that pathway knows it needs nothing else. + if c, err := fs.SafePath(pi); err != nil { + return err + } else { + push(c) + } + + return nil + } + }) + } + + // Block until all of the routines finish and have returned a value. + return cleaned, g.Wait() +} diff --git a/server/filesystem/path_test.go b/server/filesystem/path_test.go index 4d46fbf48..ecb96279b 100644 --- a/server/filesystem/path_test.go +++ b/server/filesystem/path_test.go @@ -8,8 +8,6 @@ import ( "emperror.dev/errors" . "github.com/franela/goblin" - - "github.com/pterodactyl/wings/internal/ufs" ) func TestFilesystem_Path(t *testing.T) { @@ -23,6 +21,80 @@ func TestFilesystem_Path(t *testing.T) { }) } +func TestFilesystem_SafePath(t *testing.T) { + g := Goblin(t) + fs, rfs := NewFs() + prefix := filepath.Join(rfs.root, "/server") + + g.Describe("SafePath", func() { + g.It("returns a cleaned path to a given file", func() { + p, err := fs.SafePath("test.txt") + g.Assert(err).IsNil() + g.Assert(p).Equal(prefix + "/test.txt") + + p, err = fs.SafePath("/test.txt") + g.Assert(err).IsNil() + g.Assert(p).Equal(prefix + "/test.txt") + + p, err = fs.SafePath("./test.txt") + g.Assert(err).IsNil() + g.Assert(p).Equal(prefix + "/test.txt") + + p, err = fs.SafePath("/foo/../test.txt") + g.Assert(err).IsNil() + g.Assert(p).Equal(prefix + "/test.txt") + + p, err = fs.SafePath("/foo/bar") + g.Assert(err).IsNil() + g.Assert(p).Equal(prefix + "/foo/bar") + }) + + g.It("handles root directory access", func() { + p, err := fs.SafePath("/") + g.Assert(err).IsNil() + g.Assert(p).Equal(prefix) + + p, err = fs.SafePath("") + g.Assert(err).IsNil() + g.Assert(p).Equal(prefix) + }) + + g.It("removes trailing slashes from paths", func() { + p, err := fs.SafePath("/foo/bar/") + g.Assert(err).IsNil() + g.Assert(p).Equal(prefix + "/foo/bar") + }) + + g.It("handles deeply nested directories that do not exist", func() { + p, err := fs.SafePath("/foo/bar/baz/quaz/../../ducks/testing.txt") + g.Assert(err).IsNil() + g.Assert(p).Equal(prefix + "/foo/bar/ducks/testing.txt") + }) + + g.It("blocks access to files outside the root directory", func() { + p, err := fs.SafePath("../test.txt") + g.Assert(err).IsNotNil() + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(p).Equal("") + + p, err = fs.SafePath("/../test.txt") + g.Assert(err).IsNotNil() + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(p).Equal("") + + p, err = fs.SafePath("./foo/../../test.txt") + g.Assert(err).IsNotNil() + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(p).Equal("") + + p, err = fs.SafePath("..") + g.Assert(err).IsNotNil() + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(p).Equal("") + }) + }) +} + // We test against accessing files outside the root directory in the tests, however it // is still possible for someone to mess up and not properly use this safe path call. In // order to truly confirm this, we'll try to pass in a symlinked malicious file to all of @@ -61,7 +133,7 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { err := fs.Writefile("symlinked.txt", r) g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) g.It("cannot write to a non-existent file symlinked outside the root", func() { @@ -69,7 +141,7 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { err := fs.Writefile("symlinked_does_not_exist.txt", r) g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) g.It("cannot write to chained symlinks with target that does not exist outside the root", func() { @@ -77,7 +149,7 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { err := fs.Writefile("symlinked_does_not_exist2.txt", r) g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrBadPathResolution)).IsTrue("err is not ErrBadPathResolution") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) g.It("cannot write a file to a directory symlinked outside the root", func() { @@ -85,7 +157,7 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { err := fs.Writefile("external_dir/foo.txt", r) g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrNotDirectory)).IsTrue("err is not ErrNotDirectory") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) }) @@ -93,54 +165,55 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { g.It("cannot create a directory outside the root", func() { err := fs.CreateDirectory("my_dir", "external_dir") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrNotDirectory)).IsTrue("err is not ErrNotDirectory") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) g.It("cannot create a nested directory outside the root", func() { err := fs.CreateDirectory("my/nested/dir", "external_dir/foo/bar") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrNotDirectory)).IsTrue("err is not ErrNotDirectory") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) g.It("cannot create a nested directory outside the root", func() { err := fs.CreateDirectory("my/nested/dir", "external_dir/server") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrNotDirectory)).IsTrue("err is not ErrNotDirectory") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) }) g.Describe("Rename", func() { - g.It("can rename a file symlinked outside the directory root", func() { - _, err := os.Lstat(filepath.Join(rfs.root, "server", "symlinked.txt")) - g.Assert(err).IsNil() - err = fs.Rename("symlinked.txt", "foo.txt") - g.Assert(err).IsNil() - _, err = os.Lstat(filepath.Join(rfs.root, "server", "foo.txt")) - g.Assert(err).IsNil() + g.It("cannot rename a file symlinked outside the directory root", func() { + err := fs.Rename("symlinked.txt", "foo.txt") + g.Assert(err).IsNotNil() + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) - g.It("can rename a symlinked directory outside the root", func() { - _, err := os.Lstat(filepath.Join(rfs.root, "server", "external_dir")) - g.Assert(err).IsNil() - err = fs.Rename("external_dir", "foo") - g.Assert(err).IsNil() - _, err = os.Lstat(filepath.Join(rfs.root, "server", "foo")) - g.Assert(err).IsNil() + g.It("cannot rename a symlinked directory outside the root", func() { + err := fs.Rename("external_dir", "foo") + g.Assert(err).IsNotNil() + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) g.It("cannot rename a file to a location outside the directory root", func() { - _ = rfs.CreateServerFileFromString("my_file.txt", "internal content") - t.Log(rfs.root) + rfs.CreateServerFileFromString("my_file.txt", "internal content") - st, err := os.Lstat(filepath.Join(rfs.root, "server", "foo")) - g.Assert(err).IsNil() - g.Assert(st.Mode()&ufs.ModeSymlink != 0).IsTrue() + err := fs.Rename("my_file.txt", "external_dir/my_file.txt") + g.Assert(err).IsNotNil() + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + }) + }) - err = fs.Rename("my_file.txt", "foo/my_file.txt") - g.Assert(errors.Is(err, ufs.ErrNotDirectory)).IsTrue() + g.Describe("Chown", func() { + g.It("cannot chown a file symlinked outside the directory root", func() { + err := fs.Chown("symlinked.txt") + g.Assert(err).IsNotNil() + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + }) - st, err = os.Lstat(filepath.Join(rfs.root, "malicious_dir", "my_file.txt")) - g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue() + g.It("cannot chown a directory symlinked outside the directory root", func() { + err := fs.Chown("external_dir") + g.Assert(err).IsNotNil() + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) }) @@ -148,7 +221,7 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { g.It("cannot copy a file symlinked outside the directory root", func() { err := fs.Copy("symlinked.txt") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue("err is not ErrNotExist") + g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() }) }) @@ -162,9 +235,9 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { _, err = rfs.StatServerFile("symlinked.txt") g.Assert(err).IsNotNil() - g.Assert(errors.Is(err, ufs.ErrNotExist)).IsTrue("err is not ErrNotExist") + g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue() }) }) - _ = fs.TruncateRootDirectory() + rfs.reset() } diff --git a/server/filesystem/stat.go b/server/filesystem/stat.go index 94cab60bc..ce52b6906 100644 --- a/server/filesystem/stat.go +++ b/server/filesystem/stat.go @@ -1,18 +1,16 @@ package filesystem import ( - "encoding/json" - "io" + "os" "strconv" "time" "github.com/gabriel-vasile/mimetype" - - "github.com/pterodactyl/wings/internal/ufs" + "github.com/goccy/go-json" ) type Stat struct { - ufs.FileInfo + os.FileInfo Mimetype string } @@ -33,31 +31,40 @@ func (s *Stat) MarshalJSON() ([]byte, error) { Created: s.CTime().Format(time.RFC3339), Modified: s.ModTime().Format(time.RFC3339), Mode: s.Mode().String(), - // Using `&ModePerm` on the file's mode will cause the mode to only have the permission values, and nothing else. - ModeBits: strconv.FormatUint(uint64(s.Mode()&ufs.ModePerm), 8), + // Using `&os.ModePerm` on the file's mode will cause the mode to only have the permission values, and nothing else. + ModeBits: strconv.FormatUint(uint64(s.Mode()&os.ModePerm), 8), Size: s.Size(), Directory: s.IsDir(), File: !s.IsDir(), - Symlink: s.Mode().Type()&ufs.ModeSymlink != 0, + Symlink: s.Mode().Type()&os.ModeSymlink != 0, Mime: s.Mimetype, }) } -func statFromFile(f ufs.File) (Stat, error) { - s, err := f.Stat() +// Stat stats a file or folder and returns the base stat object from go along +// with the MIME data that can be used for editing files. +func (fs *Filesystem) Stat(p string) (Stat, error) { + cleaned, err := fs.SafePath(p) + if err != nil { + return Stat{}, err + } + return fs.unsafeStat(cleaned) +} + +func (fs *Filesystem) unsafeStat(p string) (Stat, error) { + s, err := os.Stat(p) if err != nil { return Stat{}, err } + var m *mimetype.MIME if !s.IsDir() { - m, err = mimetype.DetectReader(f) + m, err = mimetype.DetectFile(p) if err != nil { return Stat{}, err } - if _, err := f.Seek(0, io.SeekStart); err != nil { - return Stat{}, err - } } + st := Stat{ FileInfo: s, Mimetype: "inode/directory", @@ -65,20 +72,6 @@ func statFromFile(f ufs.File) (Stat, error) { if m != nil { st.Mimetype = m.String() } - return st, nil -} -// Stat stats a file or folder and returns the base stat object from go along -// with the MIME data that can be used for editing files. -func (fs *Filesystem) Stat(p string) (Stat, error) { - f, err := fs.unixFS.Open(p) - if err != nil { - return Stat{}, err - } - defer f.Close() - st, err := statFromFile(f) - if err != nil { - return Stat{}, err - } return st, nil } diff --git a/server/filesystem/stat_darwin.go b/server/filesystem/stat_darwin.go new file mode 100644 index 000000000..6d0cff32b --- /dev/null +++ b/server/filesystem/stat_darwin.go @@ -0,0 +1,13 @@ +package filesystem + +import ( + "syscall" + "time" +) + +// CTime returns the time that the file/folder was created. +func (s *Stat) CTime() time.Time { + st := s.Sys().(*syscall.Stat_t) + + return time.Unix(st.Ctimespec.Sec, st.Ctimespec.Nsec) +} diff --git a/server/filesystem/stat_linux.go b/server/filesystem/stat_linux.go index 7891bafbe..a9c7fb36f 100644 --- a/server/filesystem/stat_linux.go +++ b/server/filesystem/stat_linux.go @@ -3,22 +3,12 @@ package filesystem import ( "syscall" "time" - - "golang.org/x/sys/unix" ) -// CTime returns the time that the file/folder was created. -// -// TODO: remove. Ctim is not actually ever been correct and doesn't actually -// return the creation time. +// Returns the time that the file/folder was created. func (s *Stat) CTime() time.Time { - if st, ok := s.Sys().(*unix.Stat_t); ok { - // Do not remove these "redundant" type-casts, they are required for 32-bit builds to work. - return time.Unix(int64(st.Ctim.Sec), int64(st.Ctim.Nsec)) - } - if st, ok := s.Sys().(*syscall.Stat_t); ok { - // Do not remove these "redundant" type-casts, they are required for 32-bit builds to work. - return time.Unix(int64(st.Ctim.Sec), int64(st.Ctim.Nsec)) - } - return time.Time{} + st := s.Sys().(*syscall.Stat_t) + + // Do not remove these "redundant" type-casts, they are required for 32-bit builds to work. + return time.Unix(int64(st.Ctim.Sec), int64(st.Ctim.Nsec)) } diff --git a/server/filesystem/stat_windows.go b/server/filesystem/stat_windows.go new file mode 100644 index 000000000..3652677bc --- /dev/null +++ b/server/filesystem/stat_windows.go @@ -0,0 +1,12 @@ +package filesystem + +import ( + "time" +) + +// On linux systems this will return the time that the file was created. +// However, I have no idea how to do this on windows, so we're skipping it +// for right now. +func (s *Stat) CTime() time.Time { + return s.ModTime() +} diff --git a/server/install.go b/server/install.go index 8c29f1c7a..d4454385f 100644 --- a/server/install.go +++ b/server/install.go @@ -2,6 +2,7 @@ package server import ( "bufio" + "bytes" "context" "html/template" "io" @@ -217,18 +218,30 @@ func (ip *InstallationProcess) tempDir() string { // can be properly mounted into the installation container and then executed. func (ip *InstallationProcess) writeScriptToDisk() error { // Make sure the temp directory root exists before trying to make a directory within it. The - // os.TempDir call expects this base to exist, it won't create it for you. + // ioutil.TempDir call expects this base to exist, it won't create it for you. if err := os.MkdirAll(ip.tempDir(), 0o700); err != nil { return errors.WithMessage(err, "could not create temporary directory for install process") } + f, err := os.OpenFile(filepath.Join(ip.tempDir(), "install.sh"), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) if err != nil { return errors.WithMessage(err, "failed to write server installation script to disk before mount") } defer f.Close() - if _, err := io.Copy(f, strings.NewReader(strings.ReplaceAll(ip.Script.Script, "\r\n", "\n"))); err != nil { + + w := bufio.NewWriter(f) + + scanner := bufio.NewScanner(bytes.NewReader([]byte(ip.Script.Script))) + for scanner.Scan() { + w.WriteString(scanner.Text() + "\n") + } + + if err := scanner.Err(); err != nil { return err } + + w.Flush() + return nil } diff --git a/server/manager.go b/server/manager.go index 88970f40a..9c3de8f62 100644 --- a/server/manager.go +++ b/server/manager.go @@ -196,10 +196,7 @@ func (m *Manager) InitServer(data remote.ServerConfigurationResponse) (*Server, return nil, errors.WithStackIf(err) } - s.fs, err = filesystem.New(filepath.Join(config.Get().System.Data, s.ID()), s.DiskSpace(), s.Config().Egg.FileDenylist) - if err != nil { - return nil, errors.WithStackIf(err) - } + s.fs = filesystem.New(filepath.Join(config.Get().System.Data, s.ID()), s.DiskSpace(), s.Config().Egg.FileDenylist) // Right now we only support a Docker based environment, so I'm going to hard code // this logic in. When we're ready to support other environment we'll need to make diff --git a/server/transfer/archive.go b/server/transfer/archive.go index 26cfddcaa..e5457f12b 100644 --- a/server/transfer/archive.go +++ b/server/transfer/archive.go @@ -35,8 +35,8 @@ type Archive struct { func NewArchive(t *Transfer, size uint64) *Archive { return &Archive{ archive: &filesystem.Archive{ - Filesystem: t.Server.Filesystem(), - Progress: progress.NewProgress(size), + BasePath: t.Server.Filesystem().Path(), + Progress: progress.NewProgress(size), }, } } diff --git a/sftp/handler.go b/sftp/handler.go index 870dcd4bd..6fdc77e63 100644 --- a/sftp/handler.go +++ b/sftp/handler.go @@ -2,6 +2,7 @@ package sftp import ( "io" + "io/ioutil" "os" "path/filepath" "strings" @@ -121,7 +122,7 @@ func (h *Handler) Filewrite(request *sftp.Request) (io.WriterAt, error) { if !h.can(permission) { return nil, sftp.ErrSSHFxPermissionDenied } - f, err := h.fs.Touch(request.Filepath, os.O_RDWR|os.O_TRUNC) + f, err := h.fs.Touch(request.Filepath, os.O_RDWR|os.O_CREATE|os.O_TRUNC) if err != nil { l.WithField("flags", request.Flags).WithField("error", err).Error("failed to open existing file on system") return nil, sftp.ErrSSHFxFailure @@ -219,8 +220,16 @@ func (h *Handler) Filecmd(request *sftp.Request) error { if !h.can(PermissionFileCreate) { return sftp.ErrSSHFxPermissionDenied } - if err := h.fs.Symlink(request.Filepath, request.Target); err != nil { - l.WithField("target", request.Target).WithField("error", err).Error("failed to create symlink") + source, err := h.fs.SafePath(request.Filepath) + if err != nil { + return sftp.ErrSSHFxNoSuchFile + } + target, err := h.fs.SafePath(request.Target) + if err != nil { + return sftp.ErrSSHFxNoSuchFile + } + if err := os.Symlink(source, target); err != nil { + l.WithField("target", target).WithField("error", err).Error("failed to create symlink") return sftp.ErrSSHFxFailure } break @@ -265,12 +274,16 @@ func (h *Handler) Filelist(request *sftp.Request) (sftp.ListerAt, error) { switch request.Method { case "List": - entries, err := h.fs.ReadDirStat(request.Filepath) + p, err := h.fs.SafePath(request.Filepath) + if err != nil { + return nil, sftp.ErrSSHFxNoSuchFile + } + files, err := ioutil.ReadDir(p) if err != nil { h.logger.WithField("source", request.Filepath).WithField("error", err).Error("error while listing directory") return nil, sftp.ErrSSHFxFailure } - return ListerAt(entries), nil + return ListerAt(files), nil case "Stat": st, err := h.fs.Stat(request.Filepath) if err != nil { From a0691e53ffe4197ef90f623a3955f8236446cd23 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Tue, 3 Feb 2026 17:18:38 -0800 Subject: [PATCH 02/30] fix go.mod --- go.mod | 7 +++++-- go.sum | 6 ++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index af2e23de2..bc775f6fe 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/gin-gonic/gin v1.10.1 github.com/glebarez/sqlite v1.11.0 github.com/go-co-op/gocron v1.37.0 + github.com/goccy/go-json v0.10.5 github.com/google/uuid v1.6.0 github.com/gorilla/websocket v1.5.3 github.com/iancoleman/strcase v0.3.0 @@ -36,6 +37,7 @@ require ( github.com/klauspost/pgzip v1.2.6 github.com/magiconair/properties v1.8.9 github.com/mattn/go-colorable v0.1.14 + github.com/mholt/archiver/v4 v4.0.0-alpha.9 github.com/mholt/archives v0.1.5 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db github.com/patrickmn/go-cache v2.1.0+incompatible @@ -46,6 +48,7 @@ require ( golang.org/x/crypto v0.46.0 golang.org/x/sync v0.19.0 golang.org/x/sys v0.39.0 + golang.org/x/time v0.0.0-20220922220347-f3bd1da661af gopkg.in/ini.v1 v1.67.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 @@ -80,8 +83,8 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.25.0 // indirect - github.com/goccy/go-json v0.10.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/snappy v0.0.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect @@ -116,6 +119,7 @@ require ( github.com/sorairolake/lzip-go v0.3.8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/pflag v1.0.6 // indirect + github.com/therootcompany/xz v1.0.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/ulikunitz/xz v0.5.15 // indirect @@ -133,7 +137,6 @@ require ( golang.org/x/net v0.47.0 // indirect golang.org/x/term v0.38.0 // indirect golang.org/x/text v0.32.0 // indirect - golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect google.golang.org/protobuf v1.36.5 // indirect gotest.tools/v3 v3.0.2 // indirect diff --git a/go.sum b/go.sum index 198c38245..632953dfb 100644 --- a/go.sum +++ b/go.sum @@ -163,6 +163,8 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -263,6 +265,8 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/mholt/archiver/v4 v4.0.0-alpha.9 h1:EZgAsW6DsuawxDgTtIdjCUBa2TQ6AOe9pnCidofSRtE= +github.com/mholt/archiver/v4 v4.0.0-alpha.9/go.mod h1:5D3uct315OMkMRXKwEuMB+wQi/2m5NQngKDmApqwVlo= github.com/mholt/archives v0.1.5 h1:Fh2hl1j7VEhc6DZs2DLMgiBNChUux154a1G+2esNvzQ= github.com/mholt/archives v0.1.5/go.mod h1:3TPMmBLPsgszL+1As5zECTuKwKvIfj6YcwWPpeTAXF4= github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0= @@ -356,6 +360,8 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= +github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= From 24e3f6758b23045c5695818b438ef6f9415b77e6 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Tue, 3 Feb 2026 17:45:12 -0800 Subject: [PATCH 03/30] get things compiling --- go.mod | 7 +- go.sum | 6 - parser/parser.go | 115 +++++------- server/filesystem/archiverext/compressed.go | 100 ----------- server/filesystem/compress.go | 189 +------------------- server/filesystem/disk_space.go | 17 +- 6 files changed, 54 insertions(+), 380 deletions(-) delete mode 100644 server/filesystem/archiverext/compressed.go diff --git a/go.mod b/go.mod index bc775f6fe..03f2adff3 100644 --- a/go.mod +++ b/go.mod @@ -33,11 +33,9 @@ require ( github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0 github.com/juju/ratelimit v1.0.2 github.com/karrick/godirwalk v1.17.0 - github.com/klauspost/compress v1.18.0 github.com/klauspost/pgzip v1.2.6 github.com/magiconair/properties v1.8.9 github.com/mattn/go-colorable v0.1.14 - github.com/mholt/archiver/v4 v4.0.0-alpha.9 github.com/mholt/archives v0.1.5 github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db github.com/patrickmn/go-cache v2.1.0+incompatible @@ -47,7 +45,6 @@ require ( github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.46.0 golang.org/x/sync v0.19.0 - golang.org/x/sys v0.39.0 golang.org/x/time v0.0.0-20220922220347-f3bd1da661af gopkg.in/ini.v1 v1.67.0 gopkg.in/yaml.v2 v2.4.0 @@ -84,13 +81,13 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.25.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/snappy v0.0.4 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect + github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/kr/fs v0.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect @@ -119,7 +116,6 @@ require ( github.com/sorairolake/lzip-go v0.3.8 // indirect github.com/spf13/afero v1.15.0 // indirect github.com/spf13/pflag v1.0.6 // indirect - github.com/therootcompany/xz v1.0.1 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect github.com/ugorji/go/codec v1.2.12 // indirect github.com/ulikunitz/xz v0.5.15 // indirect @@ -135,6 +131,7 @@ require ( golang.org/x/arch v0.15.0 // indirect golang.org/x/exp v0.0.0-20250305212735-054e65f0b394 // indirect golang.org/x/net v0.47.0 // indirect + golang.org/x/sys v0.39.0 // indirect golang.org/x/term v0.38.0 // indirect golang.org/x/text v0.32.0 // indirect golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect diff --git a/go.sum b/go.sum index 632953dfb..198c38245 100644 --- a/go.sum +++ b/go.sum @@ -163,8 +163,6 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= -github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -265,8 +263,6 @@ github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/mholt/archiver/v4 v4.0.0-alpha.9 h1:EZgAsW6DsuawxDgTtIdjCUBa2TQ6AOe9pnCidofSRtE= -github.com/mholt/archiver/v4 v4.0.0-alpha.9/go.mod h1:5D3uct315OMkMRXKwEuMB+wQi/2m5NQngKDmApqwVlo= github.com/mholt/archives v0.1.5 h1:Fh2hl1j7VEhc6DZs2DLMgiBNChUux154a1G+2esNvzQ= github.com/mholt/archives v0.1.5/go.mod h1:3TPMmBLPsgszL+1As5zECTuKwKvIfj6YcwWPpeTAXF4= github.com/mikelolasagasti/xz v1.0.1 h1:Q2F2jX0RYJUG3+WsM+FJknv+6eVjsjXNDV0KJXZzkD0= @@ -360,8 +356,6 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+xzw= -github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= diff --git a/parser/parser.go b/parser/parser.go index 12d1ed9d0..f309c0da1 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -2,7 +2,6 @@ package parser import ( "bufio" - "bytes" "encoding/json" "io" "os" @@ -431,18 +430,8 @@ func (f *ConfigurationFile) parseJsonFile(path string) error { return err } - if _, err := file.Seek(0, io.SeekStart); err != nil { - return err - } - if err := file.Truncate(0); err != nil { - return err - } - - // Write the data to the file. - if _, err := io.Copy(file, bytes.NewReader(data.BytesIndent("", " "))); err != nil { - return errors.Wrap(err, "parser: failed to write properties file to disk") - } - return nil + output := []byte(data.StringIndent("", " ")) + return os.WriteFile(path, output, 0o644) } // Parses a yaml file and updates any matching key/value pairs before persisting @@ -479,56 +468,35 @@ func (f *ConfigurationFile) parseYamlFile(path string) error { return err } - if _, err := file.Seek(0, io.SeekStart); err != nil { - return err - } - if err := file.Truncate(0); err != nil { - return err - } - - // Write the data to the file. - if _, err := io.Copy(file, bytes.NewReader(marshaled)); err != nil { - return errors.Wrap(err, "parser: failed to write properties file to disk") - } - return nil + return os.WriteFile(path, marshaled, 0o644) } // Parses a text file using basic find and replace. This is a highly inefficient method of // scanning a file and performing a replacement. You should attempt to use anything other // than this function where possible. -func (f *ConfigurationFile) parseTextFile(file ufs.File) error { - b := bytes.NewBuffer(nil) - s := bufio.NewScanner(file) - var replaced bool - for s.Scan() { - line := s.Bytes() - replaced = false +func (f *ConfigurationFile) parseTextFile(path string) error { + input, err := os.ReadFile(path) + if err != nil { + return err + } + + lines := strings.Split(string(input), "\n") + for i, line := range lines { for _, replace := range f.Replace { // If this line doesn't match what we expect for the replacement, move on to the next // line. Otherwise, update the line to have the replacement value. - if !bytes.HasPrefix(line, []byte(replace.Match)) { + if !strings.HasPrefix(line, replace.Match) { continue } - b.Write(replace.ReplaceWith.Bytes()) - replaced = true - } - if !replaced { - b.Write(line) + + lines[i] = replace.ReplaceWith.String() } - b.WriteByte('\n') } - if _, err := file.Seek(0, io.SeekStart); err != nil { - return err - } - if err := file.Truncate(0); err != nil { + if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")), 0o644); err != nil { return err } - // Write the data to the file. - if _, err := io.Copy(file, b); err != nil { - return errors.Wrap(err, "parser: failed to write properties file to disk") - } return nil } @@ -558,29 +526,31 @@ func (f *ConfigurationFile) parseTextFile(file ufs.File) error { // // @see https://github.com/pterodactyl/panel/issues/2308 (original) // @see https://github.com/pterodactyl/panel/issues/3009 ("bug" introduced as result) -func (f *ConfigurationFile) parsePropertiesFile(file ufs.File) error { - b, err := io.ReadAll(file) - if err != nil { - return err - } - - s := bytes.NewBuffer(nil) - scanner := bufio.NewScanner(bytes.NewReader(b)) - // Scan until we hit a line that is not a comment that actually has content - // on it. Keep appending the comments until that time. - for scanner.Scan() { - text := scanner.Bytes() - if len(text) > 0 && text[0] != '#' { - break +func (f *ConfigurationFile) parsePropertiesFile(path string) error { + var s strings.Builder + // Open the file and attempt to load any comments that currenty exist at the start + // of the file. This is kind of a hack, but should work for a majority of users for + // the time being. + if fd, err := os.Open(path); err != nil { + return errors.Wrap(err, "parser: could not open file for reading") + } else { + scanner := bufio.NewScanner(fd) + // Scan until we hit a line that is not a comment that actually has content + // on it. Keep appending the comments until that time. + for scanner.Scan() { + text := scanner.Text() + if len(text) > 0 && text[0] != '#' { + break + } + s.WriteString(text + "\n") + } + _ = fd.Close() + if err := scanner.Err(); err != nil { + return errors.WithStackIf(err) } - s.Write(text) - s.WriteByte('\n') - } - if err := scanner.Err(); err != nil { - return errors.WithStackIf(err) } - p, err := properties.Load(b, properties.UTF8) + p, err := properties.LoadFile(path, properties.UTF8) if err != nil { return errors.Wrap(err, "parser: could not load properties file for configuration update") } @@ -618,16 +588,17 @@ func (f *ConfigurationFile) parsePropertiesFile(file ufs.File) error { s.WriteString(key + "=" + strings.Trim(strconv.QuoteToASCII(value), "\"") + "\n") } - if _, err := file.Seek(0, io.SeekStart); err != nil { - return err - } - if err := file.Truncate(0); err != nil { + // Open the file for writing. + w, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { return err } + defer w.Close() // Write the data to the file. - if _, err := io.Copy(file, s); err != nil { + if _, err := w.Write([]byte(s.String())); err != nil { return errors.Wrap(err, "parser: failed to write properties file to disk") } + return nil } diff --git a/server/filesystem/archiverext/compressed.go b/server/filesystem/archiverext/compressed.go deleted file mode 100644 index 536a25f70..000000000 --- a/server/filesystem/archiverext/compressed.go +++ /dev/null @@ -1,100 +0,0 @@ -// SPDX-License-Identifier: MIT -// SPDX-FileCopyrightText: Copyright (c) 2016 Matthew Holt - -// Code in this file was derived from -// https://github.com/mholt/archiver/blob/v4.0.0-alpha.8/fs.go -// -// These modifications were necessary to allow us to use an already open file -// with archiver.FileFS. - -package archiverext - -import ( - "io" - "io/fs" - - "github.com/mholt/archives" -) - -// FileFS allows accessing a file on disk using a consistent file system interface. -// The value should be the path to a regular file, not a directory. This file will -// be the only entry in the file system and will be at its root. It can be accessed -// within the file system by the name of "." or the filename. -// -// If the file is compressed, set the Compression field so that reads from the -// file will be transparently decompressed. -type FileFS struct { - // File is the compressed file backing the FileFS. - File fs.File - - // If file is compressed, setting this field will - // transparently decompress reads. - Compression archives.Decompressor -} - -// Open opens the named file, which must be the file used to create the file system. -func (f FileFS) Open(name string) (fs.File, error) { - if err := f.checkName(name, "open"); err != nil { - return nil, err - } - if f.Compression == nil { - return f.File, nil - } - r, err := f.Compression.OpenReader(f.File) - if err != nil { - return nil, err - } - return compressedFile{f.File, r}, nil -} - -// ReadDir returns a directory listing with the file as the singular entry. -func (f FileFS) ReadDir(name string) ([]fs.DirEntry, error) { - if err := f.checkName(name, "stat"); err != nil { - return nil, err - } - info, err := f.Stat(name) - if err != nil { - return nil, err - } - return []fs.DirEntry{fs.FileInfoToDirEntry(info)}, nil -} - -// Stat stats the named file, which must be the file used to create the file system. -func (f FileFS) Stat(name string) (fs.FileInfo, error) { - if err := f.checkName(name, "stat"); err != nil { - return nil, err - } - return f.File.Stat() -} - -func (f FileFS) checkName(name, op string) error { - if !fs.ValidPath(name) { - return &fs.PathError{Op: "open", Path: name, Err: fs.ErrInvalid} - } - // TODO: we may need better name validation. - if name != "." { - return &fs.PathError{Op: op, Path: name, Err: fs.ErrNotExist} - } - return nil -} - -// compressedFile is an fs.File that specially reads -// from a decompression reader, and which closes both -// that reader and the underlying file. -type compressedFile struct { - fs.File - decomp io.ReadCloser -} - -func (cf compressedFile) Read(p []byte) (int, error) { - return cf.decomp.Read(p) -} - -func (cf compressedFile) Close() error { - err := cf.File.Close() - err2 := cf.decomp.Close() - if err2 != nil && err == nil { - err = err2 - } - return err -} diff --git a/server/filesystem/compress.go b/server/filesystem/compress.go index ed2810d67..e80421838 100644 --- a/server/filesystem/compress.go +++ b/server/filesystem/compress.go @@ -2,18 +2,10 @@ package filesystem import ( "context" - "fmt" "io" - iofs "io/fs" "os" - "path" - "path/filepath" - "strings" - "sync/atomic" - "time" "emperror.dev/errors" - "github.com/mholt/archiver/v4" ) // CompressFiles compresses all the files matching the given paths in the @@ -26,94 +18,13 @@ import ( // and the compressed file will be placed at that location named // `archive-{date}.tar.gz`. func (fs *Filesystem) CompressFiles(dir string, paths []string) (os.FileInfo, error) { - cleanedRootDir, err := fs.SafePath(dir) - if err != nil { - return nil, err - } - - // Take all the paths passed in and merge them together with the root directory we've gotten. - for i, p := range paths { - paths[i] = filepath.Join(cleanedRootDir, p) - } - - cleaned, err := fs.ParallelSafePath(paths) - if err != nil { - return nil, err - } - - a := &Archive{BasePath: cleanedRootDir, Files: cleaned} - d := path.Join( - cleanedRootDir, - fmt.Sprintf("archive-%s.tar.gz", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", "")), - ) - - if err := a.Create(context.Background(), d); err != nil { - return nil, err - } - - f, err := os.Stat(d) - if err != nil { - _ = os.Remove(d) - return nil, err - } - - if err := fs.HasSpaceFor(f.Size()); err != nil { - _ = os.Remove(d) - return nil, err - } - - fs.addDisk(f.Size()) - - return f, nil + return nil, errors.New("server/fs: not implemented") } // SpaceAvailableForDecompression looks through a given archive and determines // if decompressing it would put the server over its allocated disk space limit. func (fs *Filesystem) SpaceAvailableForDecompression(ctx context.Context, dir string, file string) error { - // Don't waste time trying to determine this if we know the server will have the space for - // it since there is no limit. - if fs.MaxDisk() <= 0 { - return nil - } - - source, err := fs.SafePath(filepath.Join(dir, file)) - if err != nil { - return err - } - - // Get the cached size in a parallel process so that if it is not cached we are not - // waiting an unnecessary amount of time on this call. - dirSize, err := fs.DiskUsage(false) - - fsys, err := archiver.FileSystem(ctx, source) - if err != nil { - if errors.Is(err, archiver.ErrNoMatch) { - return newFilesystemError(ErrCodeUnknownArchive, err) - } - return err - } - - var size int64 - return iofs.WalkDir(fsys, ".", func(path string, d iofs.DirEntry, err error) error { - if err != nil { - return err - } - - select { - case <-ctx.Done(): - // Stop walking if the context is canceled. - return ctx.Err() - default: - info, err := d.Info() - if err != nil { - return err - } - if atomic.AddInt64(&size, info.Size())+dirSize > fs.MaxDisk() { - return newFilesystemError(ErrCodeDiskSpace, nil) - } - return nil - } - }) + return nil } // DecompressFile will decompress a file in a given directory by using the @@ -122,102 +33,10 @@ func (fs *Filesystem) SpaceAvailableForDecompression(ctx context.Context, dir st // zip-slip attack being attempted by validating that the final path is within // the server data directory. func (fs *Filesystem) DecompressFile(ctx context.Context, dir string, file string) error { - source, err := fs.SafePath(filepath.Join(dir, file)) - if err != nil { - return err - } - return fs.DecompressFileUnsafe(ctx, dir, source) -} - -// DecompressFileUnsafe will decompress any file on the local disk without checking -// if it is owned by the server. The file will be SAFELY decompressed and extracted -// into the server's directory. -func (fs *Filesystem) DecompressFileUnsafe(ctx context.Context, dir string, file string) error { - // Ensure that the archive actually exists on the system. - if _, err := os.Stat(file); err != nil { - return errors.WithStack(err) - } - - f, err := os.Open(file) - if err != nil { - return err - } - defer f.Close() - - // Identify the type of archive we are dealing with. - format, input, err := archiver.Identify(filepath.Base(file), f) - if err != nil { - if errors.Is(err, archiver.ErrNoMatch) { - return newFilesystemError(ErrCodeUnknownArchive, err) - } - return err - } - - return fs.extractStream(ctx, extractStreamOptions{ - Directory: dir, - Format: format, - Reader: input, - }) + return errors.New("server/fs: not implemented") } // ExtractStreamUnsafe . func (fs *Filesystem) ExtractStreamUnsafe(ctx context.Context, dir string, r io.Reader) error { - format, input, err := archiver.Identify("archive.tar.gz", r) - if err != nil { - if errors.Is(err, archiver.ErrNoMatch) { - return newFilesystemError(ErrCodeUnknownArchive, err) - } - return err - } - - return fs.extractStream(ctx, extractStreamOptions{ - Directory: dir, - Format: format, - Reader: input, - }) -} - -type extractStreamOptions struct { - // The directory to extract the archive to. - Directory string - // File name of the archive. - FileName string - // Format of the archive. - Format archiver.Format - // Reader for the archive. - Reader io.Reader -} - -func (fs *Filesystem) extractStream(ctx context.Context, opts extractStreamOptions) error { - // Decompress and extract archive - if ex, ok := opts.Format.(archiver.Extractor); ok { - return ex.Extract(ctx, opts.Reader, nil, func(ctx context.Context, f archiver.File) error { - if f.IsDir() { - return nil - } - p := filepath.Join(opts.Directory, f.NameInArchive) - // If it is ignored, just don't do anything with the file and skip over it. - if err := fs.IsIgnored(p); err != nil { - return nil - } - r, err := f.Open() - if err != nil { - return err - } - defer r.Close() - if err := fs.Writefile(p, r); err != nil { - return wrapError(err, opts.FileName) - } - // Update the file permissions to the one set in the archive. - if err := fs.Chmod(p, f.Mode()); err != nil { - return wrapError(err, opts.FileName) - } - // Update the file modification time to the one set in the archive. - if err := fs.Chtimes(p, f.ModTime(), f.ModTime()); err != nil { - return wrapError(err, opts.FileName) - } - return nil - }) - } - return nil + return errors.New("server/fs: not implemented") } diff --git a/server/filesystem/disk_space.go b/server/filesystem/disk_space.go index 95f82b8d6..b1476ff2e 100644 --- a/server/filesystem/disk_space.go +++ b/server/filesystem/disk_space.go @@ -165,11 +165,14 @@ func (fs *Filesystem) DirectorySize(dir string) (int64, error) { var size int64 var st syscall.Stat_t - var hardLinks []uint64 err = godirwalk.Walk(d, &godirwalk.Options{ Unsorted: true, Callback: func(p string, e *godirwalk.Dirent) error { + if !e.ModeType().IsRegular() { + return nil + } + // If this is a symlink then resolve the final destination of it before trying to continue walking // over its contents. If it resolves outside the server data directory just skip everything else for // it. Otherwise, allow it to continue. @@ -188,18 +191,8 @@ func (fs *Filesystem) DirectorySize(dir string) (int64, error) { atomic.AddInt64(&size, st.Size) } - var sysFileInfo = info.Sys().(*unix.Stat_t) - if sysFileInfo.Nlink > 1 { - // Hard links have the same inode number - if slices.Contains(hardLinks, sysFileInfo.Ino) { - // Don't add hard links size twice - return nil - } else { - hardLinks = append(hardLinks, sysFileInfo.Ino) - } - } + // todo: don't count hardlinks twice - size.Add(info.Size()) return nil }, }) From a30003f4fd84731d38129171074afc20937cba49 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Tue, 3 Feb 2026 18:33:59 -0800 Subject: [PATCH 04/30] require latest go for better root support --- go.mod | 4 +- server/filesystem/chmod.go | 22 ++++ server/filesystem/chown.go | 60 +++++++++ server/filesystem/chtimes.go | 22 ++++ server/filesystem/filesystem.go | 174 ++++++--------------------- server/filesystem/filesystem_test.go | 5 +- server/filesystem/stat.go | 10 +- server/manager.go | 7 +- 8 files changed, 155 insertions(+), 149 deletions(-) create mode 100644 server/filesystem/chmod.go create mode 100644 server/filesystem/chown.go create mode 100644 server/filesystem/chtimes.go diff --git a/go.mod b/go.mod index 03f2adff3..4a4fa8c44 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/pterodactyl/wings -go 1.24.0 - -toolchain go1.24.1 +go 1.25.6 require ( emperror.dev/errors v0.8.1 diff --git a/server/filesystem/chmod.go b/server/filesystem/chmod.go new file mode 100644 index 000000000..fe3089c5e --- /dev/null +++ b/server/filesystem/chmod.go @@ -0,0 +1,22 @@ +package filesystem + +import "os" + +// todo: vulnerable to race condition with symlinks +// see: https://pkg.go.dev/os#Root +func (fs *Filesystem) Chmod(path string, mode os.FileMode) error { + cleaned, err := fs.SafePath(path) + if err != nil { + return err + } + + if fs.isTest { + return nil + } + + if err := fs.root.Chmod(cleaned, mode); err != nil { + return err + } + + return nil +} diff --git a/server/filesystem/chown.go b/server/filesystem/chown.go new file mode 100644 index 000000000..f7fe5ecfe --- /dev/null +++ b/server/filesystem/chown.go @@ -0,0 +1,60 @@ +package filesystem + +import ( + "emperror.dev/errors" + "github.com/karrick/godirwalk" + "github.com/pterodactyl/wings/config" +) + +// Recursively iterates over a file or directory and sets the permissions on all of the +// underlying files. Iterate over all of the files and directories. If it is a file just +// go ahead and perform the chown operation. Otherwise dig deeper into the directory until +// we've run out of directories to dig into. +// todo: vulnerable to race condition with symlinks +// see: https://pkg.go.dev/os#Root +func (fs *Filesystem) Chown(path string) error { + return fs.unsafeChown(path) +} + +// unsafeChown chowns the given path, without checking if the path is safe. This should only be used +// when the path has already been checked. +func (fs *Filesystem) unsafeChown(path string) error { + if fs.isTest { + return nil + } + + uid := config.Get().System.User.Uid + gid := config.Get().System.User.Gid + + // Start by just chowning the initial path that we received. + if err := fs.root.Chown(path, uid, gid); err != nil { + return errors.Wrap(err, "server/filesystem: chown: failed to chown path") + } + + // If this is not a directory we can now return from the function, there is nothing + // left that we need to do. + if st, err := fs.root.Stat(path); err != nil || !st.IsDir() { + return nil + } + + // If this was a directory, begin walking over its contents recursively and ensure that all + // of the subfiles and directories get their permissions updated as well. + err := godirwalk.Walk(path, &godirwalk.Options{ + Unsorted: true, + Callback: func(p string, e *godirwalk.Dirent) error { + // Do not attempt to chown a symlink. Go's os.Chown function will affect the symlink + // so if it points to a location outside the data directory the user would be able to + // (un)intentionally modify that files permissions. + if e.IsSymlink() { + if e.IsDir() { + return godirwalk.SkipThis + } + + return nil + } + + return fs.root.Chown(p, uid, gid) + }, + }) + return errors.Wrap(err, "server/filesystem: chown: failed to chown during walk function") +} diff --git a/server/filesystem/chtimes.go b/server/filesystem/chtimes.go new file mode 100644 index 000000000..b9e295679 --- /dev/null +++ b/server/filesystem/chtimes.go @@ -0,0 +1,22 @@ +package filesystem + +import "time" + +// todo: vulnerable to race condition with symlinks +// see: https://pkg.go.dev/os#Root +func (fs *Filesystem) Chtimes(path string, atime, mtime time.Time) error { + cleaned, err := fs.SafePath(path) + if err != nil { + return err + } + + if fs.isTest { + return nil + } + + if err := fs.root.Chtimes(cleaned, atime, mtime); err != nil { + return err + } + + return nil +} diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index ccc2daa4f..a79416b66 100644 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -2,6 +2,7 @@ package filesystem import ( "bufio" + "fmt" "io" "io/ioutil" "os" @@ -16,7 +17,6 @@ import ( "emperror.dev/errors" "github.com/gabriel-vasile/mimetype" - "github.com/karrick/godirwalk" ignore "github.com/sabhiram/go-gitignore" "github.com/pterodactyl/wings/config" @@ -35,35 +35,40 @@ type Filesystem struct { diskLimit int64 // The root data directory path for this Filesystem instance. - root string + root *os.Root + rootPath string isTest bool } // New creates a new Filesystem instance for a given server. -func New(root string, size int64, denylist []string) *Filesystem { - return &Filesystem{ - root: root, +func New(path string, size int64, denylist []string) (*Filesystem, error) { + r, err := os.OpenRoot(path) + if err != nil { + return nil, errors.WithStack(err) + } + + fs := &Filesystem{ + root: r, + rootPath: path, diskLimit: size, diskCheckInterval: time.Duration(config.Get().System.DiskCheckInterval), lastLookupTime: &usageLookupTime{}, lookupInProgress: system.NewAtomicBool(false), denylist: ignore.CompileIgnoreLines(denylist...), } + + return fs, nil } // Path returns the root path for the Filesystem instance. func (fs *Filesystem) Path() string { - return fs.root + return fs.rootPath } // File returns a reader for a file instance as well as the stat information. func (fs *Filesystem) File(p string) (*os.File, Stat, error) { - cleaned, err := fs.SafePath(p) - if err != nil { - return nil, Stat{}, errors.WithStackIf(err) - } - st, err := fs.Stat(cleaned) + st, err := fs.Stat(p) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, Stat{}, newFilesystemError(ErrNotExist, err) @@ -73,7 +78,7 @@ func (fs *Filesystem) File(p string) (*os.File, Stat, error) { if st.IsDir() { return nil, Stat{}, newFilesystemError(ErrCodeIsDirectory, nil) } - f, err := os.Open(cleaned) + f, err := fs.root.Open(p) if err != nil { return nil, Stat{}, errors.WithStackIf(err) } @@ -84,11 +89,7 @@ func (fs *Filesystem) File(p string) (*os.File, Stat, error) { // already. If it is present, the file is opened using the defaults which will truncate // the contents. The opened file is then returned to the caller. func (fs *Filesystem) Touch(p string, flag int) (*os.File, error) { - cleaned, err := fs.SafePath(p) - if err != nil { - return nil, err - } - f, err := os.OpenFile(cleaned, flag, 0o644) + f, err := fs.root.OpenFile(p, flag, 0o644) if err == nil { return f, nil } @@ -100,24 +101,24 @@ func (fs *Filesystem) Touch(p string, flag int) (*os.File, error) { return nil, errors.Wrap(err, "server/filesystem: touch: failed to open file handle") } // Only create and chown the directory if it doesn't exist. - if _, err := os.Stat(filepath.Dir(cleaned)); errors.Is(err, os.ErrNotExist) { + if _, err := fs.root.Stat(filepath.Dir(p)); errors.Is(err, os.ErrNotExist) { // Create the path leading up to the file we're trying to create, setting the final perms // on it as we go. - if err := os.MkdirAll(filepath.Dir(cleaned), 0o755); err != nil { + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { return nil, errors.Wrap(err, "server/filesystem: touch: failed to create directory tree") } - if err := fs.Chown(filepath.Dir(cleaned)); err != nil { + if err := fs.Chown(filepath.Dir(p)); err != nil { return nil, err } } - o := &fileOpener{} + o := &fileOpener{root: fs.root} // Try to open the file now that we have created the pathing necessary for it, and then // Chown that file so that the permissions don't mess with things. - f, err = o.open(cleaned, flag, 0o644) + f, err = o.open(p, flag, 0o644) if err != nil { return nil, errors.Wrap(err, "server/filesystem: touch: failed to open file with wait") } - _ = fs.Chown(cleaned) + _ = fs.Chown(p) return f, nil } @@ -125,20 +126,16 @@ func (fs *Filesystem) Touch(p string, flag int) (*os.File, error) { // will be created. This will also properly recalculate the disk space used by // the server when writing new files or modifying existing ones. func (fs *Filesystem) Writefile(p string, r io.Reader) error { - cleaned, err := fs.SafePath(p) - if err != nil { - return err - } - var currentSize int64 // If the file does not exist on the system already go ahead and create the pathway // to it and an empty file. We'll then write to it later on after this completes. - stat, err := os.Stat(cleaned) + fmt.Println(p) + stat, err := fs.root.Stat(p) if err != nil && !os.IsNotExist(err) { return errors.Wrap(err, "server/filesystem: writefile: failed to stat file") } else if err == nil { if stat.IsDir() { - return errors.WithStack(&Error{code: ErrCodeIsDirectory, resolved: cleaned}) + return errors.WithStack(&Error{code: ErrCodeIsDirectory, resolved: stat.Name()}) } currentSize = stat.Size() } @@ -153,7 +150,7 @@ func (fs *Filesystem) Writefile(p string, r io.Reader) error { // Touch the file and return the handle to it at this point. This will create the file, // any necessary directories, and set the proper owner of the file. - file, err := fs.Touch(cleaned, os.O_RDWR|os.O_CREATE|os.O_TRUNC) + file, err := fs.Touch(p, os.O_RDWR|os.O_CREATE|os.O_TRUNC) if err != nil { return err } @@ -165,7 +162,7 @@ func (fs *Filesystem) Writefile(p string, r io.Reader) error { // Adjust the disk usage to account for the old size and the new size of the file. fs.addDisk(sz - currentSize) - return fs.unsafeChown(cleaned) + return fs.unsafeChown(p) } // Creates a new directory (name) at a specified path (p) for the server. @@ -214,78 +211,6 @@ func (fs *Filesystem) Rename(from string, to string) error { return nil } -// Recursively iterates over a file or directory and sets the permissions on all of the -// underlying files. Iterate over all of the files and directories. If it is a file just -// go ahead and perform the chown operation. Otherwise dig deeper into the directory until -// we've run out of directories to dig into. -func (fs *Filesystem) Chown(path string) error { - cleaned, err := fs.SafePath(path) - if err != nil { - return err - } - return fs.unsafeChown(cleaned) -} - -// unsafeChown chowns the given path, without checking if the path is safe. This should only be used -// when the path has already been checked. -func (fs *Filesystem) unsafeChown(path string) error { - if fs.isTest { - return nil - } - - uid := config.Get().System.User.Uid - gid := config.Get().System.User.Gid - - // Start by just chowning the initial path that we received. - if err := os.Chown(path, uid, gid); err != nil { - return errors.Wrap(err, "server/filesystem: chown: failed to chown path") - } - - // If this is not a directory we can now return from the function, there is nothing - // left that we need to do. - if st, err := os.Stat(path); err != nil || !st.IsDir() { - return nil - } - - // If this was a directory, begin walking over its contents recursively and ensure that all - // of the subfiles and directories get their permissions updated as well. - err := godirwalk.Walk(path, &godirwalk.Options{ - Unsorted: true, - Callback: func(p string, e *godirwalk.Dirent) error { - // Do not attempt to chown a symlink. Go's os.Chown function will affect the symlink - // so if it points to a location outside the data directory the user would be able to - // (un)intentionally modify that files permissions. - if e.IsSymlink() { - if e.IsDir() { - return godirwalk.SkipThis - } - - return nil - } - - return os.Chown(p, uid, gid) - }, - }) - return errors.Wrap(err, "server/filesystem: chown: failed to chown during walk function") -} - -func (fs *Filesystem) Chmod(path string, mode os.FileMode) error { - cleaned, err := fs.SafePath(path) - if err != nil { - return err - } - - if fs.isTest { - return nil - } - - if err := os.Chmod(cleaned, mode); err != nil { - return err - } - - return nil -} - // Begin looping up to 50 times to try and create a unique copy file name. This will take // an input of "file.txt" and generate "file copy.txt". If that name is already taken, it will // then try to write "file copy 2.txt" and so on, until reaching 50 loops. At that point we @@ -325,12 +250,7 @@ func (fs *Filesystem) findCopySuffix(dir string, name string, extension string) // Copies a given file to the same location and appends a suffix to the file to indicate that // it has been copied. func (fs *Filesystem) Copy(p string) error { - cleaned, err := fs.SafePath(p) - if err != nil { - return err - } - - s, err := os.Stat(cleaned) + s, err := fs.root.Stat(p) if err != nil { return err } else if s.IsDir() || !s.Mode().IsRegular() { @@ -344,8 +264,8 @@ func (fs *Filesystem) Copy(p string) error { return err } - base := filepath.Base(cleaned) - relative := strings.TrimSuffix(strings.TrimPrefix(cleaned, fs.Path()), base) + base := filepath.Base(p) + relative := strings.TrimSuffix(strings.TrimPrefix(p, fs.Path()), base) extension := filepath.Ext(base) name := strings.TrimSuffix(base, extension) @@ -357,7 +277,7 @@ func (fs *Filesystem) Copy(p string) error { name = strings.TrimSuffix(name, ".tar") } - source, err := os.Open(cleaned) + source, err := fs.root.Open(p) if err != nil { return err } @@ -374,10 +294,10 @@ func (fs *Filesystem) Copy(p string) error { // TruncateRootDirectory removes _all_ files and directories from a server's // data directory and resets the used disk space to zero. func (fs *Filesystem) TruncateRootDirectory() error { - if err := os.RemoveAll(fs.Path()); err != nil { + if err := fs.root.RemoveAll("/"); err != nil { return err } - if err := os.Mkdir(fs.Path(), 0o755); err != nil { + if err := fs.root.Mkdir("/", 0o755); err != nil { return err } atomic.StoreInt64(&fs.diskUsed, 0) @@ -406,7 +326,7 @@ func (fs *Filesystem) Delete(p string) error { return errors.New("cannot delete root server directory") } - st, err := os.Lstat(resolved) + st, err := fs.root.Lstat(resolved) if err != nil { if !os.IsNotExist(err) { fs.error(err).Warn("error while attempting to stat file before deletion") @@ -466,11 +386,12 @@ func (fs *Filesystem) Delete(p string) error { fs.addDisk(-st.Size()) } - return os.RemoveAll(resolved) + return fs.root.RemoveAll(resolved) } type fileOpener struct { busy uint + root *os.Root } // Attempts to open a given file up to "attempts" number of times, using a backoff. If the file @@ -478,7 +399,7 @@ type fileOpener struct { // has been exhaused, at which point we will abort with an error. func (fo *fileOpener) open(path string, flags int, perm os.FileMode) (*os.File, error) { for { - f, err := os.OpenFile(path, flags, perm) + f, err := fo.root.OpenFile(path, flags, perm) // If there is an error because the text file is busy, go ahead and sleep for a few // hundred milliseconds and then try again up to three times before just returning the @@ -571,20 +492,3 @@ func (fs *Filesystem) ListDirectory(p string) ([]Stat, error) { return out, nil } - -func (fs *Filesystem) Chtimes(path string, atime, mtime time.Time) error { - cleaned, err := fs.SafePath(path) - if err != nil { - return err - } - - if fs.isTest { - return nil - } - - if err := os.Chtimes(cleaned, atime, mtime); err != nil { - return err - } - - return nil -} diff --git a/server/filesystem/filesystem_test.go b/server/filesystem/filesystem_test.go index 23f43bfe1..6c0a46344 100644 --- a/server/filesystem/filesystem_test.go +++ b/server/filesystem/filesystem_test.go @@ -35,7 +35,10 @@ func NewFs() (*Filesystem, *rootFs) { rfs.reset() - fs := New(filepath.Join(tmpDir, "/server"), 0, []string{}) + fs, err := New(filepath.Join(tmpDir, "/server"), 0, []string{}) + if err != nil { + panic(err) + } fs.isTest = true return fs, &rfs diff --git a/server/filesystem/stat.go b/server/filesystem/stat.go index ce52b6906..6186a21b5 100644 --- a/server/filesystem/stat.go +++ b/server/filesystem/stat.go @@ -44,15 +44,7 @@ func (s *Stat) MarshalJSON() ([]byte, error) { // Stat stats a file or folder and returns the base stat object from go along // with the MIME data that can be used for editing files. func (fs *Filesystem) Stat(p string) (Stat, error) { - cleaned, err := fs.SafePath(p) - if err != nil { - return Stat{}, err - } - return fs.unsafeStat(cleaned) -} - -func (fs *Filesystem) unsafeStat(p string) (Stat, error) { - s, err := os.Stat(p) + s, err := fs.root.Stat(p) if err != nil { return Stat{}, err } diff --git a/server/manager.go b/server/manager.go index 9c3de8f62..a834399d7 100644 --- a/server/manager.go +++ b/server/manager.go @@ -196,7 +196,12 @@ func (m *Manager) InitServer(data remote.ServerConfigurationResponse) (*Server, return nil, errors.WithStackIf(err) } - s.fs = filesystem.New(filepath.Join(config.Get().System.Data, s.ID()), s.DiskSpace(), s.Config().Egg.FileDenylist) + fs, err := filesystem.New(filepath.Join(config.Get().System.Data, s.ID()), s.DiskSpace(), s.Config().Egg.FileDenylist) + if err != nil { + return nil, errors.WithStack(err) + } + + s.fs = fs // Right now we only support a Docker based environment, so I'm going to hard code // this logic in. When we're ready to support other environment we'll need to make From e70f2f9602499945498e236ec8d8c6410131a9c3 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Fri, 6 Feb 2026 18:23:33 -0800 Subject: [PATCH 05/30] get path tests back in working order --- server/filesystem/chown.go | 22 ++----- server/filesystem/disk_space.go | 15 +++-- server/filesystem/errors.go | 10 +++ server/filesystem/filesystem.go | 113 +++++--------------------------- server/filesystem/path.go | 2 + server/filesystem/path_test.go | 104 +++++------------------------ 6 files changed, 59 insertions(+), 207 deletions(-) diff --git a/server/filesystem/chown.go b/server/filesystem/chown.go index f7fe5ecfe..c67476ba6 100644 --- a/server/filesystem/chown.go +++ b/server/filesystem/chown.go @@ -6,23 +6,11 @@ import ( "github.com/pterodactyl/wings/config" ) -// Recursively iterates over a file or directory and sets the permissions on all of the -// underlying files. Iterate over all of the files and directories. If it is a file just -// go ahead and perform the chown operation. Otherwise dig deeper into the directory until -// we've run out of directories to dig into. -// todo: vulnerable to race condition with symlinks -// see: https://pkg.go.dev/os#Root +// Chown recursively iterates over a file or directory and sets the permissions on all the +// underlying files. Iterate over all the files and directories. If it is a file go ahead +// and perform the chown operation. Otherwise dig deeper into the directory until we've run +// out of directories to dig into. func (fs *Filesystem) Chown(path string) error { - return fs.unsafeChown(path) -} - -// unsafeChown chowns the given path, without checking if the path is safe. This should only be used -// when the path has already been checked. -func (fs *Filesystem) unsafeChown(path string) error { - if fs.isTest { - return nil - } - uid := config.Get().System.User.Uid gid := config.Get().System.User.Gid @@ -38,7 +26,7 @@ func (fs *Filesystem) unsafeChown(path string) error { } // If this was a directory, begin walking over its contents recursively and ensure that all - // of the subfiles and directories get their permissions updated as well. + // the subfiles and directories get their permissions updated as well. err := godirwalk.Walk(path, &godirwalk.Options{ Unsorted: true, Callback: func(p string, e *godirwalk.Dirent) error { diff --git a/server/filesystem/disk_space.go b/server/filesystem/disk_space.go index b1476ff2e..ac862a5bc 100644 --- a/server/filesystem/disk_space.go +++ b/server/filesystem/disk_space.go @@ -158,16 +158,17 @@ func (fs *Filesystem) updateCachedDiskUsage() (int64, error) { // through all of the folders. Returns the size in bytes. This can be a fairly taxing operation // on locations with tons of files, so it is recommended that you cache the output. func (fs *Filesystem) DirectorySize(dir string) (int64, error) { - d, err := fs.SafePath(dir) - if err != nil { - return 0, err - } - var size int64 var st syscall.Stat_t - err = godirwalk.Walk(d, &godirwalk.Options{ - Unsorted: true, + // todo: safely traverse directory within root? + if _, err := fs.root.Lstat(dir); err != nil { + return 0, err + } + + err := godirwalk.Walk(dir, &godirwalk.Options{ + Unsorted: true, + FollowSymbolicLinks: false, Callback: func(p string, e *godirwalk.Dirent) error { if !e.ModeType().IsRegular() { return nil diff --git a/server/filesystem/errors.go b/server/filesystem/errors.go index afae74adc..854cc34b9 100644 --- a/server/filesystem/errors.go +++ b/server/filesystem/errors.go @@ -138,3 +138,13 @@ func wrapError(err error, resolved string) error { } return errors.WithStackDepth(&Error{code: ErrCodeUnknownError, err: err, resolved: resolved}, 1) } + +func IsPathError(err error) bool { + var pe *os.PathError + return errors.As(err, &pe) +} + +func IsLinkError(err error) bool { + var le *os.LinkError + return errors.As(err, &le) +} diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index a79416b66..5b241a5ea 100644 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -162,53 +162,39 @@ func (fs *Filesystem) Writefile(p string, r io.Reader) error { // Adjust the disk usage to account for the old size and the new size of the file. fs.addDisk(sz - currentSize) - return fs.unsafeChown(p) + return fs.Chown(p) } -// Creates a new directory (name) at a specified path (p) for the server. +// CreateDirectory creates a new directory ("name") at a specified path ("p") for the server. func (fs *Filesystem) CreateDirectory(name string, p string) error { - cleaned, err := fs.SafePath(path.Join(p, name)) - if err != nil { - return err - } - return os.MkdirAll(cleaned, 0o755) + fmt.Println(path.Join(p, name)) + return fs.root.MkdirAll(path.Join(p, name), 0o755) } // Rename moves (or renames) a file or directory. func (fs *Filesystem) Rename(from string, to string) error { - cleanedFrom, err := fs.SafePath(from) - if err != nil { - return errors.WithStack(err) - } + cleanedTo := path.Clean(to) - cleanedTo, err := fs.SafePath(to) - if err != nil { - return errors.WithStack(err) - } - - // If the target file or directory already exists the rename function will fail, so just - // bail out now. - if _, err := os.Stat(cleanedTo); err == nil { + // If the target file or directory already exists the rename function will + // fail, so just bail out now. + if _, err := fs.root.Stat(cleanedTo); err == nil { return os.ErrExist } if cleanedTo == fs.Path() { - return errors.New("attempting to rename into an invalid directory space") + return errors.New("server/filesystem: attempting to rename into an invalid directory space") } d := strings.TrimSuffix(cleanedTo, path.Base(cleanedTo)) // Ensure that the directory we're moving into exists correctly on the system. Only do this if // we're not at the root directory level. if d != fs.Path() { - if mkerr := os.MkdirAll(d, 0o755); mkerr != nil { - return errors.WithMessage(mkerr, "failed to create directory structure for file rename") + if err := fs.root.MkdirAll(d, 0o755); err != nil { + return errors.Wrap(err, "server/filesystem: failed to create directory tree") } } - if err := os.Rename(cleanedFrom, cleanedTo); err != nil { - return errors.WithStack(err) - } - return nil + return fs.root.Rename(from, cleanedTo) } // Begin looping up to 50 times to try and create a unique copy file name. This will take @@ -307,86 +293,23 @@ func (fs *Filesystem) TruncateRootDirectory() error { // Delete removes a file or folder from the system. Prevents the user from // accidentally (or maliciously) removing their root server data directory. func (fs *Filesystem) Delete(p string) error { - // This is one of the few (only?) places in the codebase where we're explicitly not using - // the SafePath functionality when working with user provided input. If we did, you would - // not be able to delete a file that is a symlink pointing to a location outside the data - // directory. - // - // We also want to avoid resolving a symlink that points _within_ the data directory and thus - // deleting the actual source file for the symlink rather than the symlink itself. For these - // purposes just resolve the actual file path using filepath.Join() and confirm that the path - // exists within the data directory. - resolved := fs.unsafeFilePath(p) - if !fs.unsafeIsInDataDirectory(resolved) { - return NewBadPathResolution(p, resolved) - } - - // Block any whoopsies. - if resolved == fs.Path() { - return errors.New("cannot delete root server directory") - } - - st, err := fs.root.Lstat(resolved) + st, err := fs.root.Lstat(p) if err != nil { - if !os.IsNotExist(err) { - fs.error(err).Warn("error while attempting to stat file before deletion") - return err - } - - // The following logic is used to handle a case where a user attempts to - // delete a file that does not exist through a directory symlink. - // We don't want to reveal that the file does not exist, so we validate - // the path of the symlink and return a bad path error if it is invalid. - - // The requested file or directory doesn't exist, so at this point we - // need to iterate up the path chain until we hit a directory that - // _does_ exist and can be validated. - parts := strings.Split(filepath.Dir(resolved), "/") - - // Range over all the path parts and form directory paths from the end - // moving up until we have a valid resolution, or we run out of paths to - // try. - for k := range parts { - try := strings.Join(parts[:(len(parts)-k)], "/") - if !fs.unsafeIsInDataDirectory(try) { - break - } - - t, err := filepath.EvalSymlinks(try) - if err == nil { - if !fs.unsafeIsInDataDirectory(t) { - return NewBadPathResolution(p, t) - } - break - } - } - - // Always return early if the file does not exist. - return nil - } - - // If the file is not a symlink, we need to check that it is not within a - // symlinked directory that points outside the data directory. - if st.Mode()&os.ModeSymlink == 0 { - ep, err := filepath.EvalSymlinks(resolved) - if err != nil { - if !os.IsNotExist(err) { - return err - } - } else if !fs.unsafeIsInDataDirectory(ep) { - return NewBadPathResolution(p, ep) + if os.IsNotExist(err) { + return nil } + return errors.Wrap(err, "server/filesystem: failed to stat file") } if st.IsDir() { - if s, err := fs.DirectorySize(resolved); err == nil { + if s, err := fs.DirectorySize(p); err == nil { fs.addDisk(-s) } } else { fs.addDisk(-st.Size()) } - return fs.root.RemoveAll(resolved) + return fs.root.RemoveAll(p) } type fileOpener struct { diff --git a/server/filesystem/path.go b/server/filesystem/path.go index 3952e5d2e..227f8f3c8 100644 --- a/server/filesystem/path.go +++ b/server/filesystem/path.go @@ -33,6 +33,8 @@ func (fs *Filesystem) IsIgnored(paths ...string) error { // // This logic is actually copied over from the SFTP server code. Ideally that eventually // either gets ported into this application, or is able to make use of this package. +// +// deprecated func (fs *Filesystem) SafePath(p string) (string, error) { // Start with a cleaned up path before checking the more complex bits. r := fs.unsafeFilePath(p) diff --git a/server/filesystem/path_test.go b/server/filesystem/path_test.go index ecb96279b..972b2e2e9 100644 --- a/server/filesystem/path_test.go +++ b/server/filesystem/path_test.go @@ -21,80 +21,6 @@ func TestFilesystem_Path(t *testing.T) { }) } -func TestFilesystem_SafePath(t *testing.T) { - g := Goblin(t) - fs, rfs := NewFs() - prefix := filepath.Join(rfs.root, "/server") - - g.Describe("SafePath", func() { - g.It("returns a cleaned path to a given file", func() { - p, err := fs.SafePath("test.txt") - g.Assert(err).IsNil() - g.Assert(p).Equal(prefix + "/test.txt") - - p, err = fs.SafePath("/test.txt") - g.Assert(err).IsNil() - g.Assert(p).Equal(prefix + "/test.txt") - - p, err = fs.SafePath("./test.txt") - g.Assert(err).IsNil() - g.Assert(p).Equal(prefix + "/test.txt") - - p, err = fs.SafePath("/foo/../test.txt") - g.Assert(err).IsNil() - g.Assert(p).Equal(prefix + "/test.txt") - - p, err = fs.SafePath("/foo/bar") - g.Assert(err).IsNil() - g.Assert(p).Equal(prefix + "/foo/bar") - }) - - g.It("handles root directory access", func() { - p, err := fs.SafePath("/") - g.Assert(err).IsNil() - g.Assert(p).Equal(prefix) - - p, err = fs.SafePath("") - g.Assert(err).IsNil() - g.Assert(p).Equal(prefix) - }) - - g.It("removes trailing slashes from paths", func() { - p, err := fs.SafePath("/foo/bar/") - g.Assert(err).IsNil() - g.Assert(p).Equal(prefix + "/foo/bar") - }) - - g.It("handles deeply nested directories that do not exist", func() { - p, err := fs.SafePath("/foo/bar/baz/quaz/../../ducks/testing.txt") - g.Assert(err).IsNil() - g.Assert(p).Equal(prefix + "/foo/bar/ducks/testing.txt") - }) - - g.It("blocks access to files outside the root directory", func() { - p, err := fs.SafePath("../test.txt") - g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() - g.Assert(p).Equal("") - - p, err = fs.SafePath("/../test.txt") - g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() - g.Assert(p).Equal("") - - p, err = fs.SafePath("./foo/../../test.txt") - g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() - g.Assert(p).Equal("") - - p, err = fs.SafePath("..") - g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() - g.Assert(p).Equal("") - }) - }) -} - // We test against accessing files outside the root directory in the tests, however it // is still possible for someone to mess up and not properly use this safe path call. In // order to truly confirm this, we'll try to pass in a symlinked malicious file to all of @@ -133,7 +59,7 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { err := fs.Writefile("symlinked.txt", r) g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) g.It("cannot write to a non-existent file symlinked outside the root", func() { @@ -141,7 +67,7 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { err := fs.Writefile("symlinked_does_not_exist.txt", r) g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) g.It("cannot write to chained symlinks with target that does not exist outside the root", func() { @@ -149,7 +75,7 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { err := fs.Writefile("symlinked_does_not_exist2.txt", r) g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) g.It("cannot write a file to a directory symlinked outside the root", func() { @@ -157,7 +83,7 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { err := fs.Writefile("external_dir/foo.txt", r) g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) }) @@ -165,19 +91,19 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { g.It("cannot create a directory outside the root", func() { err := fs.CreateDirectory("my_dir", "external_dir") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) g.It("cannot create a nested directory outside the root", func() { err := fs.CreateDirectory("my/nested/dir", "external_dir/foo/bar") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) g.It("cannot create a nested directory outside the root", func() { err := fs.CreateDirectory("my/nested/dir", "external_dir/server") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) }) @@ -185,21 +111,23 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { g.It("cannot rename a file symlinked outside the directory root", func() { err := fs.Rename("symlinked.txt", "foo.txt") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) g.It("cannot rename a symlinked directory outside the root", func() { err := fs.Rename("external_dir", "foo") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) g.It("cannot rename a file to a location outside the directory root", func() { - rfs.CreateServerFileFromString("my_file.txt", "internal content") + if err := rfs.CreateServerFileFromString("my_file.txt", "internal content"); err != nil { + panic(err) + } err := fs.Rename("my_file.txt", "external_dir/my_file.txt") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsLinkError(err)).IsTrue() }) }) @@ -207,13 +135,13 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { g.It("cannot chown a file symlinked outside the directory root", func() { err := fs.Chown("symlinked.txt") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) g.It("cannot chown a directory symlinked outside the directory root", func() { err := fs.Chown("external_dir") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) }) @@ -221,7 +149,7 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { g.It("cannot copy a file symlinked outside the directory root", func() { err := fs.Copy("symlinked.txt") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) }) From 0e0acb411c01d6ce9fc801d1a178e95ec2eb29ae Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Sat, 7 Feb 2026 10:31:31 -0800 Subject: [PATCH 06/30] Update core logic to account for how paths are handled in os.Root --- server/filesystem/chmod.go | 67 ++++++++++++++++++++++++++++----- server/filesystem/chown.go | 48 ----------------------- server/filesystem/chtimes.go | 22 ----------- server/filesystem/disk_space.go | 13 ------- server/filesystem/errors.go | 0 server/filesystem/filesystem.go | 48 +++++++++++------------ server/filesystem/path.go | 0 server/filesystem/stat.go | 15 ++++++-- 8 files changed, 93 insertions(+), 120 deletions(-) mode change 100644 => 100755 server/filesystem/chmod.go delete mode 100644 server/filesystem/chown.go delete mode 100644 server/filesystem/chtimes.go mode change 100644 => 100755 server/filesystem/errors.go mode change 100644 => 100755 server/filesystem/filesystem.go mode change 100644 => 100755 server/filesystem/path.go mode change 100644 => 100755 server/filesystem/stat.go diff --git a/server/filesystem/chmod.go b/server/filesystem/chmod.go old mode 100644 new mode 100755 index fe3089c5e..3b5fbb04a --- a/server/filesystem/chmod.go +++ b/server/filesystem/chmod.go @@ -1,21 +1,68 @@ package filesystem -import "os" +import ( + "os" + "path/filepath" + "strings" + "time" + + "emperror.dev/errors" + "github.com/karrick/godirwalk" + "github.com/pterodactyl/wings/config" +) -// todo: vulnerable to race condition with symlinks -// see: https://pkg.go.dev/os#Root func (fs *Filesystem) Chmod(path string, mode os.FileMode) error { - cleaned, err := fs.SafePath(path) - if err != nil { - return err + path = strings.TrimLeft(filepath.Clean(path), "/") + if err := fs.root.Chmod(path, mode); err != nil { + return errors.Wrap(err, "server/filesystem: chmod: failed to chmod path") + } + + return nil +} + +// Chown recursively iterates over a file or directory and sets the permissions on all the +// underlying files. Iterate over all the files and directories. If it is a file go ahead +// and perform the chown operation. Otherwise dig deeper into the directory until we've run +// out of directories to dig into. +// +// todo: insecure function due to walking on uncontrolled path +func (fs *Filesystem) Chown(path string) error { + path = strings.TrimLeft(filepath.Clean(path), "/") + + uid := config.Get().System.User.Uid + gid := config.Get().System.User.Gid + + // Start by just chowning the initial path that we received. + if err := fs.root.Chown(path, uid, gid); err != nil { + return errors.Wrap(err, "server/filesystem: chown: failed to chown path") } - if fs.isTest { - return nil + // If this is not a directory, we can now return from the function; there is nothing + // left that we need to do. + if st, err := fs.root.Stat(path); err != nil || !st.IsDir() { + if err == nil || errors.Is(err, os.ErrNotExist) { + return nil + } + + return errors.Wrap(err, "server/filesystem: chown: failed to stat path") } - if err := fs.root.Chmod(cleaned, mode); err != nil { - return err + // If this was a directory, begin walking over its contents recursively and ensure that all + // the subfiles and directories get their permissions updated as well. + err := godirwalk.Walk(filepath.Join(fs.rootPath, path), &godirwalk.Options{ + Unsorted: true, + Callback: func(p string, e *godirwalk.Dirent) error { + return fs.root.Chown(p, uid, gid) + }, + }) + + return errors.Wrap(err, "server/filesystem: chown: failed to chown during walk function") +} + +func (fs *Filesystem) Chtimes(path string, atime, mtime time.Time) error { + path = strings.TrimLeft(filepath.Clean(path), "/") + if err := fs.root.Chtimes(path, atime, mtime); err != nil { + return errors.Wrap(err, "server/filesystem: chtimes: failed to chtimes path") } return nil diff --git a/server/filesystem/chown.go b/server/filesystem/chown.go deleted file mode 100644 index c67476ba6..000000000 --- a/server/filesystem/chown.go +++ /dev/null @@ -1,48 +0,0 @@ -package filesystem - -import ( - "emperror.dev/errors" - "github.com/karrick/godirwalk" - "github.com/pterodactyl/wings/config" -) - -// Chown recursively iterates over a file or directory and sets the permissions on all the -// underlying files. Iterate over all the files and directories. If it is a file go ahead -// and perform the chown operation. Otherwise dig deeper into the directory until we've run -// out of directories to dig into. -func (fs *Filesystem) Chown(path string) error { - uid := config.Get().System.User.Uid - gid := config.Get().System.User.Gid - - // Start by just chowning the initial path that we received. - if err := fs.root.Chown(path, uid, gid); err != nil { - return errors.Wrap(err, "server/filesystem: chown: failed to chown path") - } - - // If this is not a directory we can now return from the function, there is nothing - // left that we need to do. - if st, err := fs.root.Stat(path); err != nil || !st.IsDir() { - return nil - } - - // If this was a directory, begin walking over its contents recursively and ensure that all - // the subfiles and directories get their permissions updated as well. - err := godirwalk.Walk(path, &godirwalk.Options{ - Unsorted: true, - Callback: func(p string, e *godirwalk.Dirent) error { - // Do not attempt to chown a symlink. Go's os.Chown function will affect the symlink - // so if it points to a location outside the data directory the user would be able to - // (un)intentionally modify that files permissions. - if e.IsSymlink() { - if e.IsDir() { - return godirwalk.SkipThis - } - - return nil - } - - return fs.root.Chown(p, uid, gid) - }, - }) - return errors.Wrap(err, "server/filesystem: chown: failed to chown during walk function") -} diff --git a/server/filesystem/chtimes.go b/server/filesystem/chtimes.go deleted file mode 100644 index b9e295679..000000000 --- a/server/filesystem/chtimes.go +++ /dev/null @@ -1,22 +0,0 @@ -package filesystem - -import "time" - -// todo: vulnerable to race condition with symlinks -// see: https://pkg.go.dev/os#Root -func (fs *Filesystem) Chtimes(path string, atime, mtime time.Time) error { - cleaned, err := fs.SafePath(path) - if err != nil { - return err - } - - if fs.isTest { - return nil - } - - if err := fs.root.Chtimes(cleaned, atime, mtime); err != nil { - return err - } - - return nil -} diff --git a/server/filesystem/disk_space.go b/server/filesystem/disk_space.go index ac862a5bc..352b59521 100644 --- a/server/filesystem/disk_space.go +++ b/server/filesystem/disk_space.go @@ -174,19 +174,6 @@ func (fs *Filesystem) DirectorySize(dir string) (int64, error) { return nil } - // If this is a symlink then resolve the final destination of it before trying to continue walking - // over its contents. If it resolves outside the server data directory just skip everything else for - // it. Otherwise, allow it to continue. - if e.IsSymlink() { - if _, err := fs.SafePath(p); err != nil { - if IsErrorCode(err, ErrCodePathResolution) { - return godirwalk.SkipThis - } - - return err - } - } - if !e.IsDir() { _ = syscall.Lstat(p, &st) atomic.AddInt64(&size, st.Size) diff --git a/server/filesystem/errors.go b/server/filesystem/errors.go old mode 100644 new mode 100755 diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go old mode 100644 new mode 100755 index 5b241a5ea..d3397c52f --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -2,7 +2,6 @@ package filesystem import ( "bufio" - "fmt" "io" "io/ioutil" "os" @@ -12,7 +11,6 @@ import ( "strconv" "strings" "sync" - "sync/atomic" "time" "emperror.dev/errors" @@ -68,6 +66,7 @@ func (fs *Filesystem) Path() string { // File returns a reader for a file instance as well as the stat information. func (fs *Filesystem) File(p string) (*os.File, Stat, error) { + p = strings.TrimLeft(filepath.Clean(p), "/") st, err := fs.Stat(p) if err != nil { if errors.Is(err, os.ErrNotExist) { @@ -89,6 +88,7 @@ func (fs *Filesystem) File(p string) (*os.File, Stat, error) { // already. If it is present, the file is opened using the defaults which will truncate // the contents. The opened file is then returned to the caller. func (fs *Filesystem) Touch(p string, flag int) (*os.File, error) { + p = strings.TrimLeft(filepath.Clean(p), "/") f, err := fs.root.OpenFile(p, flag, 0o644) if err == nil { return f, nil @@ -104,11 +104,11 @@ func (fs *Filesystem) Touch(p string, flag int) (*os.File, error) { if _, err := fs.root.Stat(filepath.Dir(p)); errors.Is(err, os.ErrNotExist) { // Create the path leading up to the file we're trying to create, setting the final perms // on it as we go. - if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + if err := fs.root.MkdirAll(filepath.Dir(p), 0o755); err != nil { return nil, errors.Wrap(err, "server/filesystem: touch: failed to create directory tree") } if err := fs.Chown(filepath.Dir(p)); err != nil { - return nil, err + return nil, errors.Wrap(err, "server/filesystem: touch: failed to chown directory tree") } } o := &fileOpener{root: fs.root} @@ -126,10 +126,10 @@ func (fs *Filesystem) Touch(p string, flag int) (*os.File, error) { // will be created. This will also properly recalculate the disk space used by // the server when writing new files or modifying existing ones. func (fs *Filesystem) Writefile(p string, r io.Reader) error { + p = strings.TrimLeft(filepath.Clean(p), "/") var currentSize int64 // If the file does not exist on the system already go ahead and create the pathway // to it and an empty file. We'll then write to it later on after this completes. - fmt.Println(p) stat, err := fs.root.Stat(p) if err != nil && !os.IsNotExist(err) { return errors.Wrap(err, "server/filesystem: writefile: failed to stat file") @@ -167,34 +167,35 @@ func (fs *Filesystem) Writefile(p string, r io.Reader) error { // CreateDirectory creates a new directory ("name") at a specified path ("p") for the server. func (fs *Filesystem) CreateDirectory(name string, p string) error { - fmt.Println(path.Join(p, name)) + p = strings.TrimLeft(filepath.Clean(p), "/") return fs.root.MkdirAll(path.Join(p, name), 0o755) } // Rename moves (or renames) a file or directory. func (fs *Filesystem) Rename(from string, to string) error { - cleanedTo := path.Clean(to) + to = strings.TrimLeft(filepath.Clean(to), "/") + from = strings.TrimLeft(filepath.Clean(from), "/") + + if from == "" || to == "" { + return errors.New("server/filesystem: attempting to rename into an invalid directory space") + } // If the target file or directory already exists the rename function will // fail, so just bail out now. - if _, err := fs.root.Stat(cleanedTo); err == nil { + if _, err := fs.root.Stat(to); err == nil { return os.ErrExist } - if cleanedTo == fs.Path() { - return errors.New("server/filesystem: attempting to rename into an invalid directory space") - } - - d := strings.TrimSuffix(cleanedTo, path.Base(cleanedTo)) + d := strings.TrimLeft(filepath.Dir(to), "/") // Ensure that the directory we're moving into exists correctly on the system. Only do this if // we're not at the root directory level. - if d != fs.Path() { + if d != "" { if err := fs.root.MkdirAll(d, 0o755); err != nil { return errors.Wrap(err, "server/filesystem: failed to create directory tree") } } - return fs.root.Rename(from, cleanedTo) + return fs.root.Rename(from, to) } // Begin looping up to 50 times to try and create a unique copy file name. This will take @@ -236,6 +237,7 @@ func (fs *Filesystem) findCopySuffix(dir string, name string, extension string) // Copies a given file to the same location and appends a suffix to the file to indicate that // it has been copied. func (fs *Filesystem) Copy(p string) error { + p = strings.TrimLeft(filepath.Clean(p), "/") s, err := fs.root.Stat(p) if err != nil { return err @@ -280,25 +282,23 @@ func (fs *Filesystem) Copy(p string) error { // TruncateRootDirectory removes _all_ files and directories from a server's // data directory and resets the used disk space to zero. func (fs *Filesystem) TruncateRootDirectory() error { - if err := fs.root.RemoveAll("/"); err != nil { - return err - } - if err := fs.root.Mkdir("/", 0o755); err != nil { - return err - } - atomic.StoreInt64(&fs.diskUsed, 0) - return nil + return errors.New("server/filesystem: not implemented") } // Delete removes a file or folder from the system. Prevents the user from // accidentally (or maliciously) removing their root server data directory. func (fs *Filesystem) Delete(p string) error { + p = strings.TrimLeft(filepath.Clean(p), "/") + if p == "" { + return errors.New("server/filesystem: delete: cannot delete root directory") + } + st, err := fs.root.Lstat(p) if err != nil { if os.IsNotExist(err) { return nil } - return errors.Wrap(err, "server/filesystem: failed to stat file") + return errors.Wrap(err, "server/filesystem: delete: failed to stat file") } if st.IsDir() { diff --git a/server/filesystem/path.go b/server/filesystem/path.go old mode 100644 new mode 100755 diff --git a/server/filesystem/stat.go b/server/filesystem/stat.go old mode 100644 new mode 100755 index 6186a21b5..27bae36d4 --- a/server/filesystem/stat.go +++ b/server/filesystem/stat.go @@ -2,9 +2,12 @@ package filesystem import ( "os" + "path/filepath" "strconv" + "strings" "time" + "emperror.dev/errors" "github.com/gabriel-vasile/mimetype" "github.com/goccy/go-json" ) @@ -44,16 +47,22 @@ func (s *Stat) MarshalJSON() ([]byte, error) { // Stat stats a file or folder and returns the base stat object from go along // with the MIME data that can be used for editing files. func (fs *Filesystem) Stat(p string) (Stat, error) { + p = strings.TrimLeft(filepath.Clean(p), "/") s, err := fs.root.Stat(p) if err != nil { - return Stat{}, err + return Stat{}, errors.Wrap(err, "server/filesystem: stat: failed to stat file") } var m *mimetype.MIME if !s.IsDir() { - m, err = mimetype.DetectFile(p) + f, err := fs.root.Open(p) if err != nil { - return Stat{}, err + return Stat{}, errors.Wrap(err, "server/filesystem: stat: failed to open file") + } + defer f.Close() + m, err = mimetype.DetectReader(f) + if err != nil { + return Stat{}, errors.Wrap(err, "server/filesystem: stat: failed to detect mimetype") } } From fd14a180e14f6d2fd75a17572ff0421263b62443 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Sat, 7 Feb 2026 10:52:35 -0800 Subject: [PATCH 07/30] update tests --- server/filesystem/archive_test.go | 9 +- server/filesystem/compress_test.go | 55 ------ server/filesystem/errors_test.go | 0 server/filesystem/filesystem_test.go | 272 +++++++++++++-------------- server/filesystem/path_test.go | 46 ++--- 5 files changed, 156 insertions(+), 226 deletions(-) delete mode 100644 server/filesystem/compress_test.go mode change 100644 => 100755 server/filesystem/errors_test.go mode change 100644 => 100755 server/filesystem/filesystem_test.go mode change 100644 => 100755 server/filesystem/path_test.go diff --git a/server/filesystem/archive_test.go b/server/filesystem/archive_test.go index 8f4fe6e1e..eda342558 100644 --- a/server/filesystem/archive_test.go +++ b/server/filesystem/archive_test.go @@ -15,12 +15,11 @@ import ( func TestArchive_Stream(t *testing.T) { g := Goblin(t) - fs, rfs := NewFs() + fs := NewFs() g.Describe("Archive", func() { g.AfterEach(func() { - // Reset the filesystem after each run. - rfs.reset() + fs.reset() }) g.It("throws an error when passed invalid file paths", func() { @@ -61,7 +60,7 @@ func TestArchive_Stream(t *testing.T) { } // Create the archive. - archivePath := filepath.Join(rfs.root, "archive.tar.gz") + archivePath := filepath.Join(fs.rootPath, "../archive.tar.gz") g.Assert(a.Create(context.Background(), archivePath)).IsNil() // Ensure the archive exists. @@ -119,7 +118,7 @@ func getFiles(f iofs.ReadDirFS, name string) ([]string, error) { if files == nil { return nil, nil } - + v = append(v, files...) continue } diff --git a/server/filesystem/compress_test.go b/server/filesystem/compress_test.go deleted file mode 100644 index d287424ea..000000000 --- a/server/filesystem/compress_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package filesystem - -import ( - "context" - "os" - "sync/atomic" - "testing" - - . "github.com/franela/goblin" -) - -// Given an archive named test.{ext}, with the following file structure: -// test/ -// |──inside/ -// |────finside.txt -// |──outside.txt -// this test will ensure that it's being decompressed as expected -func TestFilesystem_DecompressFile(t *testing.T) { - g := Goblin(t) - fs, rfs := NewFs() - - g.Describe("Decompress", func() { - for _, ext := range []string{"zip", "rar", "tar", "tar.gz"} { - g.It("can decompress a "+ext, func() { - // copy the file to the new FS - c, err := os.ReadFile("./testdata/test." + ext) - g.Assert(err).IsNil() - err = rfs.CreateServerFile("./test."+ext, c) - g.Assert(err).IsNil() - - // decompress - err = fs.DecompressFile(context.Background(), "/", "test."+ext) - g.Assert(err).IsNil() - - // make sure everything is where it is supposed to be - _, err = rfs.StatServerFile("test/outside.txt") - g.Assert(err).IsNil() - - st, err := rfs.StatServerFile("test/inside") - g.Assert(err).IsNil() - g.Assert(st.IsDir()).IsTrue() - - _, err = rfs.StatServerFile("test/inside/finside.txt") - g.Assert(err).IsNil() - g.Assert(st.IsDir()).IsTrue() - }) - } - - g.AfterEach(func() { - rfs.reset() - atomic.StoreInt64(&fs.diskUsed, 0) - atomic.StoreInt64(&fs.diskLimit, 0) - }) - }) -} diff --git a/server/filesystem/errors_test.go b/server/filesystem/errors_test.go old mode 100644 new mode 100755 diff --git a/server/filesystem/filesystem_test.go b/server/filesystem/filesystem_test.go old mode 100644 new mode 100755 index 6c0a46344..6ab8a374c --- a/server/filesystem/filesystem_test.go +++ b/server/filesystem/filesystem_test.go @@ -7,6 +7,7 @@ import ( "math/rand" "os" "path/filepath" + "strings" "sync/atomic" "testing" "unicode/utf8" @@ -16,7 +17,11 @@ import ( "github.com/pterodactyl/wings/config" ) -func NewFs() (*Filesystem, *rootFs) { +type testFs struct { + *Filesystem +} + +func NewFs() *testFs { config.Set(&config.Configuration{ AuthenticationToken: "abc", System: config.SystemConfiguration{ @@ -29,23 +34,17 @@ func NewFs() (*Filesystem, *rootFs) { if err != nil { panic(err) } - // defer os.RemoveAll(tmpDir) - - rfs := rootFs{root: tmpDir} - rfs.reset() - - fs, err := New(filepath.Join(tmpDir, "/server"), 0, []string{}) + fs, err := New(tmpDir, 0, []string{}) if err != nil { panic(err) } fs.isTest = true - return fs, &rfs -} + tfs := &testFs{fs} + tfs.reset() -type rootFs struct { - root string + return tfs } func getFileContent(file *os.File) string { @@ -56,40 +55,50 @@ func getFileContent(file *os.File) string { return w.String() } -func (rfs *rootFs) CreateServerFile(p string, c []byte) error { - f, err := os.Create(filepath.Join(rfs.root, "/server", p)) +func (tfs *testFs) reset() { + if err := tfs.root.Close(); err != nil { + panic(err) + } - if err == nil { - f.Write(c) - f.Close() + o := filepath.Dir(tfs.rootPath) + if !strings.HasPrefix(o, filepath.Dir(tfs.rootPath)) { + panic("filesystem_test: attempting to delete outside root directory: " + o) } - return err -} + if err := os.RemoveAll(o); err != nil { + if !os.IsNotExist(err) { + panic(err) + } + } -func (rfs *rootFs) CreateServerFileFromString(p string, c string) error { - return rfs.CreateServerFile(p, []byte(c)) -} + if err := os.Mkdir(tfs.rootPath, 0o755); err != nil { + panic(err) + } + + r, err := os.OpenRoot(tfs.rootPath) + if err != nil { + panic(err) + } -func (rfs *rootFs) StatServerFile(p string) (os.FileInfo, error) { - return os.Stat(filepath.Join(rfs.root, "/server", p)) + tfs.root = r } -func (rfs *rootFs) reset() { - if err := os.RemoveAll(filepath.Join(rfs.root, "/server")); err != nil { - if !os.IsNotExist(err) { - panic(err) - } +func (tfs *testFs) write(name string, data []byte) { + p := filepath.Clean(filepath.Join(tfs.rootPath, name)) + // Ensure we're always writing into a location that would also be cleaned up + // by the reset() function. + if !strings.HasPrefix(p, filepath.Dir(tfs.rootPath)) { + panic("filesystem_test: attempting to write outside of root directory: " + p) } - if err := os.Mkdir(filepath.Join(rfs.root, "/server"), 0o755); err != nil { + if err := os.WriteFile(filepath.Join(tfs.rootPath, name), data, 0o644); err != nil { panic(err) } } func TestFilesystem_Openfile(t *testing.T) { g := Goblin(t) - fs, rfs := NewFs() + fs := NewFs() g.Describe("File", func() { g.It("returns custom error when file does not exist", func() { @@ -100,7 +109,7 @@ func TestFilesystem_Openfile(t *testing.T) { }) g.It("returns file stat information", func() { - _ = rfs.CreateServerFile("foo.txt", []byte("hello world")) + fs.write("foo.txt", []byte("hello world")) f, st, err := fs.File("foo.txt") g.Assert(err).IsNil() @@ -111,14 +120,14 @@ func TestFilesystem_Openfile(t *testing.T) { }) g.AfterEach(func() { - rfs.reset() + fs.reset() }) }) } func TestFilesystem_Writefile(t *testing.T) { g := Goblin(t) - fs, rfs := NewFs() + fs := NewFs() g.Describe("Open and WriteFile", func() { buf := &bytes.Buffer{} @@ -167,9 +176,9 @@ func TestFilesystem_Writefile(t *testing.T) { g.It("cannot create a file outside the root directory", func() { r := bytes.NewReader([]byte("test file content")) - err := fs.Writefile("/some/../foo/../../test.txt", r) + err := fs.Writefile("/some/foo/../../../test.txt", r) g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) g.It("cannot write a file that exceeds the disk limits", func() { @@ -203,7 +212,7 @@ func TestFilesystem_Writefile(t *testing.T) { g.AfterEach(func() { buf.Truncate(0) - rfs.reset() + fs.reset() atomic.StoreInt64(&fs.diskUsed, 0) atomic.StoreInt64(&fs.diskLimit, 0) @@ -213,14 +222,14 @@ func TestFilesystem_Writefile(t *testing.T) { func TestFilesystem_CreateDirectory(t *testing.T) { g := Goblin(t) - fs, rfs := NewFs() + fs := NewFs() g.Describe("CreateDirectory", func() { g.It("should create missing directories automatically", func() { err := fs.CreateDirectory("test", "foo/bar/baz") g.Assert(err).IsNil() - st, err := rfs.StatServerFile("foo/bar/baz/test") + st, err := os.Stat(filepath.Join(fs.rootPath, "foo/bar/baz/test")) g.Assert(err).IsNil() g.Assert(st.IsDir()).IsTrue() g.Assert(st.Name()).Equal("test") @@ -230,7 +239,7 @@ func TestFilesystem_CreateDirectory(t *testing.T) { err := fs.CreateDirectory("test", "/foozie/barzie/bazzy/") g.Assert(err).IsNil() - st, err := rfs.StatServerFile("foozie/barzie/bazzy/test") + st, err := os.Stat(filepath.Join(fs.rootPath, "foozie/barzie/bazzy/test")) g.Assert(err).IsNil() g.Assert(st.IsDir()).IsTrue() g.Assert(st.Name()).Equal("test") @@ -239,7 +248,7 @@ func TestFilesystem_CreateDirectory(t *testing.T) { g.It("should not allow the creation of directories outside the root", func() { err := fs.CreateDirectory("test", "e/../../something") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) g.It("should not increment the disk usage", func() { @@ -249,27 +258,24 @@ func TestFilesystem_CreateDirectory(t *testing.T) { }) g.AfterEach(func() { - rfs.reset() + fs.reset() }) }) } func TestFilesystem_Rename(t *testing.T) { g := Goblin(t) - fs, rfs := NewFs() + fs := NewFs() g.Describe("Rename", func() { g.BeforeEach(func() { - if err := rfs.CreateServerFileFromString("source.txt", "text content"); err != nil { - panic(err) - } + fs.write("source.txt", []byte("text content")) }) g.It("returns an error if the target already exists", func() { - err := rfs.CreateServerFileFromString("target.txt", "taget content") - g.Assert(err).IsNil() + fs.write("target.txt", []byte("taget content")) - err = fs.Rename("source.txt", "target.txt") + err := fs.Rename("source.txt", "target.txt") g.Assert(err).IsNotNil() g.Assert(errors.Is(err, os.ErrExist)).IsTrue() }) @@ -281,7 +287,7 @@ func TestFilesystem_Rename(t *testing.T) { }) g.It("returns an error if the source destination is the root directory", func() { - err := fs.Rename("source.txt", "/") + err := fs.Rename("/", "destination.txt") g.Assert(err).IsNotNil() g.Assert(errors.Is(err, os.ErrExist)).IsTrue() }) @@ -289,43 +295,42 @@ func TestFilesystem_Rename(t *testing.T) { g.It("does not allow renaming to a location outside the root", func() { err := fs.Rename("source.txt", "../target.txt") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) g.It("does not allow renaming from a location outside the root", func() { - err := rfs.CreateServerFileFromString("/../ext-source.txt", "taget content") - - err = fs.Rename("/../ext-source.txt", "target.txt") + err := fs.Rename("/../ext-source.txt", "target.txt") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) g.It("allows a file to be renamed", func() { err := fs.Rename("source.txt", "target.txt") g.Assert(err).IsNil() - _, err = rfs.StatServerFile("source.txt") + _, err = os.Stat(filepath.Join(fs.rootPath, "source.txt")) g.Assert(err).IsNotNil() g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue() - st, err := rfs.StatServerFile("target.txt") + st, err := fs.Stat(filepath.Join(fs.rootPath, "target.txt")) g.Assert(err).IsNil() g.Assert(st.Name()).Equal("target.txt") g.Assert(st.Size()).IsNotZero() }) g.It("allows a folder to be renamed", func() { - err := os.Mkdir(filepath.Join(rfs.root, "/server/source_dir"), 0o755) - g.Assert(err).IsNil() + if err := os.Mkdir(filepath.Join(fs.rootPath, "/source_dir"), 0o755); err != nil { + panic(err) + } - err = fs.Rename("source_dir", "target_dir") + err := fs.Rename("source_dir", "target_dir") g.Assert(err).IsNil() - _, err = rfs.StatServerFile("source_dir") + _, err = os.Stat(filepath.Join(fs.rootPath, "source_dir")) g.Assert(err).IsNotNil() g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue() - st, err := rfs.StatServerFile("target_dir") + st, err := fs.Stat(filepath.Join(fs.rootPath, "target_dir")) g.Assert(err).IsNil() g.Assert(st.IsDir()).IsTrue() }) @@ -340,27 +345,24 @@ func TestFilesystem_Rename(t *testing.T) { err := fs.Rename("source.txt", "nested/folder/target.txt") g.Assert(err).IsNil() - st, err := rfs.StatServerFile("nested/folder/target.txt") + st, err := fs.Stat(filepath.Join(fs.rootPath, "nested/folder/target.txt")) g.Assert(err).IsNil() g.Assert(st.Name()).Equal("target.txt") }) g.AfterEach(func() { - rfs.reset() + fs.reset() }) }) } func TestFilesystem_Copy(t *testing.T) { g := Goblin(t) - fs, rfs := NewFs() + fs := NewFs() g.Describe("Copy", func() { g.BeforeEach(func() { - if err := rfs.CreateServerFileFromString("source.txt", "text content"); err != nil { - panic(err) - } - + fs.write("source.txt", []byte("text content")) atomic.StoreInt64(&fs.diskUsed, int64(utf8.RuneCountInString("test content"))) }) @@ -371,31 +373,23 @@ func TestFilesystem_Copy(t *testing.T) { }) g.It("should return an error if the source is outside the root", func() { - err := rfs.CreateServerFileFromString("/../ext-source.txt", "text content") - - err = fs.Copy("../ext-source.txt") + err := fs.Copy("../ext-source.txt") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) g.It("should return an error if the source directory is outside the root", func() { - err := os.MkdirAll(filepath.Join(rfs.root, "/nested/in/dir"), 0o755) - g.Assert(err).IsNil() - - err = rfs.CreateServerFileFromString("/../nested/in/dir/ext-source.txt", "external content") - g.Assert(err).IsNil() - - err = fs.Copy("../nested/in/dir/ext-source.txt") + err := fs.Copy("../nested/in/dir/ext-source.txt") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() err = fs.Copy("nested/in/../../../nested/in/dir/ext-source.txt") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) g.It("should return an error if the source is a directory", func() { - err := os.Mkdir(filepath.Join(rfs.root, "/server/dir"), 0o755) + err := os.Mkdir(filepath.Join(fs.rootPath, "/dir"), 0o755) g.Assert(err).IsNil() err = fs.Copy("dir") @@ -415,10 +409,10 @@ func TestFilesystem_Copy(t *testing.T) { err := fs.Copy("source.txt") g.Assert(err).IsNil() - _, err = rfs.StatServerFile("source.txt") + _, err = os.Stat(filepath.Join(fs.rootPath, "source.txt")) g.Assert(err).IsNil() - _, err = rfs.StatServerFile("source copy.txt") + _, err = os.Stat(filepath.Join(fs.rootPath, "source copy.txt")) g.Assert(err).IsNil() }) @@ -432,7 +426,7 @@ func TestFilesystem_Copy(t *testing.T) { r := []string{"source.txt", "source copy.txt", "source copy 1.txt"} for _, name := range r { - _, err = rfs.StatServerFile(name) + _, err = os.Stat(filepath.Join(fs.rootPath, name)) g.Assert(err).IsNil() } @@ -440,24 +434,24 @@ func TestFilesystem_Copy(t *testing.T) { }) g.It("should create a copy inside of a directory", func() { - err := os.MkdirAll(filepath.Join(rfs.root, "/server/nested/in/dir"), 0o755) - g.Assert(err).IsNil() + if err := os.MkdirAll(filepath.Join(fs.rootPath, "/nested/in/dir"), 0o755); err != nil { + panic(err) + } - err = rfs.CreateServerFileFromString("nested/in/dir/source.txt", "test content") - g.Assert(err).IsNil() + fs.write("nested/in/dir/source.txt", []byte("test content")) - err = fs.Copy("nested/in/dir/source.txt") + err := fs.Copy("nested/in/dir/source.txt") g.Assert(err).IsNil() - _, err = rfs.StatServerFile("nested/in/dir/source.txt") + _, err = os.Stat(filepath.Join(fs.rootPath, "nested/in/dir/source.txt")) g.Assert(err).IsNil() - _, err = rfs.StatServerFile("nested/in/dir/source copy.txt") + _, err = os.Stat(filepath.Join(fs.rootPath, "nested/in/dir/source copy.txt")) g.Assert(err).IsNil() }) g.AfterEach(func() { - rfs.reset() + fs.reset() atomic.StoreInt64(&fs.diskUsed, 0) atomic.StoreInt64(&fs.diskLimit, 0) @@ -467,23 +461,18 @@ func TestFilesystem_Copy(t *testing.T) { func TestFilesystem_Delete(t *testing.T) { g := Goblin(t) - fs, rfs := NewFs() + fs := NewFs() g.Describe("Delete", func() { g.BeforeEach(func() { - if err := rfs.CreateServerFileFromString("source.txt", "test content"); err != nil { - panic(err) - } - + fs.write("source.txt", []byte("text content")) atomic.StoreInt64(&fs.diskUsed, int64(utf8.RuneCountInString("test content"))) }) g.It("does not delete files outside the root directory", func() { - err := rfs.CreateServerFileFromString("/../ext-source.txt", "external content") - - err = fs.Delete("../ext-source.txt") + err := fs.Delete("../ext-source.txt") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) g.It("does not allow the deletion of the root directory", func() { @@ -496,7 +485,7 @@ func TestFilesystem_Delete(t *testing.T) { err := fs.Delete("missing.txt") g.Assert(err).IsNil() - st, err := rfs.StatServerFile("source.txt") + st, err := os.Stat(filepath.Join(fs.rootPath, "source.txt")) g.Assert(err).IsNil() g.Assert(st.Name()).Equal("source.txt") }) @@ -505,7 +494,7 @@ func TestFilesystem_Delete(t *testing.T) { err := fs.Delete("source.txt") g.Assert(err).IsNil() - _, err = rfs.StatServerFile("source.txt") + _, err = os.Stat(filepath.Join(fs.rootPath, "source.txt")) g.Assert(err).IsNotNil() g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue() @@ -519,103 +508,106 @@ func TestFilesystem_Delete(t *testing.T) { "foo/bar/baz/source.txt", } - err := os.MkdirAll(filepath.Join(rfs.root, "/server/foo/bar/baz"), 0o755) - g.Assert(err).IsNil() + if err := os.MkdirAll(filepath.Join(fs.rootPath, "/foo/bar/baz"), 0o755); err != nil { + panic(err) + } for _, s := range sources { - err = rfs.CreateServerFileFromString(s, "test content") - g.Assert(err).IsNil() + fs.write(s, []byte("test content")) } atomic.StoreInt64(&fs.diskUsed, int64(utf8.RuneCountInString("test content")*3)) - err = fs.Delete("foo") + err := fs.Delete("foo") g.Assert(err).IsNil() g.Assert(atomic.LoadInt64(&fs.diskUsed)).Equal(int64(0)) for _, s := range sources { - _, err = rfs.StatServerFile(s) + _, err = os.Stat(filepath.Join(fs.rootPath, s)) g.Assert(err).IsNotNil() g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue() } }) g.It("deletes a symlink but not it's target within the root directory", func() { - // Symlink to a file inside the root directory. - err := os.Symlink(filepath.Join(rfs.root, "server/source.txt"), filepath.Join(rfs.root, "server/symlink.txt")) - g.Assert(err).IsNil() + // Symlink to a file inside the root server data directory. + if err := os.Symlink(filepath.Join(fs.rootPath, "source.txt"), filepath.Join(fs.rootPath, "symlink.txt")); err != nil { + panic(err) + } // Delete the symlink itself. - err = fs.Delete("symlink.txt") + err := fs.Delete("symlink.txt") g.Assert(err).IsNil() // Ensure the symlink was deleted. - _, err = os.Lstat(filepath.Join(rfs.root, "server/symlink.txt")) + _, err = os.Lstat(filepath.Join(fs.rootPath, "symlink.txt")) g.Assert(err).IsNotNil() // Ensure the symlink target still exists. - _, err = os.Lstat(filepath.Join(rfs.root, "server/source.txt")) + _, err = os.Lstat(filepath.Join(fs.rootPath, "source.txt")) g.Assert(err).IsNil() }) g.It("does not delete files symlinked outside of the root directory", func() { // Create a file outside the root directory. - err := rfs.CreateServerFileFromString("/../source.txt", "test content") - g.Assert(err).IsNil() + fs.write("../external.txt", []byte("test content")) // Create a symlink to the file outside the root directory. - err = os.Symlink(filepath.Join(rfs.root, "source.txt"), filepath.Join(rfs.root, "/server/symlink.txt")) - g.Assert(err).IsNil() + if err := os.Symlink(filepath.Join(fs.rootPath, "../external.txt"), filepath.Join(fs.rootPath, "symlink.txt")); err != nil { + panic(err) + } - // Delete the symlink. (This should pass as we will delete the symlink itself, not it's target) - err = fs.Delete("symlink.txt") + // Delete the symlink. (This should pass as we will delete the symlink itself, not the target) + err := fs.Delete("symlink.txt") g.Assert(err).IsNil() // Ensure the file outside the root directory still exists. - _, err = os.Lstat(filepath.Join(rfs.root, "source.txt")) + _, err = os.Lstat(filepath.Join(fs.rootPath, "../external.txt")) g.Assert(err).IsNil() }) g.It("does not delete files symlinked through a directory outside of the root directory", func() { // Create a directory outside the root directory. - err := os.Mkdir(filepath.Join(rfs.root, "foo"), 0o755) - g.Assert(err).IsNil() + if err := os.Mkdir(filepath.Join(fs.rootPath, "../external"), 0o755); err != nil { + panic(err) + } - // Create a file inside the directory that is outside the root. - err = rfs.CreateServerFileFromString("/../foo/source.txt", "test content") - g.Assert(err).IsNil() + fs.write("../external/source.txt", []byte("test content")) // Symlink the directory that is outside the root to a file inside the root. - err = os.Symlink(filepath.Join(rfs.root, "foo"), filepath.Join(rfs.root, "server/symlink")) - g.Assert(err).IsNil() + if err := os.Symlink(filepath.Join(fs.rootPath, "../external"), filepath.Join(fs.rootPath, "/symlink")); err != nil { + panic(err) + } // Delete a file inside the symlinked directory. - err = fs.Delete("symlink/source.txt") + err := fs.Delete("symlink/source.txt") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() // Ensure the file outside the root directory still exists. - _, err = os.Lstat(filepath.Join(rfs.root, "foo/source.txt")) + _, err = os.Lstat(filepath.Join(fs.rootPath, "../external/source.txt")) g.Assert(err).IsNil() }) g.It("returns an error when trying to delete a non-existent file symlinked through a directory outside of the root directory", func() { // Create a directory outside the root directory. - err := os.Mkdir(filepath.Join(rfs.root, "foo2"), 0o755) - g.Assert(err).IsNil() + if err := os.Mkdir(filepath.Join(fs.rootPath, "../external"), 0o755); err != nil { + panic(err) + } // Symlink the directory that is outside the root to a file inside the root. - err = os.Symlink(filepath.Join(rfs.root, "foo2"), filepath.Join(rfs.root, "server/symlink")) - g.Assert(err).IsNil() + if err := os.Symlink(filepath.Join(fs.rootPath, "../external"), filepath.Join(fs.rootPath, "/symlink")); err != nil { + panic(err) + } // Delete a file inside the symlinked directory. - err = fs.Delete("symlink/source.txt") + err := fs.Delete("symlink/source.txt") g.Assert(err).IsNotNil() - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) g.AfterEach(func() { - rfs.reset() + fs.reset() atomic.StoreInt64(&fs.diskUsed, 0) atomic.StoreInt64(&fs.diskLimit, 0) diff --git a/server/filesystem/path_test.go b/server/filesystem/path_test.go old mode 100644 new mode 100755 index 972b2e2e9..d3f6e57cd --- a/server/filesystem/path_test.go +++ b/server/filesystem/path_test.go @@ -12,11 +12,11 @@ import ( func TestFilesystem_Path(t *testing.T) { g := Goblin(t) - fs, rfs := NewFs() + fs := NewFs() g.Describe("Path", func() { g.It("returns the root path for the instance", func() { - g.Assert(fs.Path()).Equal(filepath.Join(rfs.root, "/server")) + g.Assert(fs.Path()).Equal(fs.rootPath) }) }) } @@ -27,30 +27,24 @@ func TestFilesystem_Path(t *testing.T) { // the calls and ensure they all fail with the same reason. func TestFilesystem_Blocks_Symlinks(t *testing.T) { g := Goblin(t) - fs, rfs := NewFs() + fs := NewFs() - if err := rfs.CreateServerFileFromString("/../malicious.txt", "external content"); err != nil { + fs.write("../malicious.txt", []byte("external content")) + if err := os.Mkdir(filepath.Join(fs.rootPath, "../malicious_dir"), 0o777); err != nil { panic(err) } - if err := os.Mkdir(filepath.Join(rfs.root, "/malicious_dir"), 0o777); err != nil { - panic(err) - } - - if err := os.Symlink(filepath.Join(rfs.root, "malicious.txt"), filepath.Join(rfs.root, "/server/symlinked.txt")); err != nil { - panic(err) - } - - if err := os.Symlink(filepath.Join(rfs.root, "malicious_does_not_exist.txt"), filepath.Join(rfs.root, "/server/symlinked_does_not_exist.txt")); err != nil { - panic(err) - } - - if err := os.Symlink(filepath.Join(rfs.root, "/server/symlinked_does_not_exist.txt"), filepath.Join(rfs.root, "/server/symlinked_does_not_exist2.txt")); err != nil { - panic(err) + links := map[string]string{ + "../malicious.txt": "/symlinked.txt", + "../malicious_does_not_exist.txt": "/symlinked_does_not_exist.txt", + "/symlinked_does_not_exist.txt": "/symlinked_does_not_exist2.txt", + "../malicious_dir": "/external_dir", } - if err := os.Symlink(filepath.Join(rfs.root, "/malicious_dir"), filepath.Join(rfs.root, "/server/external_dir")); err != nil { - panic(err) + for src, dst := range links { + if err := os.Symlink(filepath.Join(fs.rootPath, src), filepath.Join(fs.rootPath, dst)); err != nil { + panic(err) + } } g.Describe("Writefile", func() { @@ -121,9 +115,7 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { }) g.It("cannot rename a file to a location outside the directory root", func() { - if err := rfs.CreateServerFileFromString("my_file.txt", "internal content"); err != nil { - panic(err) - } + fs.write("my_file.txt", []byte("internal content")) err := fs.Rename("my_file.txt", "external_dir/my_file.txt") g.Assert(err).IsNotNil() @@ -158,14 +150,16 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { err := fs.Delete("symlinked.txt") g.Assert(err).IsNil() - _, err = os.Stat(filepath.Join(rfs.root, "malicious.txt")) + _, err = os.Stat(filepath.Join(fs.rootPath, "../malicious.txt")) g.Assert(err).IsNil() - _, err = rfs.StatServerFile("symlinked.txt") + _, err = os.Stat(filepath.Join(fs.rootPath, "symlinked.txt")) g.Assert(err).IsNotNil() g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue() }) }) - rfs.reset() + g.After(func() { + fs.reset() + }) } From fb4105eb4391e825de4b325d44cf780b1fff7cff Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Sat, 7 Feb 2026 11:26:22 -0800 Subject: [PATCH 08/30] get most tests passing again --- server/filesystem/chmod.go | 13 ++++++-- server/filesystem/disk_space.go | 25 +++++++++------- server/filesystem/filesystem.go | 2 +- server/filesystem/filesystem_test.go | 36 ++++++++++++++--------- server/filesystem/path_test.go | 44 ++++++++++++++++++++-------- 5 files changed, 78 insertions(+), 42 deletions(-) diff --git a/server/filesystem/chmod.go b/server/filesystem/chmod.go index 3b5fbb04a..e62c01a00 100755 --- a/server/filesystem/chmod.go +++ b/server/filesystem/chmod.go @@ -50,13 +50,20 @@ func (fs *Filesystem) Chown(path string) error { // If this was a directory, begin walking over its contents recursively and ensure that all // the subfiles and directories get their permissions updated as well. err := godirwalk.Walk(filepath.Join(fs.rootPath, path), &godirwalk.Options{ - Unsorted: true, + Unsorted: true, + FollowSymbolicLinks: false, Callback: func(p string, e *godirwalk.Dirent) error { - return fs.root.Chown(p, uid, gid) + p = strings.TrimLeft(strings.TrimPrefix(p, fs.Path()), "/") + + if err := fs.root.Chown(p, uid, gid); err != nil { + return errors.Wrap(err, "server/filesystem: chown: failed to chown during walk") + } + + return nil }, }) - return errors.Wrap(err, "server/filesystem: chown: failed to chown during walk function") + return errors.Wrap(err, "server/filesystem: chown: failed to chown directory tree") } func (fs *Filesystem) Chtimes(path string, atime, mtime time.Time) error { diff --git a/server/filesystem/disk_space.go b/server/filesystem/disk_space.go index 352b59521..bc5d02b45 100644 --- a/server/filesystem/disk_space.go +++ b/server/filesystem/disk_space.go @@ -1,9 +1,10 @@ package filesystem import ( + "path/filepath" + "strings" "sync" "sync/atomic" - "syscall" "time" "emperror.dev/errors" @@ -158,15 +159,15 @@ func (fs *Filesystem) updateCachedDiskUsage() (int64, error) { // through all of the folders. Returns the size in bytes. This can be a fairly taxing operation // on locations with tons of files, so it is recommended that you cache the output. func (fs *Filesystem) DirectorySize(dir string) (int64, error) { - var size int64 - var st syscall.Stat_t - - // todo: safely traverse directory within root? - if _, err := fs.root.Lstat(dir); err != nil { - return 0, err + dir = strings.TrimLeft(filepath.Clean(dir), "/") + if dir != "" { + if _, err := fs.root.Lstat(dir); err != nil { + return 0, err + } } - err := godirwalk.Walk(dir, &godirwalk.Options{ + var size int64 + err := godirwalk.Walk(filepath.Join(fs.rootPath, dir), &godirwalk.Options{ Unsorted: true, FollowSymbolicLinks: false, Callback: func(p string, e *godirwalk.Dirent) error { @@ -175,12 +176,14 @@ func (fs *Filesystem) DirectorySize(dir string) (int64, error) { } if !e.IsDir() { - _ = syscall.Lstat(p, &st) - atomic.AddInt64(&size, st.Size) + st, err := fs.root.Lstat(strings.TrimLeft(strings.TrimPrefix(p, fs.rootPath), "/")) + if err != nil { + return errors.Wrap(err, "server/filesystem: directorysize: failed to stat file") + } + atomic.AddInt64(&size, st.Size()) } // todo: don't count hardlinks twice - return nil }, }) diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index d3397c52f..24d8540a6 100755 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -177,7 +177,7 @@ func (fs *Filesystem) Rename(from string, to string) error { from = strings.TrimLeft(filepath.Clean(from), "/") if from == "" || to == "" { - return errors.New("server/filesystem: attempting to rename into an invalid directory space") + return os.ErrExist } // If the target file or directory already exists the rename function will diff --git a/server/filesystem/filesystem_test.go b/server/filesystem/filesystem_test.go index 6ab8a374c..bbf60d331 100755 --- a/server/filesystem/filesystem_test.go +++ b/server/filesystem/filesystem_test.go @@ -19,6 +19,7 @@ import ( type testFs struct { *Filesystem + tmpDir string } func NewFs() *testFs { @@ -41,7 +42,7 @@ func NewFs() *testFs { } fs.isTest = true - tfs := &testFs{fs} + tfs := &testFs{Filesystem: fs, tmpDir: tmpDir} tfs.reset() return tfs @@ -59,19 +60,22 @@ func (tfs *testFs) reset() { if err := tfs.root.Close(); err != nil { panic(err) } - - o := filepath.Dir(tfs.rootPath) - if !strings.HasPrefix(o, filepath.Dir(tfs.rootPath)) { - panic("filesystem_test: attempting to delete outside root directory: " + o) + if !strings.HasPrefix(tfs.tmpDir, "/tmp/pterodactyl") { + panic("filesystem_test: attempting to delete outside root directory: " + tfs.tmpDir) } - if err := os.RemoveAll(o); err != nil { + if err := os.RemoveAll(tfs.tmpDir); err != nil { if !os.IsNotExist(err) { panic(err) } } - if err := os.Mkdir(tfs.rootPath, 0o755); err != nil { + if !strings.HasPrefix(tfs.rootPath, tfs.tmpDir) { + panic("filesystem_test: mismatch between root and tmp paths") + } + + tfs.rootPath = filepath.Join(tfs.tmpDir, "/server") + if err := os.MkdirAll(tfs.rootPath, 0o755); err != nil { panic(err) } @@ -176,7 +180,11 @@ func TestFilesystem_Writefile(t *testing.T) { g.It("cannot create a file outside the root directory", func() { r := bytes.NewReader([]byte("test file content")) - err := fs.Writefile("/some/foo/../../../test.txt", r) + err := fs.Writefile("../../etc/test.txt", r) + g.Assert(err).IsNotNil() + g.Assert(IsPathError(err)).IsTrue() + + err = fs.Writefile("a/../../../test.txt", r) g.Assert(err).IsNotNil() g.Assert(IsPathError(err)).IsTrue() }) @@ -299,9 +307,9 @@ func TestFilesystem_Rename(t *testing.T) { }) g.It("does not allow renaming from a location outside the root", func() { - err := fs.Rename("/../ext-source.txt", "target.txt") + err := fs.Rename("../ext-source.txt", "target.txt") g.Assert(err).IsNotNil() - g.Assert(IsPathError(err)).IsTrue() + g.Assert(IsLinkError(err)).IsTrue() }) g.It("allows a file to be renamed", func() { @@ -312,7 +320,7 @@ func TestFilesystem_Rename(t *testing.T) { g.Assert(err).IsNotNil() g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue() - st, err := fs.Stat(filepath.Join(fs.rootPath, "target.txt")) + st, err := os.Stat(filepath.Join(fs.rootPath, "target.txt")) g.Assert(err).IsNil() g.Assert(st.Name()).Equal("target.txt") g.Assert(st.Size()).IsNotZero() @@ -330,7 +338,7 @@ func TestFilesystem_Rename(t *testing.T) { g.Assert(err).IsNotNil() g.Assert(errors.Is(err, os.ErrNotExist)).IsTrue() - st, err := fs.Stat(filepath.Join(fs.rootPath, "target_dir")) + st, err := os.Stat(filepath.Join(fs.rootPath, "target_dir")) g.Assert(err).IsNil() g.Assert(st.IsDir()).IsTrue() }) @@ -345,7 +353,7 @@ func TestFilesystem_Rename(t *testing.T) { err := fs.Rename("source.txt", "nested/folder/target.txt") g.Assert(err).IsNil() - st, err := fs.Stat(filepath.Join(fs.rootPath, "nested/folder/target.txt")) + st, err := os.Stat(filepath.Join(fs.rootPath, "nested/folder/target.txt")) g.Assert(err).IsNil() g.Assert(st.Name()).Equal("target.txt") }) @@ -478,7 +486,7 @@ func TestFilesystem_Delete(t *testing.T) { g.It("does not allow the deletion of the root directory", func() { err := fs.Delete("/") g.Assert(err).IsNotNil() - g.Assert(err.Error()).Equal("cannot delete root server directory") + g.Assert(err.Error()).Equal("server/filesystem: delete: cannot delete root directory") }) g.It("does not return an error if the target does not exist", func() { diff --git a/server/filesystem/path_test.go b/server/filesystem/path_test.go index d3f6e57cd..f6a91b5ae 100755 --- a/server/filesystem/path_test.go +++ b/server/filesystem/path_test.go @@ -89,37 +89,57 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { }) g.It("cannot create a nested directory outside the root", func() { - err := fs.CreateDirectory("my/nested/dir", "external_dir/foo/bar") + err := fs.CreateDirectory("my/nested/dir", "../external_dir/foo/bar") g.Assert(err).IsNotNil() g.Assert(IsPathError(err)).IsTrue() }) g.It("cannot create a nested directory outside the root", func() { - err := fs.CreateDirectory("my/nested/dir", "external_dir/server") + err := fs.CreateDirectory("my/nested/dir", "../external_dir/server") g.Assert(err).IsNotNil() g.Assert(IsPathError(err)).IsTrue() }) }) g.Describe("Rename", func() { - g.It("cannot rename a file symlinked outside the directory root", func() { + // You can rename the symlink file itself, which does not impact the + // underlying symlinked target file outside the server directory. + g.It("can rename a file symlinked outside the directory root", func() { err := fs.Rename("symlinked.txt", "foo.txt") - g.Assert(err).IsNotNil() - g.Assert(IsPathError(err)).IsTrue() + g.Assert(err).IsNil() + + st, err := os.Lstat(filepath.Join(fs.rootPath, "foo.txt")) + g.Assert(err).IsNil() + g.Assert(st.Mode()&os.ModeSymlink != 0).IsTrue() + + st, err = os.Lstat(filepath.Join(fs.rootPath, "../malicious.txt")) + g.Assert(err).IsNil() + g.Assert(st.Mode()&os.ModeSymlink == 0).IsTrue() }) - g.It("cannot rename a symlinked directory outside the root", func() { + // The same as above, acts on the source directory and not the target directory, + // therefore, this is allowed. + g.It("can rename a directory symlinked outside the root", func() { err := fs.Rename("external_dir", "foo") - g.Assert(err).IsNotNil() - g.Assert(IsPathError(err)).IsTrue() + g.Assert(err).IsNil() + + st, err := os.Lstat(filepath.Join(fs.rootPath, "foo")) + g.Assert(err).IsNil() + g.Assert(st.IsDir()).IsTrue() + g.Assert(st.Mode()&os.ModeSymlink != 0).IsTrue() + + st, err = os.Lstat(filepath.Join(fs.rootPath, "../external_dir")) + g.Assert(err).IsNil() + g.Assert(st.IsDir()).IsTrue() + g.Assert(st.Mode()&os.ModeSymlink == 0).IsTrue() }) g.It("cannot rename a file to a location outside the directory root", func() { fs.write("my_file.txt", []byte("internal content")) - err := fs.Rename("my_file.txt", "external_dir/my_file.txt") + err := fs.Rename("my_file.txt", "../external_dir/my_file.txt") g.Assert(err).IsNotNil() - g.Assert(IsLinkError(err)).IsTrue() + g.Assert(IsPathError(err)).IsTrue() }) }) @@ -159,7 +179,5 @@ func TestFilesystem_Blocks_Symlinks(t *testing.T) { }) }) - g.After(func() { - fs.reset() - }) + fs.reset() } From acbc8d1f6630f1df39e5f6f39f22fbf903ab169f Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Sat, 7 Feb 2026 11:57:55 -0800 Subject: [PATCH 09/30] get things booting up correctly --- cmd/root.go | 2 +- router/router_transfer.go | 2 +- server/filesystem/filesystem.go | 2 +- server/install.go | 4 +++- server/manager.go | 10 ++++++--- server/server.go | 36 ++++++++++++++++++--------------- 6 files changed, 33 insertions(+), 23 deletions(-) diff --git a/cmd/root.go b/cmd/root.go index f411c53b7..3a8fff625 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -199,7 +199,7 @@ func rootCmdRun(cmd *cobra.Command, _ []string) { // For each server ensure the minimal environment is configured for the server. if err := s.CreateEnvironment(); err != nil { - s.Log().Error("could create base environment for server...") + s.Log().WithField("error", err).Error("could not create base environment for server...") continue } diff --git a/router/router_transfer.go b/router/router_transfer.go index 1e78a39a4..3cedaecdc 100644 --- a/router/router_transfer.go +++ b/router/router_transfer.go @@ -165,7 +165,7 @@ out: case "archive": trnsfr.Log().Debug("received archive") - if err := trnsfr.Server.EnsureDataDirectoryExists(); err != nil { + if _, err := trnsfr.Server.EnsureDataDirectoryExists(); err != nil { middleware.CaptureAndAbort(c, err) return } diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index 24d8540a6..6225ebaa2 100755 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -43,7 +43,7 @@ type Filesystem struct { func New(path string, size int64, denylist []string) (*Filesystem, error) { r, err := os.OpenRoot(path) if err != nil { - return nil, errors.WithStack(err) + return nil, errors.Wrap(err, "server/filesystem: failed to open root") } fs := &Filesystem{ diff --git a/server/install.go b/server/install.go index d4454385f..d5e90b59d 100644 --- a/server/install.go +++ b/server/install.go @@ -456,7 +456,9 @@ func (ip *InstallationProcess) Execute() (string, error) { // to trigger the reinstall of the server. It is possible the directory would // not exist when this runs if Wings boots with a missing directory and a user // triggers a reinstall before trying to start the server. - if err := ip.Server.EnsureDataDirectoryExists(); err != nil { + // todo: verify, this might be wrong now that the FS requires the directory to + // exist to boot it. + if _, err := ip.Server.EnsureDataDirectoryExists(); err != nil { return "", err } diff --git a/server/manager.go b/server/manager.go index a834399d7..1b4302094 100644 --- a/server/manager.go +++ b/server/manager.go @@ -6,7 +6,6 @@ import ( "fmt" "io" "os" - "path/filepath" "runtime" "sync" "time" @@ -196,9 +195,14 @@ func (m *Manager) InitServer(data remote.ServerConfigurationResponse) (*Server, return nil, errors.WithStackIf(err) } - fs, err := filesystem.New(filepath.Join(config.Get().System.Data, s.ID()), s.DiskSpace(), s.Config().Egg.FileDenylist) + d, err := s.EnsureDataDirectoryExists() if err != nil { - return nil, errors.WithStack(err) + return nil, err + } + + fs, err := filesystem.New(d, s.DiskSpace(), s.Config().Egg.FileDenylist) + if err != nil { + return nil, err } s.fs = fs diff --git a/server/server.go b/server/server.go index a1777b047..ce3fb7fe2 100644 --- a/server/server.go +++ b/server/server.go @@ -257,11 +257,11 @@ func (s *Server) ReadLogfile(len int) ([]string, error) { // for the server is setup, and that all of the necessary files are created. func (s *Server) CreateEnvironment() error { // Ensure the data directory exists before getting too far through this process. - if err := s.EnsureDataDirectoryExists(); err != nil { + if _, err := s.EnsureDataDirectoryExists(); err != nil { return err } - cfg := config.Get() + cfg := *config.Get() if cfg.System.MachineID.Enable { // Hytale wants a machine-id in order to encrypt tokens for the server. So // write a machine-id file for the server that contains the server's UUID @@ -269,7 +269,7 @@ func (s *Server) CreateEnvironment() error { p := filepath.Join(cfg.System.MachineID.Directory, s.ID()) machineID := append(bytes.ReplaceAll([]byte(s.ID()), []byte{'-'}, []byte{}), '\n') if err := os.WriteFile(p, machineID, 0o644); err != nil { - return fmt.Errorf("failed to write machine-id (at '%s') for server '%s': %w", p, s.ID(), err) + return errors.Wrap(err, "server: failed to write machine-id to disk") } } @@ -295,21 +295,25 @@ func (s *Server) Filesystem() *filesystem.Filesystem { // EnsureDataDirectoryExists ensures that the data directory for the server // instance exists. -func (s *Server) EnsureDataDirectoryExists() error { - if _, err := os.Lstat(s.fs.Path()); err != nil { - if os.IsNotExist(err) { - s.Log().Debug("server: creating root directory and setting permissions") - if err := os.MkdirAll(s.fs.Path(), 0o700); err != nil { - return errors.WithStack(err) - } - if err := s.fs.Chown("/"); err != nil { - s.Log().WithField("error", err).Warn("server: failed to chown server data directory") - } - } else { - return errors.WrapIf(err, "server: failed to stat server root directory") +func (s *Server) EnsureDataDirectoryExists() (string, error) { + c := *config.Get() + path := filepath.Join(c.System.Data, s.ID()) + + if _, err := os.Lstat(path); err != nil { + if !os.IsNotExist(err) { + return path, errors.Wrap(err, "server: failed to stat server root directory") + } + + if err := os.MkdirAll(path, 0o700); err != nil { + return path, errors.Wrap(err, "server: failed to create root directory") + } + + if err := os.Chown(path, c.System.User.Uid, c.System.User.Gid); err != nil { + return path, errors.Wrap(err, "server: failed to chown newly created root directory") } } - return nil + + return path, nil } // OnStateChange sets the state of the server internally. This function handles crash detection as From 863cc0fab3a900b8a4a993150161b831982427a9 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Sat, 7 Feb 2026 12:04:20 -0800 Subject: [PATCH 10/30] fix chown calls on root path --- server/filesystem/chmod.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/server/filesystem/chmod.go b/server/filesystem/chmod.go index e62c01a00..077b1a500 100755 --- a/server/filesystem/chmod.go +++ b/server/filesystem/chmod.go @@ -32,7 +32,14 @@ func (fs *Filesystem) Chown(path string) error { uid := config.Get().System.User.Uid gid := config.Get().System.User.Gid - // Start by just chowning the initial path that we received. + if path == "" { + if err := os.Chown(fs.rootPath, uid, gid); err != nil { + return errors.Wrap(err, "server/filesystem: chown: failed to chown root directory") + } + + return nil + } + if err := fs.root.Chown(path, uid, gid); err != nil { return errors.Wrap(err, "server/filesystem: chown: failed to chown path") } From 741cb5063637629de0a826dad08ca33c57f27c7f Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Sat, 7 Feb 2026 18:19:03 -0800 Subject: [PATCH 11/30] improve sftp server support using os.Root --- server/filesystem/chmod.go | 17 ++-- server/filesystem/filesystem.go | 144 +++++++++++++++++++-------- server/filesystem/filesystem_test.go | 82 +++++++++++++++ server/filesystem/stat.go | 12 ++- sftp/handler.go | 34 +++---- 5 files changed, 211 insertions(+), 78 deletions(-) diff --git a/server/filesystem/chmod.go b/server/filesystem/chmod.go index 077b1a500..3d85845cb 100755 --- a/server/filesystem/chmod.go +++ b/server/filesystem/chmod.go @@ -13,6 +13,9 @@ import ( func (fs *Filesystem) Chmod(path string, mode os.FileMode) error { path = strings.TrimLeft(filepath.Clean(path), "/") + if path == "" { + path = "." + } if err := fs.root.Chmod(path, mode); err != nil { return errors.Wrap(err, "server/filesystem: chmod: failed to chmod path") } @@ -24,22 +27,14 @@ func (fs *Filesystem) Chmod(path string, mode os.FileMode) error { // underlying files. Iterate over all the files and directories. If it is a file go ahead // and perform the chown operation. Otherwise dig deeper into the directory until we've run // out of directories to dig into. -// -// todo: insecure function due to walking on uncontrolled path func (fs *Filesystem) Chown(path string) error { path = strings.TrimLeft(filepath.Clean(path), "/") - - uid := config.Get().System.User.Uid - gid := config.Get().System.User.Gid - if path == "" { - if err := os.Chown(fs.rootPath, uid, gid); err != nil { - return errors.Wrap(err, "server/filesystem: chown: failed to chown root directory") - } - - return nil + path = "." } + uid := config.Get().System.User.Uid + gid := config.Get().System.User.Gid if err := fs.root.Chown(path, uid, gid); err != nil { return errors.Wrap(err, "server/filesystem: chown: failed to chown path") } diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index 6225ebaa2..49429ed70 100755 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -3,7 +3,7 @@ package filesystem import ( "bufio" "io" - "io/ioutil" + fs2 "io/fs" "os" "path" "path/filepath" @@ -59,6 +59,18 @@ func New(path string, size int64, denylist []string) (*Filesystem, error) { return fs, nil } +// normalize takes the input path, runs it through filepath.Clean and trims any +// leading forward slashes (since the os.Root method calls will fail otherwise). +// If the resulting path is an empty string, "." is returned which os.Root will +// understand as the base directory. +func normalize(path string) string { + c := strings.TrimLeft(filepath.Clean(path), "/") + if c == "" { + return "." + } + return c +} + // Path returns the root path for the Filesystem instance. func (fs *Filesystem) Path() string { return fs.rootPath @@ -66,7 +78,7 @@ func (fs *Filesystem) Path() string { // File returns a reader for a file instance as well as the stat information. func (fs *Filesystem) File(p string) (*os.File, Stat, error) { - p = strings.TrimLeft(filepath.Clean(p), "/") + p = normalize(p) st, err := fs.Stat(p) if err != nil { if errors.Is(err, os.ErrNotExist) { @@ -88,7 +100,7 @@ func (fs *Filesystem) File(p string) (*os.File, Stat, error) { // already. If it is present, the file is opened using the defaults which will truncate // the contents. The opened file is then returned to the caller. func (fs *Filesystem) Touch(p string, flag int) (*os.File, error) { - p = strings.TrimLeft(filepath.Clean(p), "/") + p = normalize(p) f, err := fs.root.OpenFile(p, flag, 0o644) if err == nil { return f, nil @@ -126,7 +138,7 @@ func (fs *Filesystem) Touch(p string, flag int) (*os.File, error) { // will be created. This will also properly recalculate the disk space used by // the server when writing new files or modifying existing ones. func (fs *Filesystem) Writefile(p string, r io.Reader) error { - p = strings.TrimLeft(filepath.Clean(p), "/") + p = normalize(p) var currentSize int64 // If the file does not exist on the system already go ahead and create the pathway // to it and an empty file. We'll then write to it later on after this completes. @@ -167,16 +179,15 @@ func (fs *Filesystem) Writefile(p string, r io.Reader) error { // CreateDirectory creates a new directory ("name") at a specified path ("p") for the server. func (fs *Filesystem) CreateDirectory(name string, p string) error { - p = strings.TrimLeft(filepath.Clean(p), "/") - return fs.root.MkdirAll(path.Join(p, name), 0o755) + return fs.root.MkdirAll(path.Join(normalize(p), name), 0o755) } // Rename moves (or renames) a file or directory. func (fs *Filesystem) Rename(from string, to string) error { - to = strings.TrimLeft(filepath.Clean(to), "/") - from = strings.TrimLeft(filepath.Clean(from), "/") + to = normalize(to) + from = normalize(from) - if from == "" || to == "" { + if from == "." || to == "." { return os.ErrExist } @@ -237,7 +248,7 @@ func (fs *Filesystem) findCopySuffix(dir string, name string, extension string) // Copies a given file to the same location and appends a suffix to the file to indicate that // it has been copied. func (fs *Filesystem) Copy(p string) error { - p = strings.TrimLeft(filepath.Clean(p), "/") + p = normalize(p) s, err := fs.root.Stat(p) if err != nil { return err @@ -279,6 +290,52 @@ func (fs *Filesystem) Copy(p string) error { return fs.Writefile(path.Join(relative, n), source) } +// Symlink creates a symbolic link between the source and target paths. +func (fs *Filesystem) Symlink(source, target string) error { + source = normalize(source) + target = normalize(target) + + // os.Root#Symlink allows for the creation of a symlink that targets a file outside + // the root directory. This isn't the end of the world because the read is blocked + // through this system, and within a container it would just point to something in the + // readonly filesystem. + // + // However, just to avoid this propagating everywhere, *attempt* to block anything that + // would be pointing to a location outside the root directory. + if _, err := fs.root.Stat(source); err != nil { + return errors.Wrap(err, "server/filesystem: symlink: failed to stat source") + } + + // Yes -- this gap between the stat and symlink allows a TOCTOU vulnerability to exist, + // but again we're layering this with the remaining logic that prevents this filesystem + // from reading any symlinks or acting on any file that points outside the root as defined + // by os.Root. The check above is mostly to prevent stupid mistakes or basic attempts to + // get around this. If someone *really* wants to make these symlinks, they can. They can + // also just create them from the running server process, and we still need to rely on our + // own internal FS logic to detect and block those reads, which it does. Therefore, I am + // not deeply concerned with this. + if err := fs.root.Symlink(source, target); err != nil { + return errors.Wrap(err, "server/filesystem: symlink: failed to create symlink") + } + + return nil +} + +// ReadDir returns all the contents of the given directory. +func (fs *Filesystem) ReadDir(p string) ([]fs2.DirEntry, error) { + d, ok := fs.root.FS().(fs2.ReadDirFS) + if !ok { + return []fs2.DirEntry{}, errors.New("server/filesystem: readdir: could not init root fs") + } + + e, err := d.ReadDir(normalize(p)) + if err != nil { + return []fs2.DirEntry{}, errors.Wrap(err, "server/filesystem: readdir: failed to read directory") + } + + return e, nil +} + // TruncateRootDirectory removes _all_ files and directories from a server's // data directory and resets the used disk space to zero. func (fs *Filesystem) TruncateRootDirectory() error { @@ -288,8 +345,8 @@ func (fs *Filesystem) TruncateRootDirectory() error { // Delete removes a file or folder from the system. Prevents the user from // accidentally (or maliciously) removing their root server data directory. func (fs *Filesystem) Delete(p string) error { - p = strings.TrimLeft(filepath.Clean(p), "/") - if p == "" { + p = normalize(p) + if p == "." { return errors.New("server/filesystem: delete: cannot delete root directory") } @@ -339,17 +396,14 @@ func (fo *fileOpener) open(path string, flags int, perm os.FileMode) (*os.File, } } -// ListDirectory lists the contents of a given directory and returns stat -// information about each file and folder within it. +// ListDirectory lists the contents of a given directory and returns stat information +// about each file and folder within it. If you only need to know the contents of the +// directory and do not need mimetype information, call [Filesystem.ReadDir] directly +// instead. func (fs *Filesystem) ListDirectory(p string) ([]Stat, error) { - cleaned, err := fs.SafePath(p) + files, err := fs.ReadDir(p) if err != nil { - return nil, err - } - - files, err := ioutil.ReadDir(cleaned) - if err != nil { - return nil, err + return []Stat{}, err } var wg sync.WaitGroup @@ -359,39 +413,41 @@ func (fs *Filesystem) ListDirectory(p string) ([]Stat, error) { // break the panel badly. out := make([]Stat, len(files)) - // Iterate over all of the files and directories returned and perform an async process + // Iterate over all the files and directories returned and perform an async process // to get the mime-type for them all. for i, file := range files { wg.Add(1) - go func(idx int, f os.FileInfo) { + go func(idx int, f fs2.DirEntry) { defer wg.Done() - var m *mimetype.MIME - d := "inode/directory" - if !f.IsDir() { - cleanedp := filepath.Join(cleaned, f.Name()) - if f.Mode()&os.ModeSymlink != 0 { - cleanedp, _ = fs.SafePath(filepath.Join(cleaned, f.Name())) - } + fi, err := f.Info() + if err != nil { + return + } - // Don't try to detect the type on a pipe — this will just hang the application and - // you'll never get a response back. - // - // @see https://github.com/pterodactyl/panel/issues/4059 - if cleanedp != "" && f.Mode()&os.ModeNamedPipe == 0 { - m, _ = mimetype.DetectFile(filepath.Join(cleaned, f.Name())) - } else { - // Just pass this for an unknown type because the file could not safely be resolved within - // the server data path. - d = "application/octet-stream" - } + if fi.IsDir() { + out[idx] = Stat{FileInfo: fi, Mimetype: "inode/directory"} + return } - st := Stat{FileInfo: f, Mimetype: d} - if m != nil { - st.Mimetype = m.String() + st := Stat{FileInfo: fi, Mimetype: "application/octet-stream"} + + // Don't try to detect the type on a pipe — this will just hang the application, + // and you'll never get a response back. + // + // @see https://github.com/pterodactyl/panel/issues/4059 + if fi.Mode()&os.ModeNamedPipe == 0 { + f, err := fs.root.Open(filepath.Join(p, f.Name())) + if err != nil { + return + } + defer f.Close() + if m, err := mimetype.DetectReader(f); err != nil { + st.Mimetype = m.String() + } } + out[idx] = st }(i, file) } diff --git a/server/filesystem/filesystem_test.go b/server/filesystem/filesystem_test.go index bbf60d331..3260e9643 100755 --- a/server/filesystem/filesystem_test.go +++ b/server/filesystem/filesystem_test.go @@ -467,6 +467,88 @@ func TestFilesystem_Copy(t *testing.T) { }) } +func TestFilesystem_Symlink(t *testing.T) { + g := Goblin(t) + fs := NewFs() + + g.Describe("Symlink", func() { + g.It("should create a symlink", func() { + fs.write("source.txt", []byte("text content")) + + err := fs.Symlink("source.txt", "symlink.txt") + g.Assert(err).IsNil() + + st, err := os.Lstat(filepath.Join(fs.rootPath, "symlink.txt")) + g.Assert(err).IsNil() + g.Assert(st.Mode()&os.ModeSymlink != 0).IsTrue() + }) + + g.It("should return an error if the source is outside the root", func() { + err := fs.Symlink("../source.txt", "symlink.txt") + g.Assert(err).IsNotNil() + g.Assert(IsPathError(err)).IsTrue() + }) + + g.It("should return an error if the dest is outside the root", func() { + fs.write("source.txt", []byte("text content")) + + err := fs.Symlink("source.txt", "../symlink.txt") + g.Assert(err).IsNotNil() + g.Assert(IsLinkError(err)).IsTrue() + }) + + g.AfterEach(func() { + fs.reset() + }) + }) +} + +func TestFilesystem_ReadDir(t *testing.T) { + g := Goblin(t) + fs := NewFs() + + g.Describe("ReadDir", func() { + g.Before(func() { + if err := os.Mkdir(filepath.Join(fs.rootPath, "child"), 0o755); err != nil { + panic(err) + } + + fs.write("one.txt", []byte("one")) + fs.write("two.txt", []byte("two")) + fs.write("child/three.txt", []byte("two")) + }) + + g.After(func() { + fs.reset() + }) + + g.It("should return the contents of the root directory", func() { + d, err := fs.ReadDir("/") + g.Assert(err).IsNil() + g.Assert(len(d)).Equal(3) + + // os.Root#ReadDir sorts them by name. + g.Assert(d[0].Name()).Equal("child") + g.Assert(d[0].IsDir()).IsTrue() + g.Assert(d[1].Name()).Equal("one.txt") + g.Assert(d[2].Name()).Equal("two.txt") + }) + + g.It("should return the contents of a child directory", func() { + d, err := fs.ReadDir("child") + g.Assert(err).IsNil() + g.Assert(len(d)).Equal(1) + g.Assert(d[0].Name()).Equal("three.txt") + }) + + g.It("should return an error if the directory is outside the root", func() { + _, err := fs.ReadDir("../server") + g.Assert(err).IsNotNil() + g.Assert(IsPathError(err)).IsTrue() + }) + }) +} + func TestFilesystem_Delete(t *testing.T) { g := Goblin(t) fs := NewFs() diff --git a/server/filesystem/stat.go b/server/filesystem/stat.go index 27bae36d4..c53617219 100755 --- a/server/filesystem/stat.go +++ b/server/filesystem/stat.go @@ -2,9 +2,7 @@ package filesystem import ( "os" - "path/filepath" "strconv" - "strings" "time" "emperror.dev/errors" @@ -47,7 +45,7 @@ func (s *Stat) MarshalJSON() ([]byte, error) { // Stat stats a file or folder and returns the base stat object from go along // with the MIME data that can be used for editing files. func (fs *Filesystem) Stat(p string) (Stat, error) { - p = strings.TrimLeft(filepath.Clean(p), "/") + p = normalize(p) s, err := fs.root.Stat(p) if err != nil { return Stat{}, errors.Wrap(err, "server/filesystem: stat: failed to stat file") @@ -76,3 +74,11 @@ func (fs *Filesystem) Stat(p string) (Stat, error) { return st, nil } + +func (fs *Filesystem) Stat2(p string) (os.FileInfo, error) { + st, err := fs.root.Stat(normalize(p)) + if err != nil { + return st, errors.Wrap(err, "server/filesystem: stat2: failed to stat file") + } + return st, nil +} diff --git a/sftp/handler.go b/sftp/handler.go index 6fdc77e63..683886d67 100644 --- a/sftp/handler.go +++ b/sftp/handler.go @@ -2,7 +2,6 @@ package sftp import ( "io" - "io/ioutil" "os" "path/filepath" "strings" @@ -214,22 +213,12 @@ func (h *Handler) Filecmd(request *sftp.Request) error { } h.events.MustLog(server.ActivitySftpCreateDirectory, FileAction{Entity: request.Filepath}) break - // Support creating symlinks between files. The source and target must resolve within - // the server home directory. case "Symlink": if !h.can(PermissionFileCreate) { return sftp.ErrSSHFxPermissionDenied } - source, err := h.fs.SafePath(request.Filepath) - if err != nil { - return sftp.ErrSSHFxNoSuchFile - } - target, err := h.fs.SafePath(request.Target) - if err != nil { - return sftp.ErrSSHFxNoSuchFile - } - if err := os.Symlink(source, target); err != nil { - l.WithField("target", target).WithField("error", err).Error("failed to create symlink") + if err := h.fs.Symlink(request.Filepath, request.Target); err != nil { + l.WithField("target", request.Target).WithField("error", err).Error("failed to create symlink") return sftp.ErrSSHFxFailure } break @@ -274,18 +263,23 @@ func (h *Handler) Filelist(request *sftp.Request) (sftp.ListerAt, error) { switch request.Method { case "List": - p, err := h.fs.SafePath(request.Filepath) - if err != nil { - return nil, sftp.ErrSSHFxNoSuchFile - } - files, err := ioutil.ReadDir(p) + d, err := h.fs.ReadDir(request.Filepath) if err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil, sftp.ErrSSHFxNoSuchFile + } h.logger.WithField("source", request.Filepath).WithField("error", err).Error("error while listing directory") return nil, sftp.ErrSSHFxFailure } + files := make([]os.FileInfo, len(d)) + for _, entry := range d { + if i, err := entry.Info(); err == nil { + files = append(files, i) + } + } return ListerAt(files), nil case "Stat": - st, err := h.fs.Stat(request.Filepath) + st, err := h.fs.Stat2(request.Filepath) if err != nil { if errors.Is(err, os.ErrNotExist) { return nil, sftp.ErrSSHFxNoSuchFile @@ -293,7 +287,7 @@ func (h *Handler) Filelist(request *sftp.Request) (sftp.ListerAt, error) { h.logger.WithField("source", request.Filepath).WithField("error", err).Error("error performing stat on file") return nil, sftp.ErrSSHFxFailure } - return ListerAt([]os.FileInfo{st.FileInfo}), nil + return ListerAt([]os.FileInfo{st}), nil default: return nil, sftp.ErrSSHFxOpUnsupported } From d3559ae2beeaaff38b6e5d48dd89372e7c27f92a Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Sun, 8 Feb 2026 10:08:48 -0800 Subject: [PATCH 12/30] fix panic condition --- server/filesystem/filesystem.go | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index 49429ed70..caf304fcc 100755 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -14,6 +14,7 @@ import ( "time" "emperror.dev/errors" + "github.com/apex/log" "github.com/gabriel-vasile/mimetype" ignore "github.com/sabhiram/go-gitignore" @@ -418,11 +419,12 @@ func (fs *Filesystem) ListDirectory(p string) ([]Stat, error) { for i, file := range files { wg.Add(1) - go func(idx int, f fs2.DirEntry) { + go func(idx int, d fs2.DirEntry) { defer wg.Done() - fi, err := f.Info() + fi, err := d.Info() if err != nil { + log.WithField("error", err).WithField("path", filepath.Join(p, d.Name())).Warn("failed to retrieve directory entry info") return } @@ -438,13 +440,13 @@ func (fs *Filesystem) ListDirectory(p string) ([]Stat, error) { // // @see https://github.com/pterodactyl/panel/issues/4059 if fi.Mode()&os.ModeNamedPipe == 0 { - f, err := fs.root.Open(filepath.Join(p, f.Name())) - if err != nil { - return - } - defer f.Close() - if m, err := mimetype.DetectReader(f); err != nil { - st.Mimetype = m.String() + if f, err := fs.root.Open(normalize(filepath.Join(p, d.Name()))); err != nil { + log.WithField("error", err).WithField("path", filepath.Join(p, d.Name())).Warn("error opening file for mimetype detection") + } else { + if m, err := mimetype.DetectReader(f); err != nil { + st.Mimetype = m.String() + } + _ = f.Close() } } From 7729d580a1cade408139da616c420833dbe355a0 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Sun, 8 Feb 2026 10:11:26 -0800 Subject: [PATCH 13/30] helps to actually return the mime --- server/filesystem/filesystem.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index caf304fcc..fb523acc2 100755 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -443,8 +443,10 @@ func (fs *Filesystem) ListDirectory(p string) ([]Stat, error) { if f, err := fs.root.Open(normalize(filepath.Join(p, d.Name()))); err != nil { log.WithField("error", err).WithField("path", filepath.Join(p, d.Name())).Warn("error opening file for mimetype detection") } else { - if m, err := mimetype.DetectReader(f); err != nil { + if m, err := mimetype.DetectReader(f); err == nil { st.Mimetype = m.String() + } else { + log.WithField("error", err).WithField("path", filepath.Join(p, d.Name())).Warn("failed to detect mimetype for file") } _ = f.Close() } From ca3d2a084f7bbda8009b4eedb1e3efcf0b557dcf Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Sun, 8 Feb 2026 10:15:44 -0800 Subject: [PATCH 14/30] update server file download logic --- router/router_download.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/router/router_download.go b/router/router_download.go index 59fec5194..14a063b59 100644 --- a/router/router_download.go +++ b/router/router_download.go @@ -87,27 +87,19 @@ func getDownloadFile(c *gin.Context) { return } - p, _ := s.Filesystem().SafePath(token.FilePath) - st, err := os.Stat(p) - // If there is an error or we're somehow trying to download a directory, just - // respond with the appropriate error. + f, st, err := s.Filesystem().File(token.FilePath) if err != nil { middleware.CaptureAndAbort(c, err) return - } else if st.IsDir() { + } + defer f.Close() + if st.IsDir() { c.AbortWithStatusJSON(http.StatusNotFound, gin.H{ "error": "The requested resource was not found on this server.", }) return } - f, err := os.Open(p) - if err != nil { - middleware.CaptureAndAbort(c, err) - return - } - defer f.Close() - c.Header("Content-Length", strconv.Itoa(int(st.Size()))) c.Header("Content-Disposition", "attachment; filename="+strconv.Quote(st.Name())) c.Header("Content-Type", "application/octet-stream") From c2b1dbfd18fddb44dc9dd370dd1340bff7de0736 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Sun, 8 Feb 2026 10:51:07 -0800 Subject: [PATCH 15/30] restore parser from main branch --- parser/helpers.go | 18 +-- parser/parser.go | 197 +++++++++++++++----------------- server/config_parser.go | 17 ++- server/filesystem/chmod.go | 13 ++- server/filesystem/filesystem.go | 18 +-- sftp/handler.go | 2 +- 6 files changed, 123 insertions(+), 142 deletions(-) diff --git a/parser/helpers.go b/parser/helpers.go index a8e8ec789..be09c686f 100644 --- a/parser/helpers.go +++ b/parser/helpers.go @@ -2,8 +2,6 @@ package parser import ( "bytes" - "io" - "os" "regexp" "strconv" "strings" @@ -29,24 +27,14 @@ var configMatchRegex = regexp.MustCompile(`{{\s?config\.([\w.-]+)\s?}}`) // matching: // // -// +// +// +// // // // noinspection RegExpRedundantEscape var xmlValueMatchRegex = regexp.MustCompile(`^\[([\w]+)='(.*)'\]$`) -// Gets the []byte representation of a configuration file to be passed through to other -// handler functions. If the file does not currently exist, it will be created. -func readFileBytes(path string) ([]byte, error) { - file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644) - if err != nil { - return nil, err - } - defer file.Close() - - return io.ReadAll(file) -} - // Gets the value of a key based on the value type defined. func (cfr *ConfigurationFileReplacement) getKeyValue(value string) interface{} { if cfr.ReplaceWith.Type() == jsonparser.Boolean { diff --git a/parser/parser.go b/parser/parser.go index f309c0da1..2c8303915 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -2,10 +2,10 @@ package parser import ( "bufio" + "bytes" "encoding/json" "io" "os" - "path/filepath" "strconv" "strings" @@ -161,14 +161,14 @@ func (cfr *ConfigurationFileReplacement) UnmarshalJSON(data []byte) error { iv, err := jsonparser.GetString(data, "if_value") // We only check keypath here since match & replace_with should be present on all of // them, however if_value is optional. - if err != nil && err != jsonparser.KeyPathNotFoundError { + if err != nil && !errors.Is(err, jsonparser.KeyPathNotFoundError) { return err } cfr.IfValue = iv rw, dt, _, err := jsonparser.Get(data, "replace_with") if err != nil { - if err != jsonparser.KeyPathNotFoundError { + if !errors.Is(err, jsonparser.KeyPathNotFoundError) { return err } @@ -188,11 +188,12 @@ func (cfr *ConfigurationFileReplacement) UnmarshalJSON(data []byte) error { return nil } -// Parses a given configuration file and updates all of the values within as defined -// in the API response from the Panel. -func (f *ConfigurationFile) Parse(path string, internal bool) error { - log.WithField("path", path).WithField("parser", f.Parser.String()).Debug("parsing server configuration file") +// Parse parses a given configuration file and updates all the values within +// as defined in the API response from the Panel. +func (f *ConfigurationFile) Parse(file *os.File) error { + // log.WithField("path", path).WithField("parser", f.Parser.String()).Debug("parsing server configuration file") + // What the fuck is going on here? if mb, err := json.Marshal(config.Get()); err != nil { return err } else { @@ -203,56 +204,24 @@ func (f *ConfigurationFile) Parse(path string, internal bool) error { switch f.Parser { case Properties: - err = f.parsePropertiesFile(path) - break + err = f.parsePropertiesFile(file) case File: - err = f.parseTextFile(path) - break + err = f.parseTextFile(file) case Yaml, "yml": - err = f.parseYamlFile(path) - break + err = f.parseYamlFile(file) case Json: - err = f.parseJsonFile(path) - break + err = f.parseJsonFile(file) case Ini: - err = f.parseIniFile(path) - break + err = f.parseIniFile(file) case Xml: - err = f.parseXmlFile(path) - break + err = f.parseXmlFile(file) } - - if errors.Is(err, os.ErrNotExist) { - // File doesn't exist, we tried creating it, and same error is returned? Pretty - // sure this pathway is impossible, but if not, abort here. - if internal { - return nil - } - - b := strings.TrimSuffix(path, filepath.Base(path)) - if err := os.MkdirAll(b, 0o755); err != nil { - return errors.WithMessage(err, "failed to create base directory for missing configuration file") - } else { - if _, err := os.Create(path); err != nil { - return errors.WithMessage(err, "failed to create missing configuration file") - } - } - - return f.Parse(path, true) - } - return err } // Parses an xml file. -func (f *ConfigurationFile) parseXmlFile(path string) error { +func (f *ConfigurationFile) parseXmlFile(file *os.File) error { doc := etree.NewDocument() - file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644) - if err != nil { - return err - } - defer file.Close() - if _, err := doc.ReadFrom(file); err != nil { return err } @@ -330,17 +299,9 @@ func (f *ConfigurationFile) parseXmlFile(path string) error { } // Parses an ini file. -func (f *ConfigurationFile) parseIniFile(path string) error { - // Ini package can't handle a non-existent file, so handle that automatically here - // by creating it if not exists. Then, immediately close the file since we will use - // other methods to write the new contents. - file, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR, 0o644) - if err != nil { - return err - } - file.Close() - - cfg, err := ini.Load(path) +func (f *ConfigurationFile) parseIniFile(file *os.File) error { + // Wrap the file in a NopCloser so the ini package doesn't close the file. + cfg, err := ini.Load(io.NopCloser(file)) if err != nil { return err } @@ -419,8 +380,8 @@ func (f *ConfigurationFile) parseIniFile(path string) error { // Parses a json file updating any matching key/value pairs. If a match is not found, the // value is set regardless in the file. See the commentary in parseYamlFile for more details // about what is happening during this process. -func (f *ConfigurationFile) parseJsonFile(path string) error { - b, err := readFileBytes(path) +func (f *ConfigurationFile) parseJsonFile(file *os.File) error { + b, err := io.ReadAll(file) if err != nil { return err } @@ -430,14 +391,24 @@ func (f *ConfigurationFile) parseJsonFile(path string) error { return err } - output := []byte(data.StringIndent("", " ")) - return os.WriteFile(path, output, 0o644) + if _, err := file.Seek(0, io.SeekStart); err != nil { + return err + } + if err := file.Truncate(0); err != nil { + return err + } + + // Write the data to the file. + if _, err := io.Copy(file, bytes.NewReader(data.BytesIndent("", " "))); err != nil { + return errors.Wrap(err, "parser: failed to write properties file to disk") + } + return nil } // Parses a yaml file and updates any matching key/value pairs before persisting // it back to the disk. -func (f *ConfigurationFile) parseYamlFile(path string) error { - b, err := readFileBytes(path) +func (f *ConfigurationFile) parseYamlFile(file *os.File) error { + b, err := io.ReadAll(file) if err != nil { return err } @@ -468,35 +439,56 @@ func (f *ConfigurationFile) parseYamlFile(path string) error { return err } - return os.WriteFile(path, marshaled, 0o644) + if _, err := file.Seek(0, io.SeekStart); err != nil { + return err + } + if err := file.Truncate(0); err != nil { + return err + } + + // Write the data to the file. + if _, err := io.Copy(file, bytes.NewReader(marshaled)); err != nil { + return errors.Wrap(err, "parser: failed to write properties file to disk") + } + return nil } // Parses a text file using basic find and replace. This is a highly inefficient method of // scanning a file and performing a replacement. You should attempt to use anything other // than this function where possible. -func (f *ConfigurationFile) parseTextFile(path string) error { - input, err := os.ReadFile(path) - if err != nil { - return err - } - - lines := strings.Split(string(input), "\n") - for i, line := range lines { +func (f *ConfigurationFile) parseTextFile(file *os.File) error { + b := bytes.NewBuffer(nil) + s := bufio.NewScanner(file) + var replaced bool + for s.Scan() { + line := s.Bytes() + replaced = false for _, replace := range f.Replace { // If this line doesn't match what we expect for the replacement, move on to the next // line. Otherwise, update the line to have the replacement value. - if !strings.HasPrefix(line, replace.Match) { + if !bytes.HasPrefix(line, []byte(replace.Match)) { continue } - - lines[i] = replace.ReplaceWith.String() + b.Write(replace.ReplaceWith.Bytes()) + replaced = true } + if !replaced { + b.Write(line) + } + b.WriteByte('\n') } - if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")), 0o644); err != nil { + if _, err := file.Seek(0, io.SeekStart); err != nil { + return err + } + if err := file.Truncate(0); err != nil { return err } + // Write the data to the file. + if _, err := io.Copy(file, b); err != nil { + return errors.Wrap(err, "parser: failed to write properties file to disk") + } return nil } @@ -526,31 +518,29 @@ func (f *ConfigurationFile) parseTextFile(path string) error { // // @see https://github.com/pterodactyl/panel/issues/2308 (original) // @see https://github.com/pterodactyl/panel/issues/3009 ("bug" introduced as result) -func (f *ConfigurationFile) parsePropertiesFile(path string) error { - var s strings.Builder - // Open the file and attempt to load any comments that currenty exist at the start - // of the file. This is kind of a hack, but should work for a majority of users for - // the time being. - if fd, err := os.Open(path); err != nil { - return errors.Wrap(err, "parser: could not open file for reading") - } else { - scanner := bufio.NewScanner(fd) - // Scan until we hit a line that is not a comment that actually has content - // on it. Keep appending the comments until that time. - for scanner.Scan() { - text := scanner.Text() - if len(text) > 0 && text[0] != '#' { - break - } - s.WriteString(text + "\n") - } - _ = fd.Close() - if err := scanner.Err(); err != nil { - return errors.WithStackIf(err) +func (f *ConfigurationFile) parsePropertiesFile(file *os.File) error { + b, err := io.ReadAll(file) + if err != nil { + return err + } + + s := bytes.NewBuffer(nil) + scanner := bufio.NewScanner(bytes.NewReader(b)) + // Scan until we hit a line that is not a comment that actually has content + // on it. Keep appending the comments until that time. + for scanner.Scan() { + text := scanner.Bytes() + if len(text) > 0 && text[0] != '#' { + break } + s.Write(text) + s.WriteByte('\n') + } + if err := scanner.Err(); err != nil { + return errors.WithStackIf(err) } - p, err := properties.LoadFile(path, properties.UTF8) + p, err := properties.Load(b, properties.UTF8) if err != nil { return errors.Wrap(err, "parser: could not load properties file for configuration update") } @@ -588,17 +578,16 @@ func (f *ConfigurationFile) parsePropertiesFile(path string) error { s.WriteString(key + "=" + strings.Trim(strconv.QuoteToASCII(value), "\"") + "\n") } - // Open the file for writing. - w, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) - if err != nil { + if _, err := file.Seek(0, io.SeekStart); err != nil { + return err + } + if err := file.Truncate(0); err != nil { return err } - defer w.Close() // Write the data to the file. - if _, err := w.Write([]byte(s.String())); err != nil { + if _, err := io.Copy(file, s); err != nil { return errors.Wrap(err, "parser: failed to write properties file to disk") } - return nil } diff --git a/server/config_parser.go b/server/config_parser.go index 4c51724bd..3a13edb35 100644 --- a/server/config_parser.go +++ b/server/config_parser.go @@ -1,6 +1,7 @@ package server import ( + "os" "runtime" "github.com/gammazero/workerpool" @@ -10,26 +11,24 @@ import ( // a server automatically to ensure that they always use the specified values. func (s *Server) UpdateConfigurationFiles() { pool := workerpool.New(runtime.NumCPU()) - - s.Log().Debug("acquiring process configuration files...") files := s.ProcessConfiguration().ConfigurationFiles - s.Log().Debug("acquired process configuration files") + for _, cf := range files { f := cf pool.Submit(func() { - p, err := s.Filesystem().SafePath(f.FileName) + fd, err := s.Filesystem().Touch(f.FileName, os.O_RDWR|os.O_CREATE, 0o644) if err != nil { - s.Log().WithField("error", err).Error("failed to generate safe path for configuration file") - + s.Log().WithField("file_name", f.FileName).WithField("error", err).Error("failed to open configuration file") return } + defer fd.Close() - if err := f.Parse(p, false); err != nil { - s.Log().WithField("error", err).Error("failed to parse and update server configuration file") + if err := f.Parse(fd); err != nil { + s.Log().WithField("error", err).WithField("file_name", f.FileName).Error("failed to parse and update server configuration file") } - s.Log().WithField("path", f.FileName).Debug("finished processing server configuration file") + s.Log().WithField("file_name", f.FileName).Debug("finished processing server configuration file") }) } diff --git a/server/filesystem/chmod.go b/server/filesystem/chmod.go index 3d85845cb..7b618e7f7 100755 --- a/server/filesystem/chmod.go +++ b/server/filesystem/chmod.go @@ -1,6 +1,7 @@ package filesystem import ( + "fmt" "os" "path/filepath" "strings" @@ -36,7 +37,7 @@ func (fs *Filesystem) Chown(path string) error { uid := config.Get().System.User.Uid gid := config.Get().System.User.Gid if err := fs.root.Chown(path, uid, gid); err != nil { - return errors.Wrap(err, "server/filesystem: chown: failed to chown path") + return errors.WrapIf(err, "server/filesystem: chown: failed to chown path") } // If this is not a directory, we can now return from the function; there is nothing @@ -46,9 +47,10 @@ func (fs *Filesystem) Chown(path string) error { return nil } - return errors.Wrap(err, "server/filesystem: chown: failed to stat path") + return errors.WrapIf(err, "server/filesystem: chown: failed to stat path") } + fmt.Println("walking path", filepath.Join(fs.rootPath, path)) // If this was a directory, begin walking over its contents recursively and ensure that all // the subfiles and directories get their permissions updated as well. err := godirwalk.Walk(filepath.Join(fs.rootPath, path), &godirwalk.Options{ @@ -56,16 +58,19 @@ func (fs *Filesystem) Chown(path string) error { FollowSymbolicLinks: false, Callback: func(p string, e *godirwalk.Dirent) error { p = strings.TrimLeft(strings.TrimPrefix(p, fs.Path()), "/") + if p == "" { + return godirwalk.SkipThis + } if err := fs.root.Chown(p, uid, gid); err != nil { - return errors.Wrap(err, "server/filesystem: chown: failed to chown during walk") + return errors.Wrap(err, fmt.Sprintf("server/filesystem: chown: failed to chown file")) } return nil }, }) - return errors.Wrap(err, "server/filesystem: chown: failed to chown directory tree") + return errors.WrapIf(err, "server/filesystem: chown: failed to chown directory tree") } func (fs *Filesystem) Chtimes(path string, atime, mtime time.Time) error { diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index fb523acc2..61209ffa1 100755 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -98,11 +98,12 @@ func (fs *Filesystem) File(p string) (*os.File, Stat, error) { } // Touch acts by creating the given file and path on the disk if it is not present -// already. If it is present, the file is opened using the defaults which will truncate +// already. If it is present, the file is opened using the defaults which will truncate // the contents. The opened file is then returned to the caller. -func (fs *Filesystem) Touch(p string, flag int) (*os.File, error) { +func (fs *Filesystem) Touch(p string, flag int, perm os.FileMode) (*os.File, error) { p = normalize(p) - f, err := fs.root.OpenFile(p, flag, 0o644) + o := &fileOpener{root: fs.root} + f, err := o.open(p, flag, perm) if err == nil { return f, nil } @@ -118,18 +119,17 @@ func (fs *Filesystem) Touch(p string, flag int) (*os.File, error) { // Create the path leading up to the file we're trying to create, setting the final perms // on it as we go. if err := fs.root.MkdirAll(filepath.Dir(p), 0o755); err != nil { - return nil, errors.Wrap(err, "server/filesystem: touch: failed to create directory tree") + return nil, errors.WrapIf(err, "server/filesystem: touch: failed to create directory tree") } if err := fs.Chown(filepath.Dir(p)); err != nil { - return nil, errors.Wrap(err, "server/filesystem: touch: failed to chown directory tree") + return nil, errors.WrapIf(err, "server/filesystem: touch: failed to chown directory tree") } } - o := &fileOpener{root: fs.root} // Try to open the file now that we have created the pathing necessary for it, and then // Chown that file so that the permissions don't mess with things. - f, err = o.open(p, flag, 0o644) + f, err = o.open(p, flag, perm) if err != nil { - return nil, errors.Wrap(err, "server/filesystem: touch: failed to open file with wait") + return nil, errors.Wrap(err, "server/filesystem: touch: failed to open file handle") } _ = fs.Chown(p) return f, nil @@ -163,7 +163,7 @@ func (fs *Filesystem) Writefile(p string, r io.Reader) error { // Touch the file and return the handle to it at this point. This will create the file, // any necessary directories, and set the proper owner of the file. - file, err := fs.Touch(p, os.O_RDWR|os.O_CREATE|os.O_TRUNC) + file, err := fs.Touch(p, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) if err != nil { return err } diff --git a/sftp/handler.go b/sftp/handler.go index 683886d67..680a9577e 100644 --- a/sftp/handler.go +++ b/sftp/handler.go @@ -121,7 +121,7 @@ func (h *Handler) Filewrite(request *sftp.Request) (io.WriterAt, error) { if !h.can(permission) { return nil, sftp.ErrSSHFxPermissionDenied } - f, err := h.fs.Touch(request.Filepath, os.O_RDWR|os.O_CREATE|os.O_TRUNC) + f, err := h.fs.Touch(request.Filepath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) if err != nil { l.WithField("flags", request.Flags).WithField("error", err).Error("failed to open existing file on system") return nil, sftp.ErrSSHFxFailure From 78066a6a48229f1b86ac2f63d8cdcf5174ee1a11 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Mon, 16 Feb 2026 13:52:40 -0800 Subject: [PATCH 16/30] add back archive creation support --- router/router_server_files.go | 2 +- server/backup/backup_local.go | 20 ++- server/backup/backup_s3.go | 29 ++-- server/filesystem/archive.go | 241 +++++++++++++----------------- server/filesystem/archive_test.go | 61 ++++++-- server/filesystem/compress.go | 32 +++- server/filesystem/filesystem.go | 2 +- server/transfer/archive.go | 26 ++-- 8 files changed, 228 insertions(+), 185 deletions(-) diff --git a/router/router_server_files.go b/router/router_server_files.go index 323d23bc3..3f44ede4b 100644 --- a/router/router_server_files.go +++ b/router/router_server_files.go @@ -422,7 +422,7 @@ func postServerCompressFiles(c *gin.Context) { return } - f, err := s.Filesystem().CompressFiles(data.RootPath, data.Files) + f, err := s.Filesystem().CompressFiles(c.Request.Context(), data.RootPath, data.Files) if err != nil { middleware.CaptureAndAbort(c, err) return diff --git a/server/backup/backup_local.go b/server/backup/backup_local.go index ca7f9749b..bd9e7282b 100644 --- a/server/backup/backup_local.go +++ b/server/backup/backup_local.go @@ -4,6 +4,7 @@ import ( "context" "io" "os" + "strings" "emperror.dev/errors" "github.com/juju/ratelimit" @@ -60,13 +61,24 @@ func (b *LocalBackup) WithLogContext(c map[string]interface{}) { // Generate generates a backup of the selected files and pushes it to the // defined location for this instance. func (b *LocalBackup) Generate(ctx context.Context, basePath, ignore string) (*ArchiveDetails, error) { - a := &filesystem.Archive{ - BasePath: basePath, - Ignore: ignore, + r, err := os.OpenRoot(basePath) + if err != nil { + return nil, errors.Wrap(err, "server/backup: failed to open root directory") + } + a, err := filesystem.NewArchive(r, nil, filesystem.WithIgnored(strings.Split(ignore, "\n"))) + if err != nil { + _ = r.Close() + return nil, errors.WrapIf(err, "server/backup: failed to create archive") } + defer a.Close() b.log().WithField("path", b.Path()).Info("creating backup for server") - if err := a.Create(ctx, b.Path()); err != nil { + f, err := os.OpenFile(b.Path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return nil, errors.Wrap(err, "server/backup: failed to open file for writing") + } + defer f.Close() + if err := a.Create(ctx, f); err != nil { return nil, err } b.log().Info("created backup successfully") diff --git a/server/backup/backup_s3.go b/server/backup/backup_s3.go index b651b2702..18f15c28a 100644 --- a/server/backup/backup_s3.go +++ b/server/backup/backup_s3.go @@ -7,6 +7,7 @@ import ( "net/http" "os" "strconv" + "strings" "time" "emperror.dev/errors" @@ -51,24 +52,34 @@ func (s *S3Backup) WithLogContext(c map[string]interface{}) { func (s *S3Backup) Generate(ctx context.Context, basePath, ignore string) (*ArchiveDetails, error) { defer s.Remove() - a := &filesystem.Archive{ - BasePath: basePath, - Ignore: ignore, + r, err := os.OpenRoot(basePath) + if err != nil { + return nil, errors.Wrap(err, "backup: failed to open root directory") + } + a, err := filesystem.NewArchive(r, nil, filesystem.WithIgnored(strings.Split(ignore, "\n"))) + if err != nil { + _ = r.Close() + return nil, errors.WrapIf(err, "backup: failed to create archive") } + defer a.Close() s.log().WithField("path", s.Path()).Info("creating backup for server") - if err := a.Create(ctx, s.Path()); err != nil { + f, err := os.OpenFile(s.Path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return nil, errors.Wrap(err, "backup: failed to open file for writing") + } + defer f.Close() + if err := a.Create(ctx, f); err != nil { return nil, err } s.log().Info("created backup successfully") - rc, err := os.Open(s.Path()) - if err != nil { - return nil, errors.Wrap(err, "backup: could not read archive from disk") + _ = f.Sync() + if _, err := f.Seek(0, io.SeekStart); err != nil { + return nil, errors.Wrap(err, "backup: failed to seek on file") } - defer rc.Close() - parts, err := s.generateRemoteRequest(ctx, rc) + parts, err := s.generateRemoteRequest(ctx, f) if err != nil { return nil, err } diff --git a/server/filesystem/archive.go b/server/filesystem/archive.go index c3bd65f30..1be2d06ea 100644 --- a/server/filesystem/archive.go +++ b/server/filesystem/archive.go @@ -3,7 +3,6 @@ package filesystem import ( "archive/tar" "context" - "fmt" "io" "io/fs" "os" @@ -12,9 +11,7 @@ import ( "sync" "emperror.dev/errors" - "github.com/apex/log" "github.com/juju/ratelimit" - "github.com/karrick/godirwalk" "github.com/klauspost/pgzip" ignore "github.com/sabhiram/go-gitignore" @@ -37,7 +34,8 @@ type TarProgress struct { p *progress.Progress } -// NewTarProgress . +// NewTarProgress returns a new progress writer for the tar file. This is a wrapper +// around the standard writer with a progress instance embedded. func NewTarProgress(w *tar.Writer, p *progress.Progress) *TarProgress { if p != nil { p.Writer = w @@ -56,41 +54,74 @@ func (p *TarProgress) Write(v []byte) (int, error) { return p.p.Write(v) } +type ArchiveOption func(a *Archive) error + type Archive struct { - // BasePath is the absolute path to create the archive from where Files and Ignore are - // relative to. - BasePath string - - // Ignore is a gitignore string (most likely read from a file) of files to ignore - // from the archive. - Ignore string - - // Files specifies the files to archive, this takes priority over the Ignore option, if - // unspecified, all files in the BasePath will be archived unless Ignore is set. - // - // All items in Files must be absolute within BasePath. - Files []string - - // Progress wraps the writer of the archive to pass through the progress tracker. - Progress *progress.Progress + root *os.Root + pw *TarProgress + ignored *ignore.GitIgnore + matching *ignore.GitIgnore + p *progress.Progress } -// Create creates an archive at dst with all the files defined in the -// included Files array. -// -// THIS IS UNSAFE TO USE IF `dst` IS PROVIDED BY A USER! ONLY USE THIS WITH -// CONTROLLED PATHS! -func (a *Archive) Create(ctx context.Context, dst string) error { - // Using os.OpenFile here is expected, as long as `dst` is not a user - // provided path. - f, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o600) - if err != nil { - return err +// NewArchive returns a new archive instance that can be used for generating an +// archive of files and folders within the provided os.Root. +func NewArchive(r *os.Root, p *progress.Progress, opts ...ArchiveOption) (*Archive, error) { + a := &Archive{root: r, p: p} + for _, opt := range opts { + if err := opt(a); err != nil { + return nil, errors.Wrap(err, "server/filesystem: archive: failed to apply callback option") + } } - defer f.Close() + return a, nil +} + +func WithIgnored(files []string) ArchiveOption { + return func(a *Archive) error { + if a.matching != nil { + return errors.NewPlain("cannot create an archive with both ignored and matching configurations") + } + + a.ignored = ignore.CompileIgnoreLines(files...) + + return nil + } +} + +func WithMatching(files []string) ArchiveOption { + return func(a *Archive) error { + if a.ignored != nil { + return errors.NewPlain("cannot create an archive with both ignored and matching configurations") + } + lines := make([]string, len(files)) + for _, f := range files { + // The old archiver logic just accepted an array of paths to include in the + // archive and did rudimentary logic to determine if they should be included. + // This newer logic makes use of the gitignore (flipped to make it an allowlist), + // but to do that we need to make sure all the provided values here start with a + // slash; otherwise files/folders nested deeply might be unintentionally included. + lines = append(lines, "/"+strings.TrimPrefix(f, "/")) + } + + a.matching = ignore.CompileIgnoreLines(lines...) + + return nil + } +} + +func (a *Archive) Progress() *progress.Progress { + return a.p +} + +func (a *Archive) Close() error { + return a.root.Close() +} + +// Create . +func (a *Archive) Create(ctx context.Context, f *os.File) error { // Select a writer based off of the WriteLimit configuration option. If there is no - // write limit, use the file as the writer. + // write limit use the file as the writer. var writer io.Writer if writeLimit := int64(config.Get().System.Backups.WriteLimit * 1024 * 1024); writeLimit > 0 { // Token bucket with a capacity of "writeLimit" MiB, adding "writeLimit" MiB/s @@ -103,16 +134,9 @@ func (a *Archive) Create(ctx context.Context, dst string) error { return a.Stream(ctx, writer) } -// Stream . +// Stream walks the given root directory and generates an archive from the +// provided files. func (a *Archive) Stream(ctx context.Context, w io.Writer) error { - for _, f := range a.Files { - if strings.HasPrefix(f, a.BasePath) { - continue - } - - return fmt.Errorf("archive: all entries in Files must be absolute and within BasePath: %s\n", f) - } - // Choose which compression level to use based on the compression_level configuration option var compressionLevel int switch config.Get().System.Backups.CompressionLevel { @@ -120,8 +144,6 @@ func (a *Archive) Stream(ctx context.Context, w io.Writer) error { compressionLevel = pgzip.NoCompression case "best_compression": compressionLevel = pgzip.BestCompression - case "best_speed": - fallthrough default: compressionLevel = pgzip.BestSpeed } @@ -135,107 +157,48 @@ func (a *Archive) Stream(ctx context.Context, w io.Writer) error { tw := tar.NewWriter(gw) defer tw.Close() - pw := NewTarProgress(tw, a.Progress) - - // Configure godirwalk. - options := &godirwalk.Options{ - FollowSymbolicLinks: false, - Unsorted: true, - } + a.pw = NewTarProgress(tw, a.p) + defer a.pw.Close() - // If we're specifically looking for only certain files, or have requested - // that certain files be ignored we'll update the callback function to reflect - // that request. - var callback godirwalk.WalkFunc - if len(a.Files) == 0 && len(a.Ignore) > 0 { - i := ignore.CompileIgnoreLines(strings.Split(a.Ignore, "\n")...) - - callback = a.callback(pw, func(_ string, rp string) error { - if i.MatchesPath(rp) { - return godirwalk.SkipThis - } - - return nil - }) - } else if len(a.Files) > 0 { - callback = a.withFilesCallback(pw) - } else { - callback = a.callback(pw) - } + return filepath.WalkDir(a.rootPath(), a.walker(ctx)) +} - // Set the callback function, wrapped with support for context cancellation. - options.Callback = func(path string, de *godirwalk.Dirent) error { - select { - case <-ctx.Done(): +// Callback function used to determine if a given file should be included in the archive +// being generated. +func (a *Archive) walker(ctx context.Context) fs.WalkDirFunc { + return func(path string, de fs.DirEntry, err error) error { + if ctx.Err() != nil { return ctx.Err() - default: - return callback(path, de) } - } - // Recursively walk the path we are archiving. - return godirwalk.Walk(a.BasePath, options) -} + if err != nil { + return fs.SkipDir + } -// Callback function used to determine if a given file should be included in the archive -// being generated. -func (a *Archive) callback(tw *TarProgress, opts ...func(path string, relative string) error) func(path string, de *godirwalk.Dirent) error { - return func(path string, de *godirwalk.Dirent) error { - // Skip directories because we are walking them recursively. - if de.IsDir() { + path = strings.TrimPrefix(path, a.rootPath()) + if a.ignored != nil && a.ignored.MatchesPath(path) { return nil } - relative := filepath.ToSlash(strings.TrimPrefix(path, a.BasePath+string(filepath.Separator))) - - // Call the additional options passed to this callback function. If any of them return - // a non-nil error we will exit immediately. - for _, opt := range opts { - if err := opt(path, relative); err != nil { - return err - } + if a.matching != nil && !a.matching.MatchesPath(path) { + return nil } // Add the file to the archive, if it is nested in a directory, // the directory will be automatically "created" in the archive. - return a.addToArchive(path, relative, tw) + return a.addToArchive(path) } } -// Pushes only files defined in the Files key to the final archive. -func (a *Archive) withFilesCallback(tw *TarProgress) func(path string, de *godirwalk.Dirent) error { - return a.callback(tw, func(p string, rp string) error { - for _, f := range a.Files { - // Allow exact file matches, otherwise check if file is within a parent directory. - // - // The slashes are added in the prefix checks to prevent partial name matches from being - // included in the archive. - if f != p && !strings.HasPrefix(strings.TrimSuffix(p, "/")+"/", strings.TrimSuffix(f, "/")+"/") { - continue - } - - // Once we have a match return a nil value here so that the loop stops and the - // call to this function will correctly include the file in the archive. If there - // are no matches we'll never make it to this line, and the final error returned - // will be the godirwalk.SkipThis error. - return nil - } - - return godirwalk.SkipThis - }) -} - // Adds a given file path to the final archive being created. -func (a *Archive) addToArchive(p string, rp string, w *TarProgress) error { - // Lstat the file, this will give us the same information as Stat except that it will not - // follow a symlink to its target automatically. This is important to avoid including - // files that exist outside the server root unintentionally in the backup. - s, err := os.Lstat(p) +func (a *Archive) addToArchive(p string) error { + p = normalize(p) + s, err := a.root.Lstat(p) if err != nil { if os.IsNotExist(err) { return nil } - return errors.WrapIff(err, "failed executing os.Lstat on '%s'", rp) + return errors.Wrap(err, "server/filesystem: archive: failed to stat file") } // Skip socket files as they are unsupported by archive/tar. @@ -251,30 +214,26 @@ func (a *Archive) addToArchive(p string, rp string, w *TarProgress) error { // the logs, but we're not going to stop the backup. There are far too many cases of // symlinks causing all sorts of unnecessary pain in this process. Sucks to suck if // it doesn't work. - target, err = os.Readlink(s.Name()) + target, err = a.root.Readlink(s.Name()) if err != nil { - // Ignore the not exist errors specifically, since there is nothing important about that. - if !os.IsNotExist(err) { - log.WithField("path", rp).WithField("readlink_err", err.Error()).Warn("failed reading symlink for target path; skipping...") - } return nil } } - // Get the tar FileInfoHeader in order to add the file to the archive. + // Get the tar FileInfoHeader to add the file to the archive. header, err := tar.FileInfoHeader(s, filepath.ToSlash(target)) if err != nil { - return errors.WrapIff(err, "failed to get tar#FileInfoHeader for '%s'", rp) + return errors.Wrap(err, "server/filesystem: archive: failed to get file info header") } // Fix the header name if the file is not a symlink. if s.Mode()&fs.ModeSymlink == 0 { - header.Name = rp + header.Name = p } // Write the tar FileInfoHeader to the archive. - if err := w.WriteHeader(header); err != nil { - return errors.WrapIff(err, "failed to write tar#FileInfoHeader for '%s'", rp) + if err := a.pw.WriteHeader(header); err != nil { + return errors.Wrap(err, "server/filesystem: archive: failed to write tar header") } // If the size of the file is less than 1 (most likely for symlinks), skip writing the file. @@ -295,20 +254,22 @@ func (a *Archive) addToArchive(p string, rp string, w *TarProgress) error { }() } - // Open the file. - f, err := os.Open(p) + f, err := a.root.Open(p) if err != nil { if os.IsNotExist(err) { return nil } - return errors.WrapIff(err, "failed to open '%s' for copying", header.Name) + return errors.Wrap(err, "server/filesystem: archive: failed to open file for copying") } defer f.Close() - // Copy the file's contents to the archive using our buffer. - if _, err := io.CopyBuffer(w, io.LimitReader(f, header.Size), buf); err != nil { - return errors.WrapIff(err, "failed to copy '%s' to archive", header.Name) + if _, err := io.CopyBuffer(a.pw, io.LimitReader(f, header.Size), buf); err != nil { + return errors.Wrap(err, "server/filesystem: archive: failed to copy file to archive") } return nil } + +func (a *Archive) rootPath() string { + return strings.TrimSuffix(a.root.Name(), "/.") +} diff --git a/server/filesystem/archive_test.go b/server/filesystem/archive_test.go index eda342558..a71e63553 100644 --- a/server/filesystem/archive_test.go +++ b/server/filesystem/archive_test.go @@ -23,13 +23,9 @@ func TestArchive_Stream(t *testing.T) { }) g.It("throws an error when passed invalid file paths", func() { - a := &Archive{ - BasePath: fs.Path(), - Files: []string{ - // To use the archiver properly, this needs to be filepath.Join(BasePath, "yeet") - // However, this test tests that we actually validate that behavior. - "yeet", - }, + a, err := NewArchive(fs.root, nil, WithMatching([]string{"yeet"})) + if err != nil { + panic(err) } g.Assert(a.Create(context.Background(), "")).IsNotNil() @@ -51,13 +47,7 @@ func TestArchive_Stream(t *testing.T) { err = fs.Writefile("test_file.txt.old", strings.NewReader("hello, world!\n")) g.Assert(err).IsNil() - a := &Archive{ - BasePath: fs.Path(), - Files: []string{ - filepath.Join(fs.Path(), "test"), - filepath.Join(fs.Path(), "test_file.txt"), - }, - } + a, err := NewArchive(fs.root, nil, WithMatching([]string{"test", "test_file.txt"})) // Create the archive. archivePath := filepath.Join(fs.rootPath, "../archive.tar.gz") @@ -92,6 +82,49 @@ func TestArchive_Stream(t *testing.T) { g.Assert(files).Equal(expected) }) + + g.It("does not archive files outside of root", func() { + if err := os.MkdirAll(filepath.Join(fs.rootPath, "../outer"), 0o755); err != nil { + panic(err) + } + + fs.write("test.txt", []byte("test")) + fs.write("../danger-1.txt", []byte("danger")) + fs.write("../outer/danger-2.txt", []byte("danger")) + + if err := os.Symlink("../danger-1.txt", filepath.Join(fs.rootPath, "symlink.txt")); err != nil { + panic(err) + } + + if err := os.Symlink("../outer", filepath.Join(fs.rootPath, "danger-dir")); err != nil { + panic(err) + } + + a, err := NewArchive(fs.root, nil) + if err != nil { + panic(err) + } + + archivePath := filepath.Join(fs.rootPath, "../archive.tar.gz") + err = a.Create(context.Background(), archivePath) + g.Assert(err).IsNil() + + // Open the archive. + genericFs, err := archives.FileSystem(context.Background(), archivePath, nil) + g.Assert(err).IsNil() + + // Assert that we are opening an archive. + afs, ok := genericFs.(iofs.ReadDirFS) + g.Assert(ok).IsTrue() + + // Get the names of the files recursively from the archive. + files, err := getFiles(afs, ".") + g.Assert(err).IsNil() + // We expect the actual symlinks themselves, but not the contents of the directory + // or the file itself. We're storing the symlinked file in the archive so that + // expanding it back is the same, but you won't have the inner contents. + g.Assert(files).Equal([]string{"danger-dir", "symlink.txt", "test.txt"}) + }) }) } diff --git a/server/filesystem/compress.go b/server/filesystem/compress.go index e80421838..346238082 100644 --- a/server/filesystem/compress.go +++ b/server/filesystem/compress.go @@ -2,8 +2,11 @@ package filesystem import ( "context" + "fmt" "io" "os" + "strings" + "time" "emperror.dev/errors" ) @@ -12,13 +15,36 @@ import ( // specified directory. This function also supports passing nested paths to only // compress certain files and folders when working in a larger directory. This // effectively creates a local backup, but rather than ignoring specific files -// and folders, it takes an allow-list of files and folders. +// and folders, it takes an allowlist of files and folders. // // All paths are relative to the dir that is passed in as the first argument, // and the compressed file will be placed at that location named // `archive-{date}.tar.gz`. -func (fs *Filesystem) CompressFiles(dir string, paths []string) (os.FileInfo, error) { - return nil, errors.New("server/fs: not implemented") +func (fs *Filesystem) CompressFiles(ctx context.Context, dir string, paths []string) (os.FileInfo, error) { + r, err := fs.root.OpenRoot(normalize(dir)) + if err != nil { + return nil, errors.Wrap(err, "server/filesystem: compress: failed to open root directory") + } + a, err := NewArchive(r, nil, WithMatching(paths)) + if err != nil { + _ = r.Close() + return nil, errors.WrapIf(err, "server/filesystem: compress: failed to create archive instance") + } + defer a.Close() + + n := fmt.Sprintf("archive-%s.tar.gz", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", "")) + f, err := r.OpenFile(n, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return nil, errors.Wrap(err, "server/filesystem: compress: failed to open file for writing") + } + defer f.Close() + + if err := a.Create(ctx, f); err != nil { + return nil, errors.Wrap(err, "server/filesystem: compress: failed to write to disk") + } + + // todo: disk space + return f.Stat() } // SpaceAvailableForDecompression looks through a given archive and determines diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index 61209ffa1..08a52a9b1 100755 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -65,7 +65,7 @@ func New(path string, size int64, denylist []string) (*Filesystem, error) { // If the resulting path is an empty string, "." is returned which os.Root will // understand as the base directory. func normalize(path string) string { - c := strings.TrimLeft(filepath.Clean(path), "/") + c := strings.TrimLeft(filepath.Clean(path), string(filepath.Separator)) if c == "" { return "." } diff --git a/server/transfer/archive.go b/server/transfer/archive.go index e5457f12b..86239e548 100644 --- a/server/transfer/archive.go +++ b/server/transfer/archive.go @@ -4,7 +4,9 @@ import ( "context" "fmt" "io" + "os" + "emperror.dev/errors" "github.com/pterodactyl/wings/internal/progress" "github.com/pterodactyl/wings/server/filesystem" ) @@ -19,8 +21,16 @@ func (t *Transfer) Archive() (*Archive, error) { return nil, fmt.Errorf("transfer: failed to get server disk usage: %w", err) } - // Create a new archive instance and assign it to the transfer. - t.archive = NewArchive(t, uint64(rawSize)) + r, err := os.OpenRoot(t.Server.Filesystem().Path()) + if err != nil { + return nil, errors.Wrap(err, "server/transfer: failed to open root directory") + } + a, err := filesystem.NewArchive(r, progress.NewProgress(uint64(rawSize))) + if err != nil { + _ = r.Close() + return nil, errors.WrapIf(err, "server/transfer: failed to create archive") + } + t.archive = &Archive{archive: a} } return t.archive, nil @@ -31,16 +41,6 @@ type Archive struct { archive *filesystem.Archive } -// NewArchive returns a new archive associated with the given transfer. -func NewArchive(t *Transfer, size uint64) *Archive { - return &Archive{ - archive: &filesystem.Archive{ - BasePath: t.Server.Filesystem().Path(), - Progress: progress.NewProgress(size), - }, - } -} - // Stream returns a reader that can be used to stream the contents of the archive. func (a *Archive) Stream(ctx context.Context, w io.Writer) error { return a.archive.Stream(ctx, w) @@ -48,5 +48,5 @@ func (a *Archive) Stream(ctx context.Context, w io.Writer) error { // Progress returns the current progress of the archive. func (a *Archive) Progress() *progress.Progress { - return a.archive.Progress + return a.archive.Progress() } From 2c6ca108ecffc1bd089b81bc0df9dbb08bfa1405 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Mon, 16 Feb 2026 14:00:14 -0800 Subject: [PATCH 17/30] fix error handling --- router/middleware/request_error.go | 5 +++-- server/filesystem/filesystem.go | 4 +++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/router/middleware/request_error.go b/router/middleware/request_error.go index 9ac6fd259..191ea9947 100644 --- a/router/middleware/request_error.go +++ b/router/middleware/request_error.go @@ -119,8 +119,9 @@ func (re *RequestError) asFilesystemError() (int, string) { } if filesystem.IsErrorCode(err, filesystem.ErrNotExist) || filesystem.IsErrorCode(err, filesystem.ErrCodePathResolution) || - strings.Contains(err.Error(), "resolves to a location outside the server root") { - return http.StatusNotFound, "The requested resources was not found on the system." + filesystem.IsPathError(err) || + filesystem.IsLinkError(err) { + return http.StatusNotFound, "The requested file or folder does not exist on the system." } if filesystem.IsErrorCode(err, filesystem.ErrCodeDenylistFile) || strings.Contains(err.Error(), "filesystem: file access prohibited") { return http.StatusForbidden, "This file cannot be modified: present in egg denylist." diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index 08a52a9b1..206e95f32 100755 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -441,7 +441,9 @@ func (fs *Filesystem) ListDirectory(p string) ([]Stat, error) { // @see https://github.com/pterodactyl/panel/issues/4059 if fi.Mode()&os.ModeNamedPipe == 0 { if f, err := fs.root.Open(normalize(filepath.Join(p, d.Name()))); err != nil { - log.WithField("error", err).WithField("path", filepath.Join(p, d.Name())).Warn("error opening file for mimetype detection") + if !IsPathError(err) && !IsLinkError(err) { + log.WithField("error", err).WithField("path", filepath.Join(p, d.Name())).Warn("error opening file for mimetype detection") + } } else { if m, err := mimetype.DetectReader(f); err == nil { st.Mimetype = m.String() From 5c7735c93bc6363d5c5405b98c245d6e94e5c9f7 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Mon, 16 Feb 2026 14:04:06 -0800 Subject: [PATCH 18/30] remove legacy safepath method --- router/middleware/request_error.go | 1 - router/router_server_files.go | 8 +- server/filesystem/errors.go | 47 ----------- server/filesystem/errors_test.go | 15 ---- server/filesystem/path.go | 126 +---------------------------- 5 files changed, 4 insertions(+), 193 deletions(-) diff --git a/router/middleware/request_error.go b/router/middleware/request_error.go index 191ea9947..2f84862d0 100644 --- a/router/middleware/request_error.go +++ b/router/middleware/request_error.go @@ -118,7 +118,6 @@ func (re *RequestError) asFilesystemError() (int, string) { return 0, "" } if filesystem.IsErrorCode(err, filesystem.ErrNotExist) || - filesystem.IsErrorCode(err, filesystem.ErrCodePathResolution) || filesystem.IsPathError(err) || filesystem.IsLinkError(err) { return http.StatusNotFound, "The requested file or folder does not exist on the system." diff --git a/router/router_server_files.go b/router/router_server_files.go index 3f44ede4b..ce676cf31 100644 --- a/router/router_server_files.go +++ b/router/router_server_files.go @@ -598,15 +598,9 @@ func postServerUploadFiles(c *gin.Context) { } for _, header := range headers { - p, err := s.Filesystem().SafePath(filepath.Join(directory, header.Filename)) - if err != nil { - middleware.CaptureAndAbort(c, err) - return - } - // We run this in a different method so I can use defer without any of // the consequences caused by calling it in a loop. - if err := handleFileUpload(p, s, header); err != nil { + if err := handleFileUpload(filepath.Join(directory, header.Filename), s, header); err != nil { middleware.CaptureAndAbort(c, err) return } else { diff --git a/server/filesystem/errors.go b/server/filesystem/errors.go index 854cc34b9..5b0b405a2 100755 --- a/server/filesystem/errors.go +++ b/server/filesystem/errors.go @@ -3,7 +3,6 @@ package filesystem import ( "fmt" "os" - "path/filepath" "emperror.dev/errors" "github.com/apex/log" @@ -15,7 +14,6 @@ const ( ErrCodeIsDirectory ErrorCode = "E_ISDIR" ErrCodeDiskSpace ErrorCode = "E_NODISK" ErrCodeUnknownArchive ErrorCode = "E_UNKNFMT" - ErrCodePathResolution ErrorCode = "E_BADPATH" ErrCodeDenylistFile ErrorCode = "E_DENYLIST" ErrCodeUnknownError ErrorCode = "E_UNKNOWN" ErrNotExist ErrorCode = "E_NOTEXIST" @@ -63,12 +61,6 @@ func (e *Error) Error() string { r = "" } return fmt.Sprintf("filesystem: file access prohibited: [%s] is on the denylist", r) - case ErrCodePathResolution: - r := e.resolved - if r == "" { - r = "" - } - return fmt.Sprintf("filesystem: server path [%s] resolves to a location outside the server root: %s", e.path, r) case ErrNotExist: return "filesystem: does not exist" case ErrCodeUnknownError: @@ -89,30 +81,6 @@ func (fs *Filesystem) error(err error) *log.Entry { return log.WithField("subsystem", "filesystem").WithField("root", fs.root).WithField("error", err) } -// Handle errors encountered when walking through directories. -// -// If there is a path resolution error just skip the item entirely. Only return this for a -// directory, otherwise return nil. Returning this error for a file will stop the walking -// for the remainder of the directory. This is assuming an os.FileInfo struct was even returned. -func (fs *Filesystem) handleWalkerError(err error, f os.FileInfo) error { - if !IsErrorCode(err, ErrCodePathResolution) { - return err - } - if f != nil && f.IsDir() { - return filepath.SkipDir - } - return nil -} - -// IsFilesystemError checks if the given error is one of the Filesystem errors. -func IsFilesystemError(err error) bool { - var fserr *Error - if err != nil && errors.As(err, &fserr) { - return true - } - return false -} - // IsErrorCode checks if "err" is a filesystem Error type. If so, it will then // drop in and check that the error code is the same as the provided ErrorCode // passed in "code". @@ -124,21 +92,6 @@ func IsErrorCode(err error, code ErrorCode) bool { return false } -// NewBadPathResolution returns a new BadPathResolution error. -func NewBadPathResolution(path string, resolved string) error { - return errors.WithStackDepth(&Error{code: ErrCodePathResolution, path: path, resolved: resolved}, 1) -} - -// wrapError wraps the provided error as a Filesystem error and attaches the -// provided resolved source to it. If the error is already a Filesystem error -// no action is taken. -func wrapError(err error, resolved string) error { - if err == nil || IsFilesystemError(err) { - return err - } - return errors.WithStackDepth(&Error{code: ErrCodeUnknownError, err: err, resolved: resolved}, 1) -} - func IsPathError(err error) bool { var pe *os.PathError return errors.As(err, &pe) diff --git a/server/filesystem/errors_test.go b/server/filesystem/errors_test.go index 5f933685d..55d5d2e77 100755 --- a/server/filesystem/errors_test.go +++ b/server/filesystem/errors_test.go @@ -39,19 +39,4 @@ func TestFilesystem_PathResolutionError(t *testing.T) { g.Assert(fserr.Unwrap()).Equal(underlying) }) }) - - g.Describe("NewBadPathResolutionError", func() { - g.It("is can detect itself as an error correctly", func() { - err := NewBadPathResolution("foo", "bar") - g.Assert(IsErrorCode(err, ErrCodePathResolution)).IsTrue() - g.Assert(err.Error()).Equal("filesystem: server path [foo] resolves to a location outside the server root: bar") - g.Assert(IsErrorCode(&Error{code: ErrCodeIsDirectory}, ErrCodePathResolution)).IsFalse() - }) - - g.It("returns if no destination path is provided", func() { - err := NewBadPathResolution("foo", "") - g.Assert(err).IsNotNil() - g.Assert(err.Error()).Equal("filesystem: server path [foo] resolves to a location outside the server root: ") - }) - }) } diff --git a/server/filesystem/path.go b/server/filesystem/path.go index 227f8f3c8..11decbd6c 100755 --- a/server/filesystem/path.go +++ b/server/filesystem/path.go @@ -1,136 +1,16 @@ package filesystem import ( - "context" - iofs "io/fs" - "os" - "path/filepath" - "strings" - "sync" - "emperror.dev/errors" - "golang.org/x/sync/errgroup" ) -// Checks if the given file or path is in the server's file denylist. If so, an Error +// IsIgnored checks if the given file or path is in the server's file denylist. If so, an Error // is returned, otherwise nil is returned. func (fs *Filesystem) IsIgnored(paths ...string) error { for _, p := range paths { - sp, err := fs.SafePath(p) - if err != nil { - return err - } - if fs.denylist.MatchesPath(sp) { - return errors.WithStack(&Error{code: ErrCodeDenylistFile, path: p, resolved: sp}) + if fs.denylist.MatchesPath(p) { + return errors.WithStack(&Error{code: ErrCodeDenylistFile, path: p}) } } return nil } - -// Normalizes a directory being passed in to ensure the user is not able to escape -// from their data directory. After normalization if the directory is still within their home -// path it is returned. If they managed to "escape" an error will be returned. -// -// This logic is actually copied over from the SFTP server code. Ideally that eventually -// either gets ported into this application, or is able to make use of this package. -// -// deprecated -func (fs *Filesystem) SafePath(p string) (string, error) { - // Start with a cleaned up path before checking the more complex bits. - r := fs.unsafeFilePath(p) - - // At the same time, evaluate the symlink status and determine where this file or folder - // is truly pointing to. - ep, err := filepath.EvalSymlinks(r) - if err != nil && !os.IsNotExist(err) { - return "", errors.Wrap(err, "server/filesystem: failed to evaluate symlink") - } else if os.IsNotExist(err) { - // The target of one of the symlinks (EvalSymlinks is recursive) does not exist. - // So we get what target path does not exist and check if it's within the data - // directory. If it is, we return the original path, otherwise we return an error. - pErr, ok := err.(*iofs.PathError) - if !ok { - return "", errors.Wrap(err, "server/filesystem: failed to evaluate symlink") - } - ep = pErr.Path - } - - // If the requested directory from EvalSymlinks begins with the server root directory go - // ahead and return it. If not we'll return an error which will block any further action - // on the file. - if fs.unsafeIsInDataDirectory(ep) { - // Returning the original path here instead of the resolved path ensures that - // whatever the user is trying to do will work as expected. If we returned the - // resolved path, the user would be unable to know that it is in fact a symlink. - return r, nil - } - - return "", NewBadPathResolution(p, r) -} - -// Generate a path to the file by cleaning it up and appending the root server path to it. This -// DOES NOT guarantee that the file resolves within the server data directory. You'll want to use -// the fs.unsafeIsInDataDirectory(p) function to confirm. -func (fs *Filesystem) unsafeFilePath(p string) string { - // Calling filepath.Clean on the joined directory will resolve it to the absolute path, - // removing any ../ type of resolution arguments, and leaving us with a direct path link. - // - // This will also trim the existing root path off the beginning of the path passed to - // the function since that can get a bit messy. - return filepath.Clean(filepath.Join(fs.Path(), strings.TrimPrefix(p, fs.Path()))) -} - -// Check that that path string starts with the server data directory path. This function DOES NOT -// validate that the rest of the path does not end up resolving out of this directory, or that the -// targeted file or folder is not a symlink doing the same thing. -func (fs *Filesystem) unsafeIsInDataDirectory(p string) bool { - return strings.HasPrefix(strings.TrimSuffix(p, "/")+"/", strings.TrimSuffix(fs.Path(), "/")+"/") -} - -// Executes the fs.SafePath function in parallel against an array of paths. If any of the calls -// fails an error will be returned. -func (fs *Filesystem) ParallelSafePath(paths []string) ([]string, error) { - var cleaned []string - - // Simple locker function to avoid racy appends to the array of cleaned paths. - m := new(sync.Mutex) - push := func(c string) { - m.Lock() - cleaned = append(cleaned, c) - m.Unlock() - } - - // Create an error group that we can use to run processes in parallel while retaining - // the ability to cancel the entire process immediately should any of it fail. - g, ctx := errgroup.WithContext(context.Background()) - - // Iterate over all of the paths and generate a cleaned path, if there is an error for any - // of the files, abort the process. - for _, p := range paths { - // Create copy so we can use it within the goroutine correctly. - pi := p - - // Recursively call this function to continue digging through the directory tree within - // a separate goroutine. If the context is canceled abort this process. - g.Go(func() error { - select { - case <-ctx.Done(): - return ctx.Err() - default: - // If the callback returns true, go ahead and keep walking deeper. This allows - // us to programmatically continue deeper into directories, or stop digging - // if that pathway knows it needs nothing else. - if c, err := fs.SafePath(pi); err != nil { - return err - } else { - push(c) - } - - return nil - } - }) - } - - // Block until all of the routines finish and have returned a value. - return cleaned, g.Wait() -} From 428c21a72191823e2b5396129ac3ce06e1f4cfde Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Mon, 16 Feb 2026 15:32:05 -0800 Subject: [PATCH 19/30] bring back unarchive logic Co-Authored-By: Matthew Penner --- server/filesystem/compress.go | 122 +++++++++++++++++++++++++++++++- server/filesystem/filesystem.go | 57 +++++++++++++-- 2 files changed, 173 insertions(+), 6 deletions(-) diff --git a/server/filesystem/compress.go b/server/filesystem/compress.go index 346238082..6f1da63f3 100644 --- a/server/filesystem/compress.go +++ b/server/filesystem/compress.go @@ -5,12 +5,21 @@ import ( "fmt" "io" "os" + "path/filepath" "strings" "time" "emperror.dev/errors" + "github.com/mholt/archives" ) +type extractOptions struct { + dir string + file string + format archives.Format + r io.Reader +} + // CompressFiles compresses all the files matching the given paths in the // specified directory. This function also supports passing nested paths to only // compress certain files and folders when working in a larger directory. This @@ -50,6 +59,10 @@ func (fs *Filesystem) CompressFiles(ctx context.Context, dir string, paths []str // SpaceAvailableForDecompression looks through a given archive and determines // if decompressing it would put the server over its allocated disk space limit. func (fs *Filesystem) SpaceAvailableForDecompression(ctx context.Context, dir string, file string) error { + if fs.MaxDisk() <= 0 { + return nil + } + return nil } @@ -59,7 +72,114 @@ func (fs *Filesystem) SpaceAvailableForDecompression(ctx context.Context, dir st // zip-slip attack being attempted by validating that the final path is within // the server data directory. func (fs *Filesystem) DecompressFile(ctx context.Context, dir string, file string) error { - return errors.New("server/fs: not implemented") + f, err := fs.root.Open(normalize(filepath.Join(dir, file))) + if err != nil { + return errors.Wrap(err, "server/filesystem: decompress: failed to open file") + } + defer f.Close() + + format, input, err := archives.Identify(ctx, filepath.Base(file), f) + if err != nil { + if errors.Is(err, archives.NoMatch) { + return newFilesystemError(ErrCodeUnknownArchive, err) + } + return errors.Wrap(err, "server/filesystem: decompress: failed to identify archive format") + } + + return fs.extractStream(ctx, extractOptions{dir: dir, file: file, format: format, r: input}) +} + +func (fs *Filesystem) extractStream(ctx context.Context, opts extractOptions) error { + // See if it's a compressed archive, such as TAR or a ZIP + ex, ok := opts.format.(archives.Extractor) + if !ok { + // If not, check if it's a single-file compression, such as + // .log.gz, .sql.gz, and so on + de, ok := opts.format.(archives.Decompressor) + if !ok { + return nil + } + + p := filepath.Join(opts.dir, strings.TrimSuffix(opts.file, opts.format.Extension())) + if err := fs.IsIgnored(p); err != nil { + return nil + } + + reader, err := de.OpenReader(opts.r) + if err != nil { + return errors.Wrap(err, "server/filesystem: decompress: failed to open reader") + } + defer reader.Close() + + // Open the file for creation/writing + fmt.Println("open file", normalize(p), p) + f, err := fs.root.OpenFile(normalize(p), os.O_WRONLY|os.O_CREATE, 0o644) + if err != nil { + return errors.Wrap(err, "server/filesystem: decompress: failed to open file") + } + defer f.Close() + + // Read in 4 KB chunks + buf := make([]byte, 4096) + for { + n, err := reader.Read(buf) + if n > 0 { + if err := fs.HasSpaceFor(int64(n)); err != nil { + return err + } + if _, err := f.Write(buf[:n]); err != nil { + return errors.Wrap(err, "server/filesystem: decompress: failed to write") + } + fs.addDisk(int64(n)) + } + + if err != nil { + if err == io.EOF { + break + } + return errors.Wrap(err, "server/filesystem: decompress: failed to read") + } + } + + return nil + } + + // Decompress and extract archive + return ex.Extract(ctx, opts.r, func(ctx context.Context, f archives.FileInfo) error { + if f.IsDir() { + return nil + } + p := filepath.Join(opts.dir, f.NameInArchive) + if err := fs.IsIgnored(p); err != nil { + return nil + } + r, err := f.Open() + if err != nil { + return err + } + defer r.Close() + if f.Mode()&os.ModeSymlink != 0 { + // Try to create the symlink if it is in the archive, but don't hold up the process + // if the file cannot be created. In that case just skip over it entirely. + if f.LinkTarget != "" { + if err := fs.Symlink(p, f.LinkTarget); err != nil { + if errors.Is(err, os.ErrNotExist) { + return nil + } + return errors.Wrap(err, "server/filesystem: decompress: failed to create symlink") + } + } + } else { + if err := fs.Write(p, r, f.Size(), f.Mode()); err != nil { + return errors.Wrap(err, "server/filesystem: decompress: failed to write file") + } + } + // Update the file modification time to the one set in the archive. + if err := fs.Chtimes(p, f.ModTime(), f.ModTime()); err != nil { + return errors.Wrap(err, "server/filesystem: decompress: failed to update file modification time") + } + return nil + }) } // ExtractStreamUnsafe . diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index 206e95f32..adb7a415d 100755 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -100,10 +100,10 @@ func (fs *Filesystem) File(p string) (*os.File, Stat, error) { // Touch acts by creating the given file and path on the disk if it is not present // already. If it is present, the file is opened using the defaults which will truncate // the contents. The opened file is then returned to the caller. -func (fs *Filesystem) Touch(p string, flag int, perm os.FileMode) (*os.File, error) { +func (fs *Filesystem) Touch(p string, flag int, mode os.FileMode) (*os.File, error) { p = normalize(p) o := &fileOpener{root: fs.root} - f, err := o.open(p, flag, perm) + f, err := o.open(p, flag, mode) if err == nil { return f, nil } @@ -127,7 +127,7 @@ func (fs *Filesystem) Touch(p string, flag int, perm os.FileMode) (*os.File, err } // Try to open the file now that we have created the pathing necessary for it, and then // Chown that file so that the permissions don't mess with things. - f, err = o.open(p, flag, perm) + f, err = o.open(p, flag, mode) if err != nil { return nil, errors.Wrap(err, "server/filesystem: touch: failed to open file handle") } @@ -138,6 +138,8 @@ func (fs *Filesystem) Touch(p string, flag int, perm os.FileMode) (*os.File, err // Writefile writes a file to the system. If the file does not already exist one // will be created. This will also properly recalculate the disk space used by // the server when writing new files or modifying existing ones. +// +// deprecated 1.12.1 prefer the use of Filesystem.Write() func (fs *Filesystem) Writefile(p string, r io.Reader) error { p = normalize(p) var currentSize int64 @@ -178,6 +180,51 @@ func (fs *Filesystem) Writefile(p string, r io.Reader) error { return fs.Chown(p) } +// Write writes a file to the disk. +func (fs *Filesystem) Write(p string, r io.Reader, newSize int64, mode os.FileMode) error { + st, err := fs.root.Stat(normalize(p)) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + return errors.Wrap(err, "server/filesystem: write: failed to stat file") + } + } + + var c int64 + if err == nil { + if st.IsDir() { + return errors.WithStack(&Error{code: ErrCodeIsDirectory, resolved: ""}) + } + c = st.Size() + } + + if err := fs.HasSpaceFor(newSize - c); err != nil { + return err + } + + f, err := fs.Touch(p, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) + if err != nil { + return errors.Wrap(err, "server/filesystem: write: failed to touch file") + } + defer f.Close() + + if newSize == 0 { + fs.addDisk(-c) + } else { + // Do not use CopyBuffer here; it is wasteful as the file implements + // io.ReaderFrom, which causes it to not use the buffer anyway. + n, err := io.Copy(f, io.LimitReader(r, newSize)) + // Always adjust the disk to account for cases where a partial copy occurs + // and there is some new content on the disk. + fs.addDisk(n - c) + if err != nil { + return errors.Wrap(err, "server/filesystem: write: failed to write file") + } + } + + // todo: might be unnecessary due to the `fs.Touch` call already doing this? + return fs.Chown(p) +} + // CreateDirectory creates a new directory ("name") at a specified path ("p") for the server. func (fs *Filesystem) CreateDirectory(name string, p string) error { return fs.root.MkdirAll(path.Join(normalize(p), name), 0o755) @@ -378,9 +425,9 @@ type fileOpener struct { // Attempts to open a given file up to "attempts" number of times, using a backoff. If the file // cannot be opened because of a "text file busy" error, we will attempt until the number of attempts // has been exhaused, at which point we will abort with an error. -func (fo *fileOpener) open(path string, flags int, perm os.FileMode) (*os.File, error) { +func (fo *fileOpener) open(path string, flags int, mode os.FileMode) (*os.File, error) { for { - f, err := fo.root.OpenFile(path, flags, perm) + f, err := fo.root.OpenFile(path, flags, mode) // If there is an error because the text file is busy, go ahead and sleep for a few // hundred milliseconds and then try again up to three times before just returning the From cdc6472d94e4c1716630519f7aeb3642f76aba4f Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Mon, 16 Feb 2026 16:55:18 -0800 Subject: [PATCH 20/30] fix handling of symlinks when restoring archives --- server/backup/backup_local.go | 5 +-- server/backup/backup_s3.go | 5 +-- server/filesystem/archive.go | 61 +++++++++++++++++-------------- server/filesystem/archive_test.go | 2 +- server/filesystem/chmod.go | 1 - server/filesystem/compress.go | 33 +++++++++-------- server/filesystem/filesystem.go | 10 ++++- server/transfer/archive.go | 2 +- 8 files changed, 67 insertions(+), 52 deletions(-) diff --git a/server/backup/backup_local.go b/server/backup/backup_local.go index bd9e7282b..fa8646ad4 100644 --- a/server/backup/backup_local.go +++ b/server/backup/backup_local.go @@ -65,12 +65,11 @@ func (b *LocalBackup) Generate(ctx context.Context, basePath, ignore string) (*A if err != nil { return nil, errors.Wrap(err, "server/backup: failed to open root directory") } - a, err := filesystem.NewArchive(r, nil, filesystem.WithIgnored(strings.Split(ignore, "\n"))) + defer r.Close() + a, err := filesystem.NewArchive(r, "/", filesystem.WithIgnored(strings.Split(ignore, "\n"))) if err != nil { - _ = r.Close() return nil, errors.WrapIf(err, "server/backup: failed to create archive") } - defer a.Close() b.log().WithField("path", b.Path()).Info("creating backup for server") f, err := os.OpenFile(b.Path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) diff --git a/server/backup/backup_s3.go b/server/backup/backup_s3.go index 18f15c28a..592979234 100644 --- a/server/backup/backup_s3.go +++ b/server/backup/backup_s3.go @@ -56,12 +56,11 @@ func (s *S3Backup) Generate(ctx context.Context, basePath, ignore string) (*Arch if err != nil { return nil, errors.Wrap(err, "backup: failed to open root directory") } - a, err := filesystem.NewArchive(r, nil, filesystem.WithIgnored(strings.Split(ignore, "\n"))) + defer r.Close() + a, err := filesystem.NewArchive(r, "/", filesystem.WithIgnored(strings.Split(ignore, "\n"))) if err != nil { - _ = r.Close() return nil, errors.WrapIf(err, "backup: failed to create archive") } - defer a.Close() s.log().WithField("path", s.Path()).Info("creating backup for server") f, err := os.OpenFile(s.Path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) diff --git a/server/filesystem/archive.go b/server/filesystem/archive.go index 1be2d06ea..ae0f50322 100644 --- a/server/filesystem/archive.go +++ b/server/filesystem/archive.go @@ -3,6 +3,7 @@ package filesystem import ( "archive/tar" "context" + "fmt" "io" "io/fs" "os" @@ -58,6 +59,7 @@ type ArchiveOption func(a *Archive) error type Archive struct { root *os.Root + dir string pw *TarProgress ignored *ignore.GitIgnore matching *ignore.GitIgnore @@ -65,9 +67,10 @@ type Archive struct { } // NewArchive returns a new archive instance that can be used for generating an -// archive of files and folders within the provided os.Root. -func NewArchive(r *os.Root, p *progress.Progress, opts ...ArchiveOption) (*Archive, error) { - a := &Archive{root: r, p: p} +// archive of files and folders within the provided os.Root. The "dir" value is +// a child directory within the `os.Root` instance. +func NewArchive(r *os.Root, dir string, opts ...ArchiveOption) (*Archive, error) { + a := &Archive{root: r, dir: dir} for _, opt := range opts { if err := opt(a); err != nil { return nil, errors.Wrap(err, "server/filesystem: archive: failed to apply callback option") @@ -76,6 +79,13 @@ func NewArchive(r *os.Root, p *progress.Progress, opts ...ArchiveOption) (*Archi return a, nil } +func WithProgress(p *progress.Progress) ArchiveOption { + return func(a *Archive) error { + a.p = p + return nil + } +} + func WithIgnored(files []string) ArchiveOption { return func(a *Archive) error { if a.matching != nil { @@ -114,10 +124,6 @@ func (a *Archive) Progress() *progress.Progress { return a.p } -func (a *Archive) Close() error { - return a.root.Close() -} - // Create . func (a *Archive) Create(ctx context.Context, f *os.File) error { // Select a writer based off of the WriteLimit configuration option. If there is no @@ -160,12 +166,19 @@ func (a *Archive) Stream(ctx context.Context, w io.Writer) error { a.pw = NewTarProgress(tw, a.p) defer a.pw.Close() - return filepath.WalkDir(a.rootPath(), a.walker(ctx)) + r, err := a.root.OpenRoot(normalize(a.dir)) + if err != nil { + return errors.Wrap(err, "server/filesystem: archive: failed to acquire root dir instance") + } + defer r.Close() + + base := strings.TrimRight(r.Name(), "./") + return filepath.WalkDir(base, a.walker(ctx, base)) } // Callback function used to determine if a given file should be included in the archive // being generated. -func (a *Archive) walker(ctx context.Context) fs.WalkDirFunc { +func (a *Archive) walker(ctx context.Context, base string) fs.WalkDirFunc { return func(path string, de fs.DirEntry, err error) error { if ctx.Err() != nil { return ctx.Err() @@ -175,7 +188,7 @@ func (a *Archive) walker(ctx context.Context) fs.WalkDirFunc { return fs.SkipDir } - path = strings.TrimPrefix(path, a.rootPath()) + path = strings.TrimPrefix(path, base) if a.ignored != nil && a.ignored.MatchesPath(path) { return nil } @@ -210,28 +223,26 @@ func (a *Archive) addToArchive(p string) error { // Resolve the symlink target if the file is a symlink. var target string if s.Mode()&fs.ModeSymlink != 0 { - // Read the target of the symlink. If there are any errors we will dump them out to - // the logs, but we're not going to stop the backup. There are far too many cases of - // symlinks causing all sorts of unnecessary pain in this process. Sucks to suck if - // it doesn't work. - target, err = a.root.Readlink(s.Name()) + // This intentionally uses [os.Readlink] and not the [os.Root] instance. We need to + // know the actual target for the symlink, even if outside the server directory, so + // that we can restore it properly. + // + // This target is only used for the sake of keeping everything correct in the archive; + // we never read the target file contents. + target, err = os.Readlink(filepath.Join(a.root.Name(), p)) if err != nil { - return nil + target = "" } + fmt.Println(p, " targeting ", target) } // Get the tar FileInfoHeader to add the file to the archive. - header, err := tar.FileInfoHeader(s, filepath.ToSlash(target)) + header, err := tar.FileInfoHeader(s, target) if err != nil { return errors.Wrap(err, "server/filesystem: archive: failed to get file info header") } - // Fix the header name if the file is not a symlink. - if s.Mode()&fs.ModeSymlink == 0 { - header.Name = p - } - - // Write the tar FileInfoHeader to the archive. + header.Name = p if err := a.pw.WriteHeader(header); err != nil { return errors.Wrap(err, "server/filesystem: archive: failed to write tar header") } @@ -269,7 +280,3 @@ func (a *Archive) addToArchive(p string) error { return nil } - -func (a *Archive) rootPath() string { - return strings.TrimSuffix(a.root.Name(), "/.") -} diff --git a/server/filesystem/archive_test.go b/server/filesystem/archive_test.go index a71e63553..da24f077a 100644 --- a/server/filesystem/archive_test.go +++ b/server/filesystem/archive_test.go @@ -23,7 +23,7 @@ func TestArchive_Stream(t *testing.T) { }) g.It("throws an error when passed invalid file paths", func() { - a, err := NewArchive(fs.root, nil, WithMatching([]string{"yeet"})) + a, err := NewArchive(fs.root, WithMatching([]string{"yeet"})) if err != nil { panic(err) } diff --git a/server/filesystem/chmod.go b/server/filesystem/chmod.go index 7b618e7f7..92115dbe4 100755 --- a/server/filesystem/chmod.go +++ b/server/filesystem/chmod.go @@ -50,7 +50,6 @@ func (fs *Filesystem) Chown(path string) error { return errors.WrapIf(err, "server/filesystem: chown: failed to stat path") } - fmt.Println("walking path", filepath.Join(fs.rootPath, path)) // If this was a directory, begin walking over its contents recursively and ensure that all // the subfiles and directories get their permissions updated as well. err := godirwalk.Walk(filepath.Join(fs.rootPath, path), &godirwalk.Options{ diff --git a/server/filesystem/compress.go b/server/filesystem/compress.go index 6f1da63f3..5a749c323 100644 --- a/server/filesystem/compress.go +++ b/server/filesystem/compress.go @@ -30,19 +30,13 @@ type extractOptions struct { // and the compressed file will be placed at that location named // `archive-{date}.tar.gz`. func (fs *Filesystem) CompressFiles(ctx context.Context, dir string, paths []string) (os.FileInfo, error) { - r, err := fs.root.OpenRoot(normalize(dir)) + a, err := NewArchive(fs.root, dir, WithMatching(paths)) if err != nil { - return nil, errors.Wrap(err, "server/filesystem: compress: failed to open root directory") - } - a, err := NewArchive(r, nil, WithMatching(paths)) - if err != nil { - _ = r.Close() return nil, errors.WrapIf(err, "server/filesystem: compress: failed to create archive instance") } - defer a.Close() n := fmt.Sprintf("archive-%s.tar.gz", strings.ReplaceAll(time.Now().Format(time.RFC3339), ":", "")) - f, err := r.OpenFile(n, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + f, err := fs.root.OpenFile(normalize(filepath.Join(dir, n)), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) if err != nil { return nil, errors.Wrap(err, "server/filesystem: compress: failed to open file for writing") } @@ -112,7 +106,6 @@ func (fs *Filesystem) extractStream(ctx context.Context, opts extractOptions) er defer reader.Close() // Open the file for creation/writing - fmt.Println("open file", normalize(p), p) f, err := fs.root.OpenFile(normalize(p), os.O_WRONLY|os.O_CREATE, 0o644) if err != nil { return errors.Wrap(err, "server/filesystem: decompress: failed to open file") @@ -162,22 +155,32 @@ func (fs *Filesystem) extractStream(ctx context.Context, opts extractOptions) er // Try to create the symlink if it is in the archive, but don't hold up the process // if the file cannot be created. In that case just skip over it entirely. if f.LinkTarget != "" { - if err := fs.Symlink(p, f.LinkTarget); err != nil { - if errors.Is(err, os.ErrNotExist) { + p2 := strings.TrimLeft(filepath.Clean(p), string(filepath.Separator)) + if p2 == "" { + p2 = "." + } + // We don't use [fs.Symlink] here because that normalizes the source directory for + // consistency with the codebase. In this case when decompressing we want to just + // accept the source without any normalization. + if err := fs.root.Symlink(f.LinkTarget, p2); err != nil { + if errors.Is(err, os.ErrNotExist) || IsPathError(err) || IsLinkError(err) { return nil } return errors.Wrap(err, "server/filesystem: decompress: failed to create symlink") } } - } else { - if err := fs.Write(p, r, f.Size(), f.Mode()); err != nil { - return errors.Wrap(err, "server/filesystem: decompress: failed to write file") - } + return nil } + + if err := fs.Write(p, r, f.Size(), f.Mode()); err != nil { + return errors.Wrap(err, "server/filesystem: decompress: failed to write file") + } + // Update the file modification time to the one set in the archive. if err := fs.Chtimes(p, f.ModTime(), f.ModTime()); err != nil { return errors.Wrap(err, "server/filesystem: decompress: failed to update file modification time") } + return nil }) } diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index adb7a415d..165e8e3d1 100755 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -338,7 +338,15 @@ func (fs *Filesystem) Copy(p string) error { return fs.Writefile(path.Join(relative, n), source) } -// Symlink creates a symbolic link between the source and target paths. +// Symlink creates a symbolic link between the source and target paths. [os.Root].Symlink +// allows for the creation of a symlink that targets a file outside the root directory. +// This isn't the end of the world because the read is blocked through this system, and +// within a container it would just point to something in the readonly filesystem. +// +// There are also valid use-cases where a symlink might need to point to a file outside +// the server data directory for a server to operate correctly. Since everything in the +// filesystem runs through os.Root though we're protected from accidentally reading a +// sensitive file on the _host_ OS. func (fs *Filesystem) Symlink(source, target string) error { source = normalize(source) target = normalize(target) diff --git a/server/transfer/archive.go b/server/transfer/archive.go index 86239e548..d38a5c97d 100644 --- a/server/transfer/archive.go +++ b/server/transfer/archive.go @@ -25,7 +25,7 @@ func (t *Transfer) Archive() (*Archive, error) { if err != nil { return nil, errors.Wrap(err, "server/transfer: failed to open root directory") } - a, err := filesystem.NewArchive(r, progress.NewProgress(uint64(rawSize))) + a, err := filesystem.NewArchive(r, "/", filesystem.WithProgress(progress.NewProgress(uint64(rawSize)))) if err != nil { _ = r.Close() return nil, errors.WrapIf(err, "server/transfer: failed to create archive") From bd60316b7558b5100e67aed36d0e0ed636d672a9 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Wed, 18 Feb 2026 18:18:25 -0800 Subject: [PATCH 21/30] some initial cleanup to account for drift --- environment/settings.go | 6 +-- router/downloader/downloader.go | 13 +++--- router/router_download.go | 2 + router/router_server.go | 9 +++- router/router_server_files.go | 7 ++- router/router_transfer.go | 7 ++- server/backup.go | 10 +---- server/filesystem/archive.go | 9 +++- server/filesystem/disk_space.go | 72 ++++++++++++++++++++----------- server/filesystem/filesystem.go | 8 ++++ server/filesystem/path.go | 0 server/filesystem/path_test.go | 0 server/filesystem/stat.go | 0 server/filesystem/stat_darwin.go | 13 ------ server/filesystem/stat_linux.go | 20 ++++++--- server/filesystem/stat_windows.go | 12 ------ server/server.go | 2 +- server/transfer/source.go | 1 + 18 files changed, 106 insertions(+), 85 deletions(-) mode change 100755 => 100644 server/filesystem/path.go mode change 100755 => 100644 server/filesystem/path_test.go mode change 100755 => 100644 server/filesystem/stat.go delete mode 100644 server/filesystem/stat_darwin.go delete mode 100644 server/filesystem/stat_windows.go diff --git a/environment/settings.go b/environment/settings.go index 1d57154ee..28cb77bd8 100644 --- a/environment/settings.go +++ b/environment/settings.go @@ -107,9 +107,9 @@ func (l Limits) AsContainerResources() container.Resources { Memory: l.BoundedMemoryLimit(), MemoryReservation: l.MemoryLimit * 1024 * 1024, MemorySwap: l.ConvertedSwap(), - BlkioWeight: l.IoWeight, - OomKillDisable: &l.OOMDisabled, - PidsLimit: &pids, + // BlkioWeight: l.IoWeight, + OomKillDisable: &l.OOMDisabled, + PidsLimit: &pids, } // If the CPU Limit is not set, don't send any of these fields through. Providing diff --git a/router/downloader/downloader.go b/router/downloader/downloader.go index 650222213..232c93483 100644 --- a/router/downloader/downloader.go +++ b/router/downloader/downloader.go @@ -199,13 +199,8 @@ func (dl *Download) Execute() error { return errors.New("downloader: got bad response status from endpoint: " + res.Status) } - // If there is a Content-Length header on this request go ahead and check that we can - // even write the whole file before beginning this process. If there is no header present - // we'll just have to give it a spin and see how it goes. - if res.ContentLength > 0 { - if err := dl.server.Filesystem().HasSpaceFor(res.ContentLength); err != nil { - return errors.WrapIf(err, "downloader: failed to write file: not enough space") - } + if res.ContentLength < 1 { + return errors.New("downloader: request is missing ContentLength") } if dl.req.UseHeader { @@ -232,8 +227,10 @@ func (dl *Download) Execute() error { p := dl.Path() dl.server.Log().WithField("path", p).Debug("writing remote file to disk") + // Write the file while tracking the progress, Write will check that the + // size of the file won't exceed the disk limit. r := io.TeeReader(res.Body, dl.counter(res.ContentLength)) - if err := dl.server.Filesystem().Writefile(p, r); err != nil { + if err := dl.server.Filesystem().Write(p, r, res.ContentLength, 0o644); err != nil { return errors.WrapIf(err, "downloader: failed to write file to server directory") } return nil diff --git a/router/router_download.go b/router/router_download.go index 14a063b59..4ca395a3f 100644 --- a/router/router_download.go +++ b/router/router_download.go @@ -56,6 +56,8 @@ func getDownloadBackup(c *gin.Context) { return } + // The use of `os` here is safe as backups are not stored within server access + // directories, and this path is program-controlled, not user input. f, err := os.Open(b.Path()) if err != nil { middleware.CaptureAndAbort(c, err) diff --git a/router/router_server.go b/router/router_server.go index f4090f060..bbcdd9b43 100644 --- a/router/router_server.go +++ b/router/router_server.go @@ -225,13 +225,18 @@ func deleteServer(c *gin.Context) { // done in a separate process since failure is not the end of the world and can be // manually cleaned up after the fact. // - // In addition, servers with large amounts of files can take some time to finish deleting, + // In addition, servers with large numbers of files can take some time to finish deleting, // so we don't want to block the HTTP call while waiting on this. + p := s.Filesystem().Path() go func(p string) { if err := os.RemoveAll(p); err != nil { log.WithFields(log.Fields{"path": p, "error": err}).Warn("failed to remove server files during deletion process") } - }(s.Filesystem().Path()) + }(p) + + if err := s.Filesystem().Close(); err != nil { + log.WithFields(log.Fields{"server": s.ID(), "error": err}).Warn("failed to close filesystem root") + } middleware.ExtractManager(c).Remove(func(server *server.Server) bool { return server.ID() == s.ID() diff --git a/router/router_server_files.go b/router/router_server_files.go index ce676cf31..ee8138e93 100644 --- a/router/router_server_files.go +++ b/router/router_server_files.go @@ -30,8 +30,7 @@ import ( // getServerFileContents returns the contents of a file on the server. func getServerFileContents(c *gin.Context) { s := middleware.ExtractServer(c) - p := "/" + strings.TrimLeft(c.Query("file"), "/") - f, st, err := s.Filesystem().File(p) + f, st, err := s.Filesystem().File(c.Query("file")) if err != nil { middleware.CaptureAndAbort(c, err) return @@ -248,7 +247,7 @@ func postServerWriteFile(c *gin.Context) { return } - if err := s.Filesystem().Writefile(f, c.Request.Body); err != nil { + if err := s.Filesystem().Write(f, c.Request.Body, c.Request.ContentLength, 0o644); err != nil { if filesystem.IsErrorCode(err, filesystem.ErrCodeIsDirectory) { c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{ "error": "Cannot write file, name conflicts with an existing directory by the same name.", @@ -622,7 +621,7 @@ func handleFileUpload(p string, s *server.Server, header *multipart.FileHeader) if err := s.Filesystem().IsIgnored(p); err != nil { return err } - if err := s.Filesystem().Writefile(p, file); err != nil { + if err := s.Filesystem().Write(p, file, header.Size, 0o644); err != nil { return err } return nil diff --git a/router/router_transfer.go b/router/router_transfer.go index 3cedaecdc..6ad2bb077 100644 --- a/router/router_transfer.go +++ b/router/router_transfer.go @@ -106,8 +106,11 @@ func postTransfers(c *gin.Context) { if !successful && err != nil { // Delete all extracted files. go func(trnsfr *transfer.Transfer) { - if err := os.RemoveAll(trnsfr.Server.Filesystem().Path()); err != nil && !os.IsNotExist(err) { - trnsfr.Log().WithError(err).Warn("failed to delete local server files") + _ = trnsfr.Server.Filesystem().Close() + if err := os.RemoveAll(trnsfr.Server.Filesystem().Path()); err != nil { + if !errors.Is(err, os.ErrNotExist) { + trnsfr.Log().WithError(err).Warn("failed to delete local server files") + } } }(trnsfr) } diff --git a/server/backup.go b/server/backup.go index 892a356f8..9de131029 100644 --- a/server/backup.go +++ b/server/backup.go @@ -154,17 +154,11 @@ func (s *Server) RestoreBackup(b backup.BackupInterface, reader io.ReadCloser) ( err = b.Restore(s.Context(), reader, func(file string, info fs.FileInfo, r io.ReadCloser) error { defer r.Close() s.Events().Publish(DaemonMessageEvent, "(restoring): "+file) - - if err := s.Filesystem().Writefile(file, r); err != nil { - return err - } - if err := s.Filesystem().Chmod(file, info.Mode()); err != nil { + if err := s.Filesystem().Write(file, r, info.Size(), info.Mode()); err != nil { return err } - atime := info.ModTime() - mtime := atime - return s.Filesystem().Chtimes(file, atime, mtime) + return s.Filesystem().Chtimes(file, atime, atime) }) return errors.WithStackIf(err) diff --git a/server/filesystem/archive.go b/server/filesystem/archive.go index ae0f50322..907455ffb 100644 --- a/server/filesystem/archive.go +++ b/server/filesystem/archive.go @@ -3,7 +3,6 @@ package filesystem import ( "archive/tar" "context" - "fmt" "io" "io/fs" "os" @@ -124,6 +123,13 @@ func (a *Archive) Progress() *progress.Progress { return a.p } +func (a *Archive) Close() error { + if err := a.root.Close(); err != nil { + return errors.Wrap(err, "server/filesystem: archive: failed to close root") + } + return nil +} + // Create . func (a *Archive) Create(ctx context.Context, f *os.File) error { // Select a writer based off of the WriteLimit configuration option. If there is no @@ -233,7 +239,6 @@ func (a *Archive) addToArchive(p string) error { if err != nil { target = "" } - fmt.Println(p, " targeting ", target) } // Get the tar FileInfoHeader to add the file to the archive. diff --git a/server/filesystem/disk_space.go b/server/filesystem/disk_space.go index bc5d02b45..e84472ef2 100644 --- a/server/filesystem/disk_space.go +++ b/server/filesystem/disk_space.go @@ -1,21 +1,24 @@ package filesystem import ( + fs2 "io/fs" + "os" "path/filepath" - "strings" + "slices" "sync" "sync/atomic" "time" "emperror.dev/errors" "github.com/apex/log" - "github.com/karrick/godirwalk" + "golang.org/x/sys/unix" ) type SpaceCheckingOpts struct { AllowStaleResponse bool } +// TODO: can this be replaced with some sort of atomic? Like atomic.Pointer? type usageLookupTime struct { sync.RWMutex value time.Time @@ -36,12 +39,13 @@ func (ult *usageLookupTime) Get() time.Time { return ult.value } -// Returns the maximum amount of disk space that this Filesystem instance is allowed to use. +// MaxDisk returns the maximum amount of disk space that this Filesystem +// instance is allowed to use. func (fs *Filesystem) MaxDisk() int64 { return atomic.LoadInt64(&fs.diskLimit) } -// Sets the disk space limit for this Filesystem instance. +// SetDiskLimit sets the disk space limit for this Filesystem instance. func (fs *Filesystem) SetDiskLimit(i int64) { atomic.SwapInt64(&fs.diskLimit, i) } @@ -66,7 +70,7 @@ func (fs *Filesystem) HasSpaceErr(allowStaleValue bool) error { func (fs *Filesystem) HasSpaceAvailable(allowStaleValue bool) bool { size, err := fs.DiskUsage(allowStaleValue) if err != nil { - log.WithField("root", fs.root).WithField("error", err).Warn("failed to determine root fs directory size") + log.WithField("root", fs.Path()).WithField("error", err).Warn("failed to determine root fs directory size") } // If space is -1 or 0 just return true, means they're allowed unlimited. @@ -115,7 +119,7 @@ func (fs *Filesystem) DiskUsage(allowStaleValue bool) (int64, error) { // currently performing a lookup, just do the disk usage calculation in the background. go func(fs *Filesystem) { if _, err := fs.updateCachedDiskUsage(); err != nil { - log.WithField("root", fs.root).WithField("error", err).Warn("failed to update fs disk usage from within routine") + log.WithField("root", fs.rootPath).WithField("error", err).Warn("failed to update fs disk usage from within routine") } }(fs) } @@ -155,37 +159,55 @@ func (fs *Filesystem) updateCachedDiskUsage() (int64, error) { return size, err } -// Determines the directory size of a given location by running parallel tasks to iterate -// through all of the folders. Returns the size in bytes. This can be a fairly taxing operation -// on locations with tons of files, so it is recommended that you cache the output. +// DirectorySize determines the directory size of a given location. Returns the size +// in bytes. This can be a fairly taxing operation on locations with tons of files, +// so it is recommended that you cache the output. func (fs *Filesystem) DirectorySize(dir string) (int64, error) { - dir = strings.TrimLeft(filepath.Clean(dir), "/") - if dir != "" { + dir = normalize(dir) + if dir != "." { if _, err := fs.root.Lstat(dir); err != nil { return 0, err } } + rt := fs.root + if dir != "." { + r, err := fs.root.OpenRoot(dir) + if err != nil { + return 0, errors.Wrap(err, "server/filesystem: directorysize: failed to open root directory") + } + defer r.Close() + rt = r + } + var size int64 - err := godirwalk.Walk(filepath.Join(fs.rootPath, dir), &godirwalk.Options{ - Unsorted: true, - FollowSymbolicLinks: false, - Callback: func(p string, e *godirwalk.Dirent) error { - if !e.ModeType().IsRegular() { + var links []uint64 + + err := filepath.WalkDir(rt.Name(), func(path string, d fs2.DirEntry, err error) error { + if !d.Type().IsRegular() { + return nil + } + + st, err := d.Info() + if err != nil { + if errors.Is(err, os.ErrNotExist) { return nil } + return err + } - if !e.IsDir() { - st, err := fs.root.Lstat(strings.TrimLeft(strings.TrimPrefix(p, fs.rootPath), "/")) - if err != nil { - return errors.Wrap(err, "server/filesystem: directorysize: failed to stat file") - } - atomic.AddInt64(&size, st.Size()) + s := st.Sys().(*unix.Stat_t) + if s.Nlink > 1 { + // Hard links have the same inode number, don't add them more than once. + if slices.Contains(links, s.Ino) { + return nil } + links = append(links, s.Ino) + } - // todo: don't count hardlinks twice - return nil - }, + size += st.Size() + + return nil }) return size, errors.WrapIf(err, "server/filesystem: directorysize: failed to walk directory") diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index 165e8e3d1..54af89f3d 100755 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -77,6 +77,14 @@ func (fs *Filesystem) Path() string { return fs.rootPath } +// Close closes the underlying os.Root instance for the server. +func (fs *Filesystem) Close() error { + if err := fs.root.Close(); err != nil { + return errors.Wrap(err, "server/filesystem: failed to close root") + } + return nil +} + // File returns a reader for a file instance as well as the stat information. func (fs *Filesystem) File(p string) (*os.File, Stat, error) { p = normalize(p) diff --git a/server/filesystem/path.go b/server/filesystem/path.go old mode 100755 new mode 100644 diff --git a/server/filesystem/path_test.go b/server/filesystem/path_test.go old mode 100755 new mode 100644 diff --git a/server/filesystem/stat.go b/server/filesystem/stat.go old mode 100755 new mode 100644 diff --git a/server/filesystem/stat_darwin.go b/server/filesystem/stat_darwin.go deleted file mode 100644 index 6d0cff32b..000000000 --- a/server/filesystem/stat_darwin.go +++ /dev/null @@ -1,13 +0,0 @@ -package filesystem - -import ( - "syscall" - "time" -) - -// CTime returns the time that the file/folder was created. -func (s *Stat) CTime() time.Time { - st := s.Sys().(*syscall.Stat_t) - - return time.Unix(st.Ctimespec.Sec, st.Ctimespec.Nsec) -} diff --git a/server/filesystem/stat_linux.go b/server/filesystem/stat_linux.go index a9c7fb36f..7891bafbe 100644 --- a/server/filesystem/stat_linux.go +++ b/server/filesystem/stat_linux.go @@ -3,12 +3,22 @@ package filesystem import ( "syscall" "time" + + "golang.org/x/sys/unix" ) -// Returns the time that the file/folder was created. +// CTime returns the time that the file/folder was created. +// +// TODO: remove. Ctim is not actually ever been correct and doesn't actually +// return the creation time. func (s *Stat) CTime() time.Time { - st := s.Sys().(*syscall.Stat_t) - - // Do not remove these "redundant" type-casts, they are required for 32-bit builds to work. - return time.Unix(int64(st.Ctim.Sec), int64(st.Ctim.Nsec)) + if st, ok := s.Sys().(*unix.Stat_t); ok { + // Do not remove these "redundant" type-casts, they are required for 32-bit builds to work. + return time.Unix(int64(st.Ctim.Sec), int64(st.Ctim.Nsec)) + } + if st, ok := s.Sys().(*syscall.Stat_t); ok { + // Do not remove these "redundant" type-casts, they are required for 32-bit builds to work. + return time.Unix(int64(st.Ctim.Sec), int64(st.Ctim.Nsec)) + } + return time.Time{} } diff --git a/server/filesystem/stat_windows.go b/server/filesystem/stat_windows.go deleted file mode 100644 index 3652677bc..000000000 --- a/server/filesystem/stat_windows.go +++ /dev/null @@ -1,12 +0,0 @@ -package filesystem - -import ( - "time" -) - -// On linux systems this will return the time that the file was created. -// However, I have no idea how to do this on windows, so we're skipping it -// for right now. -func (s *Stat) CTime() time.Time { - return s.ModTime() -} diff --git a/server/server.go b/server/server.go index ce3fb7fe2..643f678f4 100644 --- a/server/server.go +++ b/server/server.go @@ -300,7 +300,7 @@ func (s *Server) EnsureDataDirectoryExists() (string, error) { path := filepath.Join(c.System.Data, s.ID()) if _, err := os.Lstat(path); err != nil { - if !os.IsNotExist(err) { + if !errors.Is(err, os.ErrNotExist) { return path, errors.Wrap(err, "server: failed to stat server root directory") } diff --git a/server/transfer/source.go b/server/transfer/source.go index cdcceec16..7030b9447 100644 --- a/server/transfer/source.go +++ b/server/transfer/source.go @@ -28,6 +28,7 @@ func (t *Transfer) PushArchiveToTarget(url, token string) ([]byte, error) { t.Error(err, "Failed to get archive for transfer.") return nil, errors.New("failed to get archive for transfer") } + defer a.archive.Close() t.SendMessage("Streaming archive to destination...") From 942f212caf2351627edcccf4fb51813a9cc371a3 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Wed, 18 Feb 2026 19:19:02 -0800 Subject: [PATCH 22/30] wrap up for the night --- server/filesystem/archive.go | 7 ------- server/filesystem/compress.go | 1 + server/transfer/archive.go | 6 +----- server/transfer/source.go | 12 +++++++++--- 4 files changed, 11 insertions(+), 15 deletions(-) diff --git a/server/filesystem/archive.go b/server/filesystem/archive.go index 907455ffb..8b56667c7 100644 --- a/server/filesystem/archive.go +++ b/server/filesystem/archive.go @@ -123,13 +123,6 @@ func (a *Archive) Progress() *progress.Progress { return a.p } -func (a *Archive) Close() error { - if err := a.root.Close(); err != nil { - return errors.Wrap(err, "server/filesystem: archive: failed to close root") - } - return nil -} - // Create . func (a *Archive) Create(ctx context.Context, f *os.File) error { // Select a writer based off of the WriteLimit configuration option. If there is no diff --git a/server/filesystem/compress.go b/server/filesystem/compress.go index 5a749c323..48314c6ac 100644 --- a/server/filesystem/compress.go +++ b/server/filesystem/compress.go @@ -57,6 +57,7 @@ func (fs *Filesystem) SpaceAvailableForDecompression(ctx context.Context, dir st return nil } + // todo: rest of the owl return nil } diff --git a/server/transfer/archive.go b/server/transfer/archive.go index d38a5c97d..b8c7ad1d8 100644 --- a/server/transfer/archive.go +++ b/server/transfer/archive.go @@ -13,7 +13,7 @@ import ( // Archive returns an archive that can be used to stream the contents of the // contents of a server. -func (t *Transfer) Archive() (*Archive, error) { +func (t *Transfer) Archive(r *os.Root) (*Archive, error) { if t.archive == nil { // Get the disk usage of the server (used to calculate the progress of the archive process) rawSize, err := t.Server.Filesystem().DiskUsage(true) @@ -21,10 +21,6 @@ func (t *Transfer) Archive() (*Archive, error) { return nil, fmt.Errorf("transfer: failed to get server disk usage: %w", err) } - r, err := os.OpenRoot(t.Server.Filesystem().Path()) - if err != nil { - return nil, errors.Wrap(err, "server/transfer: failed to open root directory") - } a, err := filesystem.NewArchive(r, "/", filesystem.WithProgress(progress.NewProgress(uint64(rawSize)))) if err != nil { _ = r.Close() diff --git a/server/transfer/source.go b/server/transfer/source.go index 7030b9447..fcd6b92f1 100644 --- a/server/transfer/source.go +++ b/server/transfer/source.go @@ -4,13 +4,14 @@ import ( "context" "crypto/sha256" "encoding/hex" - "errors" "fmt" "io" "mime/multipart" "net/http" + "os" "time" + "emperror.dev/errors" "github.com/pterodactyl/wings/internal/progress" ) @@ -23,12 +24,17 @@ func (t *Transfer) PushArchiveToTarget(url, token string) ([]byte, error) { t.SendMessage("Preparing to stream server data to destination...") t.SetStatus(StatusProcessing) - a, err := t.Archive() + r, err := os.OpenRoot(t.Server.Filesystem().Path()) + if err != nil { + return nil, errors.Wrap(err, "server/transfer: failed to open root directory") + } + defer r.Close() + + a, err := t.Archive(r) if err != nil { t.Error(err, "Failed to get archive for transfer.") return nil, errors.New("failed to get archive for transfer") } - defer a.archive.Close() t.SendMessage("Streaming archive to destination...") From 20bb36be86150e3a09baf0ab2bd88c6e97be4400 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Thu, 19 Feb 2026 18:20:46 -0800 Subject: [PATCH 23/30] try to get backup logic working better --- internal/quota_writer.go | 115 ++++++++++++++++++++++++++++++++ router/router_server_files.go | 19 ++---- server/backup.go | 15 ++++- server/filesystem/archive.go | 2 + server/filesystem/compress.go | 39 ++++++----- server/filesystem/disk_space.go | 17 +++-- server/filesystem/filesystem.go | 37 +++++++++- 7 files changed, 204 insertions(+), 40 deletions(-) create mode 100644 internal/quota_writer.go diff --git a/internal/quota_writer.go b/internal/quota_writer.go new file mode 100644 index 000000000..aa6234fcd --- /dev/null +++ b/internal/quota_writer.go @@ -0,0 +1,115 @@ +// SPDX-License-Identifier: MIT +// SPDX-FileCopyrightText: Copyright (c) 2024 Matthew Penner + +package internal + +import ( + "io" + "os" + "sync/atomic" +) + +// CountedWriter is a writer that counts the amount of data written to the +// underlying writer. +type CountedWriter struct { + file *os.File + counter atomic.Int64 + err error +} + +// NewCountedWriter returns a new countedWriter that counts the amount of bytes +// written to the underlying writer. +func NewCountedWriter(f *os.File) *CountedWriter { + return &CountedWriter{file: f} +} + +// BytesWritten returns the amount of bytes that have been written to the +// underlying writer. +func (w *CountedWriter) BytesWritten() int64 { + return w.counter.Load() +} + +// Error returns the error from the writer if any. If the error is an EOF, nil +// will be returned. +func (w *CountedWriter) Error() error { + if w.err == io.EOF { + return nil + } + return w.err +} + +// Write writes bytes to the underlying writer while tracking the total amount +// of bytes written. +func (w *CountedWriter) Write(p []byte) (int, error) { + if w.err != nil { + return 0, io.EOF + } + + // Write is a very simple operation for us to handle. + n, err := w.file.Write(p) + w.counter.Add(int64(n)) + w.err = err + + // TODO: is this how we actually want to handle errors with this? + if err == io.EOF { + return n, io.EOF + } + return n, nil +} + +func (w *CountedWriter) ReadFrom(r io.Reader) (n int64, err error) { + cr := NewCountedReader(r) + n, err = w.file.ReadFrom(cr) + w.counter.Add(n) + return +} + +// CountedReader is a reader that counts the amount of data read from the +// underlying reader. +type CountedReader struct { + reader io.Reader + + counter atomic.Int64 + err error +} + +var _ io.Reader = (*CountedReader)(nil) + +// NewCountedReader returns a new countedReader that counts the amount of bytes +// read from the underlying reader. +func NewCountedReader(r io.Reader) *CountedReader { + return &CountedReader{reader: r} +} + +// BytesRead returns the amount of bytes that have been read from the underlying +// reader. +func (r *CountedReader) BytesRead() int64 { + return r.counter.Load() +} + +// Error returns the error from the reader if any. If the error is an EOF, nil +// will be returned. +func (r *CountedReader) Error() error { + if r.err == io.EOF { + return nil + } + return r.err +} + +// Read reads bytes from the underlying reader while tracking the total amount +// of bytes read. +func (r *CountedReader) Read(p []byte) (int, error) { + if r.err != nil { + return 0, io.EOF + } + + n, err := r.reader.Read(p) + r.counter.Add(int64(n)) + r.err = err + + // TODO: is this how we actually want to handle errors with this? + if err == io.EOF { + return n, io.EOF + } + return n, nil +} diff --git a/router/router_server_files.go b/router/router_server_files.go index ee8138e93..92878f89b 100644 --- a/router/router_server_files.go +++ b/router/router_server_files.go @@ -423,7 +423,13 @@ func postServerCompressFiles(c *gin.Context) { f, err := s.Filesystem().CompressFiles(c.Request.Context(), data.RootPath, data.Files) if err != nil { - middleware.CaptureAndAbort(c, err) + if errors.Is(err, filesystem.ErrNoSpaceAvailable) { + c.AbortWithStatusJSON(http.StatusConflict, gin.H{ + "error": "This server does not have enough available disk space to generate a compressed archive.", + }) + } else { + middleware.CaptureAndAbort(c, err) + } return } @@ -447,17 +453,6 @@ func postServerDecompressFiles(c *gin.Context) { s := middleware.ExtractServer(c) lg := middleware.ExtractLogger(c).WithFields(log.Fields{"root_path": data.RootPath, "file": data.File}) - lg.Debug("checking if space is available for file decompression") - err := s.Filesystem().SpaceAvailableForDecompression(context.Background(), data.RootPath, data.File) - if err != nil { - if filesystem.IsErrorCode(err, filesystem.ErrCodeUnknownArchive) { - lg.WithField("error", err).Warn("failed to decompress file: unknown archive format") - c.AbortWithStatusJSON(http.StatusBadRequest, gin.H{"error": "The archive provided is in a format Wings does not understand."}) - return - } - middleware.CaptureAndAbort(c, err) - return - } lg.Info("starting file decompression") if err := s.Filesystem().DecompressFile(context.Background(), data.RootPath, data.File); err != nil { diff --git a/server/backup.go b/server/backup.go index 9de131029..4e7d07076 100644 --- a/server/backup.go +++ b/server/backup.go @@ -153,10 +153,21 @@ func (s *Server) RestoreBackup(b backup.BackupInterface, reader io.ReadCloser) ( s.Log().Debug("starting file writing process for backup restoration") err = b.Restore(s.Context(), reader, func(file string, info fs.FileInfo, r io.ReadCloser) error { defer r.Close() + if file == "." { + return nil + } + s.Events().Publish(DaemonMessageEvent, "(restoring): "+file) - if err := s.Filesystem().Write(file, r, info.Size(), info.Mode()); err != nil { - return err + if info.IsDir() { + if err := s.Filesystem().Mkdir(file, info.Mode().Perm()); err != nil { + return errors.WithStack(err) + } + } else { + if err := s.Filesystem().Write(file, r, info.Size(), info.Mode().Perm()); err != nil { + return errors.WithStack(err) + } } + atime := info.ModTime() return s.Filesystem().Chtimes(file, atime, atime) }) diff --git a/server/filesystem/archive.go b/server/filesystem/archive.go index 8b56667c7..98afcdcd8 100644 --- a/server/filesystem/archive.go +++ b/server/filesystem/archive.go @@ -21,6 +21,8 @@ import ( const memory = 4 * 1024 +var ErrNoSpaceAvailable = errors.Sentinel("archive: no space available on disk") + var pool = sync.Pool{ New: func() interface{} { b := make([]byte, memory) diff --git a/server/filesystem/compress.go b/server/filesystem/compress.go index 48314c6ac..3d844558c 100644 --- a/server/filesystem/compress.go +++ b/server/filesystem/compress.go @@ -11,6 +11,7 @@ import ( "emperror.dev/errors" "github.com/mholt/archives" + "github.com/pterodactyl/wings/internal" ) type extractOptions struct { @@ -42,23 +43,20 @@ func (fs *Filesystem) CompressFiles(ctx context.Context, dir string, paths []str } defer f.Close() - if err := a.Create(ctx, f); err != nil { + cw := internal.NewCountedWriter(f) + // todo: eventing on the counted writer so that we can slowly increase the disk + // used value on the server as the file gets written? + if err := a.Stream(ctx, cw); err != nil { return nil, errors.Wrap(err, "server/filesystem: compress: failed to write to disk") } - - // todo: disk space - return f.Stat() -} - -// SpaceAvailableForDecompression looks through a given archive and determines -// if decompressing it would put the server over its allocated disk space limit. -func (fs *Filesystem) SpaceAvailableForDecompression(ctx context.Context, dir string, file string) error { - if fs.MaxDisk() <= 0 { - return nil + if err := fs.HasSpaceFor(cw.BytesWritten()); err != nil { + _ = fs.root.Remove(normalize(filepath.Join(dir, n))) + return nil, err } + fmt.Println("wrote bytes", cw.BytesWritten()) + fs.addDisk(cw.BytesWritten()) - // todo: rest of the owl - return nil + return f.Stat() } // DecompressFile will decompress a file in a given directory by using the @@ -173,7 +171,7 @@ func (fs *Filesystem) extractStream(ctx context.Context, opts extractOptions) er return nil } - if err := fs.Write(p, r, f.Size(), f.Mode()); err != nil { + if err := fs.Write(p, r, f.Size(), f.Mode().Perm()); err != nil { return errors.Wrap(err, "server/filesystem: decompress: failed to write file") } @@ -188,5 +186,16 @@ func (fs *Filesystem) extractStream(ctx context.Context, opts extractOptions) er // ExtractStreamUnsafe . func (fs *Filesystem) ExtractStreamUnsafe(ctx context.Context, dir string, r io.Reader) error { - return errors.New("server/fs: not implemented") + format, input, err := archives.Identify(ctx, "archive.tar.gz", r) + if err != nil { + if errors.Is(err, archives.NoMatch) { + return newFilesystemError(ErrCodeUnknownArchive, err) + } + return err + } + return fs.extractStream(ctx, extractOptions{ + dir: dir, + format: format, + r: input, + }) } diff --git a/server/filesystem/disk_space.go b/server/filesystem/disk_space.go index e84472ef2..f3eb75bc6 100644 --- a/server/filesystem/disk_space.go +++ b/server/filesystem/disk_space.go @@ -7,11 +7,11 @@ import ( "slices" "sync" "sync/atomic" + "syscall" "time" "emperror.dev/errors" "github.com/apex/log" - "golang.org/x/sys/unix" ) type SpaceCheckingOpts struct { @@ -59,14 +59,13 @@ func (fs *Filesystem) HasSpaceErr(allowStaleValue bool) error { return nil } -// Determines if the directory a file is trying to be added to has enough space available -// for the file to be written to. +// HasSpaceAvailable checks if the directory a file is trying to be added to has enough +// space available for the file to be written to. Because determining the amount of space +// being used by a server is a taxing operation, we will load it all up into a cache and +// pull from that as long as the key is not expired. // -// Because determining the amount of space being used by a server is a taxing operation we -// will load it all up into a cache and pull from that as long as the key is not expired. -// -// This operation will potentially block unless allowStaleValue is set to true. See the -// documentation on DiskUsage for how this affects the call. +// This operation will potentially be blocked unless allowStaleValue is set to true. See +// the documentation on DiskUsage for how this affects the call. func (fs *Filesystem) HasSpaceAvailable(allowStaleValue bool) bool { size, err := fs.DiskUsage(allowStaleValue) if err != nil { @@ -196,7 +195,7 @@ func (fs *Filesystem) DirectorySize(dir string) (int64, error) { return err } - s := st.Sys().(*unix.Stat_t) + s := st.Sys().(*syscall.Stat_t) if s.Nlink > 1 { // Hard links have the same inode number, don't add them more than once. if slices.Contains(links, s.Ino) { diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index 54af89f3d..3995c7235 100755 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -188,6 +188,13 @@ func (fs *Filesystem) Writefile(p string, r io.Reader) error { return fs.Chown(p) } +func (fs *Filesystem) Mkdir(p string, mode os.FileMode) error { + if err := fs.root.Mkdir(normalize(p), mode); err != nil { + return errors.Wrap(err, "server/filesystem: mkdir: failed to make directory") + } + return nil +} + // Write writes a file to the disk. func (fs *Filesystem) Write(p string, r io.Reader, newSize int64, mode os.FileMode) error { st, err := fs.root.Stat(normalize(p)) @@ -200,7 +207,7 @@ func (fs *Filesystem) Write(p string, r io.Reader, newSize int64, mode os.FileMo var c int64 if err == nil { if st.IsDir() { - return errors.WithStack(&Error{code: ErrCodeIsDirectory, resolved: ""}) + return errors.WithStack(&Error{code: ErrCodeIsDirectory, resolved: normalize(p)}) } c = st.Size() } @@ -403,7 +410,33 @@ func (fs *Filesystem) ReadDir(p string) ([]fs2.DirEntry, error) { // TruncateRootDirectory removes _all_ files and directories from a server's // data directory and resets the used disk space to zero. func (fs *Filesystem) TruncateRootDirectory() error { - return errors.New("server/filesystem: not implemented") + err := filepath.WalkDir(fs.rootPath, func(path string, d fs2.DirEntry, err error) error { + p := normalize(strings.TrimPrefix(path, fs.rootPath)) + if p == "." { + return nil + } + + if err := fs.root.RemoveAll(p); err != nil { + return err + } + + return filepath.SkipDir + }) + + if err != nil { + go func() { + // If there was an error, re-calculate the disk usage right away to account + // for any partially removed files. + _, _ = fs.updateCachedDiskUsage() + }() + + return errors.Wrap(err, "server/filesystem: truncate: failed to walk root directory") + } + + // Set the disk space back to zero. + fs.addDisk(fs.diskUsed * -1) + + return nil } // Delete removes a file or folder from the system. Prevents the user from From 48991a0ad6b2b17b6155e9c14699b5a6239298be Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Thu, 19 Feb 2026 18:44:31 -0800 Subject: [PATCH 24/30] fix backup restoration logic --- server/backup.go | 7 +++++++ server/filesystem/compress.go | 1 - server/filesystem/filesystem.go | 6 +++++- 3 files changed, 12 insertions(+), 2 deletions(-) diff --git a/server/backup.go b/server/backup.go index 4e7d07076..e8b087f43 100644 --- a/server/backup.go +++ b/server/backup.go @@ -160,9 +160,16 @@ func (s *Server) RestoreBackup(b backup.BackupInterface, reader io.ReadCloser) ( s.Events().Publish(DaemonMessageEvent, "(restoring): "+file) if info.IsDir() { if err := s.Filesystem().Mkdir(file, info.Mode().Perm()); err != nil { + if errors.Is(err, os.ErrExist) { + return nil + } return errors.WithStack(err) } } else { + if !info.Mode().IsRegular() { + return nil + } + if err := s.Filesystem().Write(file, r, info.Size(), info.Mode().Perm()); err != nil { return errors.WithStack(err) } diff --git a/server/filesystem/compress.go b/server/filesystem/compress.go index 3d844558c..7cd86c7ed 100644 --- a/server/filesystem/compress.go +++ b/server/filesystem/compress.go @@ -53,7 +53,6 @@ func (fs *Filesystem) CompressFiles(ctx context.Context, dir string, paths []str _ = fs.root.Remove(normalize(filepath.Join(dir, n))) return nil, err } - fmt.Println("wrote bytes", cw.BytesWritten()) fs.addDisk(cw.BytesWritten()) return f.Stat() diff --git a/server/filesystem/filesystem.go b/server/filesystem/filesystem.go index 3995c7235..d85cccec1 100755 --- a/server/filesystem/filesystem.go +++ b/server/filesystem/filesystem.go @@ -420,7 +420,11 @@ func (fs *Filesystem) TruncateRootDirectory() error { return err } - return filepath.SkipDir + if d.IsDir() { + return filepath.SkipDir + } + + return nil }) if err != nil { From a57a3424561c96cd08b9dfc748ee6e70dc71d261 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Thu, 19 Feb 2026 18:46:16 -0800 Subject: [PATCH 25/30] always run chtime even when the folder already exists --- server/backup.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/server/backup.go b/server/backup.go index e8b087f43..ebc816acc 100644 --- a/server/backup.go +++ b/server/backup.go @@ -160,10 +160,9 @@ func (s *Server) RestoreBackup(b backup.BackupInterface, reader io.ReadCloser) ( s.Events().Publish(DaemonMessageEvent, "(restoring): "+file) if info.IsDir() { if err := s.Filesystem().Mkdir(file, info.Mode().Perm()); err != nil { - if errors.Is(err, os.ErrExist) { - return nil + if !errors.Is(err, os.ErrExist) { + return errors.WithStack(err) } - return errors.WithStack(err) } } else { if !info.Mode().IsRegular() { From aa511f4180aed6aaef6c74fbc179bda9491adecb Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Thu, 19 Feb 2026 18:53:55 -0800 Subject: [PATCH 26/30] Update settings.go --- environment/settings.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/environment/settings.go b/environment/settings.go index 28cb77bd8..1d57154ee 100644 --- a/environment/settings.go +++ b/environment/settings.go @@ -107,9 +107,9 @@ func (l Limits) AsContainerResources() container.Resources { Memory: l.BoundedMemoryLimit(), MemoryReservation: l.MemoryLimit * 1024 * 1024, MemorySwap: l.ConvertedSwap(), - // BlkioWeight: l.IoWeight, - OomKillDisable: &l.OOMDisabled, - PidsLimit: &pids, + BlkioWeight: l.IoWeight, + OomKillDisable: &l.OOMDisabled, + PidsLimit: &pids, } // If the CPU Limit is not set, don't send any of these fields through. Providing From 0d3ae69fa2f16902ea5045b9c624d0b745829695 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Thu, 19 Feb 2026 19:00:34 -0800 Subject: [PATCH 27/30] Update chmod.go --- server/filesystem/chmod.go | 48 +++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/server/filesystem/chmod.go b/server/filesystem/chmod.go index 92115dbe4..8d02580e9 100755 --- a/server/filesystem/chmod.go +++ b/server/filesystem/chmod.go @@ -2,13 +2,13 @@ package filesystem import ( "fmt" + fs2 "io/fs" "os" "path/filepath" "strings" "time" "emperror.dev/errors" - "github.com/karrick/godirwalk" "github.com/pterodactyl/wings/config" ) @@ -28,21 +28,17 @@ func (fs *Filesystem) Chmod(path string, mode os.FileMode) error { // underlying files. Iterate over all the files and directories. If it is a file go ahead // and perform the chown operation. Otherwise dig deeper into the directory until we've run // out of directories to dig into. -func (fs *Filesystem) Chown(path string) error { - path = strings.TrimLeft(filepath.Clean(path), "/") - if path == "" { - path = "." - } - +func (fs *Filesystem) Chown(p string) error { + p = normalize(p) uid := config.Get().System.User.Uid gid := config.Get().System.User.Gid - if err := fs.root.Chown(path, uid, gid); err != nil { + if err := fs.root.Chown(p, uid, gid); err != nil { return errors.WrapIf(err, "server/filesystem: chown: failed to chown path") } // If this is not a directory, we can now return from the function; there is nothing // left that we need to do. - if st, err := fs.root.Stat(path); err != nil || !st.IsDir() { + if st, err := fs.root.Stat(p); err != nil || !st.IsDir() { if err == nil || errors.Is(err, os.ErrNotExist) { return nil } @@ -50,26 +46,30 @@ func (fs *Filesystem) Chown(path string) error { return errors.WrapIf(err, "server/filesystem: chown: failed to stat path") } + rt := fs.rootPath + if p == "." { + r, err := fs.root.OpenRoot(p) + if err != nil { + return errors.WithStack(err) + } + defer r.Close() + rt = r.Name() + } + // If this was a directory, begin walking over its contents recursively and ensure that all // the subfiles and directories get their permissions updated as well. - err := godirwalk.Walk(filepath.Join(fs.rootPath, path), &godirwalk.Options{ - Unsorted: true, - FollowSymbolicLinks: false, - Callback: func(p string, e *godirwalk.Dirent) error { - p = strings.TrimLeft(strings.TrimPrefix(p, fs.Path()), "/") - if p == "" { - return godirwalk.SkipThis - } + return filepath.WalkDir(rt, func(path string, _ fs2.DirEntry, err error) error { + path = normalize(path) + if path == "." { + return nil + } - if err := fs.root.Chown(p, uid, gid); err != nil { - return errors.Wrap(err, fmt.Sprintf("server/filesystem: chown: failed to chown file")) - } + if err := fs.root.Chown(path, uid, gid); err != nil { + return errors.Wrap(err, fmt.Sprintf("server/filesystem: chown: failed to chown file")) + } - return nil - }, + return nil }) - - return errors.WrapIf(err, "server/filesystem: chown: failed to chown directory tree") } func (fs *Filesystem) Chtimes(path string, atime, mtime time.Time) error { From 888ebb7f77fe354aa9609520eb274c7552aa2f65 Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Thu, 19 Feb 2026 19:03:34 -0800 Subject: [PATCH 28/30] bump go versions --- .github/workflows/push.yaml | 2 +- .github/workflows/release.yaml | 2 +- Dockerfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 89791d3c3..3905102fc 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -16,7 +16,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-24.04] - go: ["1.24.11", "1.25.5"] + go: ["1.25.6"] goos: [linux] goarch: [amd64, arm64] permissions: diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index c564c3dc6..e160af371 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -16,7 +16,7 @@ jobs: - name: Setup Go uses: actions/setup-go@7a3fe6cf4cb3a834922a1244abfce67bcef6a0c5 # v6.2.0 with: - go-version: 1.24.11 + go-version: 1.25.6 - name: Build release binaries env: diff --git a/Dockerfile b/Dockerfile index af6494797..8693627ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Stage 1 (Build) -FROM golang:1.24.11-alpine AS builder +FROM golang:1.25.6-alpine AS builder ARG VERSION RUN apk add --update --no-cache git make mailcap From 4e9255c1dfef52e15c9ae974e594227a5737955c Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Thu, 19 Feb 2026 19:06:04 -0800 Subject: [PATCH 29/30] upload debug builds --- .github/workflows/push.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/push.yaml b/.github/workflows/push.yaml index 3905102fc..b8b7c791a 100644 --- a/.github/workflows/push.yaml +++ b/.github/workflows/push.yaml @@ -63,14 +63,14 @@ jobs: - name: Upload Release Artifact uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - if: ${{ (github.ref == 'refs/heads/develop' || github.event_name == 'pull_request') && matrix.go == '1.24.11' }} + if: ${{ (github.ref == 'refs/heads/develop' || github.event_name == 'pull_request') }} with: name: wings_linux_${{ matrix.goarch }} path: dist/wings - name: Upload Debug Artifact uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 - if: ${{ (github.ref == 'refs/heads/develop' || github.event_name == 'pull_request') && matrix.go == '1.24.11' }} + if: ${{ (github.ref == 'refs/heads/develop' || github.event_name == 'pull_request') }} with: name: wings_linux_${{ matrix.goarch }}_debug path: dist/wings_debug From fce47504d40d4ad914cd1268b606be88b18de90d Mon Sep 17 00:00:00 2001 From: DaneEveritt Date: Thu, 19 Feb 2026 19:10:05 -0800 Subject: [PATCH 30/30] fix tests --- server/filesystem/archive_test.go | 36 +++++++++++++++---------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/server/filesystem/archive_test.go b/server/filesystem/archive_test.go index da24f077a..537b2cefd 100644 --- a/server/filesystem/archive_test.go +++ b/server/filesystem/archive_test.go @@ -22,15 +22,6 @@ func TestArchive_Stream(t *testing.T) { fs.reset() }) - g.It("throws an error when passed invalid file paths", func() { - a, err := NewArchive(fs.root, WithMatching([]string{"yeet"})) - if err != nil { - panic(err) - } - - g.Assert(a.Create(context.Background(), "")).IsNotNil() - }) - g.It("creates archive with intended files", func() { g.Assert(fs.CreateDirectory("test", "/")).IsNil() g.Assert(fs.CreateDirectory("test2", "/")).IsNil() @@ -47,15 +38,16 @@ func TestArchive_Stream(t *testing.T) { err = fs.Writefile("test_file.txt.old", strings.NewReader("hello, world!\n")) g.Assert(err).IsNil() - a, err := NewArchive(fs.root, nil, WithMatching([]string{"test", "test_file.txt"})) - - // Create the archive. archivePath := filepath.Join(fs.rootPath, "../archive.tar.gz") - g.Assert(a.Create(context.Background(), archivePath)).IsNil() + f, err := os.Create(archivePath) + if err != nil { + panic(err) + } + defer f.Close() - // Ensure the archive exists. - _, err = os.Stat(archivePath) - g.Assert(err).IsNil() + a, err := NewArchive(fs.root, ".", WithMatching([]string{"test", "test_file.txt"})) + + g.Assert(a.Create(context.Background(), f)).IsNil() // Open the archive. genericFs, err := archives.FileSystem(context.Background(), archivePath, nil) @@ -100,13 +92,19 @@ func TestArchive_Stream(t *testing.T) { panic(err) } - a, err := NewArchive(fs.root, nil) + archivePath := filepath.Join(fs.rootPath, "../archive.tar.gz") + f, err := os.Create(archivePath) if err != nil { panic(err) } + defer f.Close() - archivePath := filepath.Join(fs.rootPath, "../archive.tar.gz") - err = a.Create(context.Background(), archivePath) + a, err := NewArchive(fs.root, ".") + if err != nil { + panic(err) + } + + err = a.Create(context.Background(), f) g.Assert(err).IsNil() // Open the archive.