diff options
135 files changed, 4482 insertions, 1657 deletions
diff --git a/.travis.yml b/.travis.yml index 50b3c1c8..f9bd5eb9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -81,8 +81,9 @@ addons: branch_pattern: coverity_scan script: - - "if ./travis-should-run-test; then make TEST_LOG=dump test-cli; fi" - - "if ./travis-should-run-test; then make TEST_LOG=dump test-mkfs; fi" - - "if ./travis-should-run-test; then make TEST_LOG=dump test-check; fi" - - "if ./travis-should-run-test; then make TEST_LOG=dump test-misc; fi" + - "if travis/should-run-test; then make TEST_LOG=dump test-cli; fi" + - "if travis/should-run-test; then make TEST_LOG=dump test-mkfs; fi" + - "if travis/should-run-test; then make TEST_LOG=dump test-check; fi" + - "if travis/should-run-test; then make TEST_LOG=dump TEST_ENABLE_OVERRIDE=true TEST_ARGS_CHECK=--mode=lowmem test-check; fi" + - "if travis/should-run-test; then make TEST_LOG=dump test-misc; fi" - "if [ $TRAVIS_BRANCH = release-test ]; then make TEST_LOG=dump test-convert; fi" @@ -1,3 +1,26 @@ +btrfs-progs-4.14.1 (2018-01-05) + * dump-tree: print times of root items + * check: fix several lowmem mode bugs + * convert: fix rollback after balance + * other + * new and updated tests, enabled lowmem mode in CI + * docs updates + * fix travis CI build + * build fixes + * cleanups + +btrfs-progs-4.14 (2017-11-16) + * build: libzstd now required by default + * check: more lowmem mode repair enhancements + * subvol set-default: also accept path + * prop set: compression accepts no/none, same as "" + * filesystem usage: enable for filesystem on top of a seed device + * rescue: new command fix-device-size + * other + * new tests + * cleanups and refactoring + * doc updates + btrfs-progs-4.13.3 (2017-10-16) * check: fix --force, wrong check for a mounted block device * build: fix --with-convert parsing diff --git a/Documentation/btrfs-balance.8.gz b/Documentation/btrfs-balance.8.gz Binary files differindex 1d553317..e2344c16 100644 --- a/Documentation/btrfs-balance.8.gz +++ b/Documentation/btrfs-balance.8.gz diff --git a/Documentation/btrfs-balance.asciidoc b/Documentation/btrfs-balance.asciidoc index cc81de91..7017bed7 100644 --- a/Documentation/btrfs-balance.asciidoc +++ b/Documentation/btrfs-balance.asciidoc @@ -21,8 +21,8 @@ filesystem. The balance operation is cancellable by the user. The on-disk state of the filesystem is always consistent so an unexpected interruption (eg. system crash, reboot) does not corrupt the filesystem. The progress of the balance operation -is temporarily stored and will be resumed upon mount, unless the mount option -'skip_balance' is specified. +is temporarily stored as an internal state and will be resumed upon mount, +unless the mount option 'skip_balance' is specified. WARNING: running balance without filters will take a lot of time as it basically rewrites the entire filesystem and needs to update all block pointers. @@ -201,10 +201,11 @@ ENOSPC ------ The way balance operates, it usually needs to temporarily create a new block -group and move the old data there. For that it needs work space, otherwise -it fails for ENOSPC reasons. +group and move the old data there, before the old block group can be removed. +For that it needs the work space, otherwise it fails for ENOSPC reasons. This is not the same ENOSPC as if the free space is exhausted. This refers to -the space on the level of block groups. +the space on the level of block groups, which are bigger parts of the filesytem +that contain many file extents. The free work space can be calculated from the output of the *btrfs filesystem show* command: @@ -227,7 +228,7 @@ space. After that it might be possible to run other filters. Conversion to profiles based on striping (RAID0, RAID5/6) require the work space on each device. An interrupted balance may leave partially filled block -groups that might consume the work space. +groups that consume the work space. EXAMPLES -------- @@ -238,7 +239,7 @@ can be found in section 'TYPICAL USECASES' of `btrfs-device`(8). MAKING BLOCK GROUP LAYOUT MORE COMPACT ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -The layout of block groups is not normally visible, most tools report only +The layout of block groups is not normally visible; most tools report only summarized numbers of free or used space, but there are still some hints provided. @@ -298,7 +299,7 @@ data to the remaining blockgroups, ie. the 6GiB are now free of filesystem structures, and can be reused for new data or metadata block groups. We can do a similar exercise with the metadata block groups, but this should -not be typically necessary, unless the used/total ration is really off. Here +not typically be necessary, unless the used/total ratio is really off. Here the ratio is roughly 50% but the difference as an absolute number is "a few gigabytes", which can be considered normal for a workload with snapshots or reflinks updated frequently. diff --git a/Documentation/btrfs-check.8.gz b/Documentation/btrfs-check.8.gz Binary files differindex 9f5c4bca..475ea594 100644 --- a/Documentation/btrfs-check.8.gz +++ b/Documentation/btrfs-check.8.gz diff --git a/Documentation/btrfs-check.asciidoc b/Documentation/btrfs-check.asciidoc index fbf48847..cc76d846 100644 --- a/Documentation/btrfs-check.asciidoc +++ b/Documentation/btrfs-check.asciidoc @@ -22,10 +22,10 @@ by the option '--readonly'. *btrfsck* is an alias of *btrfs check* command and is now deprecated. -WARNING: Do not use '--repair' unless you are advised to by a developer, an -experienced user or accept the fact that 'fsck' cannot possibly fix all sorts -of damage that could happen to a filesystem because of software and hardware -bugs. +WARNING: Do not use '--repair' unless you are advised to do so by a developer +or an experienced user, and then only after having accepted that no 'fsck' +successfully repair all types of filesystem corruption. Eg. some other software +or hardware bugs can fatally damage a volume. The structural integrity check verifies if internal filesystem objects or data structures satisfy the constraints, point to the right objects or are @@ -49,9 +49,8 @@ This can be combined with '--super' if some of the superblocks are damaged. --check-data-csum:: verify checksums of data blocks + -This expects that the filesystem is otherwise -OK, so this is basically and offline 'scrub' but does not repair data from -spare copies. +This expects that the filesystem is otherwise OK, and is basically and offline +'scrub' but does not repair data from spare copies. --chunk-root <bytenr>:: use the given offset 'bytenr' for the chunk tree root @@ -88,8 +87,8 @@ the free space cache for block groups that are modified while the filesystem is mounted with that option. Thus, using this option with 'v1' makes it possible to actually clear the entire free space cache. + -For free space cache 'v2', the 'clear_cache' kernel mount option does destroy -the entire free space cache. This option with 'v2' provides an alternative +For free space cache 'v2', the 'clear_cache' kernel mount option destroys +the entire free space cache. This option, with 'v2' provides an alternative method of clearing the free space cache that doesn't require mounting the filesystem. diff --git a/Documentation/btrfs-convert.8.gz b/Documentation/btrfs-convert.8.gz Binary files differindex 9758a080..f361a835 100644 --- a/Documentation/btrfs-convert.8.gz +++ b/Documentation/btrfs-convert.8.gz diff --git a/Documentation/btrfs-convert.asciidoc b/Documentation/btrfs-convert.asciidoc index 07ff608a..41f23d61 100644 --- a/Documentation/btrfs-convert.asciidoc +++ b/Documentation/btrfs-convert.asciidoc @@ -19,7 +19,7 @@ Supported filesystems: * ext2, ext3, ext4 -- original feature, always built in -* reiserfs -- since version 4.13, opptinally built, requires libreiserfscore 3.6.27 +* reiserfs -- since version 4.13, optionally built, requires libreiserfscore 3.6.27 The list of supported source filesystem by a given binary is listed at the end of help (option '--help'). @@ -32,13 +32,14 @@ The conversion utilizes free space of the original filesystem. The exact estimate of the required space cannot be foretold. The final btrfs metadata might occupy several gigabytes on a hundreds-gigabyte filesystem. -If you decide not to rollback anymore, it is recommended to perform a few more -steps to transform the btrfs filesystem to a more compact layout. The -conversion inherits the original data block fragmentation and the metadata -blocks are bound to the original free space layout. +If the ability to rollback is no longer important, the it is recommended to +perform a few more steps to transition the btrfs filesystem to a more compact +layout. This is because the conversion inherits the original data blocks' +fragmentation, and also because the metadata blocks are bound to the original +free space layout. -Due to different constraints, it's possible to convert only filesystem that -have supported data block size (ie. the same that would be valid for +Due to different constraints, it is only possible to convert filesystems that +have a supported data block size (ie. the same that would be valid for 'mkfs.btrfs'). This is typically the system page size (4KiB on x86_64 machines). @@ -52,8 +53,8 @@ metadata of the original filesystem will be removed: # btrfs subvolume delete /mnt/ext2_saved -At this point it's not possible to do rollback. The filesystem is usable but may -be impacted by the fragmentation inherited from the original filesystem. +At this point it is not possible to do a rollback. The filesystem is usable but +may be impacted by the fragmentation inherited from the original filesystem. **MAKE FILE DATA MORE CONTIGUOUS** @@ -102,7 +103,7 @@ set filesystem label during conversion -L|--copy-label:: use label from the converted filesystem -O|--features <feature1>[,<feature2>...]:: -A list of filesystem features turned on at conversion time. Not all features +A list of filesystem features enabled the at time of conversion. Not all features are supported by old kernels. To disable a feature, prefix it with '^'. Description of the features is in section 'FILESYSTEM FEATURES' of `mkfs.btrfs`(8). diff --git a/Documentation/btrfs-device.8.gz b/Documentation/btrfs-device.8.gz Binary files differindex a457c952..785a0046 100644 --- a/Documentation/btrfs-device.8.gz +++ b/Documentation/btrfs-device.8.gz diff --git a/Documentation/btrfs-device.asciidoc b/Documentation/btrfs-device.asciidoc index 88822ece..223ade5c 100644 --- a/Documentation/btrfs-device.asciidoc +++ b/Documentation/btrfs-device.asciidoc @@ -68,13 +68,23 @@ Remove device(s) from a filesystem identified by <path> Device removal must satisfy the profile constraints, otherwise the command fails. The filesystem must be converted to profile(s) that would allow the removal. This can typically happen when going down from 2 devices to 1 and -using the RAID1 profile. See the example section below. +using the RAID1 profile. See the *TYPICAL USECASES* section below. + The operation can take long as it needs to move all data from the device. + It is possible to delete the device that was used to mount the filesystem. The -device entry in mount table will be replaced by another device name with the -lowest device id. +device entry in the mount table will be replaced by another device name with +the lowest device id. ++ +If the filesystem is mounted in degraded mode (-o degraded), special term +'missing' can be used for 'device'. In that case, the first device that is +described by the filesystem metadata, but not present at the mount time will be +removed. ++ +NOTE: In most cases, there is only one missing device in degraded mode, +otherwise mount fails. If there are two or more devices missing (e.g. possible +in RAID6), you need specify 'missing' as many times as the number of missing +devices to remove all of them. *delete* <device>|<devid> [<device>|<devid>...] <path>:: Alias of remove kept for backward compatibility @@ -114,7 +124,7 @@ statistics and the meaning. Print the stats and reset the values to zero afterwards. -c|--check:::: -Check if the stats are all zeros and return 0 it it is so. Set bit 6 of the +Check if the stats are all zeros and return 0 it this is so. Set bit 6 of the return code if any of the statistics is no-zero. The error values is 65 if reading stats from at least one device failed, otherwise it's 64. @@ -180,8 +190,8 @@ there's enough workspace for the conversion process, we can do: $ btrfs balance start -mconvert=raid1 /mnt -This operation can take a while as the metadata have to be moved and all block -pointers updated. Depending on the physical locations of the old and new +This operation can take a while, because al metadata have to be moved and all +block pointers updated. Depending on the physical locations of the old and new blocks, the disk seeking is the key factor affecting performance. You'll note that the system block group has been also converted to RAID1, this @@ -206,6 +216,19 @@ data or the block groups occupy the whole first device. The device size of '/dev/sdb' as seen by the filesystem remains unchanged, but the logical space from 50-100GiB will be unused. +==== REMOVE DEVICE ==== + +Device removal must satisfy the profile constraints, otherwise the command +fails. For example: + + $ btrfs device remove /dev/sda /mnt + ERROR: error removing device '/dev/sda': unable to go below two devices on raid1 + +In order to remove a device, you need to convert the profile in this case: + + $ btrfs balance start -mconvert=dup -dconvert=single /mnt + $ btrfs device remove /dev/sda /mnt + DEVICE STATS ------------ diff --git a/Documentation/btrfs-filesystem.8.gz b/Documentation/btrfs-filesystem.8.gz Binary files differindex b1628e4c..727f2421 100644 --- a/Documentation/btrfs-filesystem.8.gz +++ b/Documentation/btrfs-filesystem.8.gz diff --git a/Documentation/btrfs-filesystem.asciidoc b/Documentation/btrfs-filesystem.asciidoc index 41b30320..961405ba 100644 --- a/Documentation/btrfs-filesystem.asciidoc +++ b/Documentation/btrfs-filesystem.asciidoc @@ -3,7 +3,7 @@ btrfs-filesystem(8) NAME ---- -btrfs-filesystem - command group of btrfs that usually work on the whole filesystem +btrfs-filesystem - command group othat primarily does work on the whole filesystems SYNOPSIS -------- @@ -53,8 +53,9 @@ not total size of filesystem. when the filesystem is full. Its 'total' size is dynamic based on the filesystem size, usually not larger than 512MiB, 'used' may fluctuate. + -The global block reserve is accounted within Metadata. In case the filesystem -metadata are exhausted, 'GlobalReserve/total + Metadata/used = Metadata/total'. +The GlobalReserve is a portion of Metadata. In case the filesystem metadata is +exhausted, 'GlobalReserve/total + Metadata/used = Metadata/total'. Otherwise +there appears to be some unused space of Metadata. + `Options` + @@ -93,10 +94,10 @@ You can also turn on compression in defragment operations. + WARNING: Defragmenting with Linux kernel versions < 3.9 or ≥ 3.14-rc2 as well as with Linux stable kernel versions ≥ 3.10.31, ≥ 3.12.12 or ≥ 3.13.4 will break up -the ref-links of COW data (for example files copied with `cp --reflink`, +the reflinks of COW data (for example files copied with `cp --reflink`, snapshots or de-duplicated data). This may cause considerable increase of space usage depending on the broken up -ref-links. +reflinks. + NOTE: Directory arguments without '-r' do not defragment files recursively but will defragment certain internal trees (extent tree and the subvolume tree). This has been @@ -174,7 +175,7 @@ show sizes in TiB, or TB with --si. Show or update the label of a filesystem. This works on a mounted filesystem or a filesystem image. + -The 'newlabel' argument is optional. Current label is printed if the the argument +The 'newlabel' argument is optional. Current label is printed if the argument is omitted. + NOTE: the maximum allowable length shall be less than 256 chars and must not contain @@ -359,9 +360,9 @@ specify the devid though. *$ btrfs filesystem resize 1:max /path* -Let's assume that devid 1 exists, the filesystem does not occupy the whole block -device, eg. it has been enlarged and we wan the grow the filesystem. Simply using -'max' as size we will achieve that. +Let's assume that devid 1 exists and the filesystem does not occupy the whole +block device, eg. it has been enlarged and we wan the grow the filesystem. By +simply using 'max' as size we will achieve that. NOTE: There are two ways to minimize the filesystem on a given device. The *btrfs inspect-internal min-dev-size* command, or iteratively shrink in steps. diff --git a/Documentation/btrfs-find-root.8.gz b/Documentation/btrfs-find-root.8.gz Binary files differindex bbe8ad41..c90e2244 100644 --- a/Documentation/btrfs-find-root.8.gz +++ b/Documentation/btrfs-find-root.8.gz diff --git a/Documentation/btrfs-find-root.asciidoc b/Documentation/btrfs-find-root.asciidoc index 3d6af2df..652796c8 100644 --- a/Documentation/btrfs-find-root.asciidoc +++ b/Documentation/btrfs-find-root.asciidoc @@ -17,7 +17,7 @@ root tree's objectid, generation, level. OPTIONS ------- -a:: -Search through all the metadata extents, even the root is already found. +Search through all metadata extents, even the root has been already found. -g <generation>:: Filter root tree by it's original transaction id, tree root's generation in default. -o <objectid>:: diff --git a/Documentation/btrfs-image.8.gz b/Documentation/btrfs-image.8.gz Binary files differindex cc7fc7ba..e5591ab1 100644 --- a/Documentation/btrfs-image.8.gz +++ b/Documentation/btrfs-image.8.gz diff --git a/Documentation/btrfs-inspect-internal.8.gz b/Documentation/btrfs-inspect-internal.8.gz Binary files differindex 145bbb3d..b8147ecf 100644 --- a/Documentation/btrfs-inspect-internal.8.gz +++ b/Documentation/btrfs-inspect-internal.8.gz diff --git a/Documentation/btrfs-inspect-internal.asciidoc b/Documentation/btrfs-inspect-internal.asciidoc index 62b10294..e072a943 100644 --- a/Documentation/btrfs-inspect-internal.asciidoc +++ b/Documentation/btrfs-inspect-internal.asciidoc @@ -136,16 +136,16 @@ resize operation, this may be useful before executing the actual resize operatio specify the device 'id' to query, default is 1 if this option is not used *rootid* <path>:: -for a given file or directory, return the containing tree root id, for a -subvolume itself return it's own tree id (ie. subvol id) +for a given file or directory, return the containing tree root id, but for a +subvolume itself return its own tree id (ie. subvol id) + NOTE: The result is undefined for the so-called empty subvolumes (identified by -inode number 2), but such subvolume does not contain any files anyway +inode number 2), but such a subvolume does not contain any files anyway *subvolid-resolve* <subvolid> <path>:: (needs root privileges) + -resolve the absolute path of a the subvolume id 'subvolid' +resolve the absolute path of the subvolume id 'subvolid' *tree-stats* [options] <device>:: (needs root privileges) diff --git a/Documentation/btrfs-man5.asciidoc b/Documentation/btrfs-man5.asciidoc index 3981435e..1f444d73 100644 --- a/Documentation/btrfs-man5.asciidoc +++ b/Documentation/btrfs-man5.asciidoc @@ -26,6 +26,13 @@ This section describes mount options specific to BTRFS. For the generic mount options please refer to `mount`(8) manpage. The options are sorted alphabetically (discarding the 'no' prefix). +NOTE: most mount options apply to the whole filesystem and only options in the +first mounted subvolume will take effect. This is due to lack of implementation +and may change in the future. This means that (for example) you can't set +per-subvolume 'nodatacow', 'nodatasum', or 'compress' using mount options. This +should eventually be fixed, but it has proved to be difficult to implement +correctly within the Linux VFS framework. + *acl*:: *noacl*:: (default: on) @@ -51,17 +58,17 @@ location. + WARNING: Defragmenting with Linux kernel versions < 3.9 or ≥ 3.14-rc2 as well as with Linux stable kernel versions ≥ 3.10.31, ≥ 3.12.12 or -≥ 3.13.4 will break up the ref-links of CoW data (for example files +≥ 3.13.4 will break up the reflinks of COW data (for example files copied with `cp --reflink`, snapshots or de-duplicated data). This may cause considerable increase of space usage depending on the -broken up ref-links. +broken up reflinks. *barrier*:: *nobarrier*:: (default: on) + Ensure that all IO write operations make it through the device cache and are stored -permanently when the filesystem is at it's consistency checkpoint. This +permanently when the filesystem is at its consistency checkpoint. This typically means that a flush command is sent to the device that will synchronize all pending data and ordinary metadata blocks, then writes the superblock and issues another flush. @@ -83,21 +90,22 @@ supposed to make it to the permanent storage. (since: 3.0, default: off) + These debugging options control the behavior of the integrity checking -module (the BTRFS_FS_CHECK_INTEGRITY config option required). + +module (the BTRFS_FS_CHECK_INTEGRITY config option required). The main goal is +to verify that all blocks from a given transaction period are properly linked. + -`check_int` enables the integrity checker module, which examines all +'check_int' enables the integrity checker module, which examines all block write requests to ensure on-disk consistency, at a large -memory and CPU cost. + +memory and CPU cost. + -`check_int_data` includes extent data in the integrity checks, and -implies the check_int option. + +'check_int_data' includes extent data in the integrity checks, and +implies the 'check_int' option. + -`check_int_print_mask` takes a bitmask of BTRFSIC_PRINT_MASK_* values +'check_int_print_mask' takes a bitmask of BTRFSIC_PRINT_MASK_* values as defined in 'fs/btrfs/check-integrity.c', to control the integrity -checker module behavior. + +checker module behavior. + See comments at the top of 'fs/btrfs/check-integrity.c' -for more info. +for more information. *clear_cache*:: Force clearing and rebuilding of the disk space cache if something @@ -106,10 +114,11 @@ has gone wrong. See also: 'space_cache'. *commit='seconds'*:: (since: 3.12, default: 30) + -Set the interval of periodic commit. Higher -values defer data being synced to permanent storage with obvious -consequences when the system crashes. The upper bound is not forced, -but a warning is printed if it's more than 300 seconds (5 minutes). +Set the interval of periodic transaction commit when data are synchronized +to permanent storage. Higher interval values lead to larger amount of unwritten +data, which has obvious consequences when the system crashes. +The upper bound is not forced, but a warning is printed if it's more than 300 +seconds (5 minutes). Use with care. *compress*:: *compress='type'*:: @@ -120,7 +129,7 @@ but a warning is printed if it's more than 300 seconds (5 minutes). Control BTRFS file data compression. Type may be specified as 'zlib', 'lzo', 'zstd' or 'no' (for no compression, used for remounting). If no type is specified, 'zlib' is used. If 'compress-force' is specified, -the compression will allways be attempted, but the data may end up uncompressed +then compression will always be attempted, but the data may end up uncompressed if the compression would make them larger. + Otherwise some simple heuristics are applied to detect an incompressible file. @@ -141,6 +150,10 @@ Enable data copy-on-write for newly created files. under 'nodatacow' are also set the NOCOW file attribute (see `chattr`(1)). + NOTE: If 'nodatacow' or 'nodatasum' are enabled, compression is disabled. ++ +Updates in-place improve performance for workloads that do frequent overwrites, +at the cost of potential partial writes, in case the write is interruted +(system crash, device failure). *datasum*:: *nodatasum*:: @@ -152,13 +165,31 @@ under 'nodatasum' inherit the "no checksums" property, however there's no corresponding file attribute (see `chattr`(1)). + NOTE: If 'nodatacow' or 'nodatasum' are enabled, compression is disabled. ++ +There is a slight performance gain when checksums are turned off, the +correspoinding metadata blocks holding the checksums do not need to updated. +The cost of checksumming of the blocks in memory is much lower than the IO, +modern CPUs feature hardware support of the checksumming algorithm. *degraded*:: (default: off) + -Allow mounts with less devices than the raid profile constraints -require. A read-write mount (or remount) may fail with too many devices +Allow mounts with less devices than the RAID profile constraints +require. A read-write mount (or remount) may fail when there are too many devices missing, for example if a stripe member is completely missing from RAID0. ++ +Since 4.14, the constraint checks have been improved and are verified on the +chunk level, not an the device level. This allows degraded mounts of +filesystems with mixed RAID profiles for data and metadata, even if the +device number constraints would not be satisfied for some of the prifles. ++ +Example: metadata -- raid1, data -- single, devices -- /dev/sda, /dev/sdb ++ +Suppose the data are completely stored on 'sda', then missing 'sdb' will not +prevent the mount, even if 1 missing device would normally prevent (any) +'single' profile to mount. In case some of the data chunks are stored on 'sdb', +then the constraint of single/data is not satisfied and the filesystem +cannot be mounted. *device='devicepath'*:: Specify a path to a device that will be scanned for BTRFS filesystem during @@ -174,14 +205,22 @@ system at that point. *nodiscard*:: (default: off) + -Enable discarding of freed file blocks using TRIM operation. This is useful -for SSD devices, thinly provisioned LUNs or virtual machine images where the -backing device understands the operation. Depending on support of the -underlying device, the operation may severely hurt performance in case the TRIM -operation is synchronous (eg. with SATA devices up to revision 3.0). -+ +Enable discarding of freed file blocks. This is useful for SSD devices, thinly +provisioned LUNs, or virtual machine images; however, every storage layer must +support discard for it to work. if the backing device does not support +asynchronous queued TRIM, then this operation can severly degrade performance, +because a synchronous TRIM operation will be attempted instead. Queued TRIM +requires newer than SATA revision 3.1 chipsets and devices. + +If it is not necessary to immediately discard freed blocks, then the `fstrim` +tool can be used to discard all free blocks in a batch. Scheduling a TRIM +during a period of low system activity will prevent latent interference with +the performance of other operations. Also, a device may ignore the TRIM command +if the range is too small, so running a batch discard has a greater probability +of actually discarding the blocks. + If discarding is not necessary to be done at the block freeing time, there's -`fstrim` tool that lets the filesystem discard all free blocks in a batch, +`fstrim`(8) tool that lets the filesystem discard all free blocks in a batch, possibly not much interfering with other operations. Also, the the device may ignore the TRIM command if the range is too small, so running the batch discard can actually discard the blocks. @@ -215,7 +254,7 @@ This option forces any data dirtied by a write in a prior transaction to commit as part of the current commit, effectively a full filesystem sync. + This makes the committed state a fully consistent view of the file system from -the application's perspective (i.e., it includes all completed file system +the application's perspective (i.e. it includes all completed file system operations). This was previously the behavior only when a snapshot was created. + @@ -245,6 +284,14 @@ the option. + NOTE: Defaults to off due to a potential overflow problem when the free space checksums don't fit inside a single page. ++ +Don't use this option unless you really need it. The inode number limit +on 64bit system is 2^64^, which is practically enough for the whole filesystem +lifetime. Due to implemention of linux VFS layer, the inode numbers on 32bit +systems are only 32 bits wide. This lowers the limit significantly and makes +it possible to reach it. In such case, this mount option will help. +Alternatively, files with high inode numbers can be copied to a new subvolume +which will effectively start the inode numbers from the beginning again. *logreplay*:: *nologreplay*:: @@ -258,7 +305,7 @@ disable that behaviour, mount also with 'nologreplay'. *max_inline='bytes'*:: (default: min(2048, page size) ) + -Specify the maximum amount of space, in bytes, that can be inlined in +Specify the maximum amount of space, that can be inlined in a metadata B-tree leaf. The value is specified in bytes, optionally with a K suffix (case insensitive). In practice, this value is limited by the filesystem block size (named 'sectorsize' at mkfs time), @@ -319,8 +366,8 @@ the space cache consumes some resources, including a small amount of disk space. + There are two implementations of the free space cache. The original -implementation, 'v1', is the safe default. The 'v1' space cache can be disabled -at mount time with 'nospace_cache' without clearing. +one, referred to as 'v1', is the safe default. The 'v1' space cache can be +disabled at mount time with 'nospace_cache' without clearing. + On very large filesystems (many terabytes) and certain workloads, the performance of the 'v1' space cache may degrade drastically. The 'v2' @@ -329,12 +376,12 @@ this issue. Once enabled, the 'v2' space cache will always be used and cannot be disabled unless it is cleared. Use 'clear_cache,space_cache=v1' or 'clear_cache,nospace_cache' to do so. If 'v2' is enabled, kernels without 'v2' support will only be able to mount the filesystem in read-only mode. The -`btrfs(8)` command currently only has read-only support for 'v2'. A read-write +`btrfs`(8) command currently only has read-only support for 'v2'. A read-write command may be run on a 'v2' filesystem by clearing the cache, running the command, and then remounting with 'space_cache=v2'. + If a version is not explicitly specified, the default implementation will be -chosen, which is 'v1' as of 4.9. +chosen, which is 'v1'. *ssd*:: *ssd_spread*:: @@ -342,10 +389,22 @@ chosen, which is 'v1' as of 4.9. (default: SSD autodetected) + Options to control SSD allocation schemes. By default, BTRFS will -enable or disable SSD allocation heuristics depending on whether a -rotational or non-rotational device is in use (contents of -'/sys/block/DEV/queue/rotational'). If it is, the 'ssd' option is turned on. -The option 'nossd' will disable the autodetection. +enable or disable SSD optimizations depending on status of a device with +respect to rotational or non-rotational type. This is determined by the +contents of '/sys/block/DEV/queue/rotational'). If it is 1, the 'ssd' option is +turned on. The option 'nossd' will disable the autodetection. ++ +The optimizations make use of the absence of the seek penalty that's inherent +for the rotational devices. The blocks can be typically written faster and +are not offloaded to separate threads. ++ +NOTE: Since 4.14, the block layout optimizations have been dropped. This used +to help with first generations of SSD devices. Their FTL (flash translation +layer) was not effective and the optimization was supposed to improve the wear +by better aligning blocks. This is no longer true with modern SSD devices and +the optimization had no real benefit. Furthermore it caused increased +fragmentation. The layout tuning has been kept intact for the option +'ssd_spread'. + The 'ssd_spread' mount option attempts to allocate into bigger and aligned chunks of unused space, and may perform better on low-end SSDs. 'ssd_spread' @@ -354,25 +413,26 @@ will disable all SSD options. *subvol='path'*:: Mount subvolume from 'path' rather than the toplevel subvolume. The -'path' is absolute (ie. starts at the toplevel subvolume). +'path' is always treated as relative to the the toplevel subvolume. This mount option overrides the default subvolume set for the given filesystem. *subvolid='subvolid'*:: Mount subvolume specified by a 'subvolid' number rather than the toplevel -subvolume. You can use *btrfs subvolume list* to see subvolume ID numbers. +subvolume. You can use *btrfs subvolume list* of *btrfs subvolume show* to see +subvolume ID numbers. This mount option overrides the default subvolume set for the given filesystem. + NOTE: if both 'subvolid' and 'subvol' are specified, they must point at the -same subvolume, otherwise mount will fail. +same subvolume, otherwise the mount will fail. *thread_pool='number'*:: (default: min(NRCPUS + 2, 8) ) + -The number of worker threads to allocate. NRCPUS is number of on-line CPUs +The number of worker threads to start. NRCPUS is number of on-line CPUs detected at the time of mount. Small number leads to less parallelism in processing data and metadata, higher numbers could lead to a performance hit -due to increased locking contention, cache-line bouncing or costly data -transfers between local CPU memories. +due to increased locking contention, process scheduling, cache-line bouncing or +costly data transfers between local CPU memories. *treelog*:: *notreelog*:: @@ -384,13 +444,14 @@ are flushed at sync and transaction commit. If the system crashes between two such syncs, the pending tree log operations are replayed during mount. + WARNING: currently, the tree log is replayed even with a read-only mount! To -disable that behaviour, mount also with 'nologreplay'. +disable that behaviour, also mount with 'nologreplay'. + The tree log could contain new files/directories, these would not exist on a mounted filesystem if the log is not replayed. *usebackuproot*:: *nousebackuproot*:: +(since: 4.6, default: off) + Enable autorecovery attempts if a bad tree root is found at mount time. Currently this scans a backup list of several previous tree roots and tries to @@ -403,6 +464,11 @@ NOTE: This option has replaced 'recovery'. + Allow subvolumes to be deleted by their respective owner. Otherwise, only the root user can do that. ++ +NOTE: historically, any user could create a snapshot even if he was not owner +of the source subvolume, the subvolume deletion has been restricted for that +reason. The subvolume creation has been restricted but this mount option is +still required. This is a usability issue and will be addressed in the future. DEPRECATED MOUNT OPTIONS ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -428,6 +494,25 @@ but will work on 4.5+ kernels. A workaround option from times (pre 3.2) when it was not possible to mount a subvolume that did not reside directly under the toplevel subvolume. +NOTES ON GENERIC MOUNT OPTIONS +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Some of the general mount options from `mount`(8) that affect BTRFS and are +worth mentioning. + +*noatime*:: +under read intensive work-loads, specifying 'noatime' significantly improves +performance because no new access time information needs to be written. Without +this option, the default is 'relatime', which only reduces the number of +inode atime updates in comparison to the traditional 'strictatime'. The worst +case for atime updates under 'relatime' occurs when many files are read whose +atime is older than 24 h and which are freshly snapshotted. In that case the +atime is updated 'and' COW happens - for each file - in bulk. See also +https://lwn.net/Articles/499293/ - 'Atime and btrfs: a bad combination? (LWN, 2012-05-31)'. ++ +Note that 'noatime' may break applications that rely on atime uptimes like +the venerable Mutt (unless you use maildir mailboxes). + FILESYSTEM FEATURES ------------------- @@ -566,8 +651,8 @@ long as this attribute is set (obviously the exception is unsetting the attribut 'O_DSYNC' *X*:: -'no compression', permanently turn off compression on the given file, other -compression mount options will not affect that +'no compression', permanently turn off compression on the given file. Any +compression mount options will not affect this file. + When set on a directory, all newly created files will inherit this attribute. diff --git a/Documentation/btrfs-map-logical.8.gz b/Documentation/btrfs-map-logical.8.gz Binary files differindex ce04a33a..a1ae0432 100644 --- a/Documentation/btrfs-map-logical.8.gz +++ b/Documentation/btrfs-map-logical.8.gz diff --git a/Documentation/btrfs-map-logical.asciidoc b/Documentation/btrfs-map-logical.asciidoc index a3d110cb..34b22c0d 100644 --- a/Documentation/btrfs-map-logical.asciidoc +++ b/Documentation/btrfs-map-logical.asciidoc @@ -12,7 +12,7 @@ SYNOPSIS DESCRIPTION ----------- *btrfs-map-logical* can be used to find out what the physical offsets are -on the mirrors, the result is dumped into stdout in default. +on the mirrors, the result is dumped to stdout by default. Mainly used for debug purpose. diff --git a/Documentation/btrfs-property.8.gz b/Documentation/btrfs-property.8.gz Binary files differindex e8a46fc9..d8ecdf79 100644 --- a/Documentation/btrfs-property.8.gz +++ b/Documentation/btrfs-property.8.gz diff --git a/Documentation/btrfs-property.asciidoc b/Documentation/btrfs-property.asciidoc index 7ed6a7df..b562717b 100644 --- a/Documentation/btrfs-property.asciidoc +++ b/Documentation/btrfs-property.asciidoc @@ -3,7 +3,7 @@ btrfs-property(8) NAME ---- -btrfs-property - get/set/list properties for given btrfs object. +btrfs-property - get/set/list properties for given filesystem object SYNOPSIS -------- @@ -11,8 +11,9 @@ SYNOPSIS DESCRIPTION ----------- -*btrfs property* is used to get/set/list property for given btrfs object. -See the description of *get* subcommand for more information about +*btrfs property* is used to get/set/list property for given filesystem object. +The object can be an inode (file or directory), subvolume or the whole +filesystem. See the description of *get* subcommand for more information about both btrfs object and property. *btrfs property* provides an unified and user-friendly method to tune different @@ -22,28 +23,30 @@ btrfs properties instead of using the traditional method like `chattr`(1) or SUBCOMMAND ---------- *get* [-t <type>] <object> [<name>]:: -Gets a property from a btrfs object. +get property from a btrfs <object> of given <type> + A btrfs object, which is set by <object>, can be a btrfs filesystem -itself, a btrfs subvolume, an inode(file or directory) inside btrfs, +itself, a btrfs subvolume, an inode (file or directory) inside btrfs, or a device on which a btrfs exists. + The option '-t' can be used to explicitly specify what type of object you meant. This is only needed when a property could be set for more then one object type. + -Possible types are 's[ubvol]', 'f[ilesystem]', 'i[node]' and 'd[evice]'. +Possible types are 's[ubvol]', 'f[ilesystem]', 'i[node]' and 'd[evice]', where +the first lettes is a shortcut. + Set the name of property by 'name'. If no 'name' is specified, all properties for the given object are printed. 'name' is one of -the followings. +the following: ro:::: read-only flag of subvolume: true or false label:::: label of device compression:::: -compression setting for an inode: lzo, zlib, zstd, or "" (empty string) +compression algorithm set for an inode, possible values: 'lzo', 'zlib', 'zstd'. +To disable compression use "" (empty string), 'no' or 'none'. *list* [-t <type>] <object>:: Lists available properties with their descriptions for the given object. diff --git a/Documentation/btrfs-qgroup.8.gz b/Documentation/btrfs-qgroup.8.gz Binary files differindex 8201a6e9..a493240d 100644 --- a/Documentation/btrfs-qgroup.8.gz +++ b/Documentation/btrfs-qgroup.8.gz diff --git a/Documentation/btrfs-qgroup.asciidoc b/Documentation/btrfs-qgroup.asciidoc index 3053f2e6..3108457c 100644 --- a/Documentation/btrfs-qgroup.asciidoc +++ b/Documentation/btrfs-qgroup.asciidoc @@ -17,7 +17,7 @@ NOTE: To use qgroup you need to enable quota first using *btrfs quota enable* command. WARNING: Qgroup is not stable yet and will impact performance in current mainline -kernel (v3.14 so far). +kernel (v4.14). QGROUP ------ @@ -56,13 +56,13 @@ Explicitly ask not to do a rescan. Create a subvolume quota group. + For the '0/<subvolume id>' qgroup, a qgroup can be created even before the -subvolume created. +subvolume is created. *destroy* <qgroupid> <path>:: Destroy a qgroup. + -If a qgroup is no isolated,which means it is a parent or child qgroup, it -can't be destroyed. +If a qgroup is not isolated, meaning it is a parent or child qgroup, then it +can only be destroyed after the relationship is removed. *limit* [options] <size>|none [<qgroupid>] <path>:: Limit the size of a qgroup to <size> or no limit in the btrfs filesystem diff --git a/Documentation/btrfs-quota.8.gz b/Documentation/btrfs-quota.8.gz Binary files differindex 7c6c3001..5a94e29f 100644 --- a/Documentation/btrfs-quota.8.gz +++ b/Documentation/btrfs-quota.8.gz diff --git a/Documentation/btrfs-quota.asciidoc b/Documentation/btrfs-quota.asciidoc index f882647d..85ebf729 100644 --- a/Documentation/btrfs-quota.asciidoc +++ b/Documentation/btrfs-quota.asciidoc @@ -15,24 +15,24 @@ The commands under *btrfs quota* are used to affect the global status of quotas of a btrfs filesystem. The quota groups (qgroups) are managed by the subcommand `btrfs qgroup`(8). -NOTE: the qgroups are different than the traditional user quotas and designed +NOTE: Qgroups are different than the traditional user quotas and designed to track shared and exclusive data per-subvolume. Please refer to the section 'HIERARCHICAL QUOTA GROUP CONCEPTS' for a detailed description. PERFORMANCE IMPLICATIONS ~~~~~~~~~~~~~~~~~~~~~~~~ -When the quotas are turned on, they affect all extent processing, taking a -performance hit. It is not recommended to turn on qgroups unless the user +When quotas are activated, they affect all extent processing, which takes a +performance hit. Activation of qgroups is not recommended unless the user intends to actually use them. STABILITY STATUS ~~~~~~~~~~~~~~~~ The qgroup implementation has turned out to be quite difficult as it affects -the core of the filesystem operation. The users have hit various corner cases -over time, eg. wrong accounting or system instability. The situation is -gradually improving but currently (4.7) there are still issues found and fixed. +the core of the filesystem operation. Qgroup users have hit various corner cases +over time, such as incorrect accounting or system instability. The situation is +gradually improving and issues found and fixed. HIERARCHICAL QUOTA GROUP CONCEPTS --------------------------------- @@ -52,7 +52,7 @@ On the other hand, the traditional approach has only a poor solution to restrict directories. At installation time, the harddisk can be partitioned so that every directory (eg. /usr, /var/, ...) that needs a limit gets its own partition. The obvious -problem is, that those limits cannot be changed without a reinstallation. The +problem is that those limits cannot be changed without a reinstallation. The btrfs subvolume feature builds a bridge. Subvolumes correspond in many ways to partitions, as every subvolume looks like its own filesystem. With subvolume quota, it is now possible to restrict each subvolume like a partition, but keep @@ -181,7 +181,7 @@ own way how to integrate qgroups. `Replacement for partitions` The simplest use case is to use qgroups as simple replacement for partitions. -Btrfs takes the disk as a whole, and /, /usr, /var etc. are created as +Btrfs takes the disk as a whole, and /, /usr, /var, etc. are created as subvolumes. As each subvolume gets it own qgroup automatically, they can simply be restricted. No hierarchy is needed for that. diff --git a/Documentation/btrfs-receive.8.gz b/Documentation/btrfs-receive.8.gz Binary files differindex 4b53e360..c842f36e 100644 --- a/Documentation/btrfs-receive.8.gz +++ b/Documentation/btrfs-receive.8.gz diff --git a/Documentation/btrfs-receive.asciidoc b/Documentation/btrfs-receive.asciidoc index 1f6847a9..cbd88e6a 100644 --- a/Documentation/btrfs-receive.asciidoc +++ b/Documentation/btrfs-receive.asciidoc @@ -81,7 +81,7 @@ should be protected from access by users until the receive operation has completed and the subvolume is set to read-only. Additionally, receive does not currently do a very good job of validating -that an incremental send streams actually makes sense, and it is thus +that an incremental send stream actually makes sense, and it is thus possible for a specially crafted send stream to create a subvolume with reflinks to arbitrary files in the same filesystem. Because of this, users are advised to not use *btrfs receive* on send streams from diff --git a/Documentation/btrfs-replace.8.gz b/Documentation/btrfs-replace.8.gz Binary files differindex 75fce06e..c5d3d449 100644 --- a/Documentation/btrfs-replace.8.gz +++ b/Documentation/btrfs-replace.8.gz diff --git a/Documentation/btrfs-replace.asciidoc b/Documentation/btrfs-replace.asciidoc index 9a8a9ea1..35ecb1f8 100644 --- a/Documentation/btrfs-replace.asciidoc +++ b/Documentation/btrfs-replace.asciidoc @@ -34,7 +34,7 @@ from the system, you have to use the devid parameter format. The <targetdev> needs to be same size or larger than the <srcdev>. + NOTE: the filesystem has to be resized to fully take advantage of a -larger target device, this can be achieved with +larger target device; this can be achieved with `btrfs filesystem resize <devid>:max /path` + `Options` @@ -45,10 +45,10 @@ only read from <srcdev> if no other zero-defect mirror exists. slow) -f:::: force using and overwriting <targetdev> even if it looks like -containing a valid btrfs filesystem. +it contains a valid btrfs filesystem. + A valid filesystem is assumed if a btrfs superblock is found which contains a -correct checksum. Devices which are currently mounted are +correct checksum. Devices that are currently mounted are never allowed to be used as the <targetdev>. + -B:::: diff --git a/Documentation/btrfs-rescue.8.gz b/Documentation/btrfs-rescue.8.gz Binary files differindex cbf07923..11ea6215 100644 --- a/Documentation/btrfs-rescue.8.gz +++ b/Documentation/btrfs-rescue.8.gz diff --git a/Documentation/btrfs-rescue.asciidoc b/Documentation/btrfs-rescue.asciidoc index 24b619c6..743a23a6 100644 --- a/Documentation/btrfs-rescue.asciidoc +++ b/Documentation/btrfs-rescue.asciidoc @@ -15,6 +15,7 @@ DESCRIPTION SUBCOMMAND ---------- + *chunk-recover* [options] <device>:: Recover the chunk tree by scanning the devices + @@ -30,6 +31,25 @@ help. NOTE: Since *chunk-recover* will scan the whole device, it will be *VERY* slow especially executed on a large device. +*fix-device-size* <device>:: +fix device size and super block total bytes values that are do not match ++ +Kernel 4.11 starts to check the device size more strictly and this might +mismatch the stored value of total bytes. See the exact error message below. +Newer kernel will refuse to mount the filesystem where the values do not match. +This error is not fatal and can be fixed. This command will fix the device +size values if possible. ++ +---- +BTRFS error (device sdb): super_total_bytes 92017859088384 mismatch with fs_devices total_rw_bytes 92017859094528 +---- ++ +The mismatch may also exhibit as a kernel warning: ++ +---- +WARNING: CPU: 3 PID: 439 at fs/btrfs/ctree.h:1559 btrfs_update_device+0x1c5/0x1d0 [btrfs] +---- + *super-recover* [options] <device>:: Recover bad superblocks from good copies. + @@ -47,8 +67,8 @@ This command will clear the filesystem log tree. This may fix a specific set of problem when the filesystem mount fails due to the log replay. See below for sample stacktraces that may show up in system log. + -The common case where this happens has been fixed a long time ago, -so it is unlikely that you will see this particular problem, but the utility is +The common case where this happens was fixed a long time ago, +so it is unlikely that you will see this particular problem, but the command is kept around. + NOTE: clearing the log may lead to loss of changes that were made since the diff --git a/Documentation/btrfs-restore.8.gz b/Documentation/btrfs-restore.8.gz Binary files differindex 2fa149f6..c4c2b201 100644 --- a/Documentation/btrfs-restore.8.gz +++ b/Documentation/btrfs-restore.8.gz diff --git a/Documentation/btrfs-scrub.8.gz b/Documentation/btrfs-scrub.8.gz Binary files differindex edfc4390..56b3a58f 100644 --- a/Documentation/btrfs-scrub.8.gz +++ b/Documentation/btrfs-scrub.8.gz diff --git a/Documentation/btrfs-scrub.asciidoc b/Documentation/btrfs-scrub.asciidoc index eb90a1c4..d2d20627 100644 --- a/Documentation/btrfs-scrub.asciidoc +++ b/Documentation/btrfs-scrub.asciidoc @@ -21,8 +21,8 @@ structural damage in the filesystem. The user is supposed to run it manually or via a periodic system service. The recommended period is a month but could be less. The estimated device bandwidth utilization is about 80% on an idle filesystem. The IO priority class is by -default 'idle' so background scrub should not interfere with normal filesystem -operation significantly. +default 'idle' so background scrub should not significantly interfere with +normal filesystem operation. The scrubbing status is recorded in '/var/lib/btrfs/' in textual files named 'scrub.status.UUID' for a filesystem identified by the given UUID. (Progress diff --git a/Documentation/btrfs-select-super.8.gz b/Documentation/btrfs-select-super.8.gz Binary files differindex 2cf064db..e19b070f 100644 --- a/Documentation/btrfs-select-super.8.gz +++ b/Documentation/btrfs-select-super.8.gz diff --git a/Documentation/btrfs-send.8.gz b/Documentation/btrfs-send.8.gz Binary files differindex 06002bd1..e7afb09a 100644 --- a/Documentation/btrfs-send.8.gz +++ b/Documentation/btrfs-send.8.gz diff --git a/Documentation/btrfs-send.asciidoc b/Documentation/btrfs-send.asciidoc index ef345f68..cd7ada76 100644 --- a/Documentation/btrfs-send.asciidoc +++ b/Documentation/btrfs-send.asciidoc @@ -29,12 +29,13 @@ are available on both the sending and receiving side can be used to reduce the amount of information that has to be sent to reconstruct the sent snapshot on a different filesystem. -It is allowed to omit the '-p <parent>' option when '-c <clone-src>' options -are given, in which case *btrfs send* will determine a suitable parent among the -clone sources itself. +The '-p <parent>' option can be omitted when '-c <clone-src>' options are +given, in which case *btrfs send* will determine a suitable parent from among +the clone sources. You must not specify clone sources unless you guarantee that these snapshots -are exactly in the same state on both sides, the sender and the receiver. +are exactly in the same state on both sides--both for the sender and the +receiver. `Options` @@ -53,7 +54,7 @@ send in 'NO_FILE_DATA' mode + The output stream does not contain any file data and thus cannot be used to transfer changes. This mode is faster and -useful to show the differences in metadata. +is useful to show the differences in metadata. -v|--verbose:: enable verbose output, print generated commands in a readable form, (each diff --git a/Documentation/btrfs-subvolume.8.gz b/Documentation/btrfs-subvolume.8.gz Binary files differindex 3d8519a5..9ae7d367 100644 --- a/Documentation/btrfs-subvolume.8.gz +++ b/Documentation/btrfs-subvolume.8.gz diff --git a/Documentation/btrfs-subvolume.asciidoc b/Documentation/btrfs-subvolume.asciidoc index 5cfe8856..a8c4af4b 100644 --- a/Documentation/btrfs-subvolume.asciidoc +++ b/Documentation/btrfs-subvolume.asciidoc @@ -17,7 +17,7 @@ snapshots. SUBVOLUME AND SNAPSHOT ---------------------- -A subvolume is a part of filesystem with its own and independent +A subvolume is a part of filesystem with its own independent file/directory hierarchy. Subvolumes can share file extents. A snapshot is also subvolume, but with a given initial content of the original subvolume. @@ -66,7 +66,7 @@ If <subvolume> is not a subvolume, btrfs returns an error but continues if there are more arguments to process. + The corresponding directory is removed instantly but the data blocks are -removed later in the background. The command returns immediatelly. See `btrfs +removed later in the background. The command returns immediately. See `btrfs subvolume sync` how to wait until the subvolume gets completely removed. + The deletion does not involve full transaction commit by default due to @@ -142,29 +142,37 @@ you can add \'\+' or \'-' in front of each items, \'+' means ascending, for --sort you can combine some items together by \',', just like --sort=+ogen,-gen,path,rootid. -*set-default* <id> <path>:: -Set the subvolume of the filesystem <path> which is mounted as -default. +*set-default* [<subvolume>|<id> <path>]:: +Set the default subvolume for the (mounted) filesystem. + +Set the default subvolume for the (mounted) filesystem at <path>. This will hide +the top-level subvolume (ie. the one mounted with 'subvol=/' or 'subvolid=5'). +Takes action on next mount. + -The subvolume is identified by <id>, which is returned by the *subvolume list* -command. +There are two ways how to specify the subvolume, by <id> or by the <subvolume> +path. +The id can be obtained from *btrfs subvolume list*, *btrfs subvolume show* or +*btrfs inspect-internal rootid*. *show* <path>:: Show information of a given subvolume in the <path>. *snapshot* [-r] <source> <dest>|[<dest>/]<name>:: -Create a writable/readonly snapshot of the subvolume <source> with the +Create a snapshot of the subvolume <source> with the name <name> in the <dest> directory. + If only <dest> is given, the subvolume will be named the basename of <source>. If <source> is not a subvolume, btrfs returns an error. -If '-r' is given, the snapshot will be readonly. ++ +`Options` ++ +-r:::: +Make the new snapshot read only. *sync* <path> [subvolid...]:: -Wait until given subvolume(s) are completely removed from the filesystem -after deletion. If no subvolume id is given, wait until all current deletion -requests are completed, but do not wait for subvolumes deleted meanwhile. -The status of subvolume ids is checked periodically. +Wait until given subvolume(s) are completely removed from the filesystem after +deletion. If no subvolume id is given, wait until all current deletion requests +are completed, but do not wait for subvolumes deleted in the meantime. + `Options` + diff --git a/Documentation/btrfs.5.gz b/Documentation/btrfs.5.gz Binary files differindex b72078a8..f0dbd468 100644 --- a/Documentation/btrfs.5.gz +++ b/Documentation/btrfs.5.gz diff --git a/Documentation/btrfs.8.gz b/Documentation/btrfs.8.gz Binary files differindex 0330e855..13d14727 100644 --- a/Documentation/btrfs.8.gz +++ b/Documentation/btrfs.8.gz diff --git a/Documentation/btrfs.asciidoc b/Documentation/btrfs.asciidoc index 100a6adf..82530234 100644 --- a/Documentation/btrfs.asciidoc +++ b/Documentation/btrfs.asciidoc @@ -25,9 +25,9 @@ page `btrfs`(5). COMMAND SYNTAX -------------- -Any command name can be shortened as far as it stays unambiguous, -however it is recommended to use full command names in scripts. -All command groups have their manual page named *btrfs-<group>*. +Any command name can be shortened so long as the shortened form is unambiguous, +however, it is recommended to use full command names in scripts. All command +groups have their manual page named *btrfs-<group>*. For example: it is possible to run *btrfs sub snaps* instead of *btrfs subvolume snapshot*. @@ -106,10 +106,10 @@ COMMANDS STANDALONE TOOLS ---------------- -There are several standalone tools to provide certain functionality. If the -functionality proves to be useful, the standalone tools are declared obsolete -and their functionality copied to the main tool. The deprecation period is long -(years) and the obsolete binaries are still provided. +New functionality could be provided using a standalone tool. If the functionality +proves to be useful, then the standalone tool is declared obsolete and its +functionality is copied to the main tool. Obsolete tools are removed after a +long (years) depreciation period. Tools that are still in active use without an equivalent in *btrfs*: diff --git a/Documentation/btrfstune.8.gz b/Documentation/btrfstune.8.gz Binary files differindex 30a81a82..9de752e8 100644 --- a/Documentation/btrfstune.8.gz +++ b/Documentation/btrfstune.8.gz diff --git a/Documentation/btrfstune.asciidoc b/Documentation/btrfstune.asciidoc index cd7bb532..27100964 100644 --- a/Documentation/btrfstune.asciidoc +++ b/Documentation/btrfstune.asciidoc @@ -11,7 +11,7 @@ SYNOPSIS DESCRIPTION ----------- -*btrfstune* can be used to enable, disable or set various filesystem +*btrfstune* can be used to enable, disable, or set various filesystem parameters. The filesystem must be unmounted. The common usecase is to enable features that were not enabled at mkfs time. @@ -20,8 +20,8 @@ complete list of features and kernel version of their introduction at https://btrfs.wiki.kernel.org/index.php/Changelog#By_feature . Also, the manual page `mkfs.btrfs`(8) contains more details about the features. -Some of the features could be enabled on a mounted filesystem. Please refer to -the respective section in `btrfs`(5). +Some of the features could be also enabled on a mounted filesystem by other +means. Please refer to the 'FILESYSTEM FEATURES' in `btrfs`(5). OPTIONS ------- @@ -86,9 +86,10 @@ EXIT STATUS COMPATIBILITY NOTE ------------------ -This tool exists for historical reasons but is still in use today. The -functionality is about to be merged to the main tool someday and *btrfstune* -will become deprecated and removed afterwards. + +This deprecated tool exists for historical reasons but is still in use today. +Its functionality will be merged to the main tool, at which time *btrfstune* +will be declared obsolete and scheduled for removal. SEE ALSO -------- diff --git a/Documentation/fsck.btrfs.8.gz b/Documentation/fsck.btrfs.8.gz Binary files differindex 86c8bfc6..1275946e 100644 --- a/Documentation/fsck.btrfs.8.gz +++ b/Documentation/fsck.btrfs.8.gz diff --git a/Documentation/fsck.btrfs.asciidoc b/Documentation/fsck.btrfs.asciidoc index 0bad075b..9a174a60 100644 --- a/Documentation/fsck.btrfs.asciidoc +++ b/Documentation/fsck.btrfs.asciidoc @@ -13,16 +13,15 @@ DESCRIPTION ----------- *fsck.btrfs* is a type of utility that should exist for any filesystem and is called during system setup when the corresponding `/etc/fstab` entries -contain non-zero value for `fs_passno` , see `fstab`(5) for more. +contain non-zero value for `fs_passno`, see `fstab`(5) for more. Traditional filesystems need to run their respective fsck utility in case the filesystem was not unmounted cleanly and the log needs to be replayed before mount. This is not needed for BTRFS. You should set fs_passno to 0. If you wish to check the consistency of a BTRFS filesystem or repair a damaged -filesystem, see `btrfs-check`(8). By default the filesystem -consistency is checked, the repair mode is enabled via '--repair' option (use -with care!). +filesystem, see `btrfs-check`(8). By default filesystem consistency is checked, +the repair mode is enabled via the '--repair' option (use with care!). OPTIONS ------- diff --git a/Documentation/mkfs.btrfs.8.gz b/Documentation/mkfs.btrfs.8.gz Binary files differindex 11956269..c455e9ea 100644 --- a/Documentation/mkfs.btrfs.8.gz +++ b/Documentation/mkfs.btrfs.8.gz diff --git a/Documentation/mkfs.btrfs.asciidoc b/Documentation/mkfs.btrfs.asciidoc index d53d9e26..f69c529d 100644 --- a/Documentation/mkfs.btrfs.asciidoc +++ b/Documentation/mkfs.btrfs.asciidoc @@ -21,13 +21,8 @@ mount option. See section *MULTIPLE DEVICES* for more details. OPTIONS ------- -*-A|--alloc-start <offset>*:: -(An option to help debugging chunk allocator.) -Specify the (physical) offset from the start of the device at which allocations -start. The default value is zero. - *-b|--byte-count <size>*:: -Specify the size of the filesystem. If this option is not used, +Specify the size of the filesystem. If this option is not used, then mkfs.btrfs uses the entire device space for the filesystem. *-d|--data <profile>*:: @@ -79,7 +74,7 @@ default value is 16KiB (16384) or the page size, whichever is bigger. Must be a multiple of the sectorsize and a power of 2, but not larger than 64KiB (65536). Leafsize always equals nodesize and the options are aliases. + -Smaller node size increases fragmentation but lead to higher b-trees which in +Smaller node size increases fragmentation but leads to taller b-trees which in turn leads to lower locking contention. Higher node sizes give better packing and less fragmentation at the cost of more expensive memory operations while updating the metadata blocks. @@ -135,6 +130,12 @@ Print the *mkfs.btrfs* version and exit. *--help*:: Print help. +*-A|--alloc-start <offset>*:: +*deprecated, will be removed* +(An option to help debugging chunk allocator.) +Specify the (physical) offset from the start of the device at which allocations +start. The default value is zero. + SIZE UNITS ---------- The default unit is 'byte'. All size parameters accept suffixes in the 1024 @@ -166,7 +167,7 @@ root partition created with RAID1/10/5/6 profiles. The mount action can happen before all block devices are discovered. The waiting is usually done on the initramfs/initrd systems. -As of kernel 4.9, RAID5/6 is still considered experimental and shouldn't be +As of kernel 4.14, RAID5/6 is still considered experimental and shouldn't be employed for production use. FILESYSTEM FEATURES @@ -281,9 +282,9 @@ The mkfs utility will let the user create a filesystem with profiles that write the logical blocks to 2 physical locations. Whether there are really 2 physical copies highly depends on the underlying device type. -For example, a SSD drive can remap the blocks internally to a single copy thus +For example, a SSD drive can remap the blocks internally to a single copy--thus deduplicating them. This negates the purpose of increased redundancy and just -wastes filesystem space without the expected level of redundancy. +wastes filesystem space without providing the expected level of redundancy. The duplicated data/metadata may still be useful to statistically improve the chances on a device that might perform some internal optimizations. The actual @@ -7,11 +7,12 @@ The Btrfs utility programs require the following libraries/tools to build: - libblkid - block device id library - liblzo2 - LZO data compression library - zlib - ZLIB data compression library -- libzstd - ZSTD data compression library version >= 1.0.0 (optional) +- libzstd - ZSTD data compression library version >= 1.0.0 For the btrfs-convert utility: - e2fsprogs - ext2/ext3/ext4 file system libraries, or called e2fslibs +- libreiserfscore - reiserfs file system library version >= 3.6.27 Generating documentation: @@ -124,7 +124,7 @@ libbtrfs_headers = send-stream.h send-utils.h send.h kernel-lib/rbtree.h btrfs-l convert_objects = convert/main.o convert/common.o convert/source-fs.o \ convert/source-ext2.o convert/source-reiserfs.o mkfs_objects = mkfs/main.o mkfs/common.o -image_objects = image/main.o +image_objects = image/main.o image/sanitize.o all_objects = $(objects) $(cmds_objects) $(libbtrfs_objects) $(convert_objects) \ $(mkfs_objects) $(image_objects) @@ -246,6 +246,7 @@ static_cmds_objects = $(patsubst %.o, %.static.o, $(cmds_objects)) static_libbtrfs_objects = $(patsubst %.o, %.static.o, $(libbtrfs_objects)) static_convert_objects = $(patsubst %.o, %.static.o, $(convert_objects)) static_mkfs_objects = $(patsubst %.o, %.static.o, $(mkfs_objects)) +static_image_objects = $(patsubst %.o, %.static.o, $(image_objects)) libs_shared = libbtrfs.so.0.1 libs_static = libbtrfs.a @@ -424,11 +425,11 @@ btrfstune.static: btrfstune.static.o $(static_objects) $(static_libbtrfs_objects @echo " [LD] $@" $(Q)$(CC) -o $@ $^ $(STATIC_LDFLAGS) $(STATIC_LIBS) -btrfs-image: image/main.o $(objects) $(libs_static) +btrfs-image: $(image_objects) $(objects) $(libs_static) @echo " [LD] $@" $(Q)$(CC) -o $@ $^ $(LDFLAGS) $(LIBS) $(LIBS_COMP) -btrfs-image.static: image/main.static.o $(static_objects) $(static_libbtrfs_objects) +btrfs-image.static: $(static_image_objects) $(static_objects) $(static_libbtrfs_objects) @echo " [LD] $@" $(Q)$(CC) -o $@ $^ $(STATIC_LDFLAGS) $(STATIC_LIBS) $(STATIC_LIBS_COMP) @@ -206,6 +206,9 @@ static int __add_prelim_ref(struct pref_state *prefstate, u64 root_id, if (key) { ref->key_for_search = *key; head = &prefstate->pending; + } else if (parent) { + memset(&ref->key_for_search, 0, sizeof(ref->key_for_search)); + head = &prefstate->pending; } else { memset(&ref->key_for_search, 0, sizeof(ref->key_for_search)); head = &prefstate->pending_missing_keys; diff --git a/btrfs-show-super.c b/btrfs-show-super.c index f97f83ba..4273e42d 100644 --- a/btrfs-show-super.c +++ b/btrfs-show-super.c @@ -18,7 +18,7 @@ #include "utils.h" #include "commands.h" -#include "cmds-inspect-dump-super.h" +#include "help.h" int main(int argc, char **argv) { diff --git a/cmds-check.c b/cmds-check.c index 5c822b84..7fc30da8 100644 --- a/cmds-check.c +++ b/cmds-check.c @@ -130,6 +130,10 @@ struct data_backref { #define LAST_ITEM (1<<15) /* Complete this tree traversal */ #define ROOT_REF_MISSING (1<<16) /* ROOT_REF not found */ #define ROOT_REF_MISMATCH (1<<17) /* ROOT_REF found but not match */ +#define DIR_INDEX_MISSING (1<<18) /* INODE_INDEX not found */ +#define DIR_INDEX_MISMATCH (1<<19) /* INODE_INDEX found but not match */ +#define DIR_COUNT_AGAIN (1<<20) /* DIR isize should be recalculated */ +#define BG_ACCOUNTING_ERROR (1<<21) /* Block group accounting error */ static inline struct data_backref* to_data_backref(struct extent_backref *back) { @@ -1971,10 +1975,15 @@ struct node_refs { u64 bytenr[BTRFS_MAX_LEVEL]; u64 refs[BTRFS_MAX_LEVEL]; int need_check[BTRFS_MAX_LEVEL]; + /* field for checking all trees */ + int checked[BTRFS_MAX_LEVEL]; + /* the corresponding extent should be marked as full backref or not */ + int full_backref[BTRFS_MAX_LEVEL]; }; static int update_nodes_refs(struct btrfs_root *root, u64 bytenr, - struct node_refs *nrefs, u64 level); + struct extent_buffer *eb, struct node_refs *nrefs, + u64 level, int check_all); static int check_inode_item(struct btrfs_root *root, struct btrfs_path *path, unsigned int ext_ref); @@ -2017,6 +2026,9 @@ static int process_one_leaf_v2(struct btrfs_root *root, struct btrfs_path *path, again: err |= check_inode_item(root, path, ext_ref); + /* modify cur since check_inode_item may change path */ + cur = path->nodes[0]; + if (err & LAST_ITEM) goto out; @@ -2034,9 +2046,8 @@ again: if (path->nodes[i]->start == nrefs->bytenr[i]) continue; - ret = update_nodes_refs(root, - path->nodes[i]->start, - nrefs, i); + ret = update_nodes_refs(root, path->nodes[i]->start, + path->nodes[i], nrefs, i, 0); if (ret) goto out; @@ -2138,7 +2149,12 @@ static int need_check(struct btrfs_root *root, struct ulist *roots) struct rb_node *node; struct ulist_node *u; - if (roots->nnodes == 1) + /* + * @roots can be empty if it belongs to tree reloc tree + * In that case, we should always check the leaf, as we can't use + * the tree owner to ensure some other root will check it. + */ + if (roots->nnodes == 1 || roots->nnodes == 0) return 1; node = rb_first(&roots->root); @@ -2153,25 +2169,143 @@ static int need_check(struct btrfs_root *root, struct ulist *roots) return 1; } +static int calc_extent_flag_v2(struct btrfs_root *root, struct extent_buffer *eb, + u64 *flags_ret) +{ + struct btrfs_root *extent_root = root->fs_info->extent_root; + struct btrfs_root_item *ri = &root->root_item; + struct btrfs_extent_inline_ref *iref; + struct btrfs_extent_item *ei; + struct btrfs_key key; + struct btrfs_path *path = NULL; + unsigned long ptr; + unsigned long end; + u64 flags; + u64 owner = 0; + u64 offset; + int slot; + int type; + int ret = 0; + + /* + * Except file/reloc tree, we can not have FULL BACKREF MODE + */ + if (root->objectid < BTRFS_FIRST_FREE_OBJECTID) + goto normal; + + /* root node */ + if (eb->start == btrfs_root_bytenr(ri)) + goto normal; + + if (btrfs_header_flag(eb, BTRFS_HEADER_FLAG_RELOC)) + goto full_backref; + + owner = btrfs_header_owner(eb); + if (owner == root->objectid) + goto normal; + + path = btrfs_alloc_path(); + if (!path) + return -ENOMEM; + + key.objectid = btrfs_header_bytenr(eb); + key.type = (u8)-1; + key.offset = (u64)-1; + + ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0); + if (ret <= 0) { + ret = -EIO; + goto out; + } + + if (ret > 0) { + ret = btrfs_previous_extent_item(extent_root, path, + key.objectid); + if (ret) + goto full_backref; + + } + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + + eb = path->nodes[0]; + slot = path->slots[0]; + ei = btrfs_item_ptr(eb, slot, struct btrfs_extent_item); + + flags = btrfs_extent_flags(eb, ei); + if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) + goto full_backref; + + ptr = (unsigned long)(ei + 1); + end = (unsigned long)ei + btrfs_item_size_nr(eb, slot); + + if (key.type == BTRFS_EXTENT_ITEM_KEY) + ptr += sizeof(struct btrfs_tree_block_info); + +next: + /* Reached extent item ends normally */ + if (ptr == end) + goto full_backref; + + /* Beyond extent item end, wrong item size */ + if (ptr > end) { + error("extent item at bytenr %llu slot %d has wrong size", + eb->start, slot); + goto full_backref; + } + + iref = (struct btrfs_extent_inline_ref *)ptr; + offset = btrfs_extent_inline_ref_offset(eb, iref); + type = btrfs_extent_inline_ref_type(eb, iref); + + if (type == BTRFS_TREE_BLOCK_REF_KEY && offset == owner) + goto normal; + ptr += btrfs_extent_inline_ref_size(type); + goto next; + +normal: + *flags_ret &= ~BTRFS_BLOCK_FLAG_FULL_BACKREF; + goto out; + +full_backref: + *flags_ret |= BTRFS_BLOCK_FLAG_FULL_BACKREF; +out: + btrfs_free_path(path); + return ret; +} + /* * for a tree node or leaf, we record its reference count, so later if we still * process this node or leaf, don't need to compute its reference count again. + * + * @bytenr if @bytenr == (u64)-1, only update nrefs->full_backref[level] */ static int update_nodes_refs(struct btrfs_root *root, u64 bytenr, - struct node_refs *nrefs, u64 level) + struct extent_buffer *eb, struct node_refs *nrefs, + u64 level, int check_all) { - int check, ret; - u64 refs; struct ulist *roots; + u64 refs = 0; + u64 flags = 0; + int root_level = btrfs_header_level(root->node); + int check; + int ret; + + if (nrefs->bytenr[level] == bytenr) + return 0; - if (nrefs->bytenr[level] != bytenr) { + if (bytenr != (u64)-1) { + /* the return value of this function seems a mistake */ ret = btrfs_lookup_extent_info(NULL, root, bytenr, - level, 1, &refs, NULL); - if (ret < 0) + level, 1, &refs, &flags); + /* temporary fix */ + if (ret < 0 && !check_all) return ret; nrefs->bytenr[level] = bytenr; nrefs->refs[level] = refs; + nrefs->full_backref[level] = 0; + nrefs->checked[level] = 0; + if (refs > 1) { ret = btrfs_find_all_roots(NULL, root->fs_info, bytenr, 0, &roots); @@ -2182,13 +2316,58 @@ static int update_nodes_refs(struct btrfs_root *root, u64 bytenr, ulist_free(roots); nrefs->need_check[level] = check; } else { - nrefs->need_check[level] = 1; + if (!check_all) { + nrefs->need_check[level] = 1; + } else { + if (level == root_level) { + nrefs->need_check[level] = 1; + } else { + /* + * The node refs may have not been + * updated if upper needs checking (the + * lowest root_objectid) the node can + * be checked. + */ + nrefs->need_check[level] = + nrefs->need_check[level + 1]; + } + } } } + if (check_all && eb) { + calc_extent_flag_v2(root, eb, &flags); + if (flags & BTRFS_BLOCK_FLAG_FULL_BACKREF) + nrefs->full_backref[level] = 1; + } + return 0; } +/* + * @level if @level == -1 means extent data item + * else normal treeblocl. + */ +static int should_check_extent_strictly(struct btrfs_root *root, + struct node_refs *nrefs, int level) +{ + int root_level = btrfs_header_level(root->node); + + if (level > root_level || level < -1) + return 1; + if (level == root_level) + return 1; + /* + * if the upper node is marked full backref, it should contain shared + * backref of the parent (except owner == root->objectid). + */ + while (++level <= root_level) + if (nrefs->refs[level] > 1) + return 0; + + return 1; +} + static int walk_down_tree(struct btrfs_root *root, struct btrfs_path *path, struct walk_control *wc, int *level, struct node_refs *nrefs) @@ -2318,16 +2497,197 @@ out: return err; } +static int fs_root_objectid(u64 objectid); + +/* + * Update global fs information. + */ +static void account_bytes(struct btrfs_root *root, struct btrfs_path *path, + int level) +{ + u32 free_nrs; + struct extent_buffer *eb = path->nodes[level]; + + total_btree_bytes += eb->len; + if (fs_root_objectid(root->objectid)) + total_fs_tree_bytes += eb->len; + if (btrfs_header_owner(eb) == BTRFS_EXTENT_TREE_OBJECTID) + total_extent_tree_bytes += eb->len; + + if (level == 0) { + btree_space_waste += btrfs_leaf_free_space(root, eb); + } else { + free_nrs = (BTRFS_NODEPTRS_PER_BLOCK(root) - + btrfs_header_nritems(eb)); + btree_space_waste += free_nrs * sizeof(struct btrfs_key_ptr); + } +} + +/* + * This function only handles BACKREF_MISSING, + * If corresponding extent item exists, increase the ref, else insert an extent + * item and backref. + * + * Returns error bits after repair. + */ +static int repair_tree_block_ref(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct extent_buffer *node, + struct node_refs *nrefs, int level, int err) +{ + struct btrfs_fs_info *fs_info = root->fs_info; + struct btrfs_root *extent_root = fs_info->extent_root; + struct btrfs_path path; + struct btrfs_extent_item *ei; + struct btrfs_tree_block_info *bi; + struct btrfs_key key; + struct extent_buffer *eb; + u32 size = sizeof(*ei); + u32 node_size = root->fs_info->nodesize; + int insert_extent = 0; + int skinny_metadata = btrfs_fs_incompat(fs_info, SKINNY_METADATA); + int root_level = btrfs_header_level(root->node); + int generation; + int ret; + u64 owner; + u64 bytenr; + u64 flags = BTRFS_EXTENT_FLAG_TREE_BLOCK; + u64 parent = 0; + + if ((err & BACKREF_MISSING) == 0) + return err; + + WARN_ON(level > BTRFS_MAX_LEVEL); + WARN_ON(level < 0); + + btrfs_init_path(&path); + bytenr = btrfs_header_bytenr(node); + owner = btrfs_header_owner(node); + generation = btrfs_header_generation(node); + + key.objectid = bytenr; + key.type = (u8)-1; + key.offset = (u64)-1; + + /* Search for the extent item */ + ret = btrfs_search_slot(NULL, extent_root, &key, &path, 0, 0); + if (ret <= 0) { + ret = -EIO; + goto out; + } + + ret = btrfs_previous_extent_item(extent_root, &path, bytenr); + if (ret) + insert_extent = 1; + + /* calculate if the extent item flag is full backref or not */ + if (nrefs->full_backref[level] != 0) + flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; + + /* insert an extent item */ + if (insert_extent) { + struct btrfs_disk_key copy_key; + + generation = btrfs_header_generation(node); + + if (level < root_level && nrefs->full_backref[level + 1] && + owner != root->objectid) { + flags |= BTRFS_BLOCK_FLAG_FULL_BACKREF; + } + + key.objectid = bytenr; + if (!skinny_metadata) { + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = node_size; + size += sizeof(*bi); + } else { + key.type = BTRFS_METADATA_ITEM_KEY; + key.offset = level; + } + + btrfs_release_path(&path); + ret = btrfs_insert_empty_item(trans, extent_root, &path, &key, + size); + if (ret) + goto out; + + eb = path.nodes[0]; + ei = btrfs_item_ptr(eb, path.slots[0], struct btrfs_extent_item); + + btrfs_set_extent_refs(eb, ei, 0); + btrfs_set_extent_generation(eb, ei, generation); + btrfs_set_extent_flags(eb, ei, flags); + + if (!skinny_metadata) { + bi = (struct btrfs_tree_block_info *)(ei + 1); + memset_extent_buffer(eb, 0, (unsigned long)bi, + sizeof(*bi)); + btrfs_set_disk_key_objectid(©_key, root->objectid); + btrfs_set_disk_key_type(©_key, 0); + btrfs_set_disk_key_offset(©_key, 0); + + btrfs_set_tree_block_level(eb, bi, level); + btrfs_set_tree_block_key(eb, bi, ©_key); + } + btrfs_mark_buffer_dirty(eb); + printf("Added an extent item [%llu %u]\n", bytenr, node_size); + btrfs_update_block_group(trans, extent_root, bytenr, node_size, + 1, 0); + + nrefs->refs[level] = 0; + nrefs->full_backref[level] = + flags & BTRFS_BLOCK_FLAG_FULL_BACKREF; + btrfs_release_path(&path); + } + + if (level < root_level && nrefs->full_backref[level + 1] && + owner != root->objectid) + parent = nrefs->bytenr[level + 1]; + + /* increase the ref */ + ret = btrfs_inc_extent_ref(trans, extent_root, bytenr, node_size, + parent, root->objectid, level, 0); + + nrefs->refs[level]++; +out: + btrfs_release_path(&path); + if (ret) { + error( + "failed to repair tree block ref start %llu root %llu due to %s", + bytenr, root->objectid, strerror(-ret)); + } else { + printf("Added one tree block ref start %llu %s %llu\n", + bytenr, parent ? "parent" : "root", + parent ? parent : root->objectid); + err &= ~BACKREF_MISSING; + } + + return err; +} + static int check_inode_item(struct btrfs_root *root, struct btrfs_path *path, unsigned int ext_ref); +static int check_tree_block_ref(struct btrfs_root *root, + struct extent_buffer *eb, u64 bytenr, + int level, u64 owner, struct node_refs *nrefs); +static int check_leaf_items(struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct btrfs_path *path, + struct node_refs *nrefs, int account_bytes); /* + * @trans just for lowmem repair mode + * @check all if not 0 then check all tree block backrefs and items + * 0 then just check relationship of items in fs tree(s) + * * Returns >0 Found error, should continue * Returns <0 Fatal error, must exit the whole check * Returns 0 No errors found */ -static int walk_down_tree_v2(struct btrfs_root *root, struct btrfs_path *path, - int *level, struct node_refs *nrefs, int ext_ref) +static int walk_down_tree_v2(struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct btrfs_path *path, + int *level, struct node_refs *nrefs, int ext_ref, + int check_all) + { enum btrfs_tree_block_status status; u64 bytenr; @@ -2336,12 +2696,15 @@ static int walk_down_tree_v2(struct btrfs_root *root, struct btrfs_path *path, struct extent_buffer *next; struct extent_buffer *cur; int ret; + int err = 0; + int check; + int account_file_data = 0; WARN_ON(*level < 0); WARN_ON(*level >= BTRFS_MAX_LEVEL); - ret = update_nodes_refs(root, path->nodes[*level]->start, - nrefs, *level); + ret = update_nodes_refs(root, btrfs_header_bytenr(path->nodes[*level]), + path->nodes[*level], nrefs, *level, check_all); if (ret < 0) return ret; @@ -2349,36 +2712,80 @@ static int walk_down_tree_v2(struct btrfs_root *root, struct btrfs_path *path, WARN_ON(*level < 0); WARN_ON(*level >= BTRFS_MAX_LEVEL); cur = path->nodes[*level]; + bytenr = btrfs_header_bytenr(cur); + check = nrefs->need_check[*level]; if (btrfs_header_level(cur) != *level) WARN_ON(1); + /* + * Update bytes accounting and check tree block ref + * NOTE: Doing accounting and check before checking nritems + * is necessary because of empty node/leaf. + */ + if ((check_all && !nrefs->checked[*level]) || + (!check_all && nrefs->need_check[*level])) { + ret = check_tree_block_ref(root, cur, + btrfs_header_bytenr(cur), btrfs_header_level(cur), + btrfs_header_owner(cur), nrefs); + + if (repair && ret) + ret = repair_tree_block_ref(trans, root, + path->nodes[*level], nrefs, *level, ret); + err |= ret; + + if (check_all && nrefs->need_check[*level] && + nrefs->refs[*level]) { + account_bytes(root, path, *level); + account_file_data = 1; + } + nrefs->checked[*level] = 1; + } if (path->slots[*level] >= btrfs_header_nritems(cur)) break; + /* Don't forgot to check leaf/node validation */ if (*level == 0) { - ret = btrfs_check_leaf(root, NULL, cur); - if (ret != BTRFS_TREE_BLOCK_CLEAN) { - ret = -EIO; - break; + /* skip duplicate check */ + if (check || !check_all) { + ret = btrfs_check_leaf(root, NULL, cur); + if (ret != BTRFS_TREE_BLOCK_CLEAN) { + err |= -EIO; + break; + } } - ret = process_one_leaf_v2(root, path, nrefs, - level, ext_ref); + + ret = 0; + if (!check_all) + ret = process_one_leaf_v2(root, path, nrefs, + level, ext_ref); + else + ret = check_leaf_items(trans, root, path, + nrefs, account_file_data); + err |= ret; break; } else { - ret = btrfs_check_node(root, NULL, cur); - if (ret != BTRFS_TREE_BLOCK_CLEAN) { - ret = -EIO; - break; + if (check || !check_all) { + ret = btrfs_check_node(root, NULL, cur); + if (ret != BTRFS_TREE_BLOCK_CLEAN) { + err |= -EIO; + break; + } } } + bytenr = btrfs_node_blockptr(cur, path->slots[*level]); ptr_gen = btrfs_node_ptr_generation(cur, path->slots[*level]); - ret = update_nodes_refs(root, bytenr, nrefs, *level - 1); - if (ret) + ret = update_nodes_refs(root, bytenr, NULL, nrefs, *level - 1, + check_all); + if (ret < 0) break; - if (!nrefs->need_check[*level - 1]) { + /* + * check all trees in check_chunks_and_extent_v2 + * check shared node once in check_fs_roots + */ + if (!check_all && !nrefs->need_check[*level - 1]) { path->slots[*level]++; continue; } @@ -2395,16 +2802,15 @@ static int walk_down_tree_v2(struct btrfs_root *root, struct btrfs_path *path, &node_key, path->slots[*level]); btrfs_add_corrupt_extent_record(fs_info, - &node_key, - path->nodes[*level]->start, - fs_info->nodesize, - *level); - ret = -EIO; + &node_key, path->nodes[*level]->start, + fs_info->nodesize, *level); + err |= -EIO; break; } } ret = check_child_node(cur, path->slots[*level], next); + err |= ret; if (ret < 0) break; @@ -2414,7 +2820,7 @@ static int walk_down_tree_v2(struct btrfs_root *root, struct btrfs_path *path, status = btrfs_check_node(root, NULL, next); if (status != BTRFS_TREE_BLOCK_CLEAN) { free_extent_buffer(next); - ret = -EIO; + err |= -EIO; break; } @@ -2422,8 +2828,11 @@ static int walk_down_tree_v2(struct btrfs_root *root, struct btrfs_path *path, free_extent_buffer(path->nodes[*level]); path->nodes[*level] = next; path->slots[*level] = 0; + account_file_data = 0; + + update_nodes_refs(root, (u64)-1, next, nrefs, *level, check_all); } - return ret; + return err; } static int walk_up_tree(struct btrfs_root *root, struct btrfs_path *path, @@ -2685,13 +3094,54 @@ static int delete_dir_index(struct btrfs_root *root, return ret; } +static int __create_inode_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, u64 ino, u64 size, + u64 nbytes, u64 nlink, u32 mode) +{ + struct btrfs_inode_item ii; + time_t now = time(NULL); + int ret; + + btrfs_set_stack_inode_size(&ii, size); + btrfs_set_stack_inode_nbytes(&ii, nbytes); + btrfs_set_stack_inode_nlink(&ii, nlink); + btrfs_set_stack_inode_mode(&ii, mode); + btrfs_set_stack_inode_generation(&ii, trans->transid); + btrfs_set_stack_timespec_nsec(&ii.atime, 0); + btrfs_set_stack_timespec_sec(&ii.ctime, now); + btrfs_set_stack_timespec_nsec(&ii.ctime, 0); + btrfs_set_stack_timespec_sec(&ii.mtime, now); + btrfs_set_stack_timespec_nsec(&ii.mtime, 0); + btrfs_set_stack_timespec_sec(&ii.otime, 0); + btrfs_set_stack_timespec_nsec(&ii.otime, 0); + + ret = btrfs_insert_inode(trans, root, ino, &ii); + ASSERT(!ret); + + warning("root %llu inode %llu recreating inode item, this may " + "be incomplete, please check permissions and content after " + "the fsck completes.\n", (unsigned long long)root->objectid, + (unsigned long long)ino); + + return 0; +} + +static int create_inode_item_lowmem(struct btrfs_trans_handle *trans, + struct btrfs_root *root, u64 ino, + u8 filetype) +{ + u32 mode = (filetype == BTRFS_FT_DIR ? S_IFDIR : S_IFREG) | 0755; + + return __create_inode_item(trans, root, ino, 0, 0, 0, mode); +} + static int create_inode_item(struct btrfs_root *root, - struct inode_record *rec, - int root_dir) + struct inode_record *rec, int root_dir) { struct btrfs_trans_handle *trans; - struct btrfs_inode_item inode_item; - time_t now = time(NULL); + u64 nlink = 0; + u32 mode = 0; + u64 size = 0; int ret; trans = btrfs_start_transaction(root, 1); @@ -2700,18 +3150,7 @@ static int create_inode_item(struct btrfs_root *root, return ret; } - fprintf(stderr, "root %llu inode %llu recreating inode item, this may " - "be incomplete, please check permissions and content after " - "the fsck completes.\n", (unsigned long long)root->objectid, - (unsigned long long)rec->ino); - - memset(&inode_item, 0, sizeof(inode_item)); - btrfs_set_stack_inode_generation(&inode_item, trans->transid); - if (root_dir) - btrfs_set_stack_inode_nlink(&inode_item, 1); - else - btrfs_set_stack_inode_nlink(&inode_item, rec->found_link); - btrfs_set_stack_inode_nbytes(&inode_item, rec->found_size); + nlink = root_dir ? 1 : rec->found_link; if (rec->found_dir_item) { if (rec->found_file_extent) fprintf(stderr, "root %llu inode %llu has both a dir " @@ -2719,23 +3158,15 @@ static int create_inode_item(struct btrfs_root *root, "regular file so setting it as a directory\n", (unsigned long long)root->objectid, (unsigned long long)rec->ino); - btrfs_set_stack_inode_mode(&inode_item, S_IFDIR | 0755); - btrfs_set_stack_inode_size(&inode_item, rec->found_size); + mode = S_IFDIR | 0755; + size = rec->found_size; } else if (!rec->found_dir_item) { - btrfs_set_stack_inode_size(&inode_item, rec->extent_end); - btrfs_set_stack_inode_mode(&inode_item, S_IFREG | 0755); - } - btrfs_set_stack_timespec_sec(&inode_item.atime, now); - btrfs_set_stack_timespec_nsec(&inode_item.atime, 0); - btrfs_set_stack_timespec_sec(&inode_item.ctime, now); - btrfs_set_stack_timespec_nsec(&inode_item.ctime, 0); - btrfs_set_stack_timespec_sec(&inode_item.mtime, now); - btrfs_set_stack_timespec_nsec(&inode_item.mtime, 0); - btrfs_set_stack_timespec_sec(&inode_item.otime, 0); - btrfs_set_stack_timespec_nsec(&inode_item.otime, 0); - - ret = btrfs_insert_inode(trans, root, rec->ino, &inode_item); - BUG_ON(ret); + size = rec->extent_end; + mode = S_IFREG | 0755; + } + + ret = __create_inode_item(trans, root, rec->ino, size, rec->nbytes, + nlink, mode); btrfs_commit_transaction(trans, root); return 0; } @@ -2955,7 +3386,7 @@ static int reset_nlink(struct btrfs_trans_handle *trans, list_for_each_entry(backref, &rec->backrefs, list) { ret = btrfs_add_link(trans, root, rec->ino, backref->dir, backref->name, backref->namelen, - backref->filetype, &backref->index, 1); + backref->filetype, &backref->index, 1, 0); if (ret < 0) goto out; } @@ -2989,15 +3420,79 @@ static int get_highest_inode(struct btrfs_trans_handle *trans, return ret; } +/* + * Link inode to dir 'lost+found'. Increase @ref_count. + * + * Returns 0 means success. + * Returns <0 means failure. + */ +static int link_inode_to_lostfound(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path, + u64 ino, char *namebuf, u32 name_len, + u8 filetype, u64 *ref_count) +{ + char *dir_name = "lost+found"; + u64 lost_found_ino; + int ret; + u32 mode = 0700; + + btrfs_release_path(path); + ret = get_highest_inode(trans, root, path, &lost_found_ino); + if (ret < 0) + goto out; + lost_found_ino++; + + ret = btrfs_mkdir(trans, root, dir_name, strlen(dir_name), + BTRFS_FIRST_FREE_OBJECTID, &lost_found_ino, + mode); + if (ret < 0) { + error("failed to create '%s' dir: %s", dir_name, strerror(-ret)); + goto out; + } + ret = btrfs_add_link(trans, root, ino, lost_found_ino, + namebuf, name_len, filetype, NULL, 1, 0); + /* + * Add ".INO" suffix several times to handle case where + * "FILENAME.INO" is already taken by another file. + */ + while (ret == -EEXIST) { + /* + * Conflicting file name, add ".INO" as suffix * +1 for '.' + */ + if (name_len + count_digits(ino) + 1 > BTRFS_NAME_LEN) { + ret = -EFBIG; + goto out; + } + snprintf(namebuf + name_len, BTRFS_NAME_LEN - name_len, + ".%llu", ino); + name_len += count_digits(ino) + 1; + ret = btrfs_add_link(trans, root, ino, lost_found_ino, namebuf, + name_len, filetype, NULL, 1, 0); + } + if (ret < 0) { + error("failed to link the inode %llu to %s dir: %s", + ino, dir_name, strerror(-ret)); + goto out; + } + + ++*ref_count; + printf("Moving file '%.*s' to '%s' dir since it has no valid backref\n", + name_len, namebuf, dir_name); +out: + btrfs_release_path(path); + if (ret) + error("failed to move file '%.*s' to '%s' dir", name_len, + namebuf, dir_name); + return ret; +} + static int repair_inode_nlinks(struct btrfs_trans_handle *trans, struct btrfs_root *root, struct btrfs_path *path, struct inode_record *rec) { - char *dir_name = "lost+found"; char namebuf[BTRFS_NAME_LEN] = {0}; - u64 lost_found_ino; - u32 mode = 0700; u8 type = 0; int namelen = 0; int name_recovered = 0; @@ -3034,55 +3529,11 @@ static int repair_inode_nlinks(struct btrfs_trans_handle *trans, } if (rec->found_link == 0) { - ret = get_highest_inode(trans, root, path, &lost_found_ino); - if (ret < 0) - goto out; - lost_found_ino++; - ret = btrfs_mkdir(trans, root, dir_name, strlen(dir_name), - BTRFS_FIRST_FREE_OBJECTID, &lost_found_ino, - mode); - if (ret < 0) { - fprintf(stderr, "Failed to create '%s' dir: %s\n", - dir_name, strerror(-ret)); - goto out; - } - ret = btrfs_add_link(trans, root, rec->ino, lost_found_ino, - namebuf, namelen, type, NULL, 1); - /* - * Add ".INO" suffix several times to handle case where - * "FILENAME.INO" is already taken by another file. - */ - while (ret == -EEXIST) { - /* - * Conflicting file name, add ".INO" as suffix * +1 for '.' - */ - if (namelen + count_digits(rec->ino) + 1 > - BTRFS_NAME_LEN) { - ret = -EFBIG; - goto out; - } - snprintf(namebuf + namelen, BTRFS_NAME_LEN - namelen, - ".%llu", rec->ino); - namelen += count_digits(rec->ino) + 1; - ret = btrfs_add_link(trans, root, rec->ino, - lost_found_ino, namebuf, - namelen, type, NULL, 1); - } - if (ret < 0) { - fprintf(stderr, - "Failed to link the inode %llu to %s dir: %s\n", - rec->ino, dir_name, strerror(-ret)); + ret = link_inode_to_lostfound(trans, root, path, rec->ino, + namebuf, namelen, type, + (u64 *)&rec->found_link); + if (ret) goto out; - } - /* - * Just increase the found_link, don't actually add the - * backref. This will make things easier and this inode - * record will be freed after the repair is done. - * So fsck will not report problem about this inode. - */ - rec->found_link++; - printf("Moving file '%.*s' to '%s' dir since it has no valid backref\n", - namelen, namebuf, dir_name); } printf("Fixed the nlink of inode %llu\n", rec->ino); out: @@ -4236,81 +4687,149 @@ out: } /* - * Find DIR_ITEM/DIR_INDEX for the given key and check it with the specified - * INODE_REF/INODE_EXTREF match. + * Find the @index according by @ino and name. + * Notice:time efficiency is O(N) * * @root: the root of the fs/file tree - * @ref_key: the key of the INODE_REF/INODE_EXTREF - * @key: the key of the DIR_ITEM/DIR_INDEX - * @index: the index in the INODE_REF/INODE_EXTREF, be used to - * distinguish root_dir between normal dir/file - * @name: the name in the INODE_REF/INODE_EXTREF - * @namelen: the length of name in the INODE_REF/INODE_EXTREF - * @mode: the st_mode of INODE_ITEM + * @index_ret: the index as return value + * @namebuf: the name to match + * @name_len: the length of name to match + * @file_type: the file_type of INODE_ITEM to match * - * Return 0 if no error occurred. - * Return ROOT_DIR_ERROR if found DIR_ITEM/DIR_INDEX for root_dir. - * Return DIR_ITEM_MISSING if couldn't find DIR_ITEM/DIR_INDEX for normal - * dir/file. - * Return DIR_ITEM_MISMATCH if INODE_REF/INODE_EXTREF and DIR_ITEM/DIR_INDEX - * not match for normal dir/file. + * Returns 0 if found and *@index_ret will be modified with right value + * Returns< 0 not found and *@index_ret will be (u64)-1 */ -static int find_dir_item(struct btrfs_root *root, struct btrfs_key *ref_key, - struct btrfs_key *key, u64 index, char *name, - u32 namelen, u32 mode) +static int find_dir_index(struct btrfs_root *root, u64 dirid, u64 location_id, + u64 *index_ret, char *namebuf, u32 name_len, + u8 file_type) { struct btrfs_path path; struct extent_buffer *node; struct btrfs_dir_item *di; + struct btrfs_key key; struct btrfs_key location; - char namebuf[BTRFS_NAME_LEN] = {0}; + char name[BTRFS_NAME_LEN] = {0}; + u32 total; u32 cur = 0; u32 len; - u32 name_len; u32 data_len; u8 filetype; int slot; int ret; + ASSERT(index_ret); + + /* search from the last index */ + key.objectid = dirid; + key.offset = (u64)-1; + key.type = BTRFS_DIR_INDEX_KEY; + btrfs_init_path(&path); - ret = btrfs_search_slot(NULL, root, key, &path, 0, 0); - if (ret < 0) { - ret = DIR_ITEM_MISSING; + ret = btrfs_search_slot(NULL, root, &key, &path, 0, 0); + if (ret < 0) + return ret; + +loop: + ret = btrfs_previous_item(root, &path, dirid, BTRFS_DIR_INDEX_KEY); + if (ret) { + ret = -ENOENT; + *index_ret = (64)-1; goto out; } + /* Check whether inode_id/filetype/name match */ + node = path.nodes[0]; + slot = path.slots[0]; + di = btrfs_item_ptr(node, slot, struct btrfs_dir_item); + total = btrfs_item_size_nr(node, slot); + while (cur < total) { + ret = -ENOENT; + len = btrfs_dir_name_len(node, di); + data_len = btrfs_dir_data_len(node, di); - /* Process root dir and goto out*/ - if (index == 0) { - if (ret == 0) { - ret = ROOT_DIR_ERROR; - error( - "root %llu INODE %s[%llu %llu] ROOT_DIR shouldn't have %s", - root->objectid, - ref_key->type == BTRFS_INODE_REF_KEY ? - "REF" : "EXTREF", - ref_key->objectid, ref_key->offset, - key->type == BTRFS_DIR_ITEM_KEY ? - "DIR_ITEM" : "DIR_INDEX"); - } else { - ret = 0; - } + btrfs_dir_item_key_to_cpu(node, di, &location); + if (location.objectid != location_id || + location.type != BTRFS_INODE_ITEM_KEY || + location.offset != 0) + goto next; + + filetype = btrfs_dir_type(node, di); + if (file_type != filetype) + goto next; + + if (len > BTRFS_NAME_LEN) + len = BTRFS_NAME_LEN; + read_extent_buffer(node, name, (unsigned long)(di + 1), len); + if (len != name_len || strncmp(namebuf, name, len)) + goto next; + + btrfs_item_key_to_cpu(path.nodes[0], &key, path.slots[0]); + *index_ret = key.offset; + ret = 0; goto out; +next: + len += sizeof(*di) + data_len; + di = (struct btrfs_dir_item *)((char *)di + len); + cur += len; } + goto loop; - /* Process normal file/dir */ - if (ret > 0) { - ret = DIR_ITEM_MISSING; - error( - "root %llu INODE %s[%llu %llu] doesn't have related %s[%llu %llu] namelen %u filename %s filetype %d", - root->objectid, - ref_key->type == BTRFS_INODE_REF_KEY ? "REF" : "EXTREF", - ref_key->objectid, ref_key->offset, - key->type == BTRFS_DIR_ITEM_KEY ? - "DIR_ITEM" : "DIR_INDEX", - key->objectid, key->offset, namelen, name, - imode_to_type(mode)); +out: + btrfs_release_path(&path); + return ret; +} + +/* + * Find DIR_ITEM/DIR_INDEX for the given key and check it with the specified + * INODE_REF/INODE_EXTREF match. + * + * @root: the root of the fs/file tree + * @key: the key of the DIR_ITEM/DIR_INDEX, key->offset will be right + * value while find index + * @location_key: location key of the struct btrfs_dir_item to match + * @name: the name to match + * @namelen: the length of name + * @file_type: the type of file to math + * + * Return 0 if no error occurred. + * Return DIR_ITEM_MISSING/DIR_INDEX_MISSING if couldn't find + * DIR_ITEM/DIR_INDEX + * Return DIR_ITEM_MISMATCH/DIR_INDEX_MISMATCH if INODE_REF/INODE_EXTREF + * and DIR_ITEM/DIR_INDEX mismatch + */ +static int find_dir_item(struct btrfs_root *root, struct btrfs_key *key, + struct btrfs_key *location_key, char *name, + u32 namelen, u8 file_type) +{ + struct btrfs_path path; + struct extent_buffer *node; + struct btrfs_dir_item *di; + struct btrfs_key location; + char namebuf[BTRFS_NAME_LEN] = {0}; + u32 total; + u32 cur = 0; + u32 len; + u32 data_len; + u8 filetype; + int slot; + int ret; + + /* get the index by traversing all index */ + if (key->type == BTRFS_DIR_INDEX_KEY && key->offset == (u64)-1) { + ret = find_dir_index(root, key->objectid, + location_key->objectid, &key->offset, + name, namelen, file_type); + if (ret) + ret = DIR_INDEX_MISSING; + return ret; + } + + btrfs_init_path(&path); + ret = btrfs_search_slot(NULL, root, key, &path, 0, 0); + if (ret) { + ret = key->type == BTRFS_DIR_ITEM_KEY ? DIR_ITEM_MISSING : + DIR_INDEX_MISSING; goto out; } @@ -4320,132 +4839,310 @@ static int find_dir_item(struct btrfs_root *root, struct btrfs_key *ref_key, di = btrfs_item_ptr(node, slot, struct btrfs_dir_item); total = btrfs_item_size_nr(node, slot); while (cur < total) { - ret = DIR_ITEM_MISMATCH; - name_len = btrfs_dir_name_len(node, di); + ret = key->type == BTRFS_DIR_ITEM_KEY ? + DIR_ITEM_MISMATCH : DIR_INDEX_MISMATCH; + + len = btrfs_dir_name_len(node, di); data_len = btrfs_dir_data_len(node, di); btrfs_dir_item_key_to_cpu(node, di, &location); - if (location.objectid != ref_key->objectid || - location.type != BTRFS_INODE_ITEM_KEY || - location.offset != 0) + if (location.objectid != location_key->objectid || + location.type != location_key->type || + location.offset != location_key->offset) goto next; filetype = btrfs_dir_type(node, di); - if (imode_to_type(mode) != filetype) + if (file_type != filetype) goto next; - if (cur + sizeof(*di) + name_len > total || - name_len > BTRFS_NAME_LEN) { + if (len > BTRFS_NAME_LEN) { + len = BTRFS_NAME_LEN; warning("root %llu %s[%llu %llu] name too long %u, trimmed", - root->objectid, - key->type == BTRFS_DIR_ITEM_KEY ? - "DIR_ITEM" : "DIR_INDEX", - key->objectid, key->offset, name_len); - - if (cur + sizeof(*di) > total) - break; - len = min_t(u32, total - cur - sizeof(*di), - BTRFS_NAME_LEN); - } else { - len = name_len; + root->objectid, + key->type == BTRFS_DIR_ITEM_KEY ? + "DIR_ITEM" : "DIR_INDEX", + key->objectid, key->offset, len); } - - read_extent_buffer(node, namebuf, (unsigned long)(di + 1), len); + read_extent_buffer(node, namebuf, (unsigned long)(di + 1), + len); if (len != namelen || strncmp(namebuf, name, len)) goto next; ret = 0; goto out; next: - len = sizeof(*di) + name_len + data_len; + len += sizeof(*di) + data_len; di = (struct btrfs_dir_item *)((char *)di + len); cur += len; } - if (ret == DIR_ITEM_MISMATCH) + +out: + btrfs_release_path(&path); + return ret; +} + +/* + * Prints inode ref error message + */ +static void print_inode_ref_err(struct btrfs_root *root, struct btrfs_key *key, + u64 index, const char *namebuf, int name_len, + u8 filetype, int err) +{ + if (!err) + return; + + /* root dir error */ + if (key->objectid == BTRFS_FIRST_FREE_OBJECTID) { error( - "root %llu INODE %s[%llu %llu] and %s[%llu %llu] mismatch namelen %u filename %s filetype %d", - root->objectid, - ref_key->type == BTRFS_INODE_REF_KEY ? "REF" : "EXTREF", - ref_key->objectid, ref_key->offset, - key->type == BTRFS_DIR_ITEM_KEY ? - "DIR_ITEM" : "DIR_INDEX", - key->objectid, key->offset, namelen, name, - imode_to_type(mode)); + "root %llu root dir shouldn't have INODE REF[%llu %llu] name %s", + root->objectid, key->objectid, key->offset, namebuf); + return; + } + + /* normal error */ + if (err & (DIR_ITEM_MISMATCH | DIR_ITEM_MISSING)) + error("root %llu DIR ITEM[%llu %llu] %s name %s filetype %u", + root->objectid, key->offset, + btrfs_name_hash(namebuf, name_len), + err & DIR_ITEM_MISMATCH ? "mismatch" : "missing", + namebuf, filetype); + if (err & (DIR_INDEX_MISMATCH | DIR_INDEX_MISSING)) + error("root %llu DIR INDEX[%llu %llu] %s name %s filetype %u", + root->objectid, key->offset, index, + err & DIR_ITEM_MISMATCH ? "mismatch" : "missing", + namebuf, filetype); +} + +/* + * Insert the missing inode item. + * + * Returns 0 means success. + * Returns <0 means error. + */ +static int repair_inode_item_missing(struct btrfs_root *root, u64 ino, + u8 filetype) +{ + struct btrfs_key key; + struct btrfs_trans_handle *trans; + struct btrfs_path path; + int ret; + + key.objectid = ino; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + + btrfs_init_path(&path); + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) { + ret = -EIO; + goto out; + } + + ret = btrfs_search_slot(trans, root, &key, &path, 1, 1); + if (ret < 0 || !ret) + goto fail; + + /* insert inode item */ + create_inode_item_lowmem(trans, root, ino, filetype); + ret = 0; +fail: + btrfs_commit_transaction(trans, root); out: + if (ret) + error("failed to repair root %llu INODE ITEM[%llu] missing", + root->objectid, ino); btrfs_release_path(&path); return ret; } /* + * The ternary means dir item, dir index and relative inode ref. + * The function handles errs: INODE_MISSING, DIR_INDEX_MISSING + * DIR_INDEX_MISMATCH, DIR_ITEM_MISSING, DIR_ITEM_MISMATCH by the follow + * strategy: + * If two of three is missing or mismatched, delete the existing one. + * If one of three is missing or mismatched, add the missing one. + * + * returns 0 means success. + * returns not 0 means on error; + */ +int repair_ternary_lowmem(struct btrfs_root *root, u64 dir_ino, u64 ino, + u64 index, char *name, int name_len, u8 filetype, + int err) +{ + struct btrfs_trans_handle *trans; + int stage = 0; + int ret = 0; + + /* + * stage shall be one of following valild values: + * 0: Fine, nothing to do. + * 1: One of three is wrong, so add missing one. + * 2: Two of three is wrong, so delete existed one. + */ + if (err & (DIR_INDEX_MISMATCH | DIR_INDEX_MISSING)) + stage++; + if (err & (DIR_ITEM_MISMATCH | DIR_ITEM_MISSING)) + stage++; + if (err & (INODE_REF_MISSING)) + stage++; + + /* stage must be smllarer than 3 */ + ASSERT(stage < 3); + + trans = btrfs_start_transaction(root, 1); + if (stage == 2) { + ret = btrfs_unlink(trans, root, ino, dir_ino, index, name, + name_len, 0); + goto out; + } + if (stage == 1) { + ret = btrfs_add_link(trans, root, ino, dir_ino, name, name_len, + filetype, &index, 1, 1); + goto out; + } +out: + btrfs_commit_transaction(trans, root); + + if (ret) + error("fail to repair inode %llu name %s filetype %u", + ino, name, filetype); + else + printf("%s ref/dir_item of inode %llu name %s filetype %u\n", + stage == 2 ? "Delete" : "Add", + ino, name, filetype); + + return ret; +} + +/* * Traverse the given INODE_REF and call find_dir_item() to find related * DIR_ITEM/DIR_INDEX. * * @root: the root of the fs/file tree * @ref_key: the key of the INODE_REF + * @path the path provides node and slot * @refs: the count of INODE_REF * @mode: the st_mode of INODE_ITEM + * @name_ret: returns with the first ref's name + * @name_len_ret: len of the name_ret * * Return 0 if no error occurred. */ static int check_inode_ref(struct btrfs_root *root, struct btrfs_key *ref_key, - struct extent_buffer *node, int slot, u64 *refs, - int mode) + struct btrfs_path *path, char *name_ret, + u32 *namelen_ret, u64 *refs_ret, int mode) { struct btrfs_key key; + struct btrfs_key location; struct btrfs_inode_ref *ref; + struct extent_buffer *node; char namebuf[BTRFS_NAME_LEN] = {0}; u32 total; u32 cur = 0; u32 len; u32 name_len; u64 index; - int ret, err = 0; + int ret; + int err = 0; + int tmp_err; + int slot; + int need_research = 0; + u64 refs; + +begin: + err = 0; + cur = 0; + refs = *refs_ret; + + /* since after repair, path and the dir item may be changed */ + if (need_research) { + need_research = 0; + btrfs_release_path(path); + ret = btrfs_search_slot(NULL, root, ref_key, path, 0, 0); + /* the item was deleted, let path point to the last checked item */ + if (ret > 0) { + if (path->slots[0] == 0) + btrfs_prev_leaf(root, path); + else + path->slots[0]--; + } + if (ret) + goto out; + } + + location.objectid = ref_key->objectid; + location.type = BTRFS_INODE_ITEM_KEY; + location.offset = 0; + node = path->nodes[0]; + slot = path->slots[0]; + memset(namebuf, 0, sizeof(namebuf) / sizeof(*namebuf)); ref = btrfs_item_ptr(node, slot, struct btrfs_inode_ref); total = btrfs_item_size_nr(node, slot); next: /* Update inode ref count */ - (*refs)++; - + refs++; + tmp_err = 0; index = btrfs_inode_ref_index(node, ref); name_len = btrfs_inode_ref_name_len(node, ref); - if (cur + sizeof(*ref) + name_len > total || - name_len > BTRFS_NAME_LEN) { - warning("root %llu INODE_REF[%llu %llu] name too long", - root->objectid, ref_key->objectid, ref_key->offset); - if (total < cur + sizeof(*ref)) - goto out; - len = min_t(u32, total - cur - sizeof(*ref), BTRFS_NAME_LEN); - } else { + if (name_len <= BTRFS_NAME_LEN) { len = name_len; + } else { + len = BTRFS_NAME_LEN; + warning("root %llu INODE_REF[%llu %llu] name too long", + root->objectid, ref_key->objectid, ref_key->offset); } read_extent_buffer(node, namebuf, (unsigned long)(ref + 1), len); - /* Check root dir ref name */ - if (index == 0 && strncmp(namebuf, "..", name_len)) { - error("root %llu INODE_REF[%llu %llu] ROOT_DIR name shouldn't be %s", - root->objectid, ref_key->objectid, ref_key->offset, - namebuf); - err |= ROOT_DIR_ERROR; + /* copy the first name found to name_ret */ + if (refs == 1 && name_ret) { + memcpy(name_ret, namebuf, len); + *namelen_ret = len; + } + + /* Check root dir ref */ + if (ref_key->objectid == BTRFS_FIRST_FREE_OBJECTID) { + if (index != 0 || len != strlen("..") || + strncmp("..", namebuf, len) || + ref_key->offset != BTRFS_FIRST_FREE_OBJECTID) { + /* set err bits then repair will delete the ref */ + err |= DIR_INDEX_MISSING; + err |= DIR_ITEM_MISSING; + } + goto end; } /* Find related DIR_INDEX */ key.objectid = ref_key->offset; key.type = BTRFS_DIR_INDEX_KEY; key.offset = index; - ret = find_dir_item(root, ref_key, &key, index, namebuf, len, mode); - err |= ret; + tmp_err |= find_dir_item(root, &key, &location, namebuf, len, + imode_to_type(mode)); /* Find related dir_item */ key.objectid = ref_key->offset; key.type = BTRFS_DIR_ITEM_KEY; key.offset = btrfs_name_hash(namebuf, len); - ret = find_dir_item(root, ref_key, &key, index, namebuf, len, mode); - err |= ret; - + tmp_err |= find_dir_item(root, &key, &location, namebuf, len, + imode_to_type(mode)); +end: + if (tmp_err && repair) { + ret = repair_ternary_lowmem(root, ref_key->offset, + ref_key->objectid, index, namebuf, + name_len, imode_to_type(mode), + tmp_err); + if (!ret) { + need_research = 1; + goto begin; + } + } + print_inode_ref_err(root, ref_key, index, namebuf, name_len, + imode_to_type(mode), tmp_err); + err |= tmp_err; len = sizeof(*ref) + name_len; ref = (struct btrfs_inode_ref *)((char *)ref + len); cur += len; @@ -4453,6 +5150,7 @@ next: goto next; out: + *refs_ret = refs; return err; } @@ -4473,6 +5171,7 @@ static int check_inode_extref(struct btrfs_root *root, int mode) { struct btrfs_key key; + struct btrfs_key location; struct btrfs_inode_extref *extref; char namebuf[BTRFS_NAME_LEN] = {0}; u32 total; @@ -4484,6 +5183,10 @@ static int check_inode_extref(struct btrfs_root *root, int ret; int err = 0; + location.objectid = ref_key->objectid; + location.type = BTRFS_INODE_ITEM_KEY; + location.offset = 0; + extref = btrfs_item_ptr(node, slot, struct btrfs_inode_extref); total = btrfs_item_size_nr(node, slot); @@ -4514,14 +5217,14 @@ next: key.objectid = parent; key.type = BTRFS_DIR_INDEX_KEY; key.offset = index; - ret = find_dir_item(root, ref_key, &key, index, namebuf, len, mode); + ret = find_dir_item(root, &key, &location, namebuf, len, mode); err |= ret; /* find related dir_item */ key.objectid = parent; key.type = BTRFS_DIR_ITEM_KEY; key.offset = btrfs_name_hash(namebuf, len); - ret = find_dir_item(root, ref_key, &key, index, namebuf, len, mode); + ret = find_dir_item(root, &key, &location, namebuf, len, mode); err |= ret; len = sizeof(*extref) + name_len; @@ -4537,20 +5240,21 @@ next: /* * Find INODE_REF/INODE_EXTREF for the given key and check it with the specified * DIR_ITEM/DIR_INDEX match. + * Return with @index_ret. * * @root: the root of the fs/file tree * @key: the key of the INODE_REF/INODE_EXTREF * @name: the name in the INODE_REF/INODE_EXTREF * @namelen: the length of name in the INODE_REF/INODE_EXTREF - * @index: the index in the INODE_REF/INODE_EXTREF, for DIR_ITEM set index - * to (u64)-1 + * @index_ret: the index in the INODE_REF/INODE_EXTREF, + * value (64)-1 means do not check index * @ext_ref: the EXTENDED_IREF feature * * Return 0 if no error occurred. * Return >0 for error bitmap */ static int find_inode_ref(struct btrfs_root *root, struct btrfs_key *key, - char *name, int namelen, u64 index, + char *name, int namelen, u64 *index_ret, unsigned int ext_ref) { struct btrfs_path path; @@ -4568,6 +5272,8 @@ static int find_inode_ref(struct btrfs_root *root, struct btrfs_key *key, int slot; int ret; + ASSERT(index_ret); + btrfs_init_path(&path); ret = btrfs_search_slot(NULL, root, key, &path, 0, 0); if (ret) { @@ -4587,7 +5293,7 @@ static int find_inode_ref(struct btrfs_root *root, struct btrfs_key *key, ref_namelen = btrfs_inode_ref_name_len(node, ref); ref_index = btrfs_inode_ref_index(node, ref); - if (index != (u64)-1 && index != ref_index) + if (*index_ret != (u64)-1 && *index_ret != ref_index) goto next_ref; if (cur + sizeof(*ref) + ref_namelen > total || @@ -4612,6 +5318,7 @@ static int find_inode_ref(struct btrfs_root *root, struct btrfs_key *key, if (len != namelen || strncmp(ref_namebuf, name, len)) goto next_ref; + *index_ret = ref_index; ret = 0; goto out; next_ref: @@ -4652,7 +5359,7 @@ extref: ref_namelen = btrfs_inode_extref_name_len(node, extref); ref_index = btrfs_inode_extref_index(node, extref); parent = btrfs_inode_extref_parent(node, extref); - if (index != (u64)-1 && index != ref_index) + if (*index_ret != (u64)-1 && *index_ret != ref_index) goto next_extref; if (parent != dir_id) @@ -4674,6 +5381,7 @@ extref: if (len != namelen || strncmp(ref_namebuf, name, len)) goto next_extref; + *index_ret = ref_index; ret = 0; goto out; @@ -4688,25 +5396,172 @@ out: return ret; } +static void print_dir_item_err(struct btrfs_root *root, struct btrfs_key *key, + u64 ino, u64 index, const char *namebuf, + int name_len, u8 filetype, int err) +{ + if (err & (DIR_ITEM_MISMATCH | DIR_ITEM_MISSING)) { + error("root %llu DIR ITEM[%llu %llu] name %s filetype %d %s", + root->objectid, key->objectid, key->offset, namebuf, + filetype, + err & DIR_ITEM_MISMATCH ? "mismath" : "missing"); + } + + if (err & (DIR_INDEX_MISMATCH | DIR_INDEX_MISSING)) { + error("root %llu DIR INDEX[%llu %llu] name %s filetype %d %s", + root->objectid, key->objectid, index, namebuf, filetype, + err & DIR_ITEM_MISMATCH ? "mismath" : "missing"); + } + + if (err & (INODE_ITEM_MISSING | INODE_ITEM_MISMATCH)) { + error( + "root %llu INODE_ITEM[%llu] index %llu name %s filetype %d %s", + root->objectid, ino, index, namebuf, filetype, + err & INODE_ITEM_MISMATCH ? "mismath" : "missing"); + } + + if (err & INODE_REF_MISSING) + error( + "root %llu INODE REF[%llu, %llu] name %s filetype %u missing", + root->objectid, ino, key->objectid, namebuf, filetype); + +} + +/* + * Call repair_inode_item_missing and repair_ternary_lowmem to repair + * + * Returns error after repair + */ +static int repair_dir_item(struct btrfs_root *root, u64 dirid, u64 ino, + u64 index, u8 filetype, char *namebuf, u32 name_len, + int err) +{ + int ret; + + if (err & INODE_ITEM_MISSING) { + ret = repair_inode_item_missing(root, ino, filetype); + if (!ret) + err &= ~(INODE_ITEM_MISMATCH | INODE_ITEM_MISSING); + } + + if (err & ~(INODE_ITEM_MISMATCH | INODE_ITEM_MISSING)) { + ret = repair_ternary_lowmem(root, dirid, ino, index, namebuf, + name_len, filetype, err); + if (!ret) { + err &= ~(DIR_INDEX_MISMATCH | DIR_INDEX_MISSING); + err &= ~(DIR_ITEM_MISMATCH | DIR_ITEM_MISSING); + err &= ~(INODE_REF_MISSING); + } + } + return err; +} + +static int __count_dir_isize(struct btrfs_root *root, u64 ino, int type, + u64 *size_ret) +{ + struct btrfs_key key; + struct btrfs_path path; + u32 len; + struct btrfs_dir_item *di; + int ret; + int cur = 0; + int total = 0; + + ASSERT(size_ret); + *size_ret = 0; + + key.objectid = ino; + key.type = type; + key.offset = (u64)-1; + + btrfs_init_path(&path); + ret = btrfs_search_slot(NULL, root, &key, &path, 0, 0); + if (ret < 0) { + ret = -EIO; + goto out; + } + /* if found, go to spacial case */ + if (ret == 0) + goto special_case; + +loop: + ret = btrfs_previous_item(root, &path, ino, type); + + if (ret) { + ret = 0; + goto out; + } + +special_case: + di = btrfs_item_ptr(path.nodes[0], path.slots[0], struct btrfs_dir_item); + cur = 0; + total = btrfs_item_size_nr(path.nodes[0], path.slots[0]); + + while (cur < total) { + len = btrfs_dir_name_len(path.nodes[0], di); + if (len > BTRFS_NAME_LEN) + len = BTRFS_NAME_LEN; + *size_ret += len; + + len += btrfs_dir_data_len(path.nodes[0], di); + len += sizeof(*di); + di = (struct btrfs_dir_item *)((char *)di + len); + cur += len; + } + goto loop; + +out: + btrfs_release_path(&path); + return ret; +} + +static int count_dir_isize(struct btrfs_root *root, u64 ino, u64 *size) +{ + u64 item_size; + u64 index_size; + int ret; + + ASSERT(size); + ret = __count_dir_isize(root, ino, BTRFS_DIR_ITEM_KEY, &item_size); + if (ret) + goto out; + + ret = __count_dir_isize(root, ino, BTRFS_DIR_INDEX_KEY, &index_size); + if (ret) + goto out; + + *size = item_size + index_size; + +out: + if (ret) + error("failed to count root %llu INODE[%llu] root size", + root->objectid, ino); + return ret; +} + /* * Traverse the given DIR_ITEM/DIR_INDEX and check related INODE_ITEM and * call find_inode_ref() to check related INODE_REF/INODE_EXTREF. * * @root: the root of the fs/file tree * @key: the key of the INODE_REF/INODE_EXTREF + * @path: the path * @size: the st_size of the INODE_ITEM * @ext_ref: the EXTENDED_IREF feature * * Return 0 if no error occurred. + * Return DIR_COUNT_AGAIN if the isize of the inode should be recalculated. */ -static int check_dir_item(struct btrfs_root *root, struct btrfs_key *key, - struct extent_buffer *node, int slot, u64 *size, +static int check_dir_item(struct btrfs_root *root, struct btrfs_key *di_key, + struct btrfs_path *path, u64 *size, unsigned int ext_ref) { struct btrfs_dir_item *di; struct btrfs_inode_item *ii; - struct btrfs_path path; + struct btrfs_key key; struct btrfs_key location; + struct extent_buffer *node; + int slot; char namebuf[BTRFS_NAME_LEN] = {0}; u32 total; u32 cur = 0; @@ -4714,118 +5569,190 @@ static int check_dir_item(struct btrfs_root *root, struct btrfs_key *key, u32 name_len; u32 data_len; u8 filetype; - u32 mode; + u32 mode = 0; u64 index; int ret; - int err = 0; + int err; + int tmp_err; + int need_research = 0; /* * For DIR_ITEM set index to (u64)-1, so that find_inode_ref * ignore index check. */ - index = (key->type == BTRFS_DIR_INDEX_KEY) ? key->offset : (u64)-1; + if (di_key->type == BTRFS_DIR_INDEX_KEY) + index = di_key->offset; + else + index = (u64)-1; +begin: + err = 0; + cur = 0; + + /* since after repair, path and the dir item may be changed */ + if (need_research) { + need_research = 0; + err |= DIR_COUNT_AGAIN; + btrfs_release_path(path); + ret = btrfs_search_slot(NULL, root, di_key, path, 0, 0); + /* the item was deleted, let path point the last checked item */ + if (ret > 0) { + if (path->slots[0] == 0) + btrfs_prev_leaf(root, path); + else + path->slots[0]--; + } + if (ret) + goto out; + } + + node = path->nodes[0]; + slot = path->slots[0]; di = btrfs_item_ptr(node, slot, struct btrfs_dir_item); total = btrfs_item_size_nr(node, slot); + memset(namebuf, 0, sizeof(namebuf) / sizeof(*namebuf)); while (cur < total) { data_len = btrfs_dir_data_len(node, di); + tmp_err = 0; if (data_len) error("root %llu %s[%llu %llu] data_len shouldn't be %u", - root->objectid, key->type == BTRFS_DIR_ITEM_KEY ? - "DIR_ITEM" : "DIR_INDEX", - key->objectid, key->offset, data_len); + root->objectid, + di_key->type == BTRFS_DIR_ITEM_KEY ? "DIR_ITEM" : "DIR_INDEX", + di_key->objectid, di_key->offset, data_len); name_len = btrfs_dir_name_len(node, di); - if (cur + sizeof(*di) + name_len > total || - name_len > BTRFS_NAME_LEN) { + if (name_len <= BTRFS_NAME_LEN) { + len = name_len; + } else { + len = BTRFS_NAME_LEN; warning("root %llu %s[%llu %llu] name too long", root->objectid, - key->type == BTRFS_DIR_ITEM_KEY ? - "DIR_ITEM" : "DIR_INDEX", - key->objectid, key->offset); - - if (cur + sizeof(*di) > total) - break; - len = min_t(u32, total - cur - sizeof(*di), - BTRFS_NAME_LEN); - } else { - len = name_len; + di_key->type == BTRFS_DIR_ITEM_KEY ? "DIR_ITEM" : "DIR_INDEX", + di_key->objectid, di_key->offset); } (*size) += name_len; - - read_extent_buffer(node, namebuf, (unsigned long)(di + 1), len); + read_extent_buffer(node, namebuf, (unsigned long)(di + 1), + len); filetype = btrfs_dir_type(node, di); - if (key->type == BTRFS_DIR_ITEM_KEY && - key->offset != btrfs_name_hash(namebuf, len)) { + if (di_key->type == BTRFS_DIR_ITEM_KEY && + di_key->offset != btrfs_name_hash(namebuf, len)) { err |= -EIO; error("root %llu DIR_ITEM[%llu %llu] name %s namelen %u filetype %u mismatch with its hash, wanted %llu have %llu", - root->objectid, key->objectid, key->offset, - namebuf, len, filetype, key->offset, - btrfs_name_hash(namebuf, len)); + root->objectid, di_key->objectid, di_key->offset, + namebuf, len, filetype, di_key->offset, + btrfs_name_hash(namebuf, len)); } - btrfs_init_path(&path); btrfs_dir_item_key_to_cpu(node, di, &location); - /* Ignore related ROOT_ITEM check */ if (location.type == BTRFS_ROOT_ITEM_KEY) goto next; + btrfs_release_path(path); /* Check relative INODE_ITEM(existence/filetype) */ - ret = btrfs_search_slot(NULL, root, &location, &path, 0, 0); + ret = btrfs_search_slot(NULL, root, &location, path, 0, 0); if (ret) { - err |= INODE_ITEM_MISSING; - error("root %llu %s[%llu %llu] couldn't find relative INODE_ITEM[%llu] namelen %u filename %s filetype %x", - root->objectid, key->type == BTRFS_DIR_ITEM_KEY ? - "DIR_ITEM" : "DIR_INDEX", key->objectid, - key->offset, location.objectid, name_len, - namebuf, filetype); + tmp_err |= INODE_ITEM_MISSING; goto next; } - ii = btrfs_item_ptr(path.nodes[0], path.slots[0], + ii = btrfs_item_ptr(path->nodes[0], path->slots[0], struct btrfs_inode_item); - mode = btrfs_inode_mode(path.nodes[0], ii); - + mode = btrfs_inode_mode(path->nodes[0], ii); if (imode_to_type(mode) != filetype) { - err |= INODE_ITEM_MISMATCH; - error("root %llu %s[%llu %llu] relative INODE_ITEM filetype mismatch namelen %u filename %s filetype %d", - root->objectid, key->type == BTRFS_DIR_ITEM_KEY ? - "DIR_ITEM" : "DIR_INDEX", key->objectid, - key->offset, name_len, namebuf, filetype); + tmp_err |= INODE_ITEM_MISMATCH; + goto next; } /* Check relative INODE_REF/INODE_EXTREF */ - location.type = BTRFS_INODE_REF_KEY; - location.offset = key->objectid; - ret = find_inode_ref(root, &location, namebuf, len, - index, ext_ref); - err |= ret; - if (ret & INODE_REF_MISSING) - error("root %llu %s[%llu %llu] relative INODE_REF missing namelen %u filename %s filetype %d", - root->objectid, key->type == BTRFS_DIR_ITEM_KEY ? - "DIR_ITEM" : "DIR_INDEX", key->objectid, - key->offset, name_len, namebuf, filetype); + key.objectid = location.objectid; + key.type = BTRFS_INODE_REF_KEY; + key.offset = di_key->objectid; + tmp_err |= find_inode_ref(root, &key, namebuf, len, + &index, ext_ref); + + /* check relative INDEX/ITEM */ + key.objectid = di_key->objectid; + if (key.type == BTRFS_DIR_ITEM_KEY) { + key.type = BTRFS_DIR_INDEX_KEY; + key.offset = index; + } else { + key.type = BTRFS_DIR_ITEM_KEY; + key.offset = btrfs_name_hash(namebuf, name_len); + } + tmp_err |= find_dir_item(root, &key, &location, namebuf, + name_len, filetype); + /* find_dir_item may find index */ + if (key.type == BTRFS_DIR_INDEX_KEY) + index = key.offset; next: - btrfs_release_path(&path); + + if (tmp_err && repair) { + ret = repair_dir_item(root, di_key->objectid, + location.objectid, index, + imode_to_type(mode), namebuf, + name_len, tmp_err); + if (ret != tmp_err) { + need_research = 1; + goto begin; + } + } + btrfs_release_path(path); + print_dir_item_err(root, di_key, location.objectid, index, + namebuf, name_len, filetype, tmp_err); + err |= tmp_err; len = sizeof(*di) + name_len + data_len; di = (struct btrfs_dir_item *)((char *)di + len); cur += len; - if (key->type == BTRFS_DIR_INDEX_KEY && cur < total) { + if (di_key->type == BTRFS_DIR_INDEX_KEY && cur < total) { error("root %llu DIR_INDEX[%llu %llu] should contain only one entry", - root->objectid, key->objectid, key->offset); + root->objectid, di_key->objectid, + di_key->offset); break; } } - +out: + /* research path */ + btrfs_release_path(path); + ret = btrfs_search_slot(NULL, root, di_key, path, 0, 0); + if (ret) + err |= ret > 0 ? -ENOENT : ret; return err; } /* + * Wrapper function of btrfs_punch_hole. + * + * Returns 0 means success. + * Returns not 0 means error. + */ +static int punch_extent_hole(struct btrfs_root *root, u64 ino, u64 start, + u64 len) +{ + struct btrfs_trans_handle *trans; + int ret = 0; + + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) + return PTR_ERR(trans); + + ret = btrfs_punch_hole(trans, root, ino, start, len); + if (ret) + error("failed to add hole [%llu, %llu] in inode [%llu]", + start, len, ino); + else + printf("Add a hole [%llu, %llu] in inode [%llu]\n", start, len, + ino); + + btrfs_commit_transaction(trans, root); + return ret; +} + +/* * Check file extent datasum/hole, update the size of the file extents, * check and update the last offset of the file extent. * @@ -4940,9 +5867,16 @@ static int check_file_extent(struct btrfs_root *root, struct btrfs_key *fkey, /* Check EXTENT_DATA hole */ if (!no_holes && *end != fkey->offset) { - err |= FILE_EXTENT_ERROR; - error("root %llu EXTENT_DATA[%llu %llu] interrupt", - root->objectid, fkey->objectid, fkey->offset); + if (repair) + ret = punch_extent_hole(root, fkey->objectid, + *end, fkey->offset - *end); + if (!repair || ret) { + err |= FILE_EXTENT_ERROR; + error( +"root %llu EXTENT_DATA[%llu %llu] gap exists, expected: EXTENT_DATA[%llu %llu]", + root->objectid, fkey->objectid, fkey->offset, + fkey->objectid, *end); + } } *end += extent_num_bytes; @@ -4953,6 +5887,255 @@ static int check_file_extent(struct btrfs_root *root, struct btrfs_key *fkey, } /* + * Set inode item nbytes to @nbytes + * + * Returns 0 on success + * Returns != 0 on error + */ +static int repair_inode_nbytes_lowmem(struct btrfs_root *root, + struct btrfs_path *path, + u64 ino, u64 nbytes) +{ + struct btrfs_trans_handle *trans; + struct btrfs_inode_item *ii; + struct btrfs_key key; + struct btrfs_key research_key; + int err = 0; + int ret; + + btrfs_item_key_to_cpu(path->nodes[0], &research_key, path->slots[0]); + + key.objectid = ino; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + err |= ret; + goto out; + } + + btrfs_release_path(path); + ret = btrfs_search_slot(trans, root, &key, path, 0, 1); + if (ret > 0) + ret = -ENOENT; + if (ret) { + err |= ret; + goto fail; + } + + ii = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_inode_item); + btrfs_set_inode_nbytes(path->nodes[0], ii, nbytes); + btrfs_mark_buffer_dirty(path->nodes[0]); +fail: + btrfs_commit_transaction(trans, root); +out: + if (ret) + error("failed to set nbytes in inode %llu root %llu", + ino, root->root_key.objectid); + else + printf("Set nbytes in inode item %llu root %llu\n to %llu", ino, + root->root_key.objectid, nbytes); + + /* research path */ + btrfs_release_path(path); + ret = btrfs_search_slot(NULL, root, &research_key, path, 0, 0); + err |= ret; + + return err; +} + +/* + * Set directory inode isize to @isize. + * + * Returns 0 on success. + * Returns != 0 on error. + */ +static int repair_dir_isize_lowmem(struct btrfs_root *root, + struct btrfs_path *path, + u64 ino, u64 isize) +{ + struct btrfs_trans_handle *trans; + struct btrfs_inode_item *ii; + struct btrfs_key key; + struct btrfs_key research_key; + int ret; + int err = 0; + + btrfs_item_key_to_cpu(path->nodes[0], &research_key, path->slots[0]); + + key.objectid = ino; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + err |= ret; + goto out; + } + + btrfs_release_path(path); + ret = btrfs_search_slot(trans, root, &key, path, 0, 1); + if (ret > 0) + ret = -ENOENT; + if (ret) { + err |= ret; + goto fail; + } + + ii = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_inode_item); + btrfs_set_inode_size(path->nodes[0], ii, isize); + btrfs_mark_buffer_dirty(path->nodes[0]); +fail: + btrfs_commit_transaction(trans, root); +out: + if (ret) + error("failed to set isize in inode %llu root %llu", + ino, root->root_key.objectid); + else + printf("Set isize in inode %llu root %llu to %llu\n", + ino, root->root_key.objectid, isize); + + btrfs_release_path(path); + ret = btrfs_search_slot(NULL, root, &research_key, path, 0, 0); + err |= ret; + + return err; +} + +/* + * Wrapper function for btrfs_add_orphan_item(). + * + * Returns 0 on success. + * Returns != 0 on error. + */ +static int repair_inode_orphan_item_lowmem(struct btrfs_root *root, + struct btrfs_path *path, u64 ino) +{ + struct btrfs_trans_handle *trans; + struct btrfs_key research_key; + int ret; + int err = 0; + + btrfs_item_key_to_cpu(path->nodes[0], &research_key, path->slots[0]); + + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + err |= ret; + goto out; + } + + btrfs_release_path(path); + ret = btrfs_add_orphan_item(trans, root, path, ino); + err |= ret; + btrfs_commit_transaction(trans, root); +out: + if (ret) + error("failed to add inode %llu as orphan item root %llu", + ino, root->root_key.objectid); + else + printf("Added inode %llu as orphan item root %llu\n", + ino, root->root_key.objectid); + + btrfs_release_path(path); + ret = btrfs_search_slot(NULL, root, &research_key, path, 0, 0); + err |= ret; + + return err; +} + +/* Set inode_item nlink to @ref_count. + * If @ref_count == 0, move it to "lost+found" and increase @ref_count. + * + * Returns 0 on success + */ +static int repair_inode_nlinks_lowmem(struct btrfs_root *root, + struct btrfs_path *path, u64 ino, + const char *name, u32 namelen, + u64 ref_count, u8 filetype, u64 *nlink) +{ + struct btrfs_trans_handle *trans; + struct btrfs_inode_item *ii; + struct btrfs_key key; + struct btrfs_key old_key; + char namebuf[BTRFS_NAME_LEN] = {0}; + int name_len; + int ret; + int ret2; + + /* save the key */ + btrfs_item_key_to_cpu(path->nodes[0], &old_key, path->slots[0]); + + if (name && namelen) { + ASSERT(namelen <= BTRFS_NAME_LEN); + memcpy(namebuf, name, namelen); + name_len = namelen; + } else { + sprintf(namebuf, "%llu", ino); + name_len = count_digits(ino); + printf("Can't find file name for inode %llu, use %s instead\n", + ino, namebuf); + } + + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out; + } + + btrfs_release_path(path); + /* if refs is 0, put it into lostfound */ + if (ref_count == 0) { + ret = link_inode_to_lostfound(trans, root, path, ino, namebuf, + name_len, filetype, &ref_count); + if (ret) + goto fail; + } + + /* reset inode_item's nlink to ref_count */ + key.objectid = ino; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + + btrfs_release_path(path); + ret = btrfs_search_slot(trans, root, &key, path, 0, 1); + if (ret > 0) + ret = -ENOENT; + if (ret) + goto fail; + + ii = btrfs_item_ptr(path->nodes[0], path->slots[0], + struct btrfs_inode_item); + btrfs_set_inode_nlink(path->nodes[0], ii, ref_count); + btrfs_mark_buffer_dirty(path->nodes[0]); + + if (nlink) + *nlink = ref_count; +fail: + btrfs_commit_transaction(trans, root); +out: + if (ret) + error( + "fail to repair nlink of inode %llu root %llu name %s filetype %u", + root->objectid, ino, namebuf, filetype); + else + printf("Fixed nlink of inode %llu root %llu name %s filetype %u\n", + root->objectid, ino, namebuf, filetype); + + /* research */ + btrfs_release_path(path); + ret2 = btrfs_search_slot(NULL, root, &old_key, path, 0, 0); + if (ret2 < 0) + return ret |= ret2; + return ret; +} + +/* * Check INODE_ITEM and related ITEMs (the same inode number) * 1. check link count * 2. check inode ref/extref @@ -4969,6 +6152,7 @@ static int check_inode_item(struct btrfs_root *root, struct btrfs_path *path, struct extent_buffer *node; struct btrfs_inode_item *ii; struct btrfs_key key; + struct btrfs_key last_key; u64 inode_id; u32 mode; u64 nlink; @@ -4983,6 +6167,8 @@ static int check_inode_item(struct btrfs_root *root, struct btrfs_path *path, int slot; int ret; int err = 0; + char namebuf[BTRFS_NAME_LEN] = {0}; + u32 name_len = 0; node = path->nodes[0]; slot = path->slots[0]; @@ -5006,6 +6192,7 @@ static int check_inode_item(struct btrfs_root *root, struct btrfs_path *path, nodatasum = btrfs_inode_flags(node, ii) & BTRFS_INODE_NODATASUM; while (1) { + btrfs_item_key_to_cpu(path->nodes[0], &last_key, path->slots[0]); ret = btrfs_next_item(root, path); if (ret < 0) { /* out will fill 'err' rusing current statistics */ @@ -5023,8 +6210,8 @@ static int check_inode_item(struct btrfs_root *root, struct btrfs_path *path, switch (key.type) { case BTRFS_INODE_REF_KEY: - ret = check_inode_ref(root, &key, node, slot, &refs, - mode); + ret = check_inode_ref(root, &key, path, namebuf, + &name_len, &refs, mode); err |= ret; break; case BTRFS_INODE_EXTREF_KEY: @@ -5044,8 +6231,7 @@ static int check_inode_item(struct btrfs_root *root, struct btrfs_path *path, imode_to_type(mode), key.objectid, key.offset); } - ret = check_dir_item(root, &key, node, slot, &size, - ext_ref); + ret = check_dir_item(root, &key, path, &size, ext_ref); err |= ret; break; case BTRFS_EXTENT_DATA_KEY: @@ -5068,8 +6254,26 @@ static int check_inode_item(struct btrfs_root *root, struct btrfs_path *path, } out: + if (err & LAST_ITEM) { + btrfs_release_path(path); + ret = btrfs_search_slot(NULL, root, &last_key, path, 0, 0); + if (ret) + return err; + } + /* verify INODE_ITEM nlink/isize/nbytes */ if (dir) { + if (repair && (err & DIR_COUNT_AGAIN)) { + err &= ~DIR_COUNT_AGAIN; + count_dir_isize(root, inode_id, &size); + } + + if ((nlink != 1 || refs != 1) && repair) { + ret = repair_inode_nlinks_lowmem(root, path, inode_id, + namebuf, name_len, refs, imode_to_type(mode), + &nlink); + } + if (nlink != 1) { err |= LINK_COUNT_ERROR; error("root %llu DIR INODE[%llu] shouldn't have more than one link(%llu)", @@ -5087,39 +6291,148 @@ out: } if (isize != size) { - err |= ISIZE_ERROR; - error("root %llu DIR INODE [%llu] size(%llu) not equal to %llu", - root->objectid, inode_id, isize, size); + if (repair) + ret = repair_dir_isize_lowmem(root, path, + inode_id, size); + if (!repair || ret) { + err |= ISIZE_ERROR; + error( + "root %llu DIR INODE [%llu] size %llu not equal to %llu", + root->objectid, inode_id, isize, size); + } } } else { if (nlink != refs) { - err |= LINK_COUNT_ERROR; - error("root %llu INODE[%llu] nlink(%llu) not equal to inode_refs(%llu)", - root->objectid, inode_id, nlink, refs); + if (repair) + ret = repair_inode_nlinks_lowmem(root, path, + inode_id, namebuf, name_len, refs, + imode_to_type(mode), &nlink); + if (!repair || ret) { + err |= LINK_COUNT_ERROR; + error( + "root %llu INODE[%llu] nlink(%llu) not equal to inode_refs(%llu)", + root->objectid, inode_id, nlink, refs); + } } else if (!nlink) { - err |= ORPHAN_ITEM; + if (repair) + ret = repair_inode_orphan_item_lowmem(root, + path, inode_id); + if (!repair || ret) { + err |= ORPHAN_ITEM; + error("root %llu INODE[%llu] is orphan item", + root->objectid, inode_id); + } } if (!nbytes && !no_holes && extent_end < isize) { - err |= NBYTES_ERROR; - error("root %llu INODE[%llu] size (%llu) should have a file extent hole", - root->objectid, inode_id, isize); + if (repair) + ret = punch_extent_hole(root, inode_id, + extent_end, isize - extent_end); + if (!repair || ret) { + err |= NBYTES_ERROR; + error( + "root %llu INODE[%llu] size %llu should have a file extent hole", + root->objectid, inode_id, isize); + } } if (nbytes != extent_size) { - err |= NBYTES_ERROR; - error("root %llu INODE[%llu] nbytes(%llu) not equal to extent_size(%llu)", - root->objectid, inode_id, nbytes, extent_size); + if (repair) + ret = repair_inode_nbytes_lowmem(root, path, + inode_id, extent_size); + if (!repair || ret) { + err |= NBYTES_ERROR; + error( + "root %llu INODE[%llu] nbytes %llu not equal to extent_size %llu", + root->objectid, inode_id, nbytes, + extent_size); + } + } + } + + if (err & LAST_ITEM) + btrfs_next_item(root, path); + return err; +} + +/* + * Insert the missing inode item and inode ref. + * + * Normal INODE_ITEM_MISSING and INODE_REF_MISSING are handled in backref * dir. + * Root dir should be handled specially because root dir is the root of fs. + * + * returns err (>0 or 0) after repair + */ +static int repair_fs_first_inode(struct btrfs_root *root, int err) +{ + struct btrfs_trans_handle *trans; + struct btrfs_key key; + struct btrfs_path path; + int filetype = BTRFS_FT_DIR; + int ret = 0; + + btrfs_init_path(&path); + + if (err & INODE_REF_MISSING) { + key.objectid = BTRFS_FIRST_FREE_OBJECTID; + key.type = BTRFS_INODE_REF_KEY; + key.offset = BTRFS_FIRST_FREE_OBJECTID; + + trans = btrfs_start_transaction(root, 1); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + goto out; } + + btrfs_release_path(&path); + ret = btrfs_search_slot(trans, root, &key, &path, 1, 1); + if (ret) + goto trans_fail; + + ret = btrfs_insert_inode_ref(trans, root, "..", 2, + BTRFS_FIRST_FREE_OBJECTID, + BTRFS_FIRST_FREE_OBJECTID, 0); + if (ret) + goto trans_fail; + + printf("Add INODE_REF[%llu %llu] name %s\n", + BTRFS_FIRST_FREE_OBJECTID, BTRFS_FIRST_FREE_OBJECTID, + ".."); + err &= ~INODE_REF_MISSING; +trans_fail: + if (ret) + error("fail to insert first inode's ref"); + btrfs_commit_transaction(trans, root); } + if (err & INODE_ITEM_MISSING) { + ret = repair_inode_item_missing(root, + BTRFS_FIRST_FREE_OBJECTID, filetype); + if (ret) + goto out; + err &= ~INODE_ITEM_MISSING; + } +out: + if (ret) + error("fail to repair first inode"); + btrfs_release_path(&path); return err; } +/* + * check first root dir's inode_item and inode_ref + * + * returns 0 means no error + * returns >0 means error + * returns <0 means fatal error + */ static int check_fs_first_inode(struct btrfs_root *root, unsigned int ext_ref) { struct btrfs_path path; struct btrfs_key key; + struct btrfs_inode_item *ii; + u64 index; + u32 mode; int err = 0; int ret; @@ -5130,28 +6443,48 @@ static int check_fs_first_inode(struct btrfs_root *root, unsigned int ext_ref) /* For root being dropped, we don't need to check first inode */ if (btrfs_root_refs(&root->root_item) == 0 && btrfs_disk_key_objectid(&root->root_item.drop_progress) >= - key.objectid) + BTRFS_FIRST_FREE_OBJECTID) return 0; btrfs_init_path(&path); - ret = btrfs_search_slot(NULL, root, &key, &path, 0, 0); if (ret < 0) goto out; if (ret > 0) { ret = 0; err |= INODE_ITEM_MISSING; - error("first inode item of root %llu is missing", - root->objectid); + } else { + ii = btrfs_item_ptr(path.nodes[0], path.slots[0], + struct btrfs_inode_item); + mode = btrfs_inode_mode(path.nodes[0], ii); + if (imode_to_type(mode) != BTRFS_FT_DIR) + err |= INODE_ITEM_MISMATCH; } - err |= check_inode_item(root, &path, ext_ref); - err &= ~LAST_ITEM; - if (err && !ret) - ret = -EIO; + /* lookup first inode ref */ + key.offset = BTRFS_FIRST_FREE_OBJECTID; + key.type = BTRFS_INODE_REF_KEY; + /* special index value */ + index = 0; + + ret = find_inode_ref(root, &key, "..", strlen(".."), &index, ext_ref); + if (ret < 0) + goto out; + err |= ret; + out: btrfs_release_path(&path); - return ret; + + if (err && repair) + err = repair_fs_first_inode(root, err); + + if (err & (INODE_ITEM_MISSING | INODE_ITEM_MISMATCH)) + error("root dir INODE_ITEM is %s", + err & INODE_ITEM_MISMATCH ? "mismatch" : "missing"); + if (err & INODE_REF_MISSING) + error("root dir INODE_REF is missing"); + + return ret < 0 ? ret : err; } static struct tree_backref *find_tree_backref(struct extent_record *rec, @@ -5214,15 +6547,21 @@ static struct data_backref *find_data_backref(struct extent_record *rec, return back; } /* - * Iterate all item on the tree and call check_inode_item() to check. - * - * @root: the root of the tree to be checked. - * @ext_ref: the EXTENDED_IREF feature + * This function calls walk_down_tree_v2 and walk_up_tree_v2 to check tree + * blocks and integrity of fs tree items. * - * Return 0 if no error found. - * Return <0 for error. + * @root: the root of the tree to be checked. + * @ext_ref feature EXTENDED_IREF is enable or not. + * @account if NOT 0 means check the tree (including tree)'s treeblocks. + * otherwise means check fs tree(s) items relationship and + * @root MUST be a fs tree root. + * Returns 0 represents OK. + * Returns not 0 represents error. */ -static int check_fs_root_v2(struct btrfs_root *root, unsigned int ext_ref) +static int check_btrfs_root(struct btrfs_trans_handle *trans, + struct btrfs_root *root, unsigned int ext_ref, + int check_all) + { struct btrfs_path path; struct node_refs nrefs; @@ -5231,17 +6570,20 @@ static int check_fs_root_v2(struct btrfs_root *root, unsigned int ext_ref) int level; int err = 0; - /* - * We need to manually check the first inode item(256) - * As the following traversal function will only start from - * the first inode item in the leaf, if inode item(256) is missing - * we will just skip it forever. - */ - ret = check_fs_first_inode(root, ext_ref); - if (ret < 0) - return ret; - memset(&nrefs, 0, sizeof(nrefs)); + if (!check_all) { + /* + * We need to manually check the first inode item (256) + * As the following traversal function will only start from + * the first inode item in the leaf, if inode item (256) is + * missing we will skip it forever. + */ + ret = check_fs_first_inode(root, ext_ref); + if (ret < 0) + return ret; + } + + level = btrfs_header_level(root->node); btrfs_init_path(&path); @@ -5263,7 +6605,9 @@ static int check_fs_root_v2(struct btrfs_root *root, unsigned int ext_ref) } while (1) { - ret = walk_down_tree_v2(root, &path, &level, &nrefs, ext_ref); + ret = walk_down_tree_v2(trans, root, &path, &level, &nrefs, + ext_ref, check_all); + err |= !!ret; /* if ret is negative, walk shall stop */ @@ -5286,6 +6630,21 @@ out: } /* + * Iterate all items in the tree and call check_inode_item() to check. + * + * @root: the root of the tree to be checked. + * @ext_ref: the EXTENDED_IREF feature + * + * Return 0 if no error found. + * Return <0 for error. + */ +static int check_fs_root_v2(struct btrfs_root *root, unsigned int ext_ref) +{ + reset_cached_block_groups(root->fs_info); + return check_btrfs_root(NULL, root, ext_ref, 0); +} + +/* * Find the relative ref for root_ref and root_backref. * * @root: the root of the root tree. @@ -9475,6 +10834,7 @@ static int check_extent_refs(struct btrfs_root *root, struct cache_extent *cache; int ret = 0; int had_dups = 0; + int err = 0; if (repair) { /* @@ -9618,6 +10978,7 @@ static int check_extent_refs(struct btrfs_root *root, cur_err = 1; } + err = cur_err; remove_cache_extent(extent_cache, cache); free_all_extent_backrefs(rec); if (!init_extent_tree && repair && (!cur_err || fix)) @@ -9650,7 +11011,10 @@ repair_abort: } return ret; } - return 0; + + if (err) + err = -EIO; + return err; } u64 calc_stripe_length(u64 type, u64 length, int num_stripes) @@ -9879,6 +11243,66 @@ static int check_device_used(struct device_record *dev_rec, } } +/* + * Extra (optional) check for dev_item size to report possbile problem on a new + * kernel. + */ +static void check_dev_size_alignment(u64 devid, u64 total_bytes, u32 sectorsize) +{ + if (!IS_ALIGNED(total_bytes, sectorsize)) { + warning( +"unaligned total_bytes detected for devid %llu, have %llu should be aligned to %u", + devid, total_bytes, sectorsize); + warning( +"this is OK for older kernel, but may cause kernel warning for newer kernels"); + warning("this can be fixed by 'btrfs rescue fix-device-size'"); + } +} + +/* + * Unlike device size alignment check above, some super total_bytes check + * failure can lead to mount failure for newer kernel. + * + * So this function will return the error for a fatal super total_bytes problem. + */ +static bool is_super_size_valid(struct btrfs_fs_info *fs_info) +{ + struct btrfs_device *dev; + struct list_head *dev_list = &fs_info->fs_devices->devices; + u64 total_bytes = 0; + u64 super_bytes = btrfs_super_total_bytes(fs_info->super_copy); + + list_for_each_entry(dev, dev_list, dev_list) + total_bytes += dev->total_bytes; + + /* Important check, which can cause unmountable fs */ + if (super_bytes < total_bytes) { + error("super total bytes %llu smaller than real device(s) size %llu", + super_bytes, total_bytes); + error("mounting this fs may fail for newer kernels"); + error("this can be fixed by 'btrfs rescue fix-device-size'"); + return false; + } + + /* + * Optional check, just to make everything aligned and match with each + * other. + * + * For a btrfs-image restored fs, we don't need to check it anyway. + */ + if (btrfs_super_flags(fs_info->super_copy) & + (BTRFS_SUPER_FLAG_METADUMP | BTRFS_SUPER_FLAG_METADUMP_V2)) + return true; + if (!IS_ALIGNED(super_bytes, fs_info->sectorsize) || + !IS_ALIGNED(total_bytes, fs_info->sectorsize) || + super_bytes != total_bytes) { + warning("minor unaligned/mismatch device size detected"); + warning( + "recommended to use 'btrfs rescue fix-device-size' to fix it"); + } + return true; +} + /* check btrfs_dev_item -> btrfs_dev_extent */ static int check_devices(struct rb_root *dev_cache, struct device_extent_tree *dev_extent_cache) @@ -9896,6 +11320,8 @@ static int check_devices(struct rb_root *dev_cache, if (err) ret = err; + check_dev_size_alignment(dev_rec->devid, dev_rec->total_byte, + global_info->sectorsize); dev_node = rb_next(dev_node); } list_for_each_entry(dext_rec, &dev_extent_cache->no_device_orphans, @@ -10221,6 +11647,29 @@ loop: goto again; } +static int check_extent_inline_ref(struct extent_buffer *eb, + struct btrfs_key *key, struct btrfs_extent_inline_ref *iref) +{ + int ret; + u8 type = btrfs_extent_inline_ref_type(eb, iref); + + switch (type) { + case BTRFS_TREE_BLOCK_REF_KEY: + case BTRFS_EXTENT_DATA_REF_KEY: + case BTRFS_SHARED_BLOCK_REF_KEY: + case BTRFS_SHARED_DATA_REF_KEY: + ret = 0; + break; + default: + error("extent[%llu %u %llu] has unknown ref type: %d", + key->objectid, key->type, key->offset, type); + ret = UNKNOWN_TYPE; + break; + } + + return ret; +} + /* * Check backrefs of a tree block given by @bytenr or @eb. * @@ -10235,7 +11684,7 @@ loop: */ static int check_tree_block_ref(struct btrfs_root *root, struct extent_buffer *eb, u64 bytenr, - int level, u64 owner) + int level, u64 owner, struct node_refs *nrefs) { struct btrfs_key key; struct btrfs_root *extent_root = root->fs_info->extent_root; @@ -10247,18 +11696,16 @@ static int check_tree_block_ref(struct btrfs_root *root, unsigned long ptr; int slot; int skinny_level; + int root_level = btrfs_header_level(root->node); int type; u32 nodesize = root->fs_info->nodesize; u32 item_size; u64 offset; - int tree_reloc_root = 0; int found_ref = 0; int err = 0; int ret; - - if (root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID && - btrfs_header_bytenr(root->node) == bytenr) - tree_reloc_root = 1; + int strict = 1; + int parent = 0; btrfs_init_path(&path); key.objectid = bytenr; @@ -10297,10 +11744,19 @@ static int check_tree_block_ref(struct btrfs_root *root, iref = (struct btrfs_extent_inline_ref *)(info + 1); } + if (eb) { u64 header_gen; u64 extent_gen; + /* + * Due to the feature of shared tree blocks, if the upper node + * is a fs root or shared node, the extent of checked node may + * not be updated until the next CoW. + */ + if (nrefs) + strict = should_check_extent_strictly(root, nrefs, + level); if (!(btrfs_extent_flags(leaf, ei) & BTRFS_EXTENT_FLAG_TREE_BLOCK)) { error( @@ -10338,25 +11794,42 @@ static int check_tree_block_ref(struct btrfs_root *root, item_size = btrfs_item_size_nr(leaf, slot); ptr = (unsigned long)iref; end = (unsigned long)ei + item_size; + while (ptr < end) { iref = (struct btrfs_extent_inline_ref *)ptr; type = btrfs_extent_inline_ref_type(leaf, iref); offset = btrfs_extent_inline_ref_offset(leaf, iref); - if (type == BTRFS_TREE_BLOCK_REF_KEY && - (offset == root->objectid || offset == owner)) { - found_ref = 1; + ret = check_extent_inline_ref(leaf, &key, iref); + if (ret) { + err |= ret; + break; + } + if (type == BTRFS_TREE_BLOCK_REF_KEY) { + if (offset == root->objectid) + found_ref = 1; + if (!strict && owner == offset) + found_ref = 1; } else if (type == BTRFS_SHARED_BLOCK_REF_KEY) { /* * Backref of tree reloc root points to itself, no need * to check backref any more. + * + * This may be an error of loop backref, but extent tree + * checker should have already handled it. + * Here we only need to avoid infinite iteration. */ - if (tree_reloc_root) + if (offset == bytenr) { found_ref = 1; - else - /* Check if the backref points to valid referencer */ - found_ref = !check_tree_block_ref(root, NULL, - offset, level + 1, owner); + } else { + /* + * Check if the backref points to valid + * referencer + */ + found_ref = !check_tree_block_ref( root, NULL, + offset, level + 1, owner, + NULL); + } } if (found_ref) @@ -10378,13 +11851,157 @@ static int check_tree_block_ref(struct btrfs_root *root, if (!ret) found_ref = 1; } + /* + * Finally check SHARED BLOCK REF, any found will be good + * Here we're not doing comprehensive extent backref checking, + * only need to ensure there is some extent referring to this + * tree block. + */ + if (!found_ref) { + btrfs_release_path(&path); + key.objectid = bytenr; + key.type = BTRFS_SHARED_BLOCK_REF_KEY; + key.offset = (u64)-1; + + ret = btrfs_search_slot(NULL, extent_root, &key, &path, 0, 0); + if (ret < 0) { + err |= BACKREF_MISSING; + goto out; + } + ret = btrfs_previous_extent_item(extent_root, &path, bytenr); + if (ret) { + err |= BACKREF_MISSING; + goto out; + } + found_ref = 1; + } if (!found_ref) err |= BACKREF_MISSING; out: btrfs_release_path(&path); + if (nrefs && strict && + level < root_level && nrefs->full_backref[level + 1]) + parent = nrefs->bytenr[level + 1]; if (eb && (err & BACKREF_MISSING)) - error("extent[%llu %u] backref lost (owner: %llu, level: %u)", - bytenr, nodesize, owner, level); + error( + "extent[%llu %u] backref lost (owner: %llu, level: %u) %s %llu", + bytenr, nodesize, owner, level, + parent ? "parent" : "root", + parent ? parent : root->objectid); + return err; +} + +/* + * If @err contains BACKREF_MISSING then add extent of the + * file_extent_data_item. + * + * Returns error bits after reapir. + */ +static int repair_extent_data_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *pathp, + struct node_refs *nrefs, + int err) +{ + struct btrfs_file_extent_item *fi; + struct btrfs_key fi_key; + struct btrfs_key key; + struct btrfs_extent_item *ei; + struct btrfs_path path; + struct btrfs_root *extent_root = root->fs_info->extent_root; + struct extent_buffer *eb; + u64 size; + u64 disk_bytenr; + u64 num_bytes; + u64 parent; + u64 offset; + u64 extent_offset; + u64 file_offset; + int generation; + int slot; + int ret = 0; + + eb = pathp->nodes[0]; + slot = pathp->slots[0]; + btrfs_item_key_to_cpu(eb, &fi_key, slot); + fi = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item); + + if (btrfs_file_extent_type(eb, fi) == BTRFS_FILE_EXTENT_INLINE || + btrfs_file_extent_disk_bytenr(eb, fi) == 0) + return err; + + file_offset = fi_key.offset; + generation = btrfs_file_extent_generation(eb, fi); + disk_bytenr = btrfs_file_extent_disk_bytenr(eb, fi); + num_bytes = btrfs_file_extent_disk_num_bytes(eb, fi); + extent_offset = btrfs_file_extent_offset(eb, fi); + offset = file_offset - extent_offset; + + /* now repair only adds backref */ + if ((err & BACKREF_MISSING) == 0) + return err; + + /* search extent item */ + key.objectid = disk_bytenr; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = num_bytes; + + btrfs_init_path(&path); + ret = btrfs_search_slot(NULL, extent_root, &key, &path, 0, 0); + if (ret < 0) { + ret = -EIO; + goto out; + } + + /* insert an extent item */ + if (ret > 0) { + key.objectid = disk_bytenr; + key.type = BTRFS_EXTENT_ITEM_KEY; + key.offset = num_bytes; + size = sizeof(*ei); + + btrfs_release_path(&path); + ret = btrfs_insert_empty_item(trans, extent_root, &path, &key, + size); + if (ret) + goto out; + eb = path.nodes[0]; + ei = btrfs_item_ptr(eb, path.slots[0], struct btrfs_extent_item); + + btrfs_set_extent_refs(eb, ei, 0); + btrfs_set_extent_generation(eb, ei, generation); + btrfs_set_extent_flags(eb, ei, BTRFS_EXTENT_FLAG_DATA); + + btrfs_mark_buffer_dirty(eb); + ret = btrfs_update_block_group(trans, extent_root, disk_bytenr, + num_bytes, 1, 0); + btrfs_release_path(&path); + } + + if (nrefs->full_backref[0]) + parent = btrfs_header_bytenr(eb); + else + parent = 0; + + ret = btrfs_inc_extent_ref(trans, root, disk_bytenr, num_bytes, parent, + root->objectid, + parent ? BTRFS_FIRST_FREE_OBJECTID : fi_key.objectid, + offset); + if (ret) { + error( + "failed to increase extent data backref[%llu %llu] root %llu", + disk_bytenr, num_bytes, root->objectid); + goto out; + } else { + printf("Add one extent data backref [%llu %llu]\n", + disk_bytenr, num_bytes); + } + + err &= ~BACKREF_MISSING; +out: + if (ret) + error("can't repair root %llu extent data item[%llu %llu]", + root->objectid, disk_bytenr, num_bytes); return err; } @@ -10395,9 +12012,11 @@ out: * Return 0 for no error found */ static int check_extent_data_item(struct btrfs_root *root, - struct extent_buffer *eb, int slot) + struct btrfs_path *pathp, + struct node_refs *nrefs, int account_bytes) { struct btrfs_file_extent_item *fi; + struct extent_buffer *eb = pathp->nodes[0]; struct btrfs_path path; struct btrfs_root *extent_root = root->fs_info->extent_root; struct btrfs_key fi_key; @@ -10411,14 +12030,16 @@ static int check_extent_data_item(struct btrfs_root *root, u64 disk_num_bytes; u64 extent_num_bytes; u64 extent_flags; + u64 offset; u32 item_size; unsigned long end; unsigned long ptr; int type; - u64 ref_root; int found_dbackref = 0; + int slot = pathp->slots[0]; int err = 0; int ret; + int strict; btrfs_item_key_to_cpu(eb, &fi_key, slot); fi = btrfs_item_ptr(eb, slot, struct btrfs_file_extent_item); @@ -10431,6 +12052,7 @@ static int check_extent_data_item(struct btrfs_root *root, disk_bytenr = btrfs_file_extent_disk_bytenr(eb, fi); disk_num_bytes = btrfs_file_extent_disk_num_bytes(eb, fi); extent_num_bytes = btrfs_file_extent_num_bytes(eb, fi); + offset = btrfs_file_extent_offset(eb, fi); /* Check unaligned disk_num_bytes and num_bytes */ if (!IS_ALIGNED(disk_num_bytes, root->fs_info->sectorsize)) { @@ -10439,7 +12061,7 @@ static int check_extent_data_item(struct btrfs_root *root, fi_key.objectid, fi_key.offset, disk_num_bytes, root->fs_info->sectorsize); err |= BYTES_UNALIGNED; - } else { + } else if (account_bytes) { data_bytes_allocated += disk_num_bytes; } if (!IS_ALIGNED(extent_num_bytes, root->fs_info->sectorsize)) { @@ -10448,7 +12070,7 @@ static int check_extent_data_item(struct btrfs_root *root, fi_key.objectid, fi_key.offset, extent_num_bytes, root->fs_info->sectorsize); err |= BYTES_UNALIGNED; - } else { + } else if (account_bytes) { data_bytes_referenced += extent_num_bytes; } owner = btrfs_header_owner(eb); @@ -10482,19 +12104,39 @@ static int check_extent_data_item(struct btrfs_root *root, iref = (struct btrfs_extent_inline_ref *)(ei + 1); ptr = (unsigned long)iref; end = (unsigned long)ei + item_size; + strict = should_check_extent_strictly(root, nrefs, -1); + while (ptr < end) { + u64 ref_root; + u64 ref_objectid; + u64 ref_offset; + bool match = false; + iref = (struct btrfs_extent_inline_ref *)ptr; type = btrfs_extent_inline_ref_type(leaf, iref); dref = (struct btrfs_extent_data_ref *)(&iref->offset); + ret = check_extent_inline_ref(leaf, &dbref_key, iref); + if (ret) { + err |= ret; + break; + } if (type == BTRFS_EXTENT_DATA_REF_KEY) { ref_root = btrfs_extent_data_ref_root(leaf, dref); - if (ref_root == owner || ref_root == root->objectid) + ref_objectid = btrfs_extent_data_ref_objectid(leaf, dref); + ref_offset = btrfs_extent_data_ref_offset(leaf, dref); + + if (ref_objectid == fi_key.objectid && + ref_offset == fi_key.offset - offset) + match = true; + if (ref_root == root->objectid && match) + found_dbackref = 1; + else if (!strict && owner == ref_root && match) found_dbackref = 1; } else if (type == BTRFS_SHARED_DATA_REF_KEY) { found_dbackref = !check_tree_block_ref(root, NULL, btrfs_extent_inline_ref_offset(leaf, iref), - 0, owner); + 0, owner, NULL); } if (found_dbackref) @@ -10509,7 +12151,7 @@ static int check_extent_data_item(struct btrfs_root *root, dbref_key.objectid = btrfs_file_extent_disk_bytenr(eb, fi); dbref_key.type = BTRFS_EXTENT_DATA_REF_KEY; dbref_key.offset = hash_extent_data_ref(root->objectid, - fi_key.objectid, fi_key.offset); + fi_key.objectid, fi_key.offset - offset); ret = btrfs_search_slot(NULL, root->fs_info->extent_root, &dbref_key, &path, 0, 0); @@ -10866,7 +12508,8 @@ static int check_extent_data_backref(struct btrfs_fs_info *fs_info, leaf = path.nodes[0]; slot = path.slots[0]; - if (slot >= btrfs_header_nritems(leaf)) + if (slot >= btrfs_header_nritems(leaf) || + btrfs_header_owner(leaf) != root_id) goto next; btrfs_item_key_to_cpu(leaf, &key, slot); if (key.objectid != objectid || key.type != BTRFS_EXTENT_DATA_KEY) @@ -10876,11 +12519,17 @@ static int check_extent_data_backref(struct btrfs_fs_info *fs_info, * Except normal disk bytenr and disk num bytes, we still * need to do extra check on dbackref offset as * dbackref offset = file_offset - file_extent_offset + * + * Also, we must check the leaf owner. + * In case of shared tree blocks (snapshots) we can inherit + * leaves from source snapshot. + * In that case, reference from source snapshot should not + * count. */ if (btrfs_file_extent_disk_bytenr(leaf, fi) == bytenr && btrfs_file_extent_disk_num_bytes(leaf, fi) == len && (u64)(key.offset - btrfs_file_extent_offset(leaf, fi)) == - offset) + offset && btrfs_header_owner(leaf) == root_id) found_count++; next: @@ -10943,24 +12592,77 @@ out: } /* + * Only delete backref if REFERENCER_MISSING now + * + * Returns <0 the extent was deleted + * Returns >0 the backref was deleted but extent still exists, returned value + * means error after repair + * Returns 0 nothing happened + */ +static int repair_extent_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct btrfs_path *path, + u64 bytenr, u64 num_bytes, u64 parent, u64 root_objectid, + u64 owner, u64 offset, int err) +{ + struct btrfs_key old_key; + int freed = 0; + int ret; + + btrfs_item_key_to_cpu(path->nodes[0], &old_key, path->slots[0]); + + if (err & (REFERENCER_MISSING | REFERENCER_MISMATCH)) { + /* delete the backref */ + ret = btrfs_free_extent(trans, root->fs_info->fs_root, bytenr, + num_bytes, parent, root_objectid, owner, offset); + if (!ret) { + freed = 1; + err &= ~REFERENCER_MISSING; + printf("Delete backref in extent [%llu %llu]\n", + bytenr, num_bytes); + } else { + error("fail to delete backref in extent [%llu %llu]", + bytenr, num_bytes); + } + } + + /* btrfs_free_extent may delete the extent */ + btrfs_release_path(path); + ret = btrfs_search_slot(NULL, root, &old_key, path, 0, 0); + + if (ret) + ret = -ENOENT; + else if (freed) + ret = err; + return ret; +} + +/* * This function will check a given extent item, including its backref and * itself (like crossing stripe boundary and type) * * Since we don't use extent_record anymore, introduce new error bit */ -static int check_extent_item(struct btrfs_fs_info *fs_info, - struct extent_buffer *eb, int slot) +static int check_extent_item(struct btrfs_trans_handle *trans, + struct btrfs_fs_info *fs_info, + struct btrfs_path *path) { struct btrfs_extent_item *ei; struct btrfs_extent_inline_ref *iref; struct btrfs_extent_data_ref *dref; + struct extent_buffer *eb = path->nodes[0]; unsigned long end; unsigned long ptr; + int slot = path->slots[0]; int type; u32 nodesize = btrfs_super_nodesize(fs_info->super_copy); u32 item_size = btrfs_item_size_nr(eb, slot); u64 flags; u64 offset; + u64 parent; + u64 num_bytes; + u64 root_objectid; + u64 owner; + u64 owner_offset; int metadata = 0; int level; struct btrfs_key key; @@ -10968,10 +12670,13 @@ static int check_extent_item(struct btrfs_fs_info *fs_info, int err = 0; btrfs_item_key_to_cpu(eb, &key, slot); - if (key.type == BTRFS_EXTENT_ITEM_KEY) + if (key.type == BTRFS_EXTENT_ITEM_KEY) { bytes_used += key.offset; - else + num_bytes = key.offset; + } else { bytes_used += nodesize; + num_bytes = nodesize; + } if (item_size < sizeof(*ei)) { /* @@ -11023,42 +12728,63 @@ next: goto out; } + parent = 0; + root_objectid = 0; + owner = 0; + owner_offset = 0; /* Now check every backref in this extent item */ iref = (struct btrfs_extent_inline_ref *)ptr; type = btrfs_extent_inline_ref_type(eb, iref); offset = btrfs_extent_inline_ref_offset(eb, iref); switch (type) { case BTRFS_TREE_BLOCK_REF_KEY: + root_objectid = offset; + owner = level; ret = check_tree_block_backref(fs_info, offset, key.objectid, level); err |= ret; break; case BTRFS_SHARED_BLOCK_REF_KEY: + parent = offset; ret = check_shared_block_backref(fs_info, offset, key.objectid, level); err |= ret; break; case BTRFS_EXTENT_DATA_REF_KEY: dref = (struct btrfs_extent_data_ref *)(&iref->offset); - ret = check_extent_data_backref(fs_info, - btrfs_extent_data_ref_root(eb, dref), - btrfs_extent_data_ref_objectid(eb, dref), - btrfs_extent_data_ref_offset(eb, dref), - key.objectid, key.offset, - btrfs_extent_data_ref_count(eb, dref)); + root_objectid = btrfs_extent_data_ref_root(eb, dref); + owner = btrfs_extent_data_ref_objectid(eb, dref); + owner_offset = btrfs_extent_data_ref_offset(eb, dref); + ret = check_extent_data_backref(fs_info, root_objectid, owner, + owner_offset, key.objectid, key.offset, + btrfs_extent_data_ref_count(eb, dref)); err |= ret; break; case BTRFS_SHARED_DATA_REF_KEY: + parent = offset; ret = check_shared_data_backref(fs_info, offset, key.objectid); err |= ret; break; default: error("extent[%llu %d %llu] has unknown ref type: %d", key.objectid, key.type, key.offset, type); - err |= UNKNOWN_TYPE; + ret = UNKNOWN_TYPE; + err |= ret; goto out; } + if (err && repair) { + ret = repair_extent_item(trans, fs_info->extent_root, path, + key.objectid, num_bytes, parent, root_objectid, + owner, owner_offset, ret); + if (ret < 0) + goto out; + if (ret) { + goto next; + err = ret; + } + } + ptr += btrfs_extent_inline_ref_size(type); goto next; @@ -11141,6 +12867,7 @@ static int check_dev_item(struct btrfs_fs_info *fs_info, struct btrfs_path path; struct btrfs_key key; struct btrfs_dev_extent *ptr; + u64 total_bytes; u64 dev_id; u64 used; u64 total = 0; @@ -11149,6 +12876,7 @@ static int check_dev_item(struct btrfs_fs_info *fs_info, dev_item = btrfs_item_ptr(eb, slot, struct btrfs_dev_item); dev_id = btrfs_device_id(eb, dev_item); used = btrfs_device_bytes_used(eb, dev_item); + total_bytes = btrfs_device_total_bytes(eb, dev_item); key.objectid = dev_id; key.type = BTRFS_DEV_EXTENT_KEY; @@ -11193,6 +12921,8 @@ next: BTRFS_DEV_EXTENT_KEY, dev_id); return ACCOUNTING_MISMATCH; } + check_dev_size_alignment(dev_id, total_bytes, fs_info->sectorsize); + return 0; } @@ -11320,8 +13050,51 @@ out: error( "block group[%llu %llu] used %llu but extent items used %llu", bg_key.objectid, bg_key.offset, used, total); - err |= ACCOUNTING_MISMATCH; + err |= BG_ACCOUNTING_ERROR; + } + return err; +} + +/* + * Add block group item to the extent tree if @err contains REFERENCER_MISSING. + * FIXME: We still need to repair error of dev_item. + * + * Returns error after repair. + */ +static int repair_chunk_item(struct btrfs_trans_handle *trans, + struct btrfs_root *chunk_root, + struct btrfs_path *path, int err) +{ + struct btrfs_chunk *chunk; + struct btrfs_key chunk_key; + struct extent_buffer *eb = path->nodes[0]; + u64 length; + int slot = path->slots[0]; + u64 type; + int ret = 0; + + btrfs_item_key_to_cpu(eb, &chunk_key, slot); + if (chunk_key.type != BTRFS_CHUNK_ITEM_KEY) + return err; + chunk = btrfs_item_ptr(eb, slot, struct btrfs_chunk); + type = btrfs_chunk_type(path->nodes[0], chunk); + length = btrfs_chunk_length(eb, chunk); + + if (err & REFERENCER_MISSING) { + ret = btrfs_make_block_group(trans, chunk_root->fs_info, 0, + type, chunk_key.objectid, chunk_key.offset, length); + if (ret) { + error("fail to add block group item[%llu %llu]", + chunk_key.offset, length); + goto out; + } else { + err &= ~REFERENCER_MISSING; + printf("Added block group item[%llu %llu]\n", + chunk_key.offset, length); + } } + +out: return err; } @@ -11430,30 +13203,83 @@ out: return err; } +static int delete_extent_tree_item(struct btrfs_trans_handle *trans, + struct btrfs_root *root, + struct btrfs_path *path) +{ + struct btrfs_key key; + int ret = 0; + + btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]); + btrfs_release_path(path); + ret = btrfs_search_slot(trans, root, &key, path, -1, 1); + if (ret) { + ret = -ENOENT; + goto out; + } + + ret = btrfs_del_item(trans, root, path); + if (ret) + goto out; + + if (path->slots[0] == 0) + btrfs_prev_leaf(root, path); + else + path->slots[0]--; +out: + if (ret) + error("failed to delete root %llu item[%llu, %u, %llu]", + root->objectid, key.objectid, key.type, key.offset); + else + printf("Deleted root %llu item[%llu, %u, %llu]\n", + root->objectid, key.objectid, key.type, key.offset); + return ret; +} + /* * Main entry function to check known items and update related accounting info */ -static int check_leaf_items(struct btrfs_root *root, struct extent_buffer *eb) +static int check_leaf_items(struct btrfs_trans_handle *trans, + struct btrfs_root *root, struct btrfs_path *path, + struct node_refs *nrefs, int account_bytes) { struct btrfs_fs_info *fs_info = root->fs_info; struct btrfs_key key; - int slot = 0; + struct extent_buffer *eb; + int slot; int type; struct btrfs_extent_data_ref *dref; - int ret; + int ret = 0; int err = 0; -next: +again: + eb = path->nodes[0]; + slot = path->slots[0]; + if (slot >= btrfs_header_nritems(eb)) { + if (slot == 0) { + error("empty leaf [%llu %u] root %llu", eb->start, + root->fs_info->nodesize, root->objectid); + err |= EIO; + } + goto out; + } + btrfs_item_key_to_cpu(eb, &key, slot); type = key.type; switch (type) { case BTRFS_EXTENT_DATA_KEY: - ret = check_extent_data_item(root, eb, slot); + ret = check_extent_data_item(root, path, nrefs, account_bytes); + if (repair && ret) + ret = repair_extent_data_item(trans, root, path, nrefs, + ret); err |= ret; break; case BTRFS_BLOCK_GROUP_ITEM_KEY: ret = check_block_group_item(fs_info, eb, slot); + if (repair && + ret & REFERENCER_MISSING) + ret = delete_extent_tree_item(trans, root, path); err |= ret; break; case BTRFS_DEV_ITEM_KEY: @@ -11462,6 +13288,8 @@ next: break; case BTRFS_CHUNK_ITEM_KEY: ret = check_chunk_item(fs_info, eb, slot); + if (repair && ret) + ret = repair_chunk_item(trans, root, path, ret); err |= ret; break; case BTRFS_DEV_EXTENT_KEY: @@ -11470,15 +13298,19 @@ next: break; case BTRFS_EXTENT_ITEM_KEY: case BTRFS_METADATA_ITEM_KEY: - ret = check_extent_item(fs_info, eb, slot); + ret = check_extent_item(trans, fs_info, path); err |= ret; break; case BTRFS_EXTENT_CSUM_KEY: total_csum_bytes += btrfs_item_size_nr(eb, slot); + err |= ret; break; case BTRFS_TREE_BLOCK_REF_KEY: ret = check_tree_block_backref(fs_info, key.offset, key.objectid, -1); + if (repair && + ret & (REFERENCER_MISMATCH | REFERENCER_MISSING)) + ret = delete_extent_tree_item(trans, root, path); err |= ret; break; case BTRFS_EXTENT_DATA_REF_KEY: @@ -11489,194 +13321,34 @@ next: btrfs_extent_data_ref_offset(eb, dref), key.objectid, 0, btrfs_extent_data_ref_count(eb, dref)); + if (repair && + ret & (REFERENCER_MISMATCH | REFERENCER_MISSING)) + ret = delete_extent_tree_item(trans, root, path); err |= ret; break; case BTRFS_SHARED_BLOCK_REF_KEY: ret = check_shared_block_backref(fs_info, key.offset, key.objectid, -1); + if (repair && + ret & (REFERENCER_MISMATCH | REFERENCER_MISSING)) + ret = delete_extent_tree_item(trans, root, path); err |= ret; break; case BTRFS_SHARED_DATA_REF_KEY: ret = check_shared_data_backref(fs_info, key.offset, key.objectid); + if (repair && + ret & (REFERENCER_MISMATCH | REFERENCER_MISSING)) + ret = delete_extent_tree_item(trans, root, path); err |= ret; break; default: break; } - if (++slot < btrfs_header_nritems(eb)) - goto next; - - return err; -} - -/* - * Helper function for later fs/subvol tree check. To determine if a tree - * block should be checked. - * This function will ensure only the direct referencer with lowest rootid to - * check a fs/subvolume tree block. - * - * Backref check at extent tree would detect errors like missing subvolume - * tree, so we can do aggressive check to reduce duplicated checks. - */ -static int should_check(struct btrfs_root *root, struct extent_buffer *eb) -{ - struct btrfs_root *extent_root = root->fs_info->extent_root; - struct btrfs_key key; - struct btrfs_path path; - struct extent_buffer *leaf; - int slot; - struct btrfs_extent_item *ei; - unsigned long ptr; - unsigned long end; - int type; - u32 item_size; - u64 offset; - struct btrfs_extent_inline_ref *iref; - int ret; - - btrfs_init_path(&path); - key.objectid = btrfs_header_bytenr(eb); - key.type = BTRFS_METADATA_ITEM_KEY; - key.offset = (u64)-1; - - /* - * Any failure in backref resolving means we can't determine - * whom the tree block belongs to. - * So in that case, we need to check that tree block - */ - ret = btrfs_search_slot(NULL, extent_root, &key, &path, 0, 0); - if (ret < 0) - goto need_check; - - ret = btrfs_previous_extent_item(extent_root, &path, - btrfs_header_bytenr(eb)); - if (ret) - goto need_check; - - leaf = path.nodes[0]; - slot = path.slots[0]; - btrfs_item_key_to_cpu(leaf, &key, slot); - ei = btrfs_item_ptr(leaf, slot, struct btrfs_extent_item); - - if (key.type == BTRFS_METADATA_ITEM_KEY) { - iref = (struct btrfs_extent_inline_ref *)(ei + 1); - } else { - struct btrfs_tree_block_info *info; - - info = (struct btrfs_tree_block_info *)(ei + 1); - iref = (struct btrfs_extent_inline_ref *)(info + 1); - } - - item_size = btrfs_item_size_nr(leaf, slot); - ptr = (unsigned long)iref; - end = (unsigned long)ei + item_size; - while (ptr < end) { - iref = (struct btrfs_extent_inline_ref *)ptr; - type = btrfs_extent_inline_ref_type(leaf, iref); - offset = btrfs_extent_inline_ref_offset(leaf, iref); - - /* - * We only check the tree block if current root is - * the lowest referencer of it. - */ - if (type == BTRFS_TREE_BLOCK_REF_KEY && - offset < root->objectid) { - btrfs_release_path(&path); - return 0; - } - - ptr += btrfs_extent_inline_ref_size(type); - } - /* - * Normally we should also check keyed tree block ref, but that may be - * very time consuming. Inlined ref should already make us skip a lot - * of refs now. So skip search keyed tree block ref. - */ - -need_check: - btrfs_release_path(&path); - return 1; -} - -/* - * Traversal function for tree block. We will do: - * 1) Skip shared fs/subvolume tree blocks - * 2) Update related bytes accounting - * 3) Pre-order traversal - */ -static int traverse_tree_block(struct btrfs_root *root, - struct extent_buffer *node) -{ - struct extent_buffer *eb; - struct btrfs_key key; - struct btrfs_key drop_key; - int level; - u64 nr; - int i; - int err = 0; - int ret; - - /* - * Skip shared fs/subvolume tree block, in that case they will - * be checked by referencer with lowest rootid - */ - if (is_fstree(root->objectid) && !should_check(root, node)) - return 0; - - /* Update bytes accounting */ - total_btree_bytes += node->len; - if (fs_root_objectid(btrfs_header_owner(node))) - total_fs_tree_bytes += node->len; - if (btrfs_header_owner(node) == BTRFS_EXTENT_TREE_OBJECTID) - total_extent_tree_bytes += node->len; - - /* pre-order tranversal, check itself first */ - level = btrfs_header_level(node); - ret = check_tree_block_ref(root, node, btrfs_header_bytenr(node), - btrfs_header_level(node), - btrfs_header_owner(node)); - err |= ret; - if (err) - error( - "check %s failed root %llu bytenr %llu level %d, force continue check", - level ? "node":"leaf", root->objectid, - btrfs_header_bytenr(node), btrfs_header_level(node)); - - if (!level) { - btree_space_waste += btrfs_leaf_free_space(root, node); - ret = check_leaf_items(root, node); - err |= ret; - return err; - } - - nr = btrfs_header_nritems(node); - btrfs_disk_key_to_cpu(&drop_key, &root->root_item.drop_progress); - btree_space_waste += (BTRFS_NODEPTRS_PER_BLOCK(root) - nr) * - sizeof(struct btrfs_key_ptr); - - /* Then check all its children */ - for (i = 0; i < nr; i++) { - u64 blocknr = btrfs_node_blockptr(node, i); - - btrfs_node_key_to_cpu(node, &key, i); - if (level == root->root_item.drop_level && - is_dropped_key(&key, &drop_key)) - continue; - - /* - * As a btrfs tree has most 8 levels (0..7), so it's quite safe - * to call the function itself. - */ - eb = read_tree_block(root->fs_info, blocknr, 0); - if (extent_buffer_uptodate(eb)) { - ret = traverse_tree_block(root, eb); - err |= ret; - } - free_extent_buffer(eb); - } - + ++path->slots[0]; + goto again; +out: return err; } @@ -11685,7 +13357,9 @@ static int traverse_tree_block(struct btrfs_root *root, */ static int check_chunks_and_extents_v2(struct btrfs_fs_info *fs_info) { + struct btrfs_trans_handle *trans = NULL; struct btrfs_path path; + struct btrfs_key old_key; struct btrfs_key key; struct btrfs_root *root1; struct btrfs_root *root; @@ -11695,12 +13369,20 @@ static int check_chunks_and_extents_v2(struct btrfs_fs_info *fs_info) root = fs_info->fs_root; + if (repair) { + trans = btrfs_start_transaction(fs_info->extent_root, 1); + if (IS_ERR(trans)) { + error("failed to start transaction before check"); + return PTR_ERR(trans); + } + } + root1 = root->fs_info->chunk_root; - ret = traverse_tree_block(root1, root1->node); + ret = check_btrfs_root(trans, root1, 0, 1); err |= ret; root1 = root->fs_info->tree_root; - ret = traverse_tree_block(root1, root1->node); + ret = check_btrfs_root(trans, root1, 0, 1); err |= ret; btrfs_init_path(&path); @@ -11710,7 +13392,7 @@ static int check_chunks_and_extents_v2(struct btrfs_fs_info *fs_info) ret = btrfs_search_slot(NULL, root1, &key, &path, 0, 0); if (ret) { - error("cannot find extent treet in tree_root"); + error("cannot find extent tree in tree_root"); goto out; } @@ -11718,6 +13400,7 @@ static int check_chunks_and_extents_v2(struct btrfs_fs_info *fs_info) btrfs_item_key_to_cpu(path.nodes[0], &key, path.slots[0]); if (key.type != BTRFS_ROOT_ITEM_KEY) goto next; + old_key = key; key.offset = (u64)-1; if (key.objectid == BTRFS_TREE_RELOC_OBJECTID) @@ -11730,19 +13413,38 @@ static int check_chunks_and_extents_v2(struct btrfs_fs_info *fs_info) goto next; } - ret = traverse_tree_block(cur_root, cur_root->node); + ret = check_btrfs_root(trans, cur_root, 0, 1); err |= ret; if (key.objectid == BTRFS_TREE_RELOC_OBJECTID) btrfs_free_fs_root(cur_root); + + btrfs_release_path(&path); + ret = btrfs_search_slot(NULL, root->fs_info->tree_root, + &old_key, &path, 0, 0); + if (ret) + goto out; next: ret = btrfs_next_item(root1, &path); if (ret) goto out; } - out: + + /* if repair, update block accounting */ + if (repair) { + ret = btrfs_fix_block_accounting(trans, root); + if (ret) + err |= ret; + else + err &= ~BG_ACCOUNTING_ERROR; + } + + if (trans) + btrfs_commit_transaction(trans, root->fs_info->extent_root); + btrfs_release_path(&path); + return err; } @@ -11757,6 +13459,12 @@ static int do_check_chunks_and_extents(struct btrfs_fs_info *fs_info) else ret = check_chunks_and_extents(fs_info); + /* Also repair device size related problems */ + if (repair && !ret) { + ret = btrfs_fix_device_and_super_size(fs_info); + if (ret > 0) + ret = 0; + } return ret; } @@ -13072,12 +14780,10 @@ int cmd_check(int argc, char **argv) } /* - * Not supported yet + * experimental and dangerous */ - if (repair && check_mode == CHECK_MODE_LOWMEM) { - error("low memory mode doesn't support repair yet"); - exit(1); - } + if (repair && check_mode == CHECK_MODE_LOWMEM) + warning("low-memory mode repair support is only partial"); radix_tree_init(); cache_tree_init(&root_cache); @@ -13250,31 +14956,36 @@ int cmd_check(int argc, char **argv) goto close_out; } + if (!init_extent_tree) { + ret = repair_root_items(info); + if (ret < 0) { + err = !!ret; + error("failed to repair root items: %s", strerror(-ret)); + goto close_out; + } + if (repair) { + fprintf(stderr, "Fixed %d roots.\n", ret); + ret = 0; + } else if (ret > 0) { + fprintf(stderr, + "Found %d roots with an outdated root item.\n", + ret); + fprintf(stderr, + "Please run a filesystem check with the option --repair to fix them.\n"); + ret = 1; + err |= ret; + goto close_out; + } + } + ret = do_check_chunks_and_extents(info); err |= !!ret; if (ret) error( "errors found in extent allocation tree or chunk allocation"); - ret = repair_root_items(info); - err |= !!ret; - if (ret < 0) { - error("failed to repair root items: %s", strerror(-ret)); - goto close_out; - } - if (repair) { - fprintf(stderr, "Fixed %d roots.\n", ret); - ret = 0; - } else if (ret > 0) { - fprintf(stderr, - "Found %d roots with an outdated root item.\n", - ret); - fprintf(stderr, - "Please run a filesystem check with the option --repair to fix them.\n"); - ret = 1; - err |= !!ret; - goto close_out; - } + /* Only re-check super size after we checked and repaired the fs */ + err |= !is_super_size_valid(info); if (!ctx.progress_enabled) { if (btrfs_fs_compat_ro(info, FREE_SPACE_TREE)) diff --git a/cmds-device.c b/cmds-device.c index 4337eb27..f4cdb39f 100644 --- a/cmds-device.c +++ b/cmds-device.c @@ -224,9 +224,16 @@ static int _cmd_device_remove(int argc, char **argv, return !!ret; } +#define COMMON_USAGE_REMOVE_DELETE \ + "If 'missing' is specified for <device>, the first device that is", \ + "described by the filesystem metadata, but not present at the mount", \ + "time will be removed. (only in degraded mode)" + static const char * const cmd_device_remove_usage[] = { "btrfs device remove <device>|<devid> [<device>|<devid>...] <path>", "Remove a device from a filesystem", + "", + COMMON_USAGE_REMOVE_DELETE, NULL }; @@ -237,7 +244,9 @@ static int cmd_device_remove(int argc, char **argv) static const char * const cmd_device_delete_usage[] = { "btrfs device delete <device>|<devid> [<device>|<devid>...] <path>", - "Remove a device from a filesystem", + "Remove a device from a filesystem (alias of \"btrfs device remove\")", + "", + COMMON_USAGE_REMOVE_DELETE, NULL }; diff --git a/cmds-fi-usage.c b/cmds-fi-usage.c index 6c846c15..0b0e47fe 100644 --- a/cmds-fi-usage.c +++ b/cmds-fi-usage.c @@ -22,6 +22,7 @@ #include <errno.h> #include <stdarg.h> #include <getopt.h> +#include <fcntl.h> #include "utils.h" #include "kerncompat.h" @@ -29,6 +30,7 @@ #include "string-table.h" #include "cmds-fi-usage.h" #include "commands.h" +#include "disk-io.h" #include "version.h" #include "help.h" @@ -506,6 +508,33 @@ static int cmp_device_info(const void *a, const void *b) ((struct device_info *)b)->path); } +int dev_to_fsid(const char *dev, u8 *fsid) +{ + struct btrfs_super_block *disk_super; + char buf[BTRFS_SUPER_INFO_SIZE]; + int ret; + int fd; + + fd = open(dev, O_RDONLY); + if (fd < 0) { + ret = -errno; + return ret; + } + + disk_super = (struct btrfs_super_block *)buf; + ret = btrfs_read_dev_super(fd, disk_super, + BTRFS_SUPER_INFO_OFFSET, SBREAD_DEFAULT); + if (ret) + goto out; + + memcpy(fsid, disk_super->fsid, BTRFS_FSID_SIZE); + ret = 0; + +out: + close(fd); + return ret; +} + /* * This function loads the device_info structure and put them in an array */ @@ -516,6 +545,7 @@ static int load_device_info(int fd, struct device_info **device_info_ptr, struct btrfs_ioctl_fs_info_args fi_args; struct btrfs_ioctl_dev_info_args dev_info; struct device_info *info; + u8 fsid[BTRFS_UUID_SIZE]; *device_info_count = 0; *device_info_ptr = NULL; @@ -539,6 +569,8 @@ static int load_device_info(int fd, struct device_info **device_info_ptr, if (ndevs >= fi_args.num_devices) { error("unexpected number of devices: %d >= %llu", ndevs, (unsigned long long)fi_args.num_devices); + error( + "if seed device is used, try running this command as root"); goto out; } memset(&dev_info, 0, sizeof(dev_info)); @@ -551,6 +583,17 @@ static int load_device_info(int fd, struct device_info **device_info_ptr, goto out; } + /* + * Skip seed device by checking device's fsid (requires root). + * And we will skip only if dev_to_fsid is successful and dev + * is a seed device. + * Ignore any other error including -EACCES, which is seen when + * a non-root process calls dev_to_fsid(path)->open(path). + */ + ret = dev_to_fsid((const char *)dev_info.path, fsid); + if (!ret && memcmp(fi_args.fsid, fsid, BTRFS_FSID_SIZE) != 0) + continue; + info[ndevs].devid = dev_info.devid; if (!dev_info.path[0]) { strcpy(info[ndevs].path, "missing"); diff --git a/cmds-fi-usage.h b/cmds-fi-usage.h index a399517f..cab38462 100644 --- a/cmds-fi-usage.h +++ b/cmds-fi-usage.h @@ -50,5 +50,6 @@ void print_device_chunks(struct device_info *devinfo, struct chunk_info *chunks_info_ptr, int chunks_info_count, unsigned unit_mode); void print_device_sizes(struct device_info *devinfo, unsigned unit_mode); +int dev_to_fsid(const char *dev, u8 *fsid); #endif diff --git a/cmds-filesystem.c b/cmds-filesystem.c index c39f2d15..17d399d5 100644 --- a/cmds-filesystem.c +++ b/cmds-filesystem.c @@ -150,7 +150,7 @@ static int cmd_filesystem_df(int argc, char **argv) return !!ret; } -static int match_search_item_kernel(__u8 *fsid, char *mnt, char *label, +static int match_search_item_kernel(u8 *fsid, char *mnt, char *label, char *search) { char uuidbuf[BTRFS_UUID_UNPARSED_SIZE]; @@ -431,33 +431,6 @@ out: return !found; } -static int dev_to_fsid(const char *dev, __u8 *fsid) -{ - struct btrfs_super_block *disk_super; - char buf[BTRFS_SUPER_INFO_SIZE]; - int ret; - int fd; - - fd = open(dev, O_RDONLY); - if (fd < 0) { - ret = -errno; - return ret; - } - - disk_super = (struct btrfs_super_block *)buf; - ret = btrfs_read_dev_super(fd, disk_super, - BTRFS_SUPER_INFO_OFFSET, SBREAD_DEFAULT); - if (ret) - goto out; - - memcpy(fsid, disk_super->fsid, BTRFS_FSID_SIZE); - ret = 0; - -out: - close(fd); - return ret; -} - static void free_fs_devices(struct btrfs_fs_devices *fs_devices) { struct btrfs_fs_devices *cur_seed, *next_seed; @@ -705,7 +678,7 @@ static int cmd_filesystem_show(int argc, char **argv) int type = 0; char mp[PATH_MAX]; char path[PATH_MAX]; - __u8 fsid[BTRFS_FSID_SIZE]; + u8 fsid[BTRFS_FSID_SIZE]; char uuid_buf[BTRFS_UUID_UNPARSED_SIZE]; unsigned unit_mode; int found = 0; @@ -893,6 +866,11 @@ static const char * const cmd_filesystem_defrag_usage[] = { "-s start defragment only from byte onward", "-l len defragment only up to len bytes", "-t size target extent size hint (default: 32M)", + "", + "Warning: most Linux kernels will break up the ref-links of COW data", + "(e.g., files copied with 'cp --reflink', snapshots) which may cause", + "considerable increase of space usage. See btrfs-filesystem(8) for", + "more information.", NULL }; @@ -1051,23 +1029,22 @@ static int cmd_filesystem_defrag(int argc, char **argv) if (fd < 0) { error("cannot open %s: %s", argv[i], strerror(errno)); - defrag_global_errors++; - close_file_or_dir(fd, dirstream); - continue; + ret = -errno; + goto next; } - if (fstat(fd, &st)) { + + ret = fstat(fd, &st); + if (ret) { error("failed to stat %s: %s", argv[i], strerror(errno)); - defrag_global_errors++; - close_file_or_dir(fd, dirstream); - continue; + ret = -errno; + goto next; } if (!(S_ISDIR(st.st_mode) || S_ISREG(st.st_mode))) { error("%s is not a directory or a regular file", argv[i]); - defrag_global_errors++; - close_file_or_dir(fd, dirstream); - continue; + ret = -EINVAL; + goto next; } if (recursive && S_ISDIR(st.st_mode)) { ret = nftw(argv[i], defrag_callback, 10, @@ -1082,20 +1059,25 @@ static int cmd_filesystem_defrag(int argc, char **argv) ret = ioctl(fd, BTRFS_IOC_DEFRAG_RANGE, &defrag_global_range); defrag_err = errno; - } - close_file_or_dir(fd, dirstream); - if (ret && defrag_err == ENOTTY) { - error( + if (ret && defrag_err == ENOTTY) { + error( "defrag range ioctl not supported in this kernel version, 2.6.33 and newer is required"); - defrag_global_errors++; - break; + defrag_global_errors++; + close_file_or_dir(fd, dirstream); + break; + } + if (ret) { + error("defrag failed on %s: %s", argv[i], + strerror(defrag_err)); + goto next; + } } - if (ret) { - error("defrag failed on %s: %s", argv[i], - strerror(defrag_err)); +next: + if (ret) defrag_global_errors++; - } + close_file_or_dir(fd, dirstream); } + if (defrag_global_errors) fprintf(stderr, "total %d failures\n", defrag_global_errors); diff --git a/cmds-inspect-dump-tree.c b/cmds-inspect-dump-tree.c index 4e72c08a..df44bb63 100644 --- a/cmds-inspect-dump-tree.c +++ b/cmds-inspect-dump-tree.c @@ -143,7 +143,7 @@ static u64 treeid_from_string(const char *str, const char **end) { "CHUNK", BTRFS_CHUNK_TREE_OBJECTID }, { "DEVICE", BTRFS_DEV_TREE_OBJECTID }, { "DEV", BTRFS_DEV_TREE_OBJECTID }, - { "FS_TREE", BTRFS_FS_TREE_OBJECTID }, + { "FS", BTRFS_FS_TREE_OBJECTID }, { "CSUM", BTRFS_CSUM_TREE_OBJECTID }, { "CHECKSUM", BTRFS_CSUM_TREE_OBJECTID }, { "QUOTA", BTRFS_QUOTA_TREE_OBJECTID }, diff --git a/cmds-rescue.c b/cmds-rescue.c index d1bec021..c40088ad 100644 --- a/cmds-rescue.c +++ b/cmds-rescue.c @@ -20,6 +20,7 @@ #include <getopt.h> #include "ctree.h" +#include "volumes.h" #include "transaction.h" #include "disk-io.h" #include "commands.h" @@ -177,6 +178,7 @@ static int cmd_rescue_zero_log(int argc, char **argv) } else if (ret) { error("%s is currently mounted", devname); ret = -EBUSY; + goto out; } root = open_ctree(devname, 0, OPEN_CTREE_WRITES | OPEN_CTREE_PARTIAL); @@ -201,6 +203,51 @@ out: return !!ret; } +static const char * const cmd_rescue_fix_device_size_usage[] = { + "btrfs rescue fix-device-size <device>", + "Re-align device and super block sizes. Usable if newer kernel refuse to mount it due to mismatch super size", + "", + NULL +}; + +static int cmd_rescue_fix_device_size(int argc, char **argv) +{ + struct btrfs_fs_info *fs_info; + char *devname; + int ret; + + clean_args_no_options(argc, argv, cmd_rescue_fix_device_size_usage); + + if (check_argc_exact(argc, 2)) + usage(cmd_rescue_fix_device_size_usage); + + devname = argv[optind]; + ret = check_mounted(devname); + if (ret < 0) { + error("could not check mount status: %s", strerror(-ret)); + goto out; + } else if (ret) { + error("%s is currently mounted", devname); + ret = -EBUSY; + goto out; + } + + fs_info = open_ctree_fs_info(devname, 0, 0, 0, OPEN_CTREE_WRITES | + OPEN_CTREE_PARTIAL); + if (!fs_info) { + error("could not open btrfs"); + ret = -EIO; + goto out; + } + + ret = btrfs_fix_device_and_super_size(fs_info); + if (ret > 0) + ret = 0; + close_ctree(fs_info->tree_root); +out: + return !!ret; +} + static const char rescue_cmd_group_info[] = "toolbox for specific rescue operations"; @@ -211,6 +258,8 @@ const struct cmd_group rescue_cmd_group = { { "super-recover", cmd_rescue_super_recover, cmd_rescue_super_recover_usage, NULL, 0}, { "zero-log", cmd_rescue_zero_log, cmd_rescue_zero_log_usage, NULL, 0}, + { "fix-device-size", cmd_rescue_fix_device_size, + cmd_rescue_fix_device_size_usage, NULL, 0}, NULL_CMD_STRUCT } }; diff --git a/cmds-subvolume.c b/cmds-subvolume.c index e7caa05a..dc626a64 100644 --- a/cmds-subvolume.c +++ b/cmds-subvolume.c @@ -858,8 +858,11 @@ out: } static const char * const cmd_subvol_set_default_usage[] = { + "btrfs subvolume set-default <subvolume>\n" "btrfs subvolume set-default <subvolid> <path>", - "Set the default subvolume of a filesystem", + "Set the default subvolume of the filesystem mounted as default.", + "The subvolume can be specified by its path,", + "or the pair of subvolume id and path to the filesystem.", NULL }; @@ -873,17 +876,43 @@ static int cmd_subvol_set_default(int argc, char **argv) clean_args_no_options(argc, argv, cmd_subvol_set_default_usage); - if (check_argc_exact(argc - optind, 2)) + if (check_argc_min(argc - optind, 1) || + check_argc_max(argc - optind, 2)) usage(cmd_subvol_set_default_usage); - subvolid = argv[optind]; - path = argv[optind + 1]; + if (argc - optind == 1) { + /* path to the subvolume is specified */ + path = argv[optind]; - objectid = arg_strtou64(subvolid); + ret = test_issubvolume(path); + if (ret < 0) { + error("stat error: %s", strerror(-ret)); + return 1; + } else if (!ret) { + error("'%s' is not a subvolume", path); + return 1; + } - fd = btrfs_open_dir(path, &dirstream, 1); - if (fd < 0) - return 1; + fd = btrfs_open_dir(path, &dirstream, 1); + if (fd < 0) + return 1; + + ret = lookup_path_rootid(fd, &objectid); + if (ret) { + error("unable to get subvol id: %s", strerror(-ret)); + close_file_or_dir(fd, dirstream); + return 1; + } + } else { + /* subvol id and path to the filesystem are specified */ + subvolid = argv[optind]; + path = argv[optind + 1]; + objectid = arg_strtou64(subvolid); + + fd = btrfs_open_dir(path, &dirstream, 1); + if (fd < 0) + return 1; + } ret = ioctl(fd, BTRFS_IOC_DEFAULT_SUBVOL, &objectid); e = errno; diff --git a/config/config.guess b/config/config.guess index b9da8803..1bf683d0 100755 --- a/config/config.guess +++ b/config/config.guess @@ -1,8 +1,8 @@ #! /bin/sh # Attempt to guess a canonical system name. -# Copyright 1992-2014 Free Software Foundation, Inc. +# Copyright 1992-2017 Free Software Foundation, Inc. -timestamp='2014-11-04' +timestamp='2017-05-27' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -27,7 +27,7 @@ timestamp='2014-11-04' # Originally written by Per Bothner; maintained since 2000 by Ben Elliston. # # You can get the latest version of this script from: -# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess # # Please send patches to <config-patches@gnu.org>. @@ -50,7 +50,7 @@ version="\ GNU config.guess ($timestamp) Originally written by Per Bothner. -Copyright 1992-2014 Free Software Foundation, Inc. +Copyright 1992-2017 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -178,19 +178,29 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in # Note: NetBSD doesn't particularly care about the vendor # portion of the name. We always set it to "unknown". sysctl="sysctl -n hw.machine_arch" - UNAME_MACHINE_ARCH=`(/sbin/$sysctl 2>/dev/null || \ - /usr/sbin/$sysctl 2>/dev/null || echo unknown)` + UNAME_MACHINE_ARCH=`(uname -p 2>/dev/null || \ + /sbin/$sysctl 2>/dev/null || \ + /usr/sbin/$sysctl 2>/dev/null || \ + echo unknown)` case "${UNAME_MACHINE_ARCH}" in armeb) machine=armeb-unknown ;; arm*) machine=arm-unknown ;; sh3el) machine=shl-unknown ;; sh3eb) machine=sh-unknown ;; sh5el) machine=sh5le-unknown ;; - *) machine=${UNAME_MACHINE_ARCH}-unknown ;; + earmv*) + arch=`echo ${UNAME_MACHINE_ARCH} | sed -e 's,^e\(armv[0-9]\).*$,\1,'` + endian=`echo ${UNAME_MACHINE_ARCH} | sed -ne 's,^.*\(eb\)$,\1,p'` + machine=${arch}${endian}-${VENDOR}-unknown + ;; + *) machine=${UNAME_MACHINE_ARCH}-${VENDOR}-unknown ;; esac # The Operating System including object format, if it has switched - # to ELF recently, or will in the future. + # to ELF recently (or will in the future) and ABI. case "${UNAME_MACHINE_ARCH}" in + earm*) + os=netbsdelf + ;; arm*|i386|m68k|ns32k|sh3*|sparc|vax) eval $set_cc_for_build if echo __ELF__ | $CC_FOR_BUILD -E - 2>/dev/null \ @@ -207,6 +217,13 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in os=netbsd ;; esac + # Determine ABI tags. + case "${UNAME_MACHINE_ARCH}" in + earm*) + expr='s/^earmv[0-9]/-eabi/;s/eb$//' + abi=`echo ${UNAME_MACHINE_ARCH} | sed -e "$expr"` + ;; + esac # The OS release # Debian GNU/NetBSD machines have a different userland, and # thus, need a distinct triplet. However, they do not need @@ -217,13 +234,13 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in release='-gnu' ;; *) - release=`echo ${UNAME_RELEASE}|sed -e 's/[-_].*/\./'` + release=`echo ${UNAME_RELEASE} | sed -e 's/[-_].*//' | cut -d. -f1,2` ;; esac # Since CPU_TYPE-MANUFACTURER-KERNEL-OPERATING_SYSTEM: # contains redundant information, the shorter form: # CPU_TYPE-MANUFACTURER-OPERATING_SYSTEM is used. - echo "${machine}-${os}${release}" + echo "${machine}-${os}${release}${abi}" exit ;; *:Bitrig:*:*) UNAME_MACHINE_ARCH=`arch | sed 's/Bitrig.//'` @@ -233,6 +250,10 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in UNAME_MACHINE_ARCH=`arch | sed 's/OpenBSD.//'` echo ${UNAME_MACHINE_ARCH}-${VENDOR}-openbsd${UNAME_RELEASE} exit ;; + *:LibertyBSD:*:*) + UNAME_MACHINE_ARCH=`arch | sed 's/^.*BSD\.//'` + echo ${UNAME_MACHINE_ARCH}-${VENDOR}-libertybsd${UNAME_RELEASE} + exit ;; *:ekkoBSD:*:*) echo ${UNAME_MACHINE}-${VENDOR}-ekkobsd${UNAME_RELEASE} exit ;; @@ -245,6 +266,9 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in *:MirBSD:*:*) echo ${UNAME_MACHINE}-${VENDOR}-mirbsd${UNAME_RELEASE} exit ;; + *:Sortix:*:*) + echo ${UNAME_MACHINE}-${VENDOR}-sortix + exit ;; alpha:OSF1:*:*) case $UNAME_RELEASE in *4.0) @@ -261,42 +285,42 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in ALPHA_CPU_TYPE=`/usr/sbin/psrinfo -v | sed -n -e 's/^ The alpha \(.*\) processor.*$/\1/p' | head -n 1` case "$ALPHA_CPU_TYPE" in "EV4 (21064)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "EV4.5 (21064)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "LCA4 (21066/21068)") - UNAME_MACHINE="alpha" ;; + UNAME_MACHINE=alpha ;; "EV5 (21164)") - UNAME_MACHINE="alphaev5" ;; + UNAME_MACHINE=alphaev5 ;; "EV5.6 (21164A)") - UNAME_MACHINE="alphaev56" ;; + UNAME_MACHINE=alphaev56 ;; "EV5.6 (21164PC)") - UNAME_MACHINE="alphapca56" ;; + UNAME_MACHINE=alphapca56 ;; "EV5.7 (21164PC)") - UNAME_MACHINE="alphapca57" ;; + UNAME_MACHINE=alphapca57 ;; "EV6 (21264)") - UNAME_MACHINE="alphaev6" ;; + UNAME_MACHINE=alphaev6 ;; "EV6.7 (21264A)") - UNAME_MACHINE="alphaev67" ;; + UNAME_MACHINE=alphaev67 ;; "EV6.8CB (21264C)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.8AL (21264B)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.8CX (21264D)") - UNAME_MACHINE="alphaev68" ;; + UNAME_MACHINE=alphaev68 ;; "EV6.9A (21264/EV69A)") - UNAME_MACHINE="alphaev69" ;; + UNAME_MACHINE=alphaev69 ;; "EV7 (21364)") - UNAME_MACHINE="alphaev7" ;; + UNAME_MACHINE=alphaev7 ;; "EV7.9 (21364A)") - UNAME_MACHINE="alphaev79" ;; + UNAME_MACHINE=alphaev79 ;; esac # A Pn.n version is a patched version. # A Vn.n version is a released version. # A Tn.n version is a released field test version. # A Xn.n version is an unreleased experimental baselevel. # 1.2 uses "1.2" for uname -r. - echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` + echo ${UNAME_MACHINE}-dec-osf`echo ${UNAME_RELEASE} | sed -e 's/^[PVTX]//' | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` # Reset EXIT trap before exiting to avoid spurious non-zero exit code. exitcode=$? trap '' 0 @@ -369,16 +393,16 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in exit ;; i86pc:SunOS:5.*:* | i86xen:SunOS:5.*:*) eval $set_cc_for_build - SUN_ARCH="i386" + SUN_ARCH=i386 # If there is a compiler, see if it is configured for 64-bit objects. # Note that the Sun cc does not turn __LP64__ into 1 like gcc does. # This test works for both compilers. - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then if (echo '#ifdef __amd64'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ grep IS_64BIT_ARCH >/dev/null then - SUN_ARCH="x86_64" + SUN_ARCH=x86_64 fi fi echo ${SUN_ARCH}-pc-solaris2`echo ${UNAME_RELEASE}|sed -e 's/[^.]*//'` @@ -403,7 +427,7 @@ case "${UNAME_MACHINE}:${UNAME_SYSTEM}:${UNAME_RELEASE}:${UNAME_VERSION}" in exit ;; sun*:*:4.2BSD:*) UNAME_RELEASE=`(sed 1q /etc/motd | awk '{print substr($5,1,3)}') 2>/dev/null` - test "x${UNAME_RELEASE}" = "x" && UNAME_RELEASE=3 + test "x${UNAME_RELEASE}" = x && UNAME_RELEASE=3 case "`/bin/arch`" in sun3) echo m68k-sun-sunos${UNAME_RELEASE} @@ -628,13 +652,13 @@ EOF sc_cpu_version=`/usr/bin/getconf SC_CPU_VERSION 2>/dev/null` sc_kernel_bits=`/usr/bin/getconf SC_KERNEL_BITS 2>/dev/null` case "${sc_cpu_version}" in - 523) HP_ARCH="hppa1.0" ;; # CPU_PA_RISC1_0 - 528) HP_ARCH="hppa1.1" ;; # CPU_PA_RISC1_1 + 523) HP_ARCH=hppa1.0 ;; # CPU_PA_RISC1_0 + 528) HP_ARCH=hppa1.1 ;; # CPU_PA_RISC1_1 532) # CPU_PA_RISC2_0 case "${sc_kernel_bits}" in - 32) HP_ARCH="hppa2.0n" ;; - 64) HP_ARCH="hppa2.0w" ;; - '') HP_ARCH="hppa2.0" ;; # HP-UX 10.20 + 32) HP_ARCH=hppa2.0n ;; + 64) HP_ARCH=hppa2.0w ;; + '') HP_ARCH=hppa2.0 ;; # HP-UX 10.20 esac ;; esac fi @@ -673,11 +697,11 @@ EOF exit (0); } EOF - (CCOPTS= $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` + (CCOPTS="" $CC_FOR_BUILD -o $dummy $dummy.c 2>/dev/null) && HP_ARCH=`$dummy` test -z "$HP_ARCH" && HP_ARCH=hppa fi ;; esac - if [ ${HP_ARCH} = "hppa2.0w" ] + if [ ${HP_ARCH} = hppa2.0w ] then eval $set_cc_for_build @@ -690,12 +714,12 @@ EOF # $ CC_FOR_BUILD="cc +DA2.0w" ./config.guess # => hppa64-hp-hpux11.23 - if echo __LP64__ | (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | + if echo __LP64__ | (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | grep -q __LP64__ then - HP_ARCH="hppa2.0w" + HP_ARCH=hppa2.0w else - HP_ARCH="hppa64" + HP_ARCH=hppa64 fi fi echo ${HP_ARCH}-hp-hpux${HPUX_REV} @@ -800,14 +824,14 @@ EOF echo craynv-cray-unicosmp${UNAME_RELEASE} | sed -e 's/\.[^.]*$/.X/' exit ;; F30[01]:UNIX_System_V:*:* | F700:UNIX_System_V:*:*) - FUJITSU_PROC=`uname -m | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz'` - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` + FUJITSU_PROC=`uname -m | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz` + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` FUJITSU_REL=`echo ${UNAME_RELEASE} | sed -e 's/ /_/'` echo "${FUJITSU_PROC}-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; 5000:UNIX_System_V:4.*:*) - FUJITSU_SYS=`uname -p | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/\///'` - FUJITSU_REL=`echo ${UNAME_RELEASE} | tr 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' 'abcdefghijklmnopqrstuvwxyz' | sed -e 's/ /_/'` + FUJITSU_SYS=`uname -p | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/\///'` + FUJITSU_REL=`echo ${UNAME_RELEASE} | tr ABCDEFGHIJKLMNOPQRSTUVWXYZ abcdefghijklmnopqrstuvwxyz | sed -e 's/ /_/'` echo "sparc-fujitsu-${FUJITSU_SYS}${FUJITSU_REL}" exit ;; i*86:BSD/386:*:* | i*86:BSD/OS:*:* | *:Ascend\ Embedded/OS:*:*) @@ -823,10 +847,11 @@ EOF UNAME_PROCESSOR=`/usr/bin/uname -p` case ${UNAME_PROCESSOR} in amd64) - echo x86_64-${VENDOR}-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; - *) - echo ${UNAME_PROCESSOR}-${VENDOR}-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` ;; + UNAME_PROCESSOR=x86_64 ;; + i386) + UNAME_PROCESSOR=i586 ;; esac + echo ${UNAME_PROCESSOR}-${VENDOR}-freebsd`echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'` exit ;; i*:CYGWIN*:*) echo ${UNAME_MACHINE}-pc-cygwin @@ -889,7 +914,7 @@ EOF exit ;; *:GNU/*:*:*) # other systems with GNU libc and userland - echo ${UNAME_MACHINE}-${VENDOR}-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr '[A-Z]' '[a-z]'``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC} + echo ${UNAME_MACHINE}-${VENDOR}-`echo ${UNAME_SYSTEM} | sed 's,^[^/]*/,,' | tr "[:upper:]" "[:lower:]"``echo ${UNAME_RELEASE}|sed -e 's/[-(].*//'`-${LIBC} exit ;; i*86:Minix:*:*) echo ${UNAME_MACHINE}-pc-minix @@ -912,7 +937,7 @@ EOF EV68*) UNAME_MACHINE=alphaev68 ;; esac objdump --private-headers /bin/sh | grep -q ld.so.1 - if test "$?" = 0 ; then LIBC="gnulibc1" ; fi + if test "$?" = 0 ; then LIBC=gnulibc1 ; fi echo ${UNAME_MACHINE}-${VENDOR}-linux-${LIBC} exit ;; arc:Linux:*:* | arceb:Linux:*:*) @@ -943,6 +968,9 @@ EOF crisv32:Linux:*:*) echo ${UNAME_MACHINE}-axis-linux-${LIBC} exit ;; + e2k:Linux:*:*) + echo ${UNAME_MACHINE}-${VENDOR}-linux-${LIBC} + exit ;; frv:Linux:*:*) echo ${UNAME_MACHINE}-${VENDOR}-linux-${LIBC} exit ;; @@ -955,6 +983,9 @@ EOF ia64:Linux:*:*) echo ${UNAME_MACHINE}-${VENDOR}-linux-${LIBC} exit ;; + k1om:Linux:*:*) + echo ${UNAME_MACHINE}-${VENDOR}-linux-${LIBC} + exit ;; m32r*:Linux:*:*) echo ${UNAME_MACHINE}-${VENDOR}-linux-${LIBC} exit ;; @@ -980,6 +1011,9 @@ EOF eval `$CC_FOR_BUILD -E $dummy.c 2>/dev/null | grep '^CPU'` test x"${CPU}" != x && { echo "${CPU}-${VENDOR}-linux-${LIBC}"; exit; } ;; + mips64el:Linux:*:*) + echo ${UNAME_MACHINE}-${VENDOR}-linux-${LIBC} + exit ;; openrisc*:Linux:*:*) echo or1k-${VENDOR}-linux-${LIBC} exit ;; @@ -1012,9 +1046,12 @@ EOF ppcle:Linux:*:*) echo powerpcle-${VENDOR}-linux-${LIBC} exit ;; - s390:Linux:*:* | s390x:Linux:*:*) + riscv32:Linux:*:* | riscv64:Linux:*:*) echo ${UNAME_MACHINE}-${VENDOR}-linux-${LIBC} exit ;; + s390:Linux:*:* | s390x:Linux:*:*) + echo ${UNAME_MACHINE}-ibm-linux-${LIBC} + exit ;; sh64*:Linux:*:*) echo ${UNAME_MACHINE}-${VENDOR}-linux-${LIBC} exit ;; @@ -1031,7 +1068,7 @@ EOF echo ${UNAME_MACHINE}-dec-linux-${LIBC} exit ;; x86_64:Linux:*:*) - echo ${UNAME_MACHINE}-${VENDOR}-linux-${LIBC} + echo ${UNAME_MACHINE}-pc-linux-${LIBC} exit ;; xtensa*:Linux:*:*) echo ${UNAME_MACHINE}-${VENDOR}-linux-${LIBC} @@ -1110,7 +1147,7 @@ EOF # uname -m prints for DJGPP always 'pc', but it prints nothing about # the processor, so we play safe by assuming i586. # Note: whatever this is, it MUST be the same as what config.sub - # prints for the "djgpp" host, or else GDB configury will decide that + # prints for the "djgpp" host, or else GDB configure will decide that # this is a cross-build. echo i586-pc-msdosdjgpp exit ;; @@ -1259,6 +1296,9 @@ EOF SX-8R:SUPER-UX:*:*) echo sx8r-nec-superux${UNAME_RELEASE} exit ;; + SX-ACE:SUPER-UX:*:*) + echo sxace-nec-superux${UNAME_RELEASE} + exit ;; Power*:Rhapsody:*:*) echo powerpc-apple-rhapsody${UNAME_RELEASE} exit ;; @@ -1272,16 +1312,23 @@ EOF UNAME_PROCESSOR=powerpc fi if test `echo "$UNAME_RELEASE" | sed -e 's/\..*//'` -le 10 ; then - if [ "$CC_FOR_BUILD" != 'no_compiler_found' ]; then + if [ "$CC_FOR_BUILD" != no_compiler_found ]; then if (echo '#ifdef __LP64__'; echo IS_64BIT_ARCH; echo '#endif') | \ - (CCOPTS= $CC_FOR_BUILD -E - 2>/dev/null) | \ - grep IS_64BIT_ARCH >/dev/null + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_64BIT_ARCH >/dev/null then case $UNAME_PROCESSOR in i386) UNAME_PROCESSOR=x86_64 ;; powerpc) UNAME_PROCESSOR=powerpc64 ;; esac fi + # On 10.4-10.6 one might compile for PowerPC via gcc -arch ppc + if (echo '#ifdef __POWERPC__'; echo IS_PPC; echo '#endif') | \ + (CCOPTS="" $CC_FOR_BUILD -E - 2>/dev/null) | \ + grep IS_PPC >/dev/null + then + UNAME_PROCESSOR=powerpc + fi fi elif test "$UNAME_PROCESSOR" = i386 ; then # Avoid executing cc on OS X 10.9, as it ships with a stub @@ -1296,7 +1343,7 @@ EOF exit ;; *:procnto*:*:* | *:QNX:[0123456789]*:*) UNAME_PROCESSOR=`uname -p` - if test "$UNAME_PROCESSOR" = "x86"; then + if test "$UNAME_PROCESSOR" = x86; then UNAME_PROCESSOR=i386 UNAME_MACHINE=pc fi @@ -1305,15 +1352,18 @@ EOF *:QNX:*:4*) echo i386-pc-qnx exit ;; - NEO-?:NONSTOP_KERNEL:*:*) + NEO-*:NONSTOP_KERNEL:*:*) echo neo-tandem-nsk${UNAME_RELEASE} exit ;; NSE-*:NONSTOP_KERNEL:*:*) echo nse-tandem-nsk${UNAME_RELEASE} exit ;; - NSR-?:NONSTOP_KERNEL:*:*) + NSR-*:NONSTOP_KERNEL:*:*) echo nsr-tandem-nsk${UNAME_RELEASE} exit ;; + NSX-*:NONSTOP_KERNEL:*:*) + echo nsx-tandem-nsk${UNAME_RELEASE} + exit ;; *:NonStop-UX:*:*) echo mips-compaq-nonstopux exit ;; @@ -1327,7 +1377,7 @@ EOF # "uname -m" is not consistent, so use $cputype instead. 386 # is converted to i386 for consistency with other x86 # operating systems. - if test "$cputype" = "386"; then + if test "$cputype" = 386; then UNAME_MACHINE=i386 else UNAME_MACHINE="$cputype" @@ -1369,7 +1419,7 @@ EOF echo i386-pc-xenix exit ;; i*86:skyos:*:*) - echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE}` | sed -e 's/ .*$//' + echo ${UNAME_MACHINE}-pc-skyos`echo ${UNAME_RELEASE} | sed -e 's/ .*$//'` exit ;; i*86:rdos:*:*) echo ${UNAME_MACHINE}-pc-rdos @@ -1380,23 +1430,25 @@ EOF x86_64:VMkernel:*:*) echo ${UNAME_MACHINE}-${VENDOR}-esx exit ;; + amd64:Isilon\ OneFS:*:*) + echo x86_64-${VENDOR}-onefs + exit ;; esac cat >&2 <<EOF $0: unable to guess system type -This script, last modified $timestamp, has failed to recognize -the operating system you are using. It is advised that you -download the most up to date version of the config scripts from +This script (version $timestamp), has failed to recognize the +operating system you are using. If your script is old, overwrite +config.guess and config.sub with the latest versions from: - http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess;hb=HEAD + http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.guess and - http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD + http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub -If the version you run ($0) is already up to date, please -send the following data and any information you think might be -pertinent to <config-patches@gnu.org> in order to provide the needed -information to handle your system. +If $0 has already been updated, send the following data and any +information you think might be pertinent to config-patches@gnu.org to +provide the necessary information to handle your system. config.guess timestamp = $timestamp diff --git a/config/config.sub b/config/config.sub index a93c4879..0399c1a5 100755 --- a/config/config.sub +++ b/config/config.sub @@ -1,8 +1,8 @@ #! /bin/sh # Configuration validation subroutine script. -# Copyright 1992-2014 Free Software Foundation, Inc. +# Copyright 1992-2017 Free Software Foundation, Inc. -timestamp='2014-12-03' +timestamp='2017-04-02' # This file is free software; you can redistribute it and/or modify it # under the terms of the GNU General Public License as published by @@ -33,7 +33,7 @@ timestamp='2014-12-03' # Otherwise, we print the canonical config type on stdout and succeed. # You can get the latest version of this script from: -# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub;hb=HEAD +# http://git.savannah.gnu.org/gitweb/?p=config.git;a=blob_plain;f=config.sub # This file is supposed to be the same for all GNU packages # and recognize all the CPU types, system types and aliases @@ -53,8 +53,7 @@ timestamp='2014-12-03' me=`echo "$0" | sed -e 's,.*/,,'` usage="\ -Usage: $0 [OPTION] CPU-MFR-OPSYS - $0 [OPTION] ALIAS +Usage: $0 [OPTION] CPU-MFR-OPSYS or ALIAS Canonicalize a configuration name. @@ -68,7 +67,7 @@ Report bugs and patches to <config-patches@gnu.org>." version="\ GNU config.sub ($timestamp) -Copyright 1992-2014 Free Software Foundation, Inc. +Copyright 1992-2017 Free Software Foundation, Inc. This is free software; see the source for copying conditions. There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE." @@ -117,8 +116,8 @@ maybe_os=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\2/'` case $maybe_os in nto-qnx* | linux-gnu* | linux-android* | linux-dietlibc | linux-newlib* | \ linux-musl* | linux-uclibc* | uclinux-uclibc* | uclinux-gnu* | kfreebsd*-gnu* | \ - knetbsd*-gnu* | netbsd*-gnu* | \ - kopensolaris*-gnu* | \ + knetbsd*-gnu* | netbsd*-gnu* | netbsd*-eabi* | \ + kopensolaris*-gnu* | cloudabi*-eabi* | \ storm-chaos* | os2-emx* | rtmk-nova*) os=-$maybe_os basic_machine=`echo $1 | sed 's/^\(.*\)-\([^-]*-[^-]*\)$/\1/'` @@ -255,15 +254,16 @@ case $basic_machine in | arc | arceb \ | arm | arm[bl]e | arme[lb] | armv[2-8] | armv[3-8][lb] | armv7[arm] \ | avr | avr32 \ + | ba \ | be32 | be64 \ | bfin \ | c4x | c8051 | clipper \ | d10v | d30v | dlx | dsp16xx \ - | epiphany \ - | fido | fr30 | frv \ + | e2k | epiphany \ + | fido | fr30 | frv | ft32 \ | h8300 | h8500 | hppa | hppa1.[01] | hppa2.0 | hppa2.0[nw] | hppa64 \ | hexagon \ - | i370 | i860 | i960 | ia64 \ + | i370 | i860 | i960 | ia16 | ia64 \ | ip2k | iq2000 \ | k1om \ | le32 | le64 \ @@ -301,11 +301,12 @@ case $basic_machine in | open8 | or1k | or1knd | or32 \ | pdp10 | pdp11 | pj | pjl \ | powerpc | powerpc64 | powerpc64le | powerpcle \ + | pru \ | pyramid \ | riscv32 | riscv64 \ | rl78 | rx \ | score \ - | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[34]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ + | sh | sh[1234] | sh[24]a | sh[24]aeb | sh[23]e | sh[234]eb | sheb | shbe | shle | sh[1234]le | sh3ele \ | sh64 | sh64le \ | sparc | sparc64 | sparc64b | sparc64v | sparc86x | sparclet | sparclite \ | sparcv8 | sparcv9 | sparcv9b | sparcv9v \ @@ -314,6 +315,7 @@ case $basic_machine in | ubicom32 \ | v850 | v850e | v850e1 | v850e2 | v850es | v850e2v3 \ | visium \ + | wasm32 \ | we32k \ | x86 | xc16x | xstormy16 | xtensa \ | z8k | z80) @@ -376,17 +378,18 @@ case $basic_machine in | alphapca5[67]-* | alpha64pca5[67]-* | arc-* | arceb-* \ | arm-* | armbe-* | armle-* | armeb-* | armv*-* \ | avr-* | avr32-* \ + | ba-* \ | be32-* | be64-* \ | bfin-* | bs2000-* \ | c[123]* | c30-* | [cjt]90-* | c4x-* \ | c8051-* | clipper-* | craynv-* | cydra-* \ | d10v-* | d30v-* | dlx-* \ - | elxsi-* \ + | e2k-* | elxsi-* \ | f30[01]-* | f700-* | fido-* | fr30-* | frv-* | fx80-* \ | h8300-* | h8500-* \ | hppa-* | hppa1.[01]-* | hppa2.0-* | hppa2.0[nw]-* | hppa64-* \ | hexagon-* \ - | i*86-* | i860-* | i960-* | ia64-* \ + | i*86-* | i860-* | i960-* | ia16-* | ia64-* \ | ip2k-* | iq2000-* \ | k1om-* \ | le32-* | le64-* \ @@ -427,13 +430,15 @@ case $basic_machine in | orion-* \ | pdp10-* | pdp11-* | pj-* | pjl-* | pn-* | power-* \ | powerpc-* | powerpc64-* | powerpc64le-* | powerpcle-* \ + | pru-* \ | pyramid-* \ + | riscv32-* | riscv64-* \ | rl78-* | romp-* | rs6000-* | rx-* \ | sh-* | sh[1234]-* | sh[24]a-* | sh[24]aeb-* | sh[23]e-* | sh[34]eb-* | sheb-* | shbe-* \ | shle-* | sh[1234]le-* | sh3ele-* | sh64-* | sh64le-* \ | sparc-* | sparc64-* | sparc64b-* | sparc64v-* | sparc86x-* | sparclet-* \ | sparclite-* \ - | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx?-* \ + | sparcv8-* | sparcv9-* | sparcv9b-* | sparcv9v-* | sv1-* | sx*-* \ | tahoe-* \ | tic30-* | tic4x-* | tic54x-* | tic55x-* | tic6x-* | tic80-* \ | tile*-* \ @@ -442,6 +447,7 @@ case $basic_machine in | v850-* | v850e-* | v850e1-* | v850es-* | v850e2-* | v850e2v3-* \ | vax-* \ | visium-* \ + | wasm32-* \ | we32k-* \ | x86-* | x86_64-* | xc16x-* | xps100-* \ | xstormy16-* | xtensa*-* \ @@ -518,6 +524,9 @@ case $basic_machine in basic_machine=i386-pc os=-aros ;; + asmjs) + basic_machine=asmjs-unknown + ;; aux) basic_machine=m68k-apple os=-aux @@ -638,6 +647,14 @@ case $basic_machine in basic_machine=m68k-bull os=-sysv3 ;; + e500v[12]) + basic_machine=powerpc-unknown + os=$os"spe" + ;; + e500v[12]-*) + basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` + os=$os"spe" + ;; ebmon29k) basic_machine=a29k-amd os=-ebmon @@ -933,6 +950,9 @@ case $basic_machine in nsr-tandem) basic_machine=nsr-tandem ;; + nsx-tandem) + basic_machine=nsx-tandem + ;; op50n-* | op60c-*) basic_machine=hppa1.1-oki os=-proelf @@ -1017,7 +1037,7 @@ case $basic_machine in ppc-* | ppcbe-*) basic_machine=powerpc-`echo $basic_machine | sed 's/^[^-]*-//'` ;; - ppcle | powerpclittle | ppc-le | powerpc-little) + ppcle | powerpclittle) basic_machine=powerpcle-unknown ;; ppcle-* | powerpclittle-*) @@ -1027,7 +1047,7 @@ case $basic_machine in ;; ppc64-*) basic_machine=powerpc64-`echo $basic_machine | sed 's/^[^-]*-//'` ;; - ppc64le | powerpc64little | ppc64-le | powerpc64-little) + ppc64le | powerpc64little) basic_machine=powerpc64le-unknown ;; ppc64le-* | powerpc64little-*) @@ -1234,6 +1254,9 @@ case $basic_machine in basic_machine=a29k-wrs os=-vxworks ;; + wasm32) + basic_machine=wasm32-unknown + ;; w65*) basic_machine=w65-wdc os=-none @@ -1379,18 +1402,18 @@ case $os in | -hpux* | -unos* | -osf* | -luna* | -dgux* | -auroraux* | -solaris* \ | -sym* | -kopensolaris* | -plan9* \ | -amigaos* | -amigados* | -msdos* | -newsos* | -unicos* | -aof* \ - | -aos* | -aros* \ + | -aos* | -aros* | -cloudabi* | -sortix* \ | -nindy* | -vxsim* | -vxworks* | -ebmon* | -hms* | -mvs* \ | -clix* | -riscos* | -uniplus* | -iris* | -rtu* | -xenix* \ | -hiux* | -386bsd* | -knetbsd* | -mirbsd* | -netbsd* \ - | -bitrig* | -openbsd* | -solidbsd* \ + | -bitrig* | -openbsd* | -solidbsd* | -libertybsd* \ | -ekkobsd* | -kfreebsd* | -freebsd* | -riscix* | -lynxos* \ | -bosx* | -nextstep* | -cxux* | -aout* | -elf* | -oabi* \ | -ptx* | -coff* | -ecoff* | -winnt* | -domain* | -vsta* \ | -udi* | -eabi* | -lites* | -ieee* | -go32* | -aux* \ - | -chorusos* | -chorusrdb* | -cegcc* \ + | -chorusos* | -chorusrdb* | -cegcc* | -glidix* \ | -cygwin* | -msys* | -pe* | -psos* | -moss* | -proelf* | -rtems* \ - | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ + | -midipix* | -mingw32* | -mingw64* | -linux-gnu* | -linux-android* \ | -linux-newlib* | -linux-musl* | -linux-uclibc* \ | -uxpv* | -beos* | -mpeix* | -udk* | -moxiebox* \ | -interix* | -uwin* | -mks* | -rhapsody* | -darwin* | -opened* \ @@ -1399,7 +1422,8 @@ case $os in | -os2* | -vos* | -palmos* | -uclinux* | -nucleus* \ | -morphos* | -superux* | -rtmk* | -rtmk-nova* | -windiss* \ | -powermax* | -dnix* | -nx6 | -nx7 | -sei* | -dragonfly* \ - | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* | -tirtos*) + | -skyos* | -haiku* | -rdos* | -toppers* | -drops* | -es* \ + | -onefs* | -tirtos* | -phoenix* | -fuchsia* | -redox*) # Remember, each alternative MUST END IN *, to match a version number. ;; -qnx*) @@ -1531,6 +1555,8 @@ case $os in ;; -nacl*) ;; + -ios) + ;; -none) ;; *) @@ -1626,6 +1652,9 @@ case $basic_machine in sparc-* | *-sun) os=-sunos4.1.1 ;; + pru-*) + os=-elf + ;; *-be) os=-beos ;; diff --git a/config/install-sh b/config/install-sh index 0b0fdcbb..0360b79e 100755 --- a/config/install-sh +++ b/config/install-sh @@ -1,7 +1,7 @@ #!/bin/sh # install - install a program, script, or datafile -scriptversion=2013-12-25.23; # UTC +scriptversion=2016-01-11.22; # UTC # This originates from X11R5 (mit/util/scripts/install.sh), which was # later released in X11R6 (xc/config/util/install.sh) with the @@ -496,6 +496,6 @@ done # eval: (add-hook 'write-file-hooks 'time-stamp) # time-stamp-start: "scriptversion=" # time-stamp-format: "%:y-%02m-%02d.%02H" -# time-stamp-time-zone: "UTC" +# time-stamp-time-zone: "UTC0" # time-stamp-end: "; # UTC" # End: @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for btrfs-progs v4.13.3. +# Generated by GNU Autoconf 2.69 for btrfs-progs v4.14.1. # # Report bugs to <linux-btrfs@vger.kernel.org>. # @@ -580,8 +580,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='btrfs-progs' PACKAGE_TARNAME='btrfs-progs' -PACKAGE_VERSION='v4.13.3' -PACKAGE_STRING='btrfs-progs v4.13.3' +PACKAGE_VERSION='v4.14.1' +PACKAGE_STRING='btrfs-progs v4.14.1' PACKAGE_BUGREPORT='linux-btrfs@vger.kernel.org' PACKAGE_URL='http://btrfs.wiki.kernel.org' @@ -1302,7 +1302,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures btrfs-progs v4.13.3 to adapt to many kinds of systems. +\`configure' configures btrfs-progs v4.14.1 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1367,7 +1367,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of btrfs-progs v4.13.3:";; + short | recursive ) echo "Configuration of btrfs-progs v4.14.1:";; esac cat <<\_ACEOF @@ -1379,7 +1379,7 @@ Optional Features: --disable-backtrace disable btrfs backtrace --disable-documentation do not build domumentation --disable-convert do not build btrfs-convert - --enable-zstd[=auto] build with zstd support (default: auto) + --disable-zstd[=yes] build with zstd support (default: yes) Optional Packages: --with-PACKAGE[=ARG] use PACKAGE [ARG=yes] @@ -1490,7 +1490,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -btrfs-progs configure v4.13.3 +btrfs-progs configure v4.14.1 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -1859,7 +1859,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by btrfs-progs $as_me v4.13.3, which was +It was created by btrfs-progs $as_me v4.14.1, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -6312,7 +6312,7 @@ fi if test "${enable_zstd+set}" = set; then : enableval=$enable_zstd; else - enable_zstd=auto + enable_zstd=yes fi @@ -7021,7 +7021,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by btrfs-progs $as_me v4.13.3, which was +This file was extended by btrfs-progs $as_me v4.14.1, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -7084,7 +7084,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -btrfs-progs config.status v4.13.3 +btrfs-progs config.status v4.14.1 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/configure.ac b/configure.ac index 330fb0cc..bfc010c2 100644 --- a/configure.ac +++ b/configure.ac @@ -183,8 +183,8 @@ PKG_CHECK_MODULES(ZLIB, [zlib]) PKG_STATIC(ZLIB_LIBS_STATIC, [zlib]) AC_ARG_ENABLE([zstd], - AS_HELP_STRING([--enable-zstd@<:@=auto@:>@], [build with zstd support (default: auto)]), - [], [enable_zstd=auto] + AS_HELP_STRING([--disable-zstd@<:@=yes@:>@], [build with zstd support (default: yes)]), + [], [enable_zstd=yes] ) if test "x$enable_zstd" = xauto; then diff --git a/convert/main.c b/convert/main.c index 882daf7c..89f92611 100644 --- a/convert/main.c +++ b/convert/main.c @@ -768,7 +768,7 @@ static int create_image(struct btrfs_root *root, if (ret < 0) goto out; ret = btrfs_add_link(trans, root, ino, BTRFS_FIRST_FREE_OBJECTID, name, - strlen(name), BTRFS_FT_REG_FILE, NULL, 1); + strlen(name), BTRFS_FT_REG_FILE, NULL, 1, 0); if (ret < 0) goto out; @@ -1443,6 +1443,8 @@ next: } } btrfs_release_path(&path); + if (ret) + return ret; /* * For HOLES mode (without NO_HOLES), we must ensure file extents * cover the whole range of the image @@ -1529,7 +1531,13 @@ static int do_rollback(const char *devname) goto free_mem; } fsize = lseek(fd, 0, SEEK_END); - root = open_ctree_fd(fd, devname, 0, OPEN_CTREE_WRITES); + + /* + * For rollback, we don't really need to write anything so open it + * read-only. The write part will happen after we close the + * filesystem. + */ + root = open_ctree_fd(fd, devname, 0, 0); if (!root) { error("unable to open ctree"); ret = -EIO; diff --git a/convert/source-fs.h b/convert/source-fs.h index 23f33567..4e5babef 100644 --- a/convert/source-fs.h +++ b/convert/source-fs.h @@ -19,6 +19,7 @@ #include "kerncompat.h" #include <linux/kdev_t.h> +#include <sys/types.h> #include <pthread.h> #define CONV_IMAGE_SUBVOL_OBJECTID BTRFS_FIRST_FREE_OBJECTID @@ -2072,6 +2072,38 @@ BTRFS_SETGET_STACK_FUNCS(root_stransid, struct btrfs_root_item, BTRFS_SETGET_STACK_FUNCS(root_rtransid, struct btrfs_root_item, rtransid, 64); +static inline struct btrfs_timespec* btrfs_root_ctime( + struct btrfs_root_item *root_item) +{ + unsigned long ptr = (unsigned long)root_item; + ptr += offsetof(struct btrfs_root_item, ctime); + return (struct btrfs_timespec *)ptr; +} + +static inline struct btrfs_timespec* btrfs_root_otime( + struct btrfs_root_item *root_item) +{ + unsigned long ptr = (unsigned long)root_item; + ptr += offsetof(struct btrfs_root_item, otime); + return (struct btrfs_timespec *)ptr; +} + +static inline struct btrfs_timespec* btrfs_root_stime( + struct btrfs_root_item *root_item) +{ + unsigned long ptr = (unsigned long)root_item; + ptr += offsetof(struct btrfs_root_item, stime); + return (struct btrfs_timespec *)ptr; +} + +static inline struct btrfs_timespec* btrfs_root_rtime( + struct btrfs_root_item *root_item) +{ + unsigned long ptr = (unsigned long)root_item; + ptr += offsetof(struct btrfs_root_item, rtime); + return (struct btrfs_timespec *)ptr; +} + /* struct btrfs_root_backup */ BTRFS_SETGET_STACK_FUNCS(backup_tree_root, struct btrfs_root_backup, tree_root, 64); @@ -2738,7 +2770,7 @@ int btrfs_change_inode_flags(struct btrfs_trans_handle *trans, struct btrfs_root *root, u64 ino, u64 flags); int btrfs_add_link(struct btrfs_trans_handle *trans, struct btrfs_root *root, u64 ino, u64 parent_ino, char *name, int namelen, - u8 type, u64 *index, int add_backref); + u8 type, u64 *index, int add_backref, int ignore_existed); int btrfs_unlink(struct btrfs_trans_handle *trans, struct btrfs_root *root, u64 ino, u64 parent_ino, u64 index, const char *name, int namelen, int add_orphan); diff --git a/debian/changelog b/debian/changelog index 15da1e2c..70bb7210 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,9 @@ +btrfs-progs (4.14.1-1) unstable; urgency=medium + + * New upstream release. + + -- Dimitri John Ledkov <xnox@ubuntu.com> Thu, 11 Jan 2018 15:43:20 +0000 + btrfs-progs (4.13.3-1) unstable; urgency=medium [ Dimitri John Ledkov ] @@ -135,7 +135,14 @@ int btrfs_insert_dir_item(struct btrfs_trans_handle *trans, struct btrfs_root name, name_len); if (IS_ERR(dir_item)) { ret = PTR_ERR(dir_item); - goto out; + + /* Continue to insert item if existed */ + if (ret == -EEXIST) { + ret = 0; + goto insert; + } else { + goto out; + } } leaf = path->nodes[0]; @@ -149,6 +156,7 @@ int btrfs_insert_dir_item(struct btrfs_trans_handle *trans, struct btrfs_root write_extent_buffer(leaf, name, name_ptr, name_len); btrfs_mark_buffer_dirty(leaf); +insert: /* FIXME, use some real flag for selecting the extra index */ if (root == root->fs_info->tree_root) { ret = 0; @@ -162,6 +170,8 @@ int btrfs_insert_dir_item(struct btrfs_trans_handle *trans, struct btrfs_root name, name_len); if (IS_ERR(dir_item)) { ret2 = PTR_ERR(dir_item); + if (ret2 == -EEXIST) + ret = 0; goto out; } leaf = path->nodes[0]; diff --git a/extent-tree.c b/extent-tree.c index 525a237e..055582c3 100644 --- a/extent-tree.c +++ b/extent-tree.c @@ -2690,7 +2690,8 @@ int btrfs_reserve_extent(struct btrfs_trans_handle *trans, search_start, search_end, hint_byte, ins, trans->alloc_exclude_start, trans->alloc_exclude_nr, data); - BUG_ON(ret); + if (ret < 0) + return ret; clear_extent_dirty(&info->free_space_cache, ins->objectid, ins->objectid + ins->offset - 1); return ret; @@ -128,12 +128,32 @@ static int do_usage_one_command(const char * const *usagestr, unsigned int flags, FILE *outf) { int pad = 4; + const char *prefix = "usage: "; + const char *pad_listing = " "; if (!usagestr || !*usagestr) return -1; - fprintf(outf, "%s%s", (flags & USAGE_LISTING) ? " " : "usage: ", - *usagestr++); + if (flags & USAGE_LISTING) + prefix = pad_listing; + + fputs(prefix, outf); + if (strchr(*usagestr, '\n') == NULL) { + fputs(*usagestr, outf); + } else { + const char *c = *usagestr; + const char *nprefix = " "; + + if (flags & USAGE_LISTING) + nprefix = pad_listing; + + for (c = *usagestr; *c; c++) { + fputc(*c, outf); + if (*c == '\n') + fputs(nprefix, outf); + } + } + usagestr++; /* a short one-line description (mandatory) */ if ((flags & USAGE_SHORT) == 0) diff --git a/image/main.c b/image/main.c index 4cffbdba..54a2d7d5 100644 --- a/image/main.c +++ b/image/main.c @@ -36,54 +36,11 @@ #include "volumes.h" #include "extent_io.h" #include "help.h" - -#define HEADER_MAGIC 0xbd5c25e27295668bULL -#define MAX_PENDING_SIZE (256 * 1024) -#define BLOCK_SIZE 1024 -#define BLOCK_MASK (BLOCK_SIZE - 1) - -#define COMPRESS_NONE 0 -#define COMPRESS_ZLIB 1 +#include "image/metadump.h" +#include "image/sanitize.h" #define MAX_WORKER_THREADS (32) -struct meta_cluster_item { - __le64 bytenr; - __le32 size; -} __attribute__ ((__packed__)); - -struct meta_cluster_header { - __le64 magic; - __le64 bytenr; - __le32 nritems; - u8 compress; -} __attribute__ ((__packed__)); - -/* cluster header + index items + buffers */ -struct meta_cluster { - struct meta_cluster_header header; - struct meta_cluster_item items[]; -} __attribute__ ((__packed__)); - -#define ITEMS_PER_CLUSTER ((BLOCK_SIZE - sizeof(struct meta_cluster)) / \ - sizeof(struct meta_cluster_item)) - -struct fs_chunk { - u64 logical; - u64 physical; - /* - * physical_dup only store additonal physical for BTRFS_BLOCK_GROUP_DUP - * currently restore only support single and DUP - * TODO: modify this structure and the function related to this - * structure for support RAID* - */ - u64 physical_dup; - u64 bytes; - struct rb_node l; - struct rb_node p; - struct list_head list; -}; - struct async_work { struct list_head list; struct list_head ordered; @@ -120,18 +77,11 @@ struct metadump_struct { int compress_level; int done; int data; - int sanitize_names; + enum sanitize_mode sanitize_names; int error; }; -struct name { - struct rb_node n; - char *val; - char *sub; - u32 len; -}; - struct mdrestore_struct { FILE *in; FILE *out; @@ -192,35 +142,6 @@ static int has_name(struct btrfs_key *key) return 0; } -static char *generate_garbage(u32 name_len) -{ - char *buf = malloc(name_len); - int i; - - if (!buf) - return NULL; - - for (i = 0; i < name_len; i++) { - char c = rand_range(94) + 33; - - if (c == '/') - c++; - buf[i] = c; - } - - return buf; -} - -static int name_cmp(struct rb_node *a, struct rb_node *b, int fuzz) -{ - struct name *entry = rb_entry(a, struct name, n); - struct name *ins = rb_entry(b, struct name, n); - u32 len; - - len = min(ins->len, entry->len); - return memcmp(ins->val, entry->val, len); -} - static int chunk_cmp(struct rb_node *a, struct rb_node *b, int fuzz) { struct fs_chunk *entry = rb_entry(a, struct fs_chunk, l); @@ -339,369 +260,6 @@ static u64 logical_to_physical(struct mdrestore_struct *mdres, u64 logical, } /* - * Reverse CRC-32C table - */ -static const u32 crc32c_rev_table[256] = { - 0x00000000L,0x05EC76F1L,0x0BD8EDE2L,0x0E349B13L, - 0x17B1DBC4L,0x125DAD35L,0x1C693626L,0x198540D7L, - 0x2F63B788L,0x2A8FC179L,0x24BB5A6AL,0x21572C9BL, - 0x38D26C4CL,0x3D3E1ABDL,0x330A81AEL,0x36E6F75FL, - 0x5EC76F10L,0x5B2B19E1L,0x551F82F2L,0x50F3F403L, - 0x4976B4D4L,0x4C9AC225L,0x42AE5936L,0x47422FC7L, - 0x71A4D898L,0x7448AE69L,0x7A7C357AL,0x7F90438BL, - 0x6615035CL,0x63F975ADL,0x6DCDEEBEL,0x6821984FL, - 0xBD8EDE20L,0xB862A8D1L,0xB65633C2L,0xB3BA4533L, - 0xAA3F05E4L,0xAFD37315L,0xA1E7E806L,0xA40B9EF7L, - 0x92ED69A8L,0x97011F59L,0x9935844AL,0x9CD9F2BBL, - 0x855CB26CL,0x80B0C49DL,0x8E845F8EL,0x8B68297FL, - 0xE349B130L,0xE6A5C7C1L,0xE8915CD2L,0xED7D2A23L, - 0xF4F86AF4L,0xF1141C05L,0xFF208716L,0xFACCF1E7L, - 0xCC2A06B8L,0xC9C67049L,0xC7F2EB5AL,0xC21E9DABL, - 0xDB9BDD7CL,0xDE77AB8DL,0xD043309EL,0xD5AF466FL, - 0x7EF1CAB1L,0x7B1DBC40L,0x75292753L,0x70C551A2L, - 0x69401175L,0x6CAC6784L,0x6298FC97L,0x67748A66L, - 0x51927D39L,0x547E0BC8L,0x5A4A90DBL,0x5FA6E62AL, - 0x4623A6FDL,0x43CFD00CL,0x4DFB4B1FL,0x48173DEEL, - 0x2036A5A1L,0x25DAD350L,0x2BEE4843L,0x2E023EB2L, - 0x37877E65L,0x326B0894L,0x3C5F9387L,0x39B3E576L, - 0x0F551229L,0x0AB964D8L,0x048DFFCBL,0x0161893AL, - 0x18E4C9EDL,0x1D08BF1CL,0x133C240FL,0x16D052FEL, - 0xC37F1491L,0xC6936260L,0xC8A7F973L,0xCD4B8F82L, - 0xD4CECF55L,0xD122B9A4L,0xDF1622B7L,0xDAFA5446L, - 0xEC1CA319L,0xE9F0D5E8L,0xE7C44EFBL,0xE228380AL, - 0xFBAD78DDL,0xFE410E2CL,0xF075953FL,0xF599E3CEL, - 0x9DB87B81L,0x98540D70L,0x96609663L,0x938CE092L, - 0x8A09A045L,0x8FE5D6B4L,0x81D14DA7L,0x843D3B56L, - 0xB2DBCC09L,0xB737BAF8L,0xB90321EBL,0xBCEF571AL, - 0xA56A17CDL,0xA086613CL,0xAEB2FA2FL,0xAB5E8CDEL, - 0xFDE39562L,0xF80FE393L,0xF63B7880L,0xF3D70E71L, - 0xEA524EA6L,0xEFBE3857L,0xE18AA344L,0xE466D5B5L, - 0xD28022EAL,0xD76C541BL,0xD958CF08L,0xDCB4B9F9L, - 0xC531F92EL,0xC0DD8FDFL,0xCEE914CCL,0xCB05623DL, - 0xA324FA72L,0xA6C88C83L,0xA8FC1790L,0xAD106161L, - 0xB49521B6L,0xB1795747L,0xBF4DCC54L,0xBAA1BAA5L, - 0x8C474DFAL,0x89AB3B0BL,0x879FA018L,0x8273D6E9L, - 0x9BF6963EL,0x9E1AE0CFL,0x902E7BDCL,0x95C20D2DL, - 0x406D4B42L,0x45813DB3L,0x4BB5A6A0L,0x4E59D051L, - 0x57DC9086L,0x5230E677L,0x5C047D64L,0x59E80B95L, - 0x6F0EFCCAL,0x6AE28A3BL,0x64D61128L,0x613A67D9L, - 0x78BF270EL,0x7D5351FFL,0x7367CAECL,0x768BBC1DL, - 0x1EAA2452L,0x1B4652A3L,0x1572C9B0L,0x109EBF41L, - 0x091BFF96L,0x0CF78967L,0x02C31274L,0x072F6485L, - 0x31C993DAL,0x3425E52BL,0x3A117E38L,0x3FFD08C9L, - 0x2678481EL,0x23943EEFL,0x2DA0A5FCL,0x284CD30DL, - 0x83125FD3L,0x86FE2922L,0x88CAB231L,0x8D26C4C0L, - 0x94A38417L,0x914FF2E6L,0x9F7B69F5L,0x9A971F04L, - 0xAC71E85BL,0xA99D9EAAL,0xA7A905B9L,0xA2457348L, - 0xBBC0339FL,0xBE2C456EL,0xB018DE7DL,0xB5F4A88CL, - 0xDDD530C3L,0xD8394632L,0xD60DDD21L,0xD3E1ABD0L, - 0xCA64EB07L,0xCF889DF6L,0xC1BC06E5L,0xC4507014L, - 0xF2B6874BL,0xF75AF1BAL,0xF96E6AA9L,0xFC821C58L, - 0xE5075C8FL,0xE0EB2A7EL,0xEEDFB16DL,0xEB33C79CL, - 0x3E9C81F3L,0x3B70F702L,0x35446C11L,0x30A81AE0L, - 0x292D5A37L,0x2CC12CC6L,0x22F5B7D5L,0x2719C124L, - 0x11FF367BL,0x1413408AL,0x1A27DB99L,0x1FCBAD68L, - 0x064EEDBFL,0x03A29B4EL,0x0D96005DL,0x087A76ACL, - 0x605BEEE3L,0x65B79812L,0x6B830301L,0x6E6F75F0L, - 0x77EA3527L,0x720643D6L,0x7C32D8C5L,0x79DEAE34L, - 0x4F38596BL,0x4AD42F9AL,0x44E0B489L,0x410CC278L, - 0x588982AFL,0x5D65F45EL,0x53516F4DL,0x56BD19BCL -}; - -/* - * Calculate a 4-byte suffix to match desired CRC32C - * - * @current_crc: CRC32C checksum of all bytes before the suffix - * @desired_crc: the checksum that we want to get after adding the suffix - * - * Outputs: @suffix: pointer to where the suffix will be written (4-bytes) - */ -static void find_collision_calc_suffix(unsigned long current_crc, - unsigned long desired_crc, - char *suffix) -{ - int i; - - for(i = 3; i >= 0; i--) { - desired_crc = (desired_crc << 8) - ^ crc32c_rev_table[desired_crc >> 24 & 0xFF] - ^ ((current_crc >> i * 8) & 0xFF); - } - for (i = 0; i < 4; i++) - suffix[i] = (desired_crc >> i * 8) & 0xFF; -} - -/* - * Check if suffix is valid according to our file name conventions - */ -static int find_collision_is_suffix_valid(const char *suffix) -{ - int i; - char c; - - for (i = 0; i < 4; i++) { - c = suffix[i]; - if (c < ' ' || c > 126 || c == '/') - return 0; - } - return 1; -} - -static int find_collision_reverse_crc32c(struct name *val, u32 name_len) -{ - unsigned long checksum; - unsigned long current_checksum; - int found = 0; - int i; - - /* There are no same length collisions of 4 or less bytes */ - if (name_len <= 4) - return 0; - checksum = crc32c(~1, val->val, name_len); - name_len -= 4; - memset(val->sub, ' ', name_len); - i = 0; - while (1) { - current_checksum = crc32c(~1, val->sub, name_len); - find_collision_calc_suffix(current_checksum, - checksum, - val->sub + name_len); - if (find_collision_is_suffix_valid(val->sub + name_len) && - memcmp(val->sub, val->val, val->len)) { - found = 1; - break; - } - - if (val->sub[i] == 126) { - do { - i++; - if (i >= name_len) - break; - } while (val->sub[i] == 126); - - if (i >= name_len) - break; - val->sub[i]++; - if (val->sub[i] == '/') - val->sub[i]++; - memset(val->sub, ' ', i); - i = 0; - continue; - } else { - val->sub[i]++; - if (val->sub[i] == '/') - val->sub[i]++; - } - } - return found; -} - -static char *find_collision(struct metadump_struct *md, char *name, - u32 name_len) -{ - struct name *val; - struct rb_node *entry; - struct name tmp; - int found; - int i; - - tmp.val = name; - tmp.len = name_len; - entry = tree_search(&md->name_tree, &tmp.n, name_cmp, 0); - if (entry) { - val = rb_entry(entry, struct name, n); - free(name); - return val->sub; - } - - val = malloc(sizeof(struct name)); - if (!val) { - error("cannot sanitize name, not enough memory"); - free(name); - return NULL; - } - - memset(val, 0, sizeof(*val)); - - val->val = name; - val->len = name_len; - val->sub = malloc(name_len); - if (!val->sub) { - error("cannot sanitize name, not enough memory"); - free(val); - free(name); - return NULL; - } - - found = find_collision_reverse_crc32c(val, name_len); - - if (!found) { - warning( -"cannot find a hash collision for '%.*s', generating garbage, it won't match indexes", - val->len, val->val); - for (i = 0; i < name_len; i++) { - char c = rand_range(94) + 33; - - if (c == '/') - c++; - val->sub[i] = c; - } - } - - tree_insert(&md->name_tree, &val->n, name_cmp); - return val->sub; -} - -static void sanitize_dir_item(struct metadump_struct *md, struct extent_buffer *eb, - int slot) -{ - struct btrfs_dir_item *dir_item; - char *buf; - char *garbage; - unsigned long name_ptr; - u32 total_len; - u32 cur = 0; - u32 this_len; - u32 name_len; - int free_garbage = (md->sanitize_names == 1); - - dir_item = btrfs_item_ptr(eb, slot, struct btrfs_dir_item); - total_len = btrfs_item_size_nr(eb, slot); - while (cur < total_len) { - this_len = sizeof(*dir_item) + - btrfs_dir_name_len(eb, dir_item) + - btrfs_dir_data_len(eb, dir_item); - name_ptr = (unsigned long)(dir_item + 1); - name_len = btrfs_dir_name_len(eb, dir_item); - - if (md->sanitize_names > 1) { - buf = malloc(name_len); - if (!buf) { - error("cannot sanitize name, not enough memory"); - return; - } - read_extent_buffer(eb, buf, name_ptr, name_len); - garbage = find_collision(md, buf, name_len); - } else { - garbage = generate_garbage(name_len); - } - if (!garbage) { - error("cannot sanitize name, not enough memory"); - return; - } - write_extent_buffer(eb, garbage, name_ptr, name_len); - cur += this_len; - dir_item = (struct btrfs_dir_item *)((char *)dir_item + - this_len); - if (free_garbage) - free(garbage); - } -} - -static void sanitize_inode_ref(struct metadump_struct *md, - struct extent_buffer *eb, int slot, int ext) -{ - struct btrfs_inode_extref *extref; - struct btrfs_inode_ref *ref; - char *garbage, *buf; - unsigned long ptr; - unsigned long name_ptr; - u32 item_size; - u32 cur_offset = 0; - int len; - int free_garbage = (md->sanitize_names == 1); - - item_size = btrfs_item_size_nr(eb, slot); - ptr = btrfs_item_ptr_offset(eb, slot); - while (cur_offset < item_size) { - if (ext) { - extref = (struct btrfs_inode_extref *)(ptr + - cur_offset); - name_ptr = (unsigned long)(&extref->name); - len = btrfs_inode_extref_name_len(eb, extref); - cur_offset += sizeof(*extref); - } else { - ref = (struct btrfs_inode_ref *)(ptr + cur_offset); - len = btrfs_inode_ref_name_len(eb, ref); - name_ptr = (unsigned long)(ref + 1); - cur_offset += sizeof(*ref); - } - cur_offset += len; - - if (md->sanitize_names > 1) { - buf = malloc(len); - if (!buf) { - error("cannot sanitize name, not enough memory"); - return; - } - read_extent_buffer(eb, buf, name_ptr, len); - garbage = find_collision(md, buf, len); - } else { - garbage = generate_garbage(len); - } - - if (!garbage) { - error("cannot sanitize name, not enough memory"); - return; - } - write_extent_buffer(eb, garbage, name_ptr, len); - if (free_garbage) - free(garbage); - } -} - -static void sanitize_xattr(struct metadump_struct *md, - struct extent_buffer *eb, int slot) -{ - struct btrfs_dir_item *dir_item; - unsigned long data_ptr; - u32 data_len; - - dir_item = btrfs_item_ptr(eb, slot, struct btrfs_dir_item); - data_len = btrfs_dir_data_len(eb, dir_item); - - data_ptr = (unsigned long)((char *)(dir_item + 1) + - btrfs_dir_name_len(eb, dir_item)); - memset_extent_buffer(eb, 0, data_ptr, data_len); -} - -static void sanitize_name(struct metadump_struct *md, u8 *dst, - struct extent_buffer *src, struct btrfs_key *key, - int slot) -{ - struct extent_buffer *eb; - - eb = alloc_dummy_eb(src->start, src->len); - if (!eb) { - error("cannot sanitize name, not enough memory"); - return; - } - - memcpy(eb->data, src->data, src->len); - - switch (key->type) { - case BTRFS_DIR_ITEM_KEY: - case BTRFS_DIR_INDEX_KEY: - sanitize_dir_item(md, eb, slot); - break; - case BTRFS_INODE_REF_KEY: - sanitize_inode_ref(md, eb, slot, 0); - break; - case BTRFS_INODE_EXTREF_KEY: - sanitize_inode_ref(md, eb, slot, 1); - break; - case BTRFS_XATTR_ITEM_KEY: - sanitize_xattr(md, eb, slot); - break; - default: - break; - } - - memcpy(dst, eb->data, eb->len); - free(eb); -} - -/* * zero inline extents and csum items */ static void zero_items(struct metadump_struct *md, u8 *dst, @@ -726,7 +284,8 @@ static void zero_items(struct metadump_struct *md, u8 *dst, } if (md->sanitize_names && has_name(&key)) { - sanitize_name(md, dst, src, &key, i); + sanitize_name(md->sanitize_names, &md->name_tree, dst, + src, &key, i); continue; } @@ -872,7 +431,7 @@ static void metadump_destroy(struct metadump_struct *md, int num_threads) static int metadump_init(struct metadump_struct *md, struct btrfs_root *root, FILE *out, int num_threads, int compress_level, - int sanitize_names) + enum sanitize_mode sanitize_names) { int i, ret = 0; @@ -884,7 +443,7 @@ static int metadump_init(struct metadump_struct *md, struct btrfs_root *root, md->pending_start = (u64)-1; md->compress_level = compress_level; md->sanitize_names = sanitize_names; - if (sanitize_names > 1) + if (sanitize_names == SANITIZE_COLLISIONS) crc32c_optimization_init(); md->name_tree.rb_node = NULL; @@ -1449,7 +1008,8 @@ static int copy_from_extent_tree(struct metadump_struct *metadump, } static int create_metadump(const char *input, FILE *out, int num_threads, - int compress_level, int sanitize, int walk_trees) + int compress_level, enum sanitize_mode sanitize, + int walk_trees) { struct btrfs_root *root; struct btrfs_path path; @@ -2865,7 +2425,7 @@ int main(int argc, char *argv[]) int walk_trees = 0; int multi_devices = 0; int ret; - int sanitize = 0; + enum sanitize_mode sanitize = SANITIZE_NONE; int dev_cnt = 0; int usage_error = 0; FILE *out; @@ -2903,7 +2463,10 @@ int main(int argc, char *argv[]) old_restore = 1; break; case 's': - sanitize++; + if (sanitize == SANITIZE_NONE) + sanitize = SANITIZE_NAMES; + else if (sanitize == SANITIZE_NAMES) + sanitize = SANITIZE_COLLISIONS; break; case 'w': walk_trees = 1; @@ -2931,7 +2494,7 @@ int main(int argc, char *argv[]) usage_error++; } } else { - if (walk_trees || sanitize || compress_level) { + if (walk_trees || sanitize != SANITIZE_NONE || compress_level) { error( "useing -w, -s, -c options for restore makes no sense"); usage_error++; diff --git a/image/metadump.h b/image/metadump.h new file mode 100644 index 00000000..b9275e95 --- /dev/null +++ b/image/metadump.h @@ -0,0 +1,70 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 021110-1307, USA. + */ + +#ifndef __BTRFS_IMAGE_METADUMP_H__ +#define __BTRFS_IMAGE_METADUMP_H__ + +#include "kerncompat.h" +#include "kernel-lib/rbtree.h" +#include "kernel-lib/list.h" +#include "ctree.h" + +#define HEADER_MAGIC 0xbd5c25e27295668bULL +#define MAX_PENDING_SIZE (256 * 1024) +#define BLOCK_SIZE 1024 +#define BLOCK_MASK (BLOCK_SIZE - 1) + +#define ITEMS_PER_CLUSTER ((BLOCK_SIZE - sizeof(struct meta_cluster)) / \ + sizeof(struct meta_cluster_item)) + +#define COMPRESS_NONE 0 +#define COMPRESS_ZLIB 1 + +struct meta_cluster_item { + __le64 bytenr; + __le32 size; +} __attribute__ ((__packed__)); + +struct meta_cluster_header { + __le64 magic; + __le64 bytenr; + __le32 nritems; + u8 compress; +} __attribute__ ((__packed__)); + +/* cluster header + index items + buffers */ +struct meta_cluster { + struct meta_cluster_header header; + struct meta_cluster_item items[]; +} __attribute__ ((__packed__)); + +struct fs_chunk { + u64 logical; + u64 physical; + /* + * physical_dup only store additonal physical for BTRFS_BLOCK_GROUP_DUP + * currently restore only support single and DUP + * TODO: modify this structure and the function related to this + * structure for support RAID* + */ + u64 physical_dup; + u64 bytes; + struct rb_node l; + struct rb_node p; + struct list_head list; +}; + +#endif diff --git a/image/sanitize.c b/image/sanitize.c new file mode 100644 index 00000000..e00c8b35 --- /dev/null +++ b/image/sanitize.c @@ -0,0 +1,475 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 021110-1307, USA. + */ + +#include "kerncompat.h" +#include "internal.h" +#include "messages.h" +#include "utils.h" +#include "kernel-lib/crc32c.h" +#include "image/sanitize.h" +#include "extent_io.h" + +/* + * Reverse CRC-32C table + */ +static const u32 crc32c_rev_table[256] = { + 0x00000000L,0x05EC76F1L,0x0BD8EDE2L,0x0E349B13L, + 0x17B1DBC4L,0x125DAD35L,0x1C693626L,0x198540D7L, + 0x2F63B788L,0x2A8FC179L,0x24BB5A6AL,0x21572C9BL, + 0x38D26C4CL,0x3D3E1ABDL,0x330A81AEL,0x36E6F75FL, + 0x5EC76F10L,0x5B2B19E1L,0x551F82F2L,0x50F3F403L, + 0x4976B4D4L,0x4C9AC225L,0x42AE5936L,0x47422FC7L, + 0x71A4D898L,0x7448AE69L,0x7A7C357AL,0x7F90438BL, + 0x6615035CL,0x63F975ADL,0x6DCDEEBEL,0x6821984FL, + 0xBD8EDE20L,0xB862A8D1L,0xB65633C2L,0xB3BA4533L, + 0xAA3F05E4L,0xAFD37315L,0xA1E7E806L,0xA40B9EF7L, + 0x92ED69A8L,0x97011F59L,0x9935844AL,0x9CD9F2BBL, + 0x855CB26CL,0x80B0C49DL,0x8E845F8EL,0x8B68297FL, + 0xE349B130L,0xE6A5C7C1L,0xE8915CD2L,0xED7D2A23L, + 0xF4F86AF4L,0xF1141C05L,0xFF208716L,0xFACCF1E7L, + 0xCC2A06B8L,0xC9C67049L,0xC7F2EB5AL,0xC21E9DABL, + 0xDB9BDD7CL,0xDE77AB8DL,0xD043309EL,0xD5AF466FL, + 0x7EF1CAB1L,0x7B1DBC40L,0x75292753L,0x70C551A2L, + 0x69401175L,0x6CAC6784L,0x6298FC97L,0x67748A66L, + 0x51927D39L,0x547E0BC8L,0x5A4A90DBL,0x5FA6E62AL, + 0x4623A6FDL,0x43CFD00CL,0x4DFB4B1FL,0x48173DEEL, + 0x2036A5A1L,0x25DAD350L,0x2BEE4843L,0x2E023EB2L, + 0x37877E65L,0x326B0894L,0x3C5F9387L,0x39B3E576L, + 0x0F551229L,0x0AB964D8L,0x048DFFCBL,0x0161893AL, + 0x18E4C9EDL,0x1D08BF1CL,0x133C240FL,0x16D052FEL, + 0xC37F1491L,0xC6936260L,0xC8A7F973L,0xCD4B8F82L, + 0xD4CECF55L,0xD122B9A4L,0xDF1622B7L,0xDAFA5446L, + 0xEC1CA319L,0xE9F0D5E8L,0xE7C44EFBL,0xE228380AL, + 0xFBAD78DDL,0xFE410E2CL,0xF075953FL,0xF599E3CEL, + 0x9DB87B81L,0x98540D70L,0x96609663L,0x938CE092L, + 0x8A09A045L,0x8FE5D6B4L,0x81D14DA7L,0x843D3B56L, + 0xB2DBCC09L,0xB737BAF8L,0xB90321EBL,0xBCEF571AL, + 0xA56A17CDL,0xA086613CL,0xAEB2FA2FL,0xAB5E8CDEL, + 0xFDE39562L,0xF80FE393L,0xF63B7880L,0xF3D70E71L, + 0xEA524EA6L,0xEFBE3857L,0xE18AA344L,0xE466D5B5L, + 0xD28022EAL,0xD76C541BL,0xD958CF08L,0xDCB4B9F9L, + 0xC531F92EL,0xC0DD8FDFL,0xCEE914CCL,0xCB05623DL, + 0xA324FA72L,0xA6C88C83L,0xA8FC1790L,0xAD106161L, + 0xB49521B6L,0xB1795747L,0xBF4DCC54L,0xBAA1BAA5L, + 0x8C474DFAL,0x89AB3B0BL,0x879FA018L,0x8273D6E9L, + 0x9BF6963EL,0x9E1AE0CFL,0x902E7BDCL,0x95C20D2DL, + 0x406D4B42L,0x45813DB3L,0x4BB5A6A0L,0x4E59D051L, + 0x57DC9086L,0x5230E677L,0x5C047D64L,0x59E80B95L, + 0x6F0EFCCAL,0x6AE28A3BL,0x64D61128L,0x613A67D9L, + 0x78BF270EL,0x7D5351FFL,0x7367CAECL,0x768BBC1DL, + 0x1EAA2452L,0x1B4652A3L,0x1572C9B0L,0x109EBF41L, + 0x091BFF96L,0x0CF78967L,0x02C31274L,0x072F6485L, + 0x31C993DAL,0x3425E52BL,0x3A117E38L,0x3FFD08C9L, + 0x2678481EL,0x23943EEFL,0x2DA0A5FCL,0x284CD30DL, + 0x83125FD3L,0x86FE2922L,0x88CAB231L,0x8D26C4C0L, + 0x94A38417L,0x914FF2E6L,0x9F7B69F5L,0x9A971F04L, + 0xAC71E85BL,0xA99D9EAAL,0xA7A905B9L,0xA2457348L, + 0xBBC0339FL,0xBE2C456EL,0xB018DE7DL,0xB5F4A88CL, + 0xDDD530C3L,0xD8394632L,0xD60DDD21L,0xD3E1ABD0L, + 0xCA64EB07L,0xCF889DF6L,0xC1BC06E5L,0xC4507014L, + 0xF2B6874BL,0xF75AF1BAL,0xF96E6AA9L,0xFC821C58L, + 0xE5075C8FL,0xE0EB2A7EL,0xEEDFB16DL,0xEB33C79CL, + 0x3E9C81F3L,0x3B70F702L,0x35446C11L,0x30A81AE0L, + 0x292D5A37L,0x2CC12CC6L,0x22F5B7D5L,0x2719C124L, + 0x11FF367BL,0x1413408AL,0x1A27DB99L,0x1FCBAD68L, + 0x064EEDBFL,0x03A29B4EL,0x0D96005DL,0x087A76ACL, + 0x605BEEE3L,0x65B79812L,0x6B830301L,0x6E6F75F0L, + 0x77EA3527L,0x720643D6L,0x7C32D8C5L,0x79DEAE34L, + 0x4F38596BL,0x4AD42F9AL,0x44E0B489L,0x410CC278L, + 0x588982AFL,0x5D65F45EL,0x53516F4DL,0x56BD19BCL +}; + +/* + * Calculate a 4-byte suffix to match desired CRC32C + * + * @current_crc: CRC32C checksum of all bytes before the suffix + * @desired_crc: the checksum that we want to get after adding the suffix + * + * Outputs: @suffix: pointer to where the suffix will be written (4-bytes) + */ +static void find_collision_calc_suffix(unsigned long current_crc, + unsigned long desired_crc, + char *suffix) +{ + int i; + + for(i = 3; i >= 0; i--) { + desired_crc = (desired_crc << 8) + ^ crc32c_rev_table[desired_crc >> 24 & 0xFF] + ^ ((current_crc >> i * 8) & 0xFF); + } + for (i = 0; i < 4; i++) + suffix[i] = (desired_crc >> i * 8) & 0xFF; +} + +/* + * Check if suffix is valid according to our file name conventions + */ +static int find_collision_is_suffix_valid(const char *suffix) +{ + int i; + char c; + + for (i = 0; i < 4; i++) { + c = suffix[i]; + if (c < ' ' || c > 126 || c == '/') + return 0; + } + return 1; +} + +static int find_collision_reverse_crc32c(struct name *val, u32 name_len) +{ + unsigned long checksum; + unsigned long current_checksum; + int found = 0; + int i; + + /* There are no same length collisions of 4 or less bytes */ + if (name_len <= 4) + return 0; + checksum = crc32c(~1, val->val, name_len); + name_len -= 4; + memset(val->sub, ' ', name_len); + i = 0; + while (1) { + current_checksum = crc32c(~1, val->sub, name_len); + find_collision_calc_suffix(current_checksum, + checksum, + val->sub + name_len); + if (find_collision_is_suffix_valid(val->sub + name_len) && + memcmp(val->sub, val->val, val->len)) { + found = 1; + break; + } + + if (val->sub[i] == 126) { + do { + i++; + if (i >= name_len) + break; + } while (val->sub[i] == 126); + + if (i >= name_len) + break; + val->sub[i]++; + if (val->sub[i] == '/') + val->sub[i]++; + memset(val->sub, ' ', i); + i = 0; + continue; + } else { + val->sub[i]++; + if (val->sub[i] == '/') + val->sub[i]++; + } + } + return found; +} + +static void tree_insert(struct rb_root *root, struct rb_node *ins, + int (*cmp)(struct rb_node *a, struct rb_node *b, + int fuzz)) +{ + struct rb_node ** p = &root->rb_node; + struct rb_node * parent = NULL; + int dir; + + while(*p) { + parent = *p; + + dir = cmp(*p, ins, 1); + if (dir < 0) + p = &(*p)->rb_left; + else if (dir > 0) + p = &(*p)->rb_right; + else + BUG(); + } + + rb_link_node(ins, parent, p); + rb_insert_color(ins, root); +} + +static struct rb_node *tree_search(struct rb_root *root, + struct rb_node *search, + int (*cmp)(struct rb_node *a, + struct rb_node *b, int fuzz), + int fuzz) +{ + struct rb_node *n = root->rb_node; + int dir; + + while (n) { + dir = cmp(n, search, fuzz); + if (dir < 0) + n = n->rb_left; + else if (dir > 0) + n = n->rb_right; + else + return n; + } + + return NULL; +} + +static int name_cmp(struct rb_node *a, struct rb_node *b, int fuzz) +{ + struct name *entry = rb_entry(a, struct name, n); + struct name *ins = rb_entry(b, struct name, n); + u32 len; + + len = min(ins->len, entry->len); + return memcmp(ins->val, entry->val, len); +} + +static char *find_collision(struct rb_root *name_tree, char *name, + u32 name_len) +{ + struct name *val; + struct rb_node *entry; + struct name tmp; + int found; + int i; + + tmp.val = name; + tmp.len = name_len; + entry = tree_search(name_tree, &tmp.n, name_cmp, 0); + if (entry) { + val = rb_entry(entry, struct name, n); + free(name); + return val->sub; + } + + val = malloc(sizeof(struct name)); + if (!val) { + error("cannot sanitize name, not enough memory"); + free(name); + return NULL; + } + + memset(val, 0, sizeof(*val)); + + val->val = name; + val->len = name_len; + val->sub = malloc(name_len); + if (!val->sub) { + error("cannot sanitize name, not enough memory"); + free(val); + free(name); + return NULL; + } + + found = find_collision_reverse_crc32c(val, name_len); + + if (!found) { + warning( +"cannot find a hash collision for '%.*s', generating garbage, it won't match indexes", + val->len, val->val); + for (i = 0; i < name_len; i++) { + char c = rand_range(94) + 33; + + if (c == '/') + c++; + val->sub[i] = c; + } + } + + tree_insert(name_tree, &val->n, name_cmp); + return val->sub; +} + +static char *generate_garbage(u32 name_len) +{ + char *buf = malloc(name_len); + int i; + + if (!buf) + return NULL; + + for (i = 0; i < name_len; i++) { + char c = rand_range(94) + 33; + + if (c == '/') + c++; + buf[i] = c; + } + + return buf; +} + +static void sanitize_dir_item(enum sanitize_mode sanitize, + struct rb_root *name_tree, struct extent_buffer *eb, int slot) +{ + struct btrfs_dir_item *dir_item; + char *buf; + char *garbage; + unsigned long name_ptr; + u32 total_len; + u32 cur = 0; + u32 this_len; + u32 name_len; + int free_garbage = (sanitize == SANITIZE_NAMES); + + dir_item = btrfs_item_ptr(eb, slot, struct btrfs_dir_item); + total_len = btrfs_item_size_nr(eb, slot); + while (cur < total_len) { + this_len = sizeof(*dir_item) + + btrfs_dir_name_len(eb, dir_item) + + btrfs_dir_data_len(eb, dir_item); + name_ptr = (unsigned long)(dir_item + 1); + name_len = btrfs_dir_name_len(eb, dir_item); + + if (sanitize == SANITIZE_COLLISIONS) { + buf = malloc(name_len); + if (!buf) { + error("cannot sanitize name, not enough memory"); + return; + } + read_extent_buffer(eb, buf, name_ptr, name_len); + garbage = find_collision(name_tree, buf, name_len); + } else { + garbage = generate_garbage(name_len); + } + if (!garbage) { + error("cannot sanitize name, not enough memory"); + return; + } + write_extent_buffer(eb, garbage, name_ptr, name_len); + cur += this_len; + dir_item = (struct btrfs_dir_item *)((char *)dir_item + + this_len); + if (free_garbage) + free(garbage); + } +} + +static void sanitize_inode_ref(enum sanitize_mode sanitize, + struct rb_root *name_tree, struct extent_buffer *eb, int slot, + int ext) +{ + struct btrfs_inode_extref *extref; + struct btrfs_inode_ref *ref; + char *garbage, *buf; + unsigned long ptr; + unsigned long name_ptr; + u32 item_size; + u32 cur_offset = 0; + int len; + int free_garbage = (sanitize == SANITIZE_NAMES); + + item_size = btrfs_item_size_nr(eb, slot); + ptr = btrfs_item_ptr_offset(eb, slot); + while (cur_offset < item_size) { + if (ext) { + extref = (struct btrfs_inode_extref *)(ptr + + cur_offset); + name_ptr = (unsigned long)(&extref->name); + len = btrfs_inode_extref_name_len(eb, extref); + cur_offset += sizeof(*extref); + } else { + ref = (struct btrfs_inode_ref *)(ptr + cur_offset); + len = btrfs_inode_ref_name_len(eb, ref); + name_ptr = (unsigned long)(ref + 1); + cur_offset += sizeof(*ref); + } + cur_offset += len; + + if (sanitize == SANITIZE_COLLISIONS) { + buf = malloc(len); + if (!buf) { + error("cannot sanitize name, not enough memory"); + return; + } + read_extent_buffer(eb, buf, name_ptr, len); + garbage = find_collision(name_tree, buf, len); + } else { + garbage = generate_garbage(len); + } + + if (!garbage) { + error("cannot sanitize name, not enough memory"); + return; + } + write_extent_buffer(eb, garbage, name_ptr, len); + if (free_garbage) + free(garbage); + } +} + +static void sanitize_xattr(struct extent_buffer *eb, int slot) +{ + struct btrfs_dir_item *dir_item; + unsigned long data_ptr; + u32 data_len; + + dir_item = btrfs_item_ptr(eb, slot, struct btrfs_dir_item); + data_len = btrfs_dir_data_len(eb, dir_item); + + data_ptr = (unsigned long)((char *)(dir_item + 1) + + btrfs_dir_name_len(eb, dir_item)); + memset_extent_buffer(eb, 0, data_ptr, data_len); +} + +static struct extent_buffer *alloc_dummy_eb(u64 bytenr, u32 size) +{ + struct extent_buffer *eb; + + eb = calloc(1, sizeof(struct extent_buffer) + size); + if (!eb) + return NULL; + + eb->start = bytenr; + eb->len = size; + return eb; +} + +void sanitize_name(enum sanitize_mode sanitize, struct rb_root *name_tree, + u8 *dst, struct extent_buffer *src, struct btrfs_key *key, + int slot) +{ + struct extent_buffer *eb; + + eb = alloc_dummy_eb(src->start, src->len); + if (!eb) { + error("cannot sanitize name, not enough memory"); + return; + } + + memcpy(eb->data, src->data, src->len); + + switch (key->type) { + case BTRFS_DIR_ITEM_KEY: + case BTRFS_DIR_INDEX_KEY: + sanitize_dir_item(sanitize, name_tree, eb, slot); + break; + case BTRFS_INODE_REF_KEY: + sanitize_inode_ref(sanitize, name_tree, eb, slot, 0); + break; + case BTRFS_INODE_EXTREF_KEY: + sanitize_inode_ref(sanitize, name_tree, eb, slot, 1); + break; + case BTRFS_XATTR_ITEM_KEY: + sanitize_xattr(eb, slot); + break; + default: + break; + } + + memcpy(dst, eb->data, eb->len); + free(eb); +} + diff --git a/image/sanitize.h b/image/sanitize.h new file mode 100644 index 00000000..fc07a8a9 --- /dev/null +++ b/image/sanitize.h @@ -0,0 +1,48 @@ +/* + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 021110-1307, USA. + */ + +#ifndef __BTRFS_IMAGE_SANITIZE_H__ +#define __BTRFS_IMAGE_SANITIZE_H__ + +#include "kerncompat.h" +#include "image/metadump.h" + +struct name { + struct rb_node n; + char *val; + char *sub; + u32 len; +}; + +/* + * Filenames and xattrs can be obfuscated so they don't appear in the image + * dump. In basic mode (NAMES) a random string will be generated but such names + * do not match the direntry hashes. The advanced mode (COLLISIONS) tries to + * generate names that appear random but also match the hashes. This however + * may take significantly more time than the basic mode. And may not even + * succeed. + */ +enum sanitize_mode { + SANITIZE_NONE, + SANITIZE_NAMES, + SANITIZE_COLLISIONS +}; + +void sanitize_name(enum sanitize_mode sanitize, struct rb_root *name_tree, + u8 *dst, struct extent_buffer *src, struct btrfs_key *key, + int slot); + +#endif @@ -162,7 +162,7 @@ out: */ int btrfs_add_link(struct btrfs_trans_handle *trans, struct btrfs_root *root, u64 ino, u64 parent_ino, char *name, int namelen, - u8 type, u64 *index, int add_backref) + u8 type, u64 *index, int add_backref, int ignore_existed) { struct btrfs_path *path; struct btrfs_key key; @@ -185,33 +185,37 @@ int btrfs_add_link(struct btrfs_trans_handle *trans, struct btrfs_root *root, } ret = check_dir_conflict(root, name, namelen, parent_ino, ret_index); - if (ret < 0) + if (ret < 0 && !(ignore_existed && ret == -EEXIST)) goto out; /* Add inode ref */ if (add_backref) { ret = btrfs_insert_inode_ref(trans, root, name, namelen, ino, parent_ino, ret_index); - if (ret < 0) + if (ret < 0 && !(ignore_existed && ret == -EEXIST)) goto out; - /* Update nlinks for the inode */ - key.objectid = ino; - key.type = BTRFS_INODE_ITEM_KEY; - key.offset = 0; - ret = btrfs_search_slot(trans, root, &key, path, 1, 1); - if (ret) { - if (ret > 0) - ret = -ENOENT; - goto out; + /* do not update nlinks if existed */ + if (!ret) { + /* Update nlinks for the inode */ + key.objectid = ino; + key.type = BTRFS_INODE_ITEM_KEY; + key.offset = 0; + ret = btrfs_search_slot(trans, root, &key, path, 1, 1); + if (ret) { + if (ret > 0) + ret = -ENOENT; + goto out; + } + inode_item = btrfs_item_ptr(path->nodes[0], + path->slots[0], struct btrfs_inode_item); + nlink = btrfs_inode_nlink(path->nodes[0], inode_item); + nlink++; + btrfs_set_inode_nlink(path->nodes[0], inode_item, + nlink); + btrfs_mark_buffer_dirty(path->nodes[0]); + btrfs_release_path(path); } - inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0], - struct btrfs_inode_item); - nlink = btrfs_inode_nlink(path->nodes[0], inode_item); - nlink++; - btrfs_set_inode_nlink(path->nodes[0], inode_item, nlink); - btrfs_mark_buffer_dirty(path->nodes[0]); - btrfs_release_path(path); } /* Add dir_item and dir_index */ @@ -562,7 +566,7 @@ int btrfs_mkdir(struct btrfs_trans_handle *trans, struct btrfs_root *root, if (ret) goto out; ret = btrfs_add_link(trans, root, ret_ino, parent_ino, name, namelen, - BTRFS_FT_DIR, NULL, 1); + BTRFS_FT_DIR, NULL, 1, 0); if (ret) goto out; out: diff --git a/mkfs/common.c b/mkfs/common.c index c9ce10d4..dd5e7ecf 100644 --- a/mkfs/common.c +++ b/mkfs/common.c @@ -438,12 +438,6 @@ out: return ret; } -u64 btrfs_min_dev_size(u32 nodesize) -{ - return 2 * (BTRFS_MKFS_SYSTEM_GROUP_SIZE + - btrfs_min_global_blk_rsv_size(nodesize)); -} - /* * Btrfs minimum size calculation is complicated, it should include at least: * 1. system group size @@ -454,11 +448,72 @@ u64 btrfs_min_dev_size(u32 nodesize) * To avoid the overkill calculation, (system group + global block rsv) * 2 * for *EACH* device should be good enough. */ -u64 btrfs_min_global_blk_rsv_size(u32 nodesize) +static u64 btrfs_min_global_blk_rsv_size(u32 nodesize) { return (u64)nodesize << 10; } +u64 btrfs_min_dev_size(u32 nodesize, int mixed, u64 meta_profile, + u64 data_profile) +{ + u64 reserved = 0; + u64 meta_size; + u64 data_size; + + if (mixed) + return 2 * (BTRFS_MKFS_SYSTEM_GROUP_SIZE + + btrfs_min_global_blk_rsv_size(nodesize)); + + /* + * Minimal size calculation is complex due to several factors: + * 1) Temporary chunk reuse + * If specified chunk profile is SINGLE, we can reuse + * temporary chunks, no need to allocate new chunks. + * + * 2) Different minimal chunk size for different profiles: + * For initial sys chunk, chunk size is fixed to 4M. + * For single profile, minimal chunk size is 8M for all. + * For other profiles, minimal chunk and stripe size ranges from 8M + * to 64M. + * + * To calculate it a little easier, here we assume we don't reuse any + * temporary chunk, and calculate the size completely by ourselves. + * + * Temporary chunks sizes are always fixed: + * One initial sys chunk, one SINGLE meta, and one SINGLE data. + * The latter two are all 8M, accroding to @calc_size of + * btrfs_alloc_chunk(). + */ + reserved += BTRFS_MKFS_SYSTEM_GROUP_SIZE + SZ_8M * 2; + + /* + * For real chunks, we need to select different sizes: + * For SINGLE, it's still fixed to 8M (@calc_size). + * For other profiles, refer to max(@min_stripe_size, @calc_size). + * + * And use the stripe size to calculate its physical used space. + */ + if (meta_profile & BTRFS_BLOCK_GROUP_PROFILE_MASK) + meta_size = SZ_8M + SZ_32M; + else + meta_size = SZ_8M + SZ_8M; + /* For DUP/metadata, 2 stripes on one disk */ + if (meta_profile & BTRFS_BLOCK_GROUP_DUP) + meta_size *= 2; + reserved += meta_size; + + if (data_profile & BTRFS_BLOCK_GROUP_PROFILE_MASK) + data_size = SZ_64M; + else + data_size = SZ_8M; + /* For DUP/data, 2 stripes on one disk */ + if (data_profile & BTRFS_BLOCK_GROUP_DUP) + data_size *= 2; + reserved += data_size; + + return reserved; +} + #define isoctal(c) (((c) & ~7) == '0') static inline void translate(char *f, char *t) @@ -632,23 +687,9 @@ int test_dev_for_mkfs(const char *file, int force_overwrite) error("%s is a swap device", file); return 1; } - if (!force_overwrite) { - if (check_overwrite(file)) { - error("use the -f option to force overwrite of %s", - file); - return 1; - } - } - ret = check_mounted(file); - if (ret < 0) { - error("cannot check mount status of %s: %s", file, - strerror(-ret)); + ret = test_status_for_mkfs(file, force_overwrite); + if (ret) return 1; - } - if (ret == 1) { - error("%s is mounted", file); - return 1; - } /* check if the device is busy */ fd = open(file, O_RDWR|O_EXCL); if (fd < 0) { @@ -669,6 +710,34 @@ int test_dev_for_mkfs(const char *file, int force_overwrite) return 0; } +/* + * check if the file (device) is formatted or mounted + */ +int test_status_for_mkfs(const char *file, bool force_overwrite) +{ + int ret; + + if (!force_overwrite) { + if (check_overwrite(file)) { + error("use the -f option to force overwrite of %s", + file); + return 1; + } + } + ret = check_mounted(file); + if (ret < 0) { + error("cannot check mount status of %s: %s", file, + strerror(-ret)); + return 1; + } + if (ret == 1) { + error("%s is mounted", file); + return 1; + } + + return 0; +} + int is_vol_small(const char *file) { int fd = -1; @@ -698,7 +767,7 @@ int is_vol_small(const char *file) } } -int test_minimum_size(const char *file, u32 nodesize) +int test_minimum_size(const char *file, u64 min_dev_size) { int fd; struct stat statbuf; @@ -710,7 +779,7 @@ int test_minimum_size(const char *file, u32 nodesize) close(fd); return -errno; } - if (btrfs_device_size(fd, &statbuf) < btrfs_min_dev_size(nodesize)) { + if (btrfs_device_size(fd, &statbuf) < min_dev_size) { close(fd); return 1; } diff --git a/mkfs/common.h b/mkfs/common.h index dee0ea97..f6b60c28 100644 --- a/mkfs/common.h +++ b/mkfs/common.h @@ -66,12 +66,13 @@ struct btrfs_mkfs_config { }; int make_btrfs(int fd, struct btrfs_mkfs_config *cfg); -u64 btrfs_min_dev_size(u32 nodesize); -u64 btrfs_min_global_blk_rsv_size(u32 nodesize); -int test_minimum_size(const char *file, u32 nodesize); +u64 btrfs_min_dev_size(u32 nodesize, int mixed, u64 meta_profile, + u64 data_profile); +int test_minimum_size(const char *file, u64 min_dev_size); int is_vol_small(const char *file); int test_num_disk_vs_raid(u64 metadata_profile, u64 data_profile, u64 dev_cnt, int mixed, int ssd); +int test_status_for_mkfs(const char *file, bool force_overwrite); int test_dev_for_mkfs(const char *file, int force_overwrite); #endif diff --git a/mkfs/main.c b/mkfs/main.c index 1b4cabc1..d817ad8d 100644 --- a/mkfs/main.c +++ b/mkfs/main.c @@ -1073,6 +1073,19 @@ static int make_image(const char *source_dir, struct btrfs_root *root) printf("Making image is completed.\n"); return 0; fail: + /* + * Since we don't have btrfs_abort_transaction() yet, uncommitted trans + * will trigger a BUG_ON(). + * + * However before mkfs is fully finished, the magic number is invalid, + * so even we commit transaction here, the fs still can't be mounted. + * + * To do a graceful error out, here we commit transaction as a + * workaround. + * Since we have already hit some problem, the return value doesn't + * matter now. + */ + btrfs_commit_transaction(trans, root); while (!list_empty(&dir_head.list)) { dir_entry = list_entry(dir_head.list.next, struct directory_name_entry, list); @@ -1148,18 +1161,25 @@ static int zero_output_file(int out_fd, u64 size) { int loop_num; u64 location = 0; - char buf[4096]; + char buf[SZ_4K]; int ret = 0, i; ssize_t written; - memset(buf, 0, 4096); - loop_num = size / 4096; + memset(buf, 0, SZ_4K); + + /* Only zero out the first 1M */ + loop_num = SZ_1M / SZ_4K; for (i = 0; i < loop_num; i++) { - written = pwrite64(out_fd, buf, 4096, location); - if (written != 4096) + written = pwrite64(out_fd, buf, SZ_4K, location); + if (written != SZ_4K) ret = -EIO; - location += 4096; + location += SZ_4K; } + + /* Then enlarge the file to size */ + written = pwrite64(out_fd, buf, 1, size - 1); + if (written < 1) + ret = -EIO; return ret; } @@ -1350,6 +1370,9 @@ static int cleanup_temp_chunks(struct btrfs_fs_info *fs_info, ret = btrfs_search_slot(trans, root, &key, &path, 0, 0); if (ret < 0) goto out; + /* Don't pollute ret for >0 case */ + if (ret > 0) + ret = 0; btrfs_item_key_to_cpu(path.nodes[0], &found_key, path.slots[0]); @@ -1423,6 +1446,7 @@ int main(int argc, char **argv) int zero_end = 1; int fd = -1; int ret; + int close_ret; int i; int mixed = 0; int nodesize_forced = 0; @@ -1436,6 +1460,7 @@ int main(int argc, char **argv) u64 num_of_meta_chunks = 0; u64 size_of_data = 0; u64 source_dir_size = 0; + u64 min_dev_size; int dev_cnt = 0; int saved_optind; char fs_uuid[BTRFS_UUID_UNPARSED_SIZE] = { 0 }; @@ -1580,8 +1605,12 @@ int main(int argc, char **argv) while (dev_cnt-- > 0) { file = argv[optind++]; if (is_block_device(file) == 1) - if (test_dev_for_mkfs(file, force_overwrite)) - goto error; + ret = test_dev_for_mkfs(file, force_overwrite); + else + ret = test_status_for_mkfs(file, force_overwrite); + + if (ret) + goto error; } optind = saved_optind; @@ -1646,19 +1675,21 @@ int main(int argc, char **argv) goto error; } + min_dev_size = btrfs_min_dev_size(nodesize, mixed, metadata_profile, + data_profile); /* Check device/block_count after the nodesize is determined */ - if (block_count && block_count < btrfs_min_dev_size(nodesize)) { + if (block_count && block_count < min_dev_size) { error("size %llu is too small to make a usable filesystem", block_count); error("minimum size for btrfs filesystem is %llu", - btrfs_min_dev_size(nodesize)); + min_dev_size); goto error; } for (i = saved_optind; i < saved_optind + dev_cnt; i++) { char *path; path = argv[i]; - ret = test_minimum_size(path, nodesize); + ret = test_minimum_size(path, min_dev_size); if (ret < 0) { error("failed to check size for %s: %s", path, strerror(-ret)); @@ -1668,7 +1699,7 @@ int main(int argc, char **argv) error("'%s' is too small to make a usable filesystem", path); error("minimum size for each btrfs device is %llu", - btrfs_min_dev_size(nodesize)); + min_dev_size); goto error; } } @@ -1938,9 +1969,9 @@ raid_groups: */ fs_info->finalize_on_close = 1; out: - ret = close_ctree(root); + close_ret = close_ctree(root); - if (!ret) { + if (!close_ret) { optind = saved_optind; dev_cnt = argc - optind; while (dev_cnt-- > 0) { diff --git a/print-tree.c b/print-tree.c index 3c585e31..d3fa8621 100644 --- a/print-tree.c +++ b/print-tree.c @@ -551,6 +551,26 @@ static void root_flags_to_str(u64 flags, char *ret) strcat(ret, "none"); } +static void print_timespec(struct extent_buffer *eb, + struct btrfs_timespec *timespec, const char *prefix, + const char *suffix) +{ + struct tm tm; + u64 tmp_u64; + u32 tmp_u32; + time_t tmp_time; + char timestamp[256]; + + tmp_u64 = btrfs_timespec_sec(eb, timespec); + tmp_u32 = btrfs_timespec_nsec(eb, timespec); + tmp_time = tmp_u64; + localtime_r(&tmp_time, &tm); + strftime(timestamp, sizeof(timestamp), + "%Y-%m-%d %H:%M:%S", &tm); + printf("%s%llu.%u (%s)%s", prefix, (unsigned long long)tmp_u64, tmp_u32, + timestamp, suffix); +} + static void print_root_item(struct extent_buffer *leaf, int slot) { struct btrfs_root_item *ri; @@ -598,6 +618,18 @@ static void print_root_item(struct extent_buffer *leaf, int slot) btrfs_root_stransid(&root_item), btrfs_root_rtransid(&root_item)); } + if (btrfs_timespec_sec(leaf, btrfs_root_ctime(ri))) + print_timespec(leaf, btrfs_root_ctime(ri), + "\t\tctime ", "\n"); + if (btrfs_timespec_sec(leaf, btrfs_root_otime(ri))) + print_timespec(leaf, btrfs_root_otime(ri), + "\t\totime ", "\n"); + if (btrfs_timespec_sec(leaf, btrfs_root_stime(ri))) + print_timespec(leaf, btrfs_root_stime(ri), + "\t\tstime ", "\n"); + if (btrfs_timespec_sec(leaf, btrfs_root_rtime(ri))) + print_timespec(leaf, btrfs_root_rtime(ri), + "\t\trtime ", "\n"); } btrfs_disk_key_to_cpu(&drop_key, &root_item.drop_progress); @@ -794,6 +826,17 @@ void btrfs_print_key(struct btrfs_disk_key *disk_key) case BTRFS_UUID_KEY_RECEIVED_SUBVOL: printf(" 0x%016llx)", (unsigned long long)offset); break; + + /* + * Key offsets of ROOT_ITEM point to tree root, print them in human + * readable format. Especially useful for trees like data/tree reloc + * tree, whose tree id can be negative. + */ + case BTRFS_ROOT_ITEM_KEY: + printf(" "); + print_objectid(stdout, offset, type); + printf(")"); + break; default: if (offset == (u64)-1) printf(" -1)"); @@ -856,26 +899,6 @@ static void inode_flags_to_str(u64 flags, char *ret) strcat(ret, "none"); } -static void print_timespec(struct extent_buffer *eb, - struct btrfs_timespec *timespec, const char *prefix, - const char *suffix) -{ - struct tm tm; - u64 tmp_u64; - u32 tmp_u32; - time_t tmp_time; - char timestamp[256]; - - tmp_u64 = btrfs_timespec_sec(eb, timespec); - tmp_u32 = btrfs_timespec_nsec(eb, timespec); - tmp_time = tmp_u64; - localtime_r(&tmp_time, &tm); - strftime(timestamp, sizeof(timestamp), - "%Y-%m-%d %H:%M:%S", &tm); - printf("%s%llu.%u (%s)%s", prefix, (unsigned long long)tmp_u64, tmp_u32, - timestamp, suffix); -} - static void print_inode_item(struct extent_buffer *eb, struct btrfs_inode_item *ii) { @@ -142,10 +142,13 @@ static int prop_compression(enum prop_object_type type, memcpy(xattr_name + XATTR_BTRFS_PREFIX_LEN, name, strlen(name)); xattr_name[XATTR_BTRFS_PREFIX_LEN + strlen(name)] = '\0'; - if (value) + if (value) { + if (strcmp(value, "no") == 0 || strcmp(value, "none") == 0) + value = ""; sret = fsetxattr(fd, xattr_name, value, strlen(value), 0); - else + } else { sret = fgetxattr(fd, xattr_name, NULL, 0); + } if (sret < 0) { ret = -errno; if (ret != -ENOATTR) @@ -578,113 +578,119 @@ static struct btrfs_qgroup *qgroup_tree_search(struct qgroup_lookup *root_tree, return NULL; } -static int update_qgroup(struct qgroup_lookup *qgroup_lookup, u64 qgroupid, - u64 generation, u64 rfer, u64 rfer_cmpr, u64 excl, - u64 excl_cmpr, u64 flags, u64 max_rfer, u64 max_excl, - u64 rsv_rfer, u64 rsv_excl, struct btrfs_qgroup *pa, - struct btrfs_qgroup *child) +/* + * Lookup or insert btrfs_qgroup into qgroup_lookup. + * + * Search a btrfs_qgroup with @qgroupid from the @qgroup_lookup. If not found, + * initialize a btrfs_qgroup with the given qgroupid and insert it to the + * @qgroup_lookup. + * + * Return the pointer to the btrfs_qgroup if found or if inserted successfully. + * Return ERR_PTR if any error occurred. + */ +static struct btrfs_qgroup *get_or_add_qgroup( + struct qgroup_lookup *qgroup_lookup, u64 qgroupid) { struct btrfs_qgroup *bq; - struct btrfs_qgroup_list *list; + int ret; bq = qgroup_tree_search(qgroup_lookup, qgroupid); - if (!bq || bq->qgroupid != qgroupid) - return -ENOENT; + if (bq) + return bq; - if (generation) - bq->generation = generation; - if (rfer) - bq->rfer = rfer; - if (rfer_cmpr) - bq->rfer_cmpr = rfer_cmpr; - if (excl) - bq->excl = excl; - if (excl_cmpr) - bq->excl_cmpr = excl_cmpr; - if (flags) - bq->flags = flags; - if (max_rfer) - bq->max_rfer = max_rfer; - if (max_excl) - bq->max_excl = max_excl; - if (rsv_rfer) - bq->rsv_rfer = rsv_rfer; - if (pa && child) { - list = malloc(sizeof(*list)); - if (!list) { - error("memory allocation failed"); - exit(1); - } - list->qgroup = pa; - list->member = child; - list_add_tail(&list->next_qgroup, &child->qgroups); - list_add_tail(&list->next_member, &pa->members); + bq = calloc(1, sizeof(*bq)); + if (!bq) { + error("memory allocation failed"); + return ERR_PTR(-ENOMEM); } + + bq->qgroupid = qgroupid; + INIT_LIST_HEAD(&bq->qgroups); + INIT_LIST_HEAD(&bq->members); + + ret = qgroup_tree_insert(qgroup_lookup, bq); + if (ret) { + error("failed to insert %llu into tree: %s", + (unsigned long long)bq->qgroupid, strerror(-ret)); + free(bq); + return ERR_PTR(ret); + } + + return bq; +} + +static int update_qgroup_info(struct qgroup_lookup *qgroup_lookup, u64 qgroupid, + struct btrfs_qgroup_info_item *info) +{ + struct btrfs_qgroup *bq; + + bq = get_or_add_qgroup(qgroup_lookup, qgroupid); + if (IS_ERR_OR_NULL(bq)) + return PTR_ERR(bq); + + bq->generation = btrfs_stack_qgroup_info_generation(info); + bq->rfer = btrfs_stack_qgroup_info_referenced(info); + bq->rfer_cmpr = btrfs_stack_qgroup_info_referenced_compressed(info); + bq->excl = btrfs_stack_qgroup_info_exclusive(info); + bq->excl_cmpr = btrfs_stack_qgroup_info_exclusive_compressed(info); + return 0; } -static int add_qgroup(struct qgroup_lookup *qgroup_lookup, u64 qgroupid, - u64 generation, u64 rfer, u64 rfer_cmpr, u64 excl, - u64 excl_cmpr, u64 flags, u64 max_rfer, u64 max_excl, - u64 rsv_rfer, u64 rsv_excl, struct btrfs_qgroup *parent, - struct btrfs_qgroup *child) +static int update_qgroup_limit(struct qgroup_lookup *qgroup_lookup, + u64 qgroupid, + struct btrfs_qgroup_limit_item *limit) { struct btrfs_qgroup *bq; - struct btrfs_qgroup_list *list; - int ret; - ret = update_qgroup(qgroup_lookup, qgroupid, generation, rfer, - rfer_cmpr, excl, excl_cmpr, flags, max_rfer, - max_excl, rsv_rfer, rsv_excl, parent, child); - if (!ret) - return 0; + bq = get_or_add_qgroup(qgroup_lookup, qgroupid); + if (IS_ERR_OR_NULL(bq)) + return PTR_ERR(bq); - bq = calloc(1, sizeof(*bq)); - if (!bq) { - error("memory allocation failed"); - exit(1); - } - if (qgroupid) { - bq->qgroupid = qgroupid; - INIT_LIST_HEAD(&bq->qgroups); - INIT_LIST_HEAD(&bq->members); + bq->flags = btrfs_stack_qgroup_limit_flags(limit); + bq->max_rfer = btrfs_stack_qgroup_limit_max_referenced(limit); + bq->max_excl = btrfs_stack_qgroup_limit_max_exclusive(limit); + bq->rsv_rfer = btrfs_stack_qgroup_limit_rsv_referenced(limit); + bq->rsv_excl = btrfs_stack_qgroup_limit_rsv_exclusive(limit); + + return 0; +} + +static int update_qgroup_relation(struct qgroup_lookup *qgroup_lookup, + u64 child_id, u64 parent_id) +{ + struct btrfs_qgroup *child; + struct btrfs_qgroup *parent; + struct btrfs_qgroup_list *list; + + child = qgroup_tree_search(qgroup_lookup, child_id); + if (!child) { + error("cannot find the qgroup %llu/%llu", + btrfs_qgroup_level(child_id), + btrfs_qgroup_subvid(child_id)); + return -ENOENT; } - if (generation) - bq->generation = generation; - if (rfer) - bq->rfer = rfer; - if (rfer_cmpr) - bq->rfer_cmpr = rfer_cmpr; - if (excl) - bq->excl = excl; - if (excl_cmpr) - bq->excl_cmpr = excl_cmpr; - if (flags) - bq->flags = flags; - if (max_rfer) - bq->max_rfer = max_rfer; - if (max_excl) - bq->max_excl = max_excl; - if (rsv_rfer) - bq->rsv_rfer = rsv_rfer; - if (parent && child) { - list = malloc(sizeof(*list)); - if (!list) { - error("memory allocation failed"); - exit(1); - } - list->qgroup = parent; - list->member = child; - list_add_tail(&list->next_qgroup, &child->qgroups); - list_add_tail(&list->next_member, &parent->members); + + parent = qgroup_tree_search(qgroup_lookup, parent_id); + if (!parent) { + error("cannot find the qgroup %llu/%llu", + btrfs_qgroup_level(parent_id), + btrfs_qgroup_subvid(parent_id)); + return -ENOENT; } - ret = qgroup_tree_insert(qgroup_lookup, bq); - if (ret) { - error("failed to insert %llu into tree: %s", - (unsigned long long)bq->qgroupid, strerror(-ret)); - exit(1); + + list = malloc(sizeof(*list)); + if (!list) { + error("memory allocation failed"); + return -ENOMEM; } - return ret; + + list->qgroup = parent; + list->member = child; + list_add_tail(&list->next_qgroup, &child->qgroups); + list_add_tail(&list->next_member, &parent->members); + + return 0; } static void __free_btrfs_qgroup(struct btrfs_qgroup *bq) @@ -1042,13 +1048,8 @@ static int __qgroups_search(int fd, struct qgroup_lookup *qgroup_lookup) unsigned int i; struct btrfs_qgroup_info_item *info; struct btrfs_qgroup_limit_item *limit; - struct btrfs_qgroup *bq; - struct btrfs_qgroup *bq1; - u64 a1; - u64 a2; - u64 a3; - u64 a4; - u64 a5; + u64 qgroupid; + u64 qgroupid1; memset(&args, 0, sizeof(args)); @@ -1092,55 +1093,30 @@ static int __qgroups_search(int fd, struct qgroup_lookup *qgroup_lookup) print_status_flag_warning(flags); } else if (btrfs_search_header_type(sh) == BTRFS_QGROUP_INFO_KEY) { + qgroupid = btrfs_search_header_offset(sh); info = (struct btrfs_qgroup_info_item *) (args.buf + off); - a1 = btrfs_stack_qgroup_info_generation(info); - a2 = btrfs_stack_qgroup_info_referenced(info); - a3 = - btrfs_stack_qgroup_info_referenced_compressed - (info); - a4 = btrfs_stack_qgroup_info_exclusive(info); - a5 = - btrfs_stack_qgroup_info_exclusive_compressed - (info); - add_qgroup(qgroup_lookup, - btrfs_search_header_offset(sh), a1, - a2, a3, a4, a5, 0, 0, 0, 0, 0, NULL, - NULL); + + update_qgroup_info(qgroup_lookup, qgroupid, + info); } else if (btrfs_search_header_type(sh) == BTRFS_QGROUP_LIMIT_KEY) { + qgroupid = btrfs_search_header_offset(sh); limit = (struct btrfs_qgroup_limit_item *) - (args.buf + off); - - a1 = btrfs_stack_qgroup_limit_flags(limit); - a2 = btrfs_stack_qgroup_limit_max_referenced - (limit); - a3 = btrfs_stack_qgroup_limit_max_exclusive - (limit); - a4 = btrfs_stack_qgroup_limit_rsv_referenced - (limit); - a5 = btrfs_stack_qgroup_limit_rsv_exclusive - (limit); - add_qgroup(qgroup_lookup, - btrfs_search_header_offset(sh), 0, - 0, 0, 0, 0, a1, a2, a3, a4, a5, - NULL, NULL); + (args.buf + off); + + update_qgroup_limit(qgroup_lookup, qgroupid, + limit); } else if (btrfs_search_header_type(sh) == BTRFS_QGROUP_RELATION_KEY) { - if (btrfs_search_header_offset(sh) - < btrfs_search_header_objectid(sh)) - goto skip; - bq = qgroup_tree_search(qgroup_lookup, - btrfs_search_header_offset(sh)); - if (!bq) - goto skip; - bq1 = qgroup_tree_search(qgroup_lookup, - btrfs_search_header_objectid(sh)); - if (!bq1) + qgroupid = btrfs_search_header_offset(sh); + qgroupid1 = btrfs_search_header_objectid(sh); + + if (qgroupid < qgroupid1) goto skip; - add_qgroup(qgroup_lookup, - btrfs_search_header_offset(sh), 0, - 0, 0, 0, 0, 0, 0, 0, 0, 0, bq, bq1); + + update_qgroup_relation(qgroup_lookup, qgroupid, + qgroupid1); } else goto done; skip: @@ -1222,7 +1198,8 @@ int btrfs_qgroup_parse_sort_string(const char *opt_arg, if (!opt_tmp) return -ENOMEM; - while ((p = strtok(opt_tmp, ",")) != NULL) { + p = strtok(opt_tmp, ","); + while (p) { flag = 0; ptr_argv = all_sort_items; @@ -1262,8 +1239,7 @@ int btrfs_qgroup_parse_sort_string(const char *opt_arg, } btrfs_qgroup_setup_comparer(comps, what_to_sort, order); } - free(opt_tmp); - opt_tmp = NULL; + p = strtok(NULL, ","); } out: diff --git a/tests/cli-tests/002-balance-full-no-filters/test.sh b/tests/cli-tests/002-balance-full-no-filters/test.sh index 0501aad2..0475ea73 100755 --- a/tests/cli-tests/002-balance-full-no-filters/test.sh +++ b/tests/cli-tests/002-balance-full-no-filters/test.sh @@ -8,7 +8,7 @@ check_prereq mkfs.btrfs check_prereq btrfs setup_root_helper -prepare_test_dev 2g +prepare_test_dev run_check "$TOP/mkfs.btrfs" -f "$TEST_DEV" run_check_mount_test_dev diff --git a/tests/cli-tests/004-send-parent-multi-subvol/test.sh b/tests/cli-tests/004-send-parent-multi-subvol/test.sh index 49226f9b..c1348b50 100755 --- a/tests/cli-tests/004-send-parent-multi-subvol/test.sh +++ b/tests/cli-tests/004-send-parent-multi-subvol/test.sh @@ -8,7 +8,7 @@ check_prereq mkfs.btrfs check_prereq btrfs setup_root_helper -prepare_test_dev 2g +prepare_test_dev run_check "$TOP/mkfs.btrfs" -f "$TEST_DEV" run_check_mount_test_dev diff --git a/tests/cli-tests/005-qgroup-show/test.sh b/tests/cli-tests/005-qgroup-show/test.sh index 2af13033..d9a91831 100755 --- a/tests/cli-tests/005-qgroup-show/test.sh +++ b/tests/cli-tests/005-qgroup-show/test.sh @@ -8,7 +8,7 @@ check_prereq mkfs.btrfs check_prereq btrfs setup_root_helper -prepare_test_dev 2g +prepare_test_dev run_check "$TOP/mkfs.btrfs" -f "$TEST_DEV" run_check_mount_test_dev diff --git a/tests/cli-tests/008-subvolume-get-set-default/test.sh b/tests/cli-tests/008-subvolume-get-set-default/test.sh new file mode 100755 index 00000000..9318002e --- /dev/null +++ b/tests/cli-tests/008-subvolume-get-set-default/test.sh @@ -0,0 +1,47 @@ +#!/bin/bash +# test for "subvolume get-default/set-default" + +check_default_id() +{ + id=$(run_check_stdout $SUDO_HELPER "$TOP/btrfs" subvolume get-default .) \ + || { echo "$id"; exit 1; } + if $(echo "$id" | grep -vq "ID $1"); then + _fail "subvolume get-default: default id is not $1, but $id" + fi +} + +source "$TOP/tests/common" + +check_prereq mkfs.btrfs +check_prereq btrfs + +setup_root_helper +prepare_test_dev + +run_check "$TOP/mkfs.btrfs" -f "$TEST_DEV" +run_check_mount_test_dev +cd "$TEST_MNT" + +check_default_id 5 + +# check "subvol set-default <subvolid> <path>" +run_check $SUDO_HELPER "$TOP/btrfs" subvolume create sub +id=$(run_check_stdout "$TOP/btrfs" inspect-internal rootid sub) +run_check $SUDO_HELPER "$TOP/btrfs" subvolume set-default "$id" . +check_default_id "$id" + +run_mustfail "set-default to non existent id" \ + $SUDO_HELPER "$TOP/btrfs" subvolume set-default 100 . + +# check "subvol set-default <subvolume>" +run_check $SUDO_HELPER "$TOP/btrfs" subvolume create sub2 +id=$(run_check_stdout "$TOP/btrfs" inspect-internal rootid sub2) +run_check $SUDO_HELPER "$TOP/btrfs" subvolume set-default ./sub2 +check_default_id "$id" + +run_check $SUDO_HELPER mkdir sub2/dir +run_mustfail "set-default to normal directory" \ + $SUDO_HELPER "$TOP/btrfs" subvolume set-default ./sub2/dir + +cd .. +run_check_umount_test_dev diff --git a/tests/common b/tests/common index eb525a4d..734cd171 100644 --- a/tests/common +++ b/tests/common @@ -236,6 +236,58 @@ run_mustfail() fi } +# The first parameter is error message to print if it fails, just like +# run_must_fail(). +# NOTE: we don't use pipefail to avoid disturbing other script, so here we +# use a temporary output file. +# So it doesn't support pipeline in the @cmd +run_mustfail_stdout() +{ + local spec + local ins + local cmd + local msg + local ret + local tmp_output + + tmp_output=$(mktemp --tmpdir btrfs-progs-test--mustfail-stdtout.XXXXXX) + + msg="$1" + shift + + if _is_file_or_command "$msg"; then + echo "ASSERTION FAIL: 1st argument of run_mustfail_stdout must be a message" + exit 1 + fi + + ins=$(_get_spec_ins "$@") + spec=$(($ins-1)) + cmd=$(eval echo "\${$spec}") + spec=$(_cmd_spec "${@:$spec}") + set -- "${@:1:$(($ins-1))}" $spec "${@: $ins}" + echo "############### $@" >> "$RESULTS" 2>&1 + if [[ $TEST_LOG =~ tty ]]; then echo "CMD(mustfail): $@" > /dev/tty; fi + if [ "$1" = 'root_helper' ]; then + "$@" 2>&1 > "$tmp_output" + else + $INSTRUMENT "$@" 2>&1 > "$tmp_output" + fi + ret=$? + + cat "$tmp_output" >> "$RESULTS" + cat "$tmp_output" + rm "$tmp_output" + + if [ "$ret" != 0 ]; then + echo "failed (expected): $@" >> "$RESULTS" + return 0 + else + echo "succeeded (unexpected!): $@" >> "$RESULTS" + _fail "unexpected success: $msg" + return 1 + fi +} + check_prereq() { if ! [ -f "$TOP/$1" ]; then @@ -389,8 +441,12 @@ prepare_test_dev() # num[K/M/G/T...] local size="$1" - [[ "$TEST_DEV" ]] && return [[ "$size" ]] || size='2G' + # Still truncate it to new size + if [ -n "$TEST_DEV" ]; then + truncate -s "$size" "$TEST_DEV" + return; + fi echo "\$TEST_DEV not given, use $TOP/test/test.img as fallback" >> \ "$RESULTS" @@ -428,8 +484,12 @@ run_check_umount_test_dev() check_kernel_support() { if ! grep -iq 'btrfs' /proc/filesystems; then - echo "WARNING: btrfs filesystem not listed in /proc/filesystems, some tests might fail" - return 1 + run_check $SUDO_HELPER modprobe btrfs + if ! grep -iq 'btrfs' /proc/filesystems; then + echo \ +"WARNING: btrfs filesystem not found in /proc/filesystems, some tests might fail" + return 1 + fi fi return 0 } diff --git a/tests/common.local b/tests/common.local index d2b8d073..f5e96f5b 100644 --- a/tests/common.local +++ b/tests/common.local @@ -14,11 +14,24 @@ fi # gets arguments of a current command and can decide if the argument insertion # should happen, eg. if some option combination does not make sense or would # break tests +# +# Return 0 if we need to skip option override +# Return 1 if we don't need to skip option override _skip_spec() { + local beacon + + beacon=.lowmem_repairable + + # For lowmem repair, only support fs tree repair for now + # So we place lowmem repair beacon in the same dir of the test case if echo "$TEST_ARGS_CHECK" | grep -q 'mode=lowmem' && echo "$@" | grep -q -- '--repair'; then - return 0 + dir="$(dirname ${@: -1})" + if [ -f ${dir}/${beacon} ]; then + return 1; + fi + return 0; fi return 1 } diff --git a/tests/convert-tests/001-ext2-basic/test.sh b/tests/convert-tests/001-ext2-basic/test.sh index d94bf0b6..af75d948 100755 --- a/tests/convert-tests/001-ext2-basic/test.sh +++ b/tests/convert-tests/001-ext2-basic/test.sh @@ -4,7 +4,7 @@ source "$TOP/tests/common" source "$TOP/tests/common.convert" setup_root_helper -prepare_test_dev 512M +prepare_test_dev check_prereq btrfs-convert check_global_prereq mke2fs diff --git a/tests/convert-tests/002-ext3-basic/test.sh b/tests/convert-tests/002-ext3-basic/test.sh index f00e0e82..233e2d94 100755 --- a/tests/convert-tests/002-ext3-basic/test.sh +++ b/tests/convert-tests/002-ext3-basic/test.sh @@ -4,7 +4,7 @@ source "$TOP/tests/common" source "$TOP/tests/common.convert" setup_root_helper -prepare_test_dev 512M +prepare_test_dev check_prereq btrfs-convert check_global_prereq mke2fs diff --git a/tests/convert-tests/003-ext4-basic/test.sh b/tests/convert-tests/003-ext4-basic/test.sh index d33f1d9a..baf6115c 100755 --- a/tests/convert-tests/003-ext4-basic/test.sh +++ b/tests/convert-tests/003-ext4-basic/test.sh @@ -4,7 +4,7 @@ source "$TOP/tests/common" source "$TOP/tests/common.convert" setup_root_helper -prepare_test_dev 512M +prepare_test_dev check_prereq btrfs-convert check_global_prereq mke2fs diff --git a/tests/convert-tests/004-ext2-backup-superblock-ranges/test.sh b/tests/convert-tests/004-ext2-backup-superblock-ranges/test.sh index 0ce62f78..cf354d40 100755 --- a/tests/convert-tests/004-ext2-backup-superblock-ranges/test.sh +++ b/tests/convert-tests/004-ext2-backup-superblock-ranges/test.sh @@ -18,7 +18,7 @@ check_global_prereq e2fsck check_global_prereq xzcat setup_root_helper -prepare_test_dev 512M +prepare_test_dev # override common function function check_image() { diff --git a/tests/convert-tests/005-delete-all-rollback/test.sh b/tests/convert-tests/005-delete-all-rollback/test.sh index 19aa76d4..31fa2c4b 100755 --- a/tests/convert-tests/005-delete-all-rollback/test.sh +++ b/tests/convert-tests/005-delete-all-rollback/test.sh @@ -6,7 +6,7 @@ source "$TOP/tests/common" source "$TOP/tests/common.convert" setup_root_helper -prepare_test_dev 512M +prepare_test_dev check_prereq btrfs-convert check_global_prereq mke2fs diff --git a/tests/convert-tests/006-large-hole-extent/test.sh b/tests/convert-tests/006-large-hole-extent/test.sh index 0edb6280..38e97055 100755 --- a/tests/convert-tests/006-large-hole-extent/test.sh +++ b/tests/convert-tests/006-large-hole-extent/test.sh @@ -9,7 +9,7 @@ source "$TOP/tests/common" source "$TOP/tests/common.convert" setup_root_helper -prepare_test_dev 512M +prepare_test_dev check_prereq btrfs-convert check_global_prereq mke2fs diff --git a/tests/convert-tests/007-unsupported-block-sizes/test.sh b/tests/convert-tests/007-unsupported-block-sizes/test.sh index f1b29726..ef010202 100755 --- a/tests/convert-tests/007-unsupported-block-sizes/test.sh +++ b/tests/convert-tests/007-unsupported-block-sizes/test.sh @@ -5,7 +5,7 @@ source "$TOP/tests/common" source "$TOP/tests/common.convert" setup_root_helper -prepare_test_dev 512M +prepare_test_dev check_prereq btrfs-convert check_global_prereq mke2fs diff --git a/tests/convert-tests/008-readonly-image/test.sh b/tests/convert-tests/008-readonly-image/test.sh index 8551fb9b..064bc271 100755 --- a/tests/convert-tests/008-readonly-image/test.sh +++ b/tests/convert-tests/008-readonly-image/test.sh @@ -5,7 +5,7 @@ source "$TOP/tests/common" source "$TOP/tests/common.convert" setup_root_helper -prepare_test_dev 512M +prepare_test_dev check_prereq btrfs-convert check_global_prereq mke2fs diff --git a/tests/convert-tests/009-common-inode-flags/test.sh b/tests/convert-tests/009-common-inode-flags/test.sh index cd7b5111..6d159993 100755 --- a/tests/convert-tests/009-common-inode-flags/test.sh +++ b/tests/convert-tests/009-common-inode-flags/test.sh @@ -5,7 +5,7 @@ source "$TOP/tests/common" source "$TOP/tests/common.convert" setup_root_helper -prepare_test_dev 512M +prepare_test_dev check_prereq btrfs-convert check_global_prereq mke2fs check_global_prereq lsattr diff --git a/tests/convert-tests/010-reiserfs-basic/test.sh b/tests/convert-tests/010-reiserfs-basic/test.sh index 261f8704..87008f15 100755 --- a/tests/convert-tests/010-reiserfs-basic/test.sh +++ b/tests/convert-tests/010-reiserfs-basic/test.sh @@ -8,7 +8,7 @@ if ! check_kernel_support_reiserfs >/dev/null; then fi setup_root_helper -prepare_test_dev 512M +prepare_test_dev check_prereq btrfs-convert check_global_prereq mkreiserfs diff --git a/tests/convert-tests/011-reiserfs-delete-all-rollback/test.sh b/tests/convert-tests/011-reiserfs-delete-all-rollback/test.sh index c6c3119e..0b8366c8 100755 --- a/tests/convert-tests/011-reiserfs-delete-all-rollback/test.sh +++ b/tests/convert-tests/011-reiserfs-delete-all-rollback/test.sh @@ -9,7 +9,7 @@ if ! check_kernel_support_reiserfs >/dev/null; then fi setup_root_helper -prepare_test_dev 512M +prepare_test_dev check_prereq btrfs-convert check_global_prereq mkreiserfs diff --git a/tests/convert-tests/012-reiserfs-large-hole-extent/test.sh b/tests/convert-tests/012-reiserfs-large-hole-extent/test.sh index 7a8d4972..dde1b3eb 100755 --- a/tests/convert-tests/012-reiserfs-large-hole-extent/test.sh +++ b/tests/convert-tests/012-reiserfs-large-hole-extent/test.sh @@ -13,7 +13,7 @@ if ! check_kernel_support_reiserfs >/dev/null; then fi setup_root_helper -prepare_test_dev 512M +prepare_test_dev check_prereq btrfs-convert check_global_prereq mkreiserfs diff --git a/tests/convert-tests/013-reiserfs-common-inode-flags/test.sh b/tests/convert-tests/013-reiserfs-common-inode-flags/test.sh index e242a3b0..a15240ce 100755 --- a/tests/convert-tests/013-reiserfs-common-inode-flags/test.sh +++ b/tests/convert-tests/013-reiserfs-common-inode-flags/test.sh @@ -9,7 +9,7 @@ if ! check_kernel_support_reiserfs >/dev/null; then fi setup_root_helper -prepare_test_dev 512M +prepare_test_dev check_prereq btrfs-convert check_global_prereq mkreiserfs check_global_prereq chattr diff --git a/tests/convert-tests/014-reiserfs-tail-handling/test.sh b/tests/convert-tests/014-reiserfs-tail-handling/test.sh index f6131d96..335c0091 100755 --- a/tests/convert-tests/014-reiserfs-tail-handling/test.sh +++ b/tests/convert-tests/014-reiserfs-tail-handling/test.sh @@ -14,7 +14,7 @@ if ! check_kernel_support_reiserfs >/dev/null; then fi setup_root_helper -prepare_test_dev 512M +prepare_test_dev check_prereq btrfs-convert check_global_prereq md5sum check_global_prereq mkreiserfs diff --git a/tests/convert-tests/015-no-rollback-after-balance/test.sh b/tests/convert-tests/015-no-rollback-after-balance/test.sh new file mode 100755 index 00000000..47c9c6fa --- /dev/null +++ b/tests/convert-tests/015-no-rollback-after-balance/test.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Check if btrfs-convert refuses to rollback the filesystem, and leave the fs +# and the convert image untouched + +source "$TOP/tests/common" +source "$TOP/tests/common.convert" + +setup_root_helper +prepare_test_dev +check_prereq btrfs-convert +check_global_prereq mke2fs + +# convert_test_prep_fs() will create large enough file inside the test device, +# that's good enough for us to test rollback failure. +convert_test_prep_fs ext4 mke2fs -t ext4 -b 4096 +run_check_umount_test_dev +convert_test_do_convert "" 4096 + +run_check_mount_test_dev + +# Now the fs is converted, balance it so later rollback should fail +run_check $SUDO_HELPER "$TOP/btrfs" balance start --full-balance "$TEST_MNT" +run_check_umount_test_dev + +# rollback should fail +run_mustfail "rollback fs after balance" "$TOP/btrfs-convert" -r "$TEST_DEV" + +# Ensure the fs and convert image can pass the check +run_check "$TOP/btrfs" check "$TEST_DEV" + +run_check_mount_test_dev +run_check $SUDO_HELPER e2fsck -fn "$TEST_MNT/ext2_saved/image" +run_check_umount_test_dev diff --git a/tests/fsck-tests/004-no-dir-index/.lowmem_repairable b/tests/fsck-tests/004-no-dir-index/.lowmem_repairable new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/fsck-tests/004-no-dir-index/.lowmem_repairable diff --git a/tests/fsck-tests/009-no-dir-item-or-index/.lowmem_repairable b/tests/fsck-tests/009-no-dir-item-or-index/.lowmem_repairable new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/fsck-tests/009-no-dir-item-or-index/.lowmem_repairable diff --git a/tests/fsck-tests/010-no-rootdir-inode-item/.lowmem_repairable b/tests/fsck-tests/010-no-rootdir-inode-item/.lowmem_repairable new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/fsck-tests/010-no-rootdir-inode-item/.lowmem_repairable diff --git a/tests/fsck-tests/011-no-inode-item/.lowmem_repairable b/tests/fsck-tests/011-no-inode-item/.lowmem_repairable new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/fsck-tests/011-no-inode-item/.lowmem_repairable diff --git a/tests/fsck-tests/013-extent-tree-rebuild/test.sh b/tests/fsck-tests/013-extent-tree-rebuild/test.sh index 90fe2e83..d71c1b2e 100755 --- a/tests/fsck-tests/013-extent-tree-rebuild/test.sh +++ b/tests/fsck-tests/013-extent-tree-rebuild/test.sh @@ -7,7 +7,7 @@ check_prereq mkfs.btrfs check_prereq btrfs setup_root_helper -prepare_test_dev 1G +prepare_test_dev # test whether fsck can rebuild a corrupted extent tree test_extent_tree_rebuild() diff --git a/tests/fsck-tests/016-wrong-inode-nbytes/.lowmem_repairable b/tests/fsck-tests/016-wrong-inode-nbytes/.lowmem_repairable new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/fsck-tests/016-wrong-inode-nbytes/.lowmem_repairable diff --git a/tests/fsck-tests/017-missing-all-file-extent/.lowmem_repairable b/tests/fsck-tests/017-missing-all-file-extent/.lowmem_repairable new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/tests/fsck-tests/017-missing-all-file-extent/.lowmem_repairable diff --git a/tests/fsck-tests/020-extent-ref-cases/inline_regular_coexist.img b/tests/fsck-tests/020-extent-ref-cases/inline_regular_coexist.img Binary files differnew file mode 100644 index 00000000..cf15cc14 --- /dev/null +++ b/tests/fsck-tests/020-extent-ref-cases/inline_regular_coexist.img diff --git a/tests/fsck-tests/020-extent-ref-cases/keyed_data_ref_only.img b/tests/fsck-tests/020-extent-ref-cases/keyed_data_ref_only.img Binary files differnew file mode 100644 index 00000000..668589f4 --- /dev/null +++ b/tests/fsck-tests/020-extent-ref-cases/keyed_data_ref_only.img diff --git a/tests/fsck-tests/020-extent-ref-cases/keyed_data_ref_with_shared_leaf.img b/tests/fsck-tests/020-extent-ref-cases/keyed_data_ref_with_shared_leaf.img Binary files differnew file mode 100644 index 00000000..2ce5068f --- /dev/null +++ b/tests/fsck-tests/020-extent-ref-cases/keyed_data_ref_with_shared_leaf.img diff --git a/tests/fsck-tests/020-extent-ref-cases/ref_count_mismatch_false_alert.img b/tests/fsck-tests/020-extent-ref-cases/ref_count_mismatch_false_alert.img Binary files differnew file mode 100644 index 00000000..85110a81 --- /dev/null +++ b/tests/fsck-tests/020-extent-ref-cases/ref_count_mismatch_false_alert.img diff --git a/tests/fsck-tests/020-extent-ref-cases/shared_block_ref_only.raw.xz b/tests/fsck-tests/020-extent-ref-cases/shared_block_ref_only.raw.xz Binary files differnew file mode 100644 index 00000000..538fb4f2 --- /dev/null +++ b/tests/fsck-tests/020-extent-ref-cases/shared_block_ref_only.raw.xz diff --git a/tests/fsck-tests/020-extent-ref-cases/test.sh b/tests/fsck-tests/020-extent-ref-cases/test.sh index 1e1e4e23..0c4f7848 100755 --- a/tests/fsck-tests/020-extent-ref-cases/test.sh +++ b/tests/fsck-tests/020-extent-ref-cases/test.sh @@ -1,6 +1,6 @@ #!/bin/bash -# In order to confirm that btrfsck supports to check a variety of refs, add the -# following cases: +# In order to confirm that 'btrfs check' supports checking a variety of refs, +# add the following cases: # # * keyed_block_ref # * keyed_data_ref @@ -19,12 +19,11 @@ source "$TOP/tests/common" check_prereq btrfs -for img in *.img *.raw.xz -do - image=$(extract_image "$img") +check_image() { + local image - # Since the return value bug is already fixed, we don't need - # the old grep hack to detect bug. + image=$1 run_check "$TOP/btrfs" check "$image" - rm -f "$image" -done +} + +check_all_images diff --git a/tests/fsck-tests/021-partially-dropped-snapshot-case/test.sh b/tests/fsck-tests/021-partially-dropped-snapshot-case/test.sh index 44a33a63..5d997e24 100755 --- a/tests/fsck-tests/021-partially-dropped-snapshot-case/test.sh +++ b/tests/fsck-tests/021-partially-dropped-snapshot-case/test.sh @@ -1,18 +1,22 @@ #!/bin/bash -# confirm whether btrfsck supports to check a partially dropped snapshot +# confirm whether 'btrfs check' supports check ing of a partially dropped +# snapshot source "$TOP/tests/common" check_prereq btrfs -for img in *.img -do - image=$(extract_image "$img") +check_image() +{ + local image + + image=$1 run_check_stdout "$TOP/btrfs" check "$image" 2>&1 | grep -q "Errors found in extent allocation tree or chunk allocation" if [ $? -eq 0 ]; then rm -f "$image" _fail "unexpected error occurred when checking $img" fi - rm -f "$image" -done +} + +check_all_images diff --git a/tests/fsck-tests/024-clear-space-cache/test.sh b/tests/fsck-tests/024-clear-space-cache/test.sh index 6cf8440b..76ebcb6b 100755 --- a/tests/fsck-tests/024-clear-space-cache/test.sh +++ b/tests/fsck-tests/024-clear-space-cache/test.sh @@ -7,7 +7,7 @@ check_prereq btrfs check_prereq mkfs.btrfs setup_root_helper -prepare_test_dev 1G +prepare_test_dev run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f "$TEST_DEV" run_check_mount_test_dev diff --git a/tests/fsck-tests/027-bad-extent-inline-ref-type/bad-extent-inline-ref-type.raw.xz b/tests/fsck-tests/027-bad-extent-inline-ref-type/bad-extent-inline-ref-type.raw.xz Binary files differnew file mode 100644 index 00000000..09d82019 --- /dev/null +++ b/tests/fsck-tests/027-bad-extent-inline-ref-type/bad-extent-inline-ref-type.raw.xz diff --git a/tests/fsck-tests/027-tree-reloc-tree/test.sh b/tests/fsck-tests/027-tree-reloc-tree/test.sh new file mode 100755 index 00000000..afad1e8d --- /dev/null +++ b/tests/fsck-tests/027-tree-reloc-tree/test.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Make sure btrfs check won't report any false alerts for valid image with +# reloc tree. +# +# Also due to the short life span of reloc tree, save the as dump example for +# later usage. + +source "$TOP/tests/common" + +check_prereq btrfs + +check_image() { + local image + + image=$1 + run_check "$TOP/btrfs" check "$image" +} + +check_all_images diff --git a/tests/fsck-tests/027-tree-reloc-tree/tree_reloc_for_data_reloc.img.xz b/tests/fsck-tests/027-tree-reloc-tree/tree_reloc_for_data_reloc.img.xz Binary files differnew file mode 100644 index 00000000..66d8bde6 --- /dev/null +++ b/tests/fsck-tests/027-tree-reloc-tree/tree_reloc_for_data_reloc.img.xz diff --git a/tests/fsck-tests/027-tree-reloc-tree/tree_reloc_for_fs_tree.img.xz b/tests/fsck-tests/027-tree-reloc-tree/tree_reloc_for_fs_tree.img.xz Binary files differnew file mode 100644 index 00000000..22af324b --- /dev/null +++ b/tests/fsck-tests/027-tree-reloc-tree/tree_reloc_for_fs_tree.img.xz diff --git a/tests/fsck-tests/028-unaligned-super-dev-sizes/dev_and_super_mismatch_unaligned.raw.xz b/tests/fsck-tests/028-unaligned-super-dev-sizes/dev_and_super_mismatch_unaligned.raw.xz Binary files differnew file mode 100644 index 00000000..153e514a --- /dev/null +++ b/tests/fsck-tests/028-unaligned-super-dev-sizes/dev_and_super_mismatch_unaligned.raw.xz diff --git a/tests/fsck-tests/028-unaligned-super-dev-sizes/test.sh b/tests/fsck-tests/028-unaligned-super-dev-sizes/test.sh new file mode 100755 index 00000000..6f315fae --- /dev/null +++ b/tests/fsck-tests/028-unaligned-super-dev-sizes/test.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# +# An image with mis-aligned superblock total_bytes, that will be found and +# fixed by 'check' or fixed by 'rescue fix-device-size' + +source "$TOP/tests/common" + +check_prereq btrfs +prepare_test_dev +setup_root_helper + +check_all_images + +image=$(extract_image "./dev_and_super_mismatch_unaligned.raw.xz") + +# detect and fix +run_check "$TOP/btrfs" rescue fix-device-size "$image" +# no problem found +run_check "$TOP/btrfs" rescue fix-device-size "$image" +# check if fix-device-size worked +run_check "$TOP/btrfs" check "$image" +# mount test +run_check_mount_test_dev +run_check_umount_test_dev + +rm -f "$image" diff --git a/tests/misc-tests/005-convert-progress-thread-crash/test.sh b/tests/misc-tests/005-convert-progress-thread-crash/test.sh index 054069c2..bc71e1fd 100755 --- a/tests/misc-tests/005-convert-progress-thread-crash/test.sh +++ b/tests/misc-tests/005-convert-progress-thread-crash/test.sh @@ -6,7 +6,7 @@ source $TOP/tests/common check_prereq btrfs-convert mkfs.ext4 -V &>/dev/null || _not_run "mkfs.ext4 not found" -prepare_test_dev 1G +prepare_test_dev for ((i = 0; i < 20; i++)); do echo "loop $i" >>$RESULTS diff --git a/tests/misc-tests/016-send-clone-src/test.sh b/tests/misc-tests/016-send-clone-src/test.sh index 479da677..2780ebbd 100755 --- a/tests/misc-tests/016-send-clone-src/test.sh +++ b/tests/misc-tests/016-send-clone-src/test.sh @@ -10,7 +10,7 @@ check_prereq btrfs setup_root_helper -prepare_test_dev 1g +prepare_test_dev run_check "$TOP/mkfs.btrfs" -f "$TEST_DEV" run_check_mount_test_dev diff --git a/tests/misc-tests/017-recv-stream-malformatted/test.sh b/tests/misc-tests/017-recv-stream-malformatted/test.sh index 3756be27..d199a72e 100755 --- a/tests/misc-tests/017-recv-stream-malformatted/test.sh +++ b/tests/misc-tests/017-recv-stream-malformatted/test.sh @@ -9,7 +9,7 @@ check_prereq btrfs setup_root_helper -prepare_test_dev 1g +prepare_test_dev run_check "$TOP/mkfs.btrfs" -f "$TEST_DEV" run_check_mount_test_dev diff --git a/tests/misc-tests/018-recv-end-of-stream/test.sh b/tests/misc-tests/018-recv-end-of-stream/test.sh index 3b8a0319..9ca035f7 100755 --- a/tests/misc-tests/018-recv-end-of-stream/test.sh +++ b/tests/misc-tests/018-recv-end-of-stream/test.sh @@ -9,7 +9,7 @@ check_prereq mkfs.btrfs check_prereq btrfs setup_root_helper -prepare_test_dev 1g +prepare_test_dev here=`pwd` diff --git a/tests/misc-tests/019-receive-clones-on-munted-subvol/test.sh b/tests/misc-tests/019-receive-clones-on-mounted-subvol/test.sh index 182b0cf9..182b0cf9 100755 --- a/tests/misc-tests/019-receive-clones-on-munted-subvol/test.sh +++ b/tests/misc-tests/019-receive-clones-on-mounted-subvol/test.sh diff --git a/tests/mkfs-tests/008-secorsize-nodesize-combination/test.sh b/tests/mkfs-tests/008-sectorsize-nodesize-combination/test.sh index 151e7b77..955cd2b1 100755 --- a/tests/mkfs-tests/008-secorsize-nodesize-combination/test.sh +++ b/tests/mkfs-tests/008-sectorsize-nodesize-combination/test.sh @@ -19,7 +19,7 @@ do_test() { sectorsize=$1 nodesize=$2 - run_mayfail $TOP/mkfs.btrfs -O $features -n $nodesize -s $sectorsize \ + run_mayfail $TOP/mkfs.btrfs -f -O $features -n $nodesize -s $sectorsize \ $TEST_DEV ret=$? if [ $ret == 0 ]; then @@ -6,7 +6,7 @@ # Copyright 2008, Oracle # Released under the GNU GPLv2 -v="v4.13.3" +v="v4.14.1" opt=$1 @@ -138,7 +138,21 @@ static int device_list_add(const char *path, list_add(&device->dev_list, &fs_devices->devices); device->fs_devices = fs_devices; } else if (!device->name || strcmp(device->name, path)) { - char *name = strdup(path); + char *name; + + /* + * The existing device has newer generation, so this one could + * be a stale one, don't add it. + */ + if (found_transid < device->generation) { + warning( + "adding device %s gen %llu but found an existing device %s gen %llu", + path, found_transid, device->name, + device->generation); + return -EEXIST; + } + + name = strdup(path); if (!name) return -ENOMEM; kfree(device->name); @@ -1032,11 +1046,13 @@ again: info->chunk_root->root_key.objectid, BTRFS_FIRST_CHUNK_TREE_OBJECTID, key.offset, calc_size, &dev_offset, 0); - BUG_ON(ret); + if (ret < 0) + goto out_chunk_map; device->bytes_used += calc_size; ret = btrfs_update_device(trans, device); - BUG_ON(ret); + if (ret < 0) + goto out_chunk_map; map->stripes[index].dev = device; map->stripes[index].physical = dev_offset; @@ -1075,16 +1091,24 @@ again: map->ce.size = *num_bytes; ret = insert_cache_extent(&info->mapping_tree.cache_tree, &map->ce); - BUG_ON(ret); + if (ret < 0) + goto out_chunk_map; if (type & BTRFS_BLOCK_GROUP_SYSTEM) { ret = btrfs_add_system_chunk(info, &key, chunk, btrfs_chunk_item_size(num_stripes)); - BUG_ON(ret); + if (ret < 0) + goto out_chunk; } kfree(chunk); return ret; + +out_chunk_map: + kfree(map); +out_chunk: + kfree(chunk); + return ret; } /* @@ -2332,3 +2356,181 @@ u64 btrfs_stripe_length(struct btrfs_fs_info *fs_info, } return stripe_len; } + +/* + * Return 0 if size of @device is already good + * Return >0 if size of @device is not aligned but fixed without problems + * Return <0 if something wrong happened when aligning the size of @device + */ +int btrfs_fix_device_size(struct btrfs_fs_info *fs_info, + struct btrfs_device *device) +{ + struct btrfs_trans_handle *trans; + struct btrfs_key key; + struct btrfs_path path; + struct btrfs_root *chunk_root = fs_info->chunk_root; + struct btrfs_dev_item *di; + u64 old_bytes = device->total_bytes; + int ret; + + if (IS_ALIGNED(old_bytes, fs_info->sectorsize)) + return 0; + + /* Align the in-memory total_bytes first, and use it as correct size */ + device->total_bytes = round_down(device->total_bytes, + fs_info->sectorsize); + + key.objectid = BTRFS_DEV_ITEMS_OBJECTID; + key.type = BTRFS_DEV_ITEM_KEY; + key.offset = device->devid; + + trans = btrfs_start_transaction(chunk_root, 1); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + error("error starting transaction: %d (%s)", + ret, strerror(-ret)); + return ret; + } + + btrfs_init_path(&path); + ret = btrfs_search_slot(trans, chunk_root, &key, &path, 0, 1); + if (ret > 0) { + error("failed to find DEV_ITEM for devid %llu", device->devid); + ret = -ENOENT; + goto err; + } + if (ret < 0) { + error("failed to search chunk root: %d (%s)", + ret, strerror(-ret)); + goto err; + } + di = btrfs_item_ptr(path.nodes[0], path.slots[0], struct btrfs_dev_item); + btrfs_set_device_total_bytes(path.nodes[0], di, device->total_bytes); + btrfs_mark_buffer_dirty(path.nodes[0]); + ret = btrfs_commit_transaction(trans, chunk_root); + if (ret < 0) { + error("failed to commit current transaction: %d (%s)", + ret, strerror(-ret)); + btrfs_release_path(&path); + return ret; + } + btrfs_release_path(&path); + printf("Fixed device size for devid %llu, old size: %llu new size: %llu\n", + device->devid, old_bytes, device->total_bytes); + return 1; + +err: + /* We haven't modified anything, it's OK to commit current trans */ + btrfs_commit_transaction(trans, chunk_root); + btrfs_release_path(&path); + return ret; +} + +/* + * Return 0 if super block total_bytes matches all devices' total_bytes + * Return >0 if super block total_bytes mismatch but fixed without problem + * Return <0 if we failed to fix super block total_bytes + */ +int btrfs_fix_super_size(struct btrfs_fs_info *fs_info) +{ + struct btrfs_trans_handle *trans; + struct btrfs_device *device; + struct list_head *dev_list = &fs_info->fs_devices->devices; + u64 total_bytes = 0; + u64 old_bytes = btrfs_super_total_bytes(fs_info->super_copy); + int ret; + + list_for_each_entry(device, dev_list, dev_list) { + /* + * Caller should ensure this function is called after aligning + * all devices' total_bytes. + */ + if (!IS_ALIGNED(device->total_bytes, fs_info->sectorsize)) { + error("device %llu total_bytes %llu not aligned to %u", + device->devid, device->total_bytes, + fs_info->sectorsize); + return -EUCLEAN; + } + total_bytes += device->total_bytes; + } + + if (total_bytes == old_bytes) + return 0; + + btrfs_set_super_total_bytes(fs_info->super_copy, total_bytes); + + /* Commit transaction to update all super blocks */ + trans = btrfs_start_transaction(fs_info->tree_root, 1); + if (IS_ERR(trans)) { + ret = PTR_ERR(trans); + error("error starting transaction: %d (%s)", + ret, strerror(-ret)); + return ret; + } + ret = btrfs_commit_transaction(trans, fs_info->tree_root); + if (ret < 0) { + error("failed to commit current transaction: %d (%s)", + ret, strerror(-ret)); + return ret; + } + printf("Fixed super total bytes, old size: %llu new size: %llu\n", + old_bytes, total_bytes); + return 1; +} + +/* + * Return 0 if all devices and super block sizes are good + * Return >0 if any device/super size problem was found, but fixed + * Return <0 if something wrong happened during fixing + */ +int btrfs_fix_device_and_super_size(struct btrfs_fs_info *fs_info) +{ + struct btrfs_device *device; + struct list_head *dev_list = &fs_info->fs_devices->devices; + bool have_bad_value = false; + int ret; + + /* Seed device is not supported yet */ + if (fs_info->fs_devices->seed) { + error("fixing device size with seed device is not supported yet"); + return -EOPNOTSUPP; + } + + /* All devices must be set up before repairing */ + if (list_empty(dev_list)) { + error("no device found"); + return -ENODEV; + } + list_for_each_entry(device, dev_list, dev_list) { + if (device->fd == -1 || !device->writeable) { + error("devid %llu is missing or not writeable", + device->devid); + error( + "fixing device size needs all device(s) to be present and writeable"); + return -ENODEV; + } + } + + /* Repair total_bytes of each device */ + list_for_each_entry(device, dev_list, dev_list) { + ret = btrfs_fix_device_size(fs_info, device); + if (ret < 0) + return ret; + if (ret > 0) + have_bad_value = true; + } + + /* Repair super total_byte */ + ret = btrfs_fix_super_size(fs_info); + if (ret > 0) + have_bad_value = true; + if (have_bad_value) { + printf( + "Fixed unaligned/mismatched total_bytes for super block and device items\n"); + ret = 1; + } else { + printf("No device size related problem found\n"); + ret = 0; + } + return ret; +} @@ -245,4 +245,8 @@ int btrfs_check_chunk_valid(struct btrfs_fs_info *fs_info, u64 btrfs_stripe_length(struct btrfs_fs_info *fs_info, struct extent_buffer *leaf, struct btrfs_chunk *chunk); +int btrfs_fix_device_size(struct btrfs_fs_info *fs_info, + struct btrfs_device *device); +int btrfs_fix_super_size(struct btrfs_fs_info *fs_info); +int btrfs_fix_device_and_super_size(struct btrfs_fs_info *fs_info); #endif |