diff options
Diffstat (limited to 'tests')
51 files changed, 2347 insertions, 157 deletions
diff --git a/tests/README.md b/tests/README.md index bb2846a1..a8d3a2ba 100644 --- a/tests/README.md +++ b/tests/README.md @@ -182,3 +182,37 @@ description of the problem or the stack trace. If you have a fix for the problem, please submit it prior to the test image, so the fuzz tests always succeed when run on random checked out. This helps bisectability. + + +# Coding style, best practices + +## do + +* quote all variables by default, any path, even the TOP could need that, and + we use it everywhere + * there are exceptions: + * `$SUDO_HELPER` as it might be intentionally unset + * the variable is obviously set to a value that does not require it +* use `#!/bin/bash` explicitly +* check for all external dependencies (`check_prereq_global`) +* check for internal dependencies (`check_prereq`), though the basic set is + always built when the tests are started through make +* use functions instead of repeating code + * generic helpers could be factored to the `common` script +* cleanup after successful test +* use common helpers and variables + +## do not + +* pull external dependencies if we can find a way to replace them: example is + `xfs_io` that's conveniently used in fstests but we'd require `xfsprogs`, + so use `dd` instead +* throw away (redirect to */dev/null*) output of commands unless it's justified + (ie. really too much text, unnecessary slowdown) -- the test output log is + regenerated all the time and we need to be able to analyze test failures or + just observe how the tests progress +* cleanup after failed test -- the testsuite stops on first failure and the + developer can eg. access the environment that the test created and do further + debugging + * this might change in the future so the tests cover as much as possible, but + this would require to enhance all tests with a cleanup phase diff --git a/tests/build-tests.sh b/tests/build-tests.sh index 04e3fd19..4dc8744e 100755 --- a/tests/build-tests.sh +++ b/tests/build-tests.sh @@ -63,6 +63,9 @@ function build_make_targets() { # defaults, library target="library-test" buildme + # defaults, static library + target="library-test.static" + buildme } # main() diff --git a/tests/cli-tests/002-balance-full-no-filters/test.sh b/tests/cli-tests/002-balance-full-no-filters/test.sh index 81a719eb..0501aad2 100755 --- a/tests/cli-tests/002-balance-full-no-filters/test.sh +++ b/tests/cli-tests/002-balance-full-no-filters/test.sh @@ -10,7 +10,7 @@ check_prereq btrfs setup_root_helper prepare_test_dev 2g -run_check "$TOP/mkfs.btrfs" -f "$IMAGE" +run_check "$TOP/mkfs.btrfs" -f "$TEST_DEV" run_check_mount_test_dev run_check $SUDO_HELPER "$TOP/btrfs" balance start --full-balance "$TEST_MNT" diff --git a/tests/cli-tests/003-fi-resize-args/test.sh b/tests/cli-tests/003-fi-resize-args/test.sh index b835e078..e4f262b6 100755 --- a/tests/cli-tests/003-fi-resize-args/test.sh +++ b/tests/cli-tests/003-fi-resize-args/test.sh @@ -10,7 +10,7 @@ check_prereq btrfs setup_root_helper prepare_test_dev 2g -run_check "$TOP/mkfs.btrfs" -f "$IMAGE" +run_check "$TOP/mkfs.btrfs" -f "$TEST_DEV" run_check_mount_test_dev # missing the one of the required arguments diff --git a/tests/cli-tests/004-send-parent-multi-subvol/test.sh b/tests/cli-tests/004-send-parent-multi-subvol/test.sh index 72a9eb36..49226f9b 100755 --- a/tests/cli-tests/004-send-parent-multi-subvol/test.sh +++ b/tests/cli-tests/004-send-parent-multi-subvol/test.sh @@ -10,7 +10,7 @@ check_prereq btrfs setup_root_helper prepare_test_dev 2g -run_check "$TOP/mkfs.btrfs" -f "$IMAGE" +run_check "$TOP/mkfs.btrfs" -f "$TEST_DEV" run_check_mount_test_dev here=`pwd` diff --git a/tests/cli-tests/005-qgroup-show/test.sh b/tests/cli-tests/005-qgroup-show/test.sh index 46d3c3a7..2af13033 100755 --- a/tests/cli-tests/005-qgroup-show/test.sh +++ b/tests/cli-tests/005-qgroup-show/test.sh @@ -10,7 +10,7 @@ check_prereq btrfs setup_root_helper prepare_test_dev 2g -run_check "$TOP/mkfs.btrfs" -f "$IMAGE" +run_check "$TOP/mkfs.btrfs" -f "$TEST_DEV" run_check_mount_test_dev run_mayfail "$TOP/btrfs" qgroup show "$TEST_MNT" run_mayfail $SUDO_HELPER "$TOP/btrfs" qgroup show "$TEST_MNT" diff --git a/tests/cli-tests/006-qgroup-show-sync/test.sh b/tests/cli-tests/006-qgroup-show-sync/test.sh index 30d0a9a1..d552b8b9 100755 --- a/tests/cli-tests/006-qgroup-show-sync/test.sh +++ b/tests/cli-tests/006-qgroup-show-sync/test.sh @@ -10,7 +10,7 @@ check_prereq btrfs setup_root_helper prepare_test_dev 1g -run_check "$TOP/mkfs.btrfs" -f "$IMAGE" +run_check "$TOP/mkfs.btrfs" -f "$TEST_DEV" run_check_mount_test_dev run_check $SUDO_HELPER "$TOP/btrfs" subvolume create "$TEST_MNT/Sub" diff --git a/tests/common b/tests/common index 51c2e267..bed60094 100644 --- a/tests/common +++ b/tests/common @@ -106,7 +106,7 @@ run_check() ins=$(_get_spec_ins "$@") spec=$(($ins-1)) cmd=$(eval echo "\${$spec}") - spec=$(_cmd_spec "$cmd") + spec=$(_cmd_spec "${@:$spec}") set -- "${@:1:$(($ins-1))}" $spec "${@: $ins}" echo "############### $@" >> "$RESULTS" 2>&1 if [[ $TEST_LOG =~ tty ]]; then echo "CMD: $@" > /dev/tty; fi @@ -128,7 +128,7 @@ run_check_stdout() ins=$(_get_spec_ins "$@") spec=$(($ins-1)) cmd=$(eval echo "\${$spec}") - spec=$(_cmd_spec "$cmd") + spec=$(_cmd_spec "${@:$spec}") set -- "${@:1:$(($ins-1))}" $spec "${@: $ins}" echo "############### $@" >> "$RESULTS" 2>&1 if [[ $TEST_LOG =~ tty ]]; then echo "CMD(stdout): $@" > /dev/tty; fi @@ -152,7 +152,7 @@ run_mayfail() ins=$(_get_spec_ins "$@") spec=$(($ins-1)) cmd=$(eval echo "\${$spec}") - spec=$(_cmd_spec "$cmd") + spec=$(_cmd_spec "${@:$spec}") set -- "${@:1:$(($ins-1))}" $spec "${@: $ins}" echo "############### $@" >> "$RESULTS" 2>&1 if [[ $TEST_LOG =~ tty ]]; then echo "CMD(mayfail): $@" > /dev/tty; fi @@ -188,7 +188,7 @@ run_mustfail() ins=$(_get_spec_ins "$@") spec=$(($ins-1)) cmd=$(eval echo "\${$spec}") - spec=$(_cmd_spec "$cmd") + spec=$(_cmd_spec "${@:$spec}") set -- "${@:1:$(($ins-1))}" $spec "${@: $ins}" echo "############### $@" >> "$RESULTS" 2>&1 if [[ $TEST_LOG =~ tty ]]; then echo "CMD(mustfail): $@" > /dev/tty; fi @@ -387,7 +387,7 @@ run_check_mount_test_dev() _fail "Invalid \$TEST_MNT: $TEST_MNT" } - run_check $SUDO_HELPER mount $loop_opt "$@" "$TEST_DEV" "$TEST_MNT" + run_check $SUDO_HELPER mount -t btrfs $loop_opt "$@" "$TEST_DEV" "$TEST_MNT" } run_check_umount_test_dev() diff --git a/tests/common.convert b/tests/common.convert index 8c9242e5..2c19a4be 100644 --- a/tests/common.convert +++ b/tests/common.convert @@ -1,6 +1,39 @@ #!/bin/bash # helpers for btrfs-convert tests +# mount image of converted filesystem of a given type +# $1: type of the filesystem +run_check_mount_convert_dev() +{ + local fstype + local loop_opt + + setup_root_helper + + fstype="$1" + shift + if [ -z "$fstype" ]; then + _fail "Missing source filesystem type" + fi + if [ "$fstype" = 'btrfs' ]; then + _fail "Incorrect type for converted filesystem: btrfs" + fi + + if [[ -b "$TEST_DEV" ]]; then + loop_opt="" + elif [[ -f "$TEST_DEV" ]]; then + loop_opt="-o loop" + else + _fail "Invalid \$TEST_DEV: $TEST_DEV" + fi + + [[ -d "$TEST_MNT" ]] || { + _fail "Invalid \$TEST_MNT: $TEST_MNT" + } + + run_check $SUDO_HELPER mount $loop_opt -t "$fstype" "$@" "$TEST_DEV" "$TEST_MNT" +} + populate_fs() { for dataset_type in 'small' 'hardlink' 'fast_symlink' 'brokenlink' 'perm' 'sparse' 'acls' 'fifo' 'slow_symlink'; do @@ -22,8 +55,13 @@ convert_test_preamble() { # prepare TEST_DEV before conversion, create filesystem and mount it, image # size is 512MB -# $@: free form, command to create the filesystem, with appended -F +# $1: type of the filesystem +# $2+: free form, command to create the filesystem, with appended -F convert_test_prep_fs() { + local fstype + + fstype="$1" + shift # TEST_DEV not removed as the file might have special permissions, eg. # when test image is on NFS and would not be writable for root run_check truncate -s 0 "$TEST_DEV" @@ -32,7 +70,7 @@ convert_test_prep_fs() { run_check "$@" -F "$TEST_DEV" # create a file to check btrfs-convert can convert regular file correct - run_check_mount_test_dev + run_check_mount_convert_dev "$fstype" # create a file inside the fs before convert, to make sure there is # data covering btrfs backup superblock range (64M) @@ -173,11 +211,13 @@ convert_test_post_rollback() { } # simple wrapper for a convert test -# $1: btrfs features, argument to -O -# $2: description of the test "ext2 8k nodesize" -# $3: nodesize value -# $4 + rest: command to create the ext2 image +# $1: type of the converted filesystem +# $2: btrfs features, argument to -O +# $3: description of the test "ext2 8k nodesize" +# $4: nodesize value +# $5 + rest: command to create the ext2 image convert_test() { + local fstype local features local nodesize local msg @@ -185,12 +225,13 @@ convert_test() { local EXT_PERMTMP local EXT_ACLTMP - features="$1" - msg="$2" - nodesize="$3" - shift 3 + fstype="$1" + features="$2" + msg="$3" + nodesize="$4" + shift 4 convert_test_preamble "$features" "$msg" "$nodesize" "$@" - convert_test_prep_fs "$@" + convert_test_prep_fs "$fstype" "$@" populate_fs CHECKSUMTMP=$(mktemp --tmpdir btrfs-progs-convert.XXXXXXXXXX) EXT_PERMTMP=$(mktemp --tmpdir btrfs-progs-convert.permXXXXXX) diff --git a/tests/common.local b/tests/common.local index 9f567c27..4f56bb08 100644 --- a/tests/common.local +++ b/tests/common.local @@ -17,7 +17,7 @@ TEST_ARGS_CHECK=--mode=lowmem # break tests _skip_spec() { - if echo "$TEST_CHECK" | grep -q 'mode=lowmem' && + if echo "$TEST_ARGS_CHECK" | grep -q 'mode=lowmem' && echo "$@" | grep -q -- '--repair'; then return 0 fi diff --git a/tests/convert-tests/001-ext2-basic/test.sh b/tests/convert-tests/001-ext2-basic/test.sh index 8f4f935d..7d8e87d8 100755 --- a/tests/convert-tests/001-ext2-basic/test.sh +++ b/tests/convert-tests/001-ext2-basic/test.sh @@ -1,16 +1,16 @@ #!/bin/bash -source $TOP/tests/common -source $TOP/tests/common.convert +source "$TOP/tests/common" +source "$TOP/tests/common.convert" setup_root_helper prepare_test_dev 512M check_prereq btrfs-convert for feature in '' 'extref' 'skinny-metadata' 'no-holes'; do - convert_test "$feature" "ext2 4k nodesize" 4096 mke2fs -b 4096 - convert_test "$feature" "ext2 8k nodesize" 8192 mke2fs -b 4096 - convert_test "$feature" "ext2 16k nodesize" 16384 mke2fs -b 4096 - convert_test "$feature" "ext2 32k nodesize" 32768 mke2fs -b 4096 - convert_test "$feature" "ext2 64k nodesize" 65536 mke2fs -b 4096 + convert_test ext2 "$feature" "ext2 4k nodesize" 4096 mke2fs -b 4096 + convert_test ext2 "$feature" "ext2 8k nodesize" 8192 mke2fs -b 4096 + convert_test ext2 "$feature" "ext2 16k nodesize" 16384 mke2fs -b 4096 + convert_test ext2 "$feature" "ext2 32k nodesize" 32768 mke2fs -b 4096 + convert_test ext2 "$feature" "ext2 64k nodesize" 65536 mke2fs -b 4096 done diff --git a/tests/convert-tests/002-ext3-basic/test.sh b/tests/convert-tests/002-ext3-basic/test.sh index aeb111eb..5a33c2ca 100755 --- a/tests/convert-tests/002-ext3-basic/test.sh +++ b/tests/convert-tests/002-ext3-basic/test.sh @@ -1,16 +1,16 @@ #!/bin/bash -source $TOP/tests/common -source $TOP/tests/common.convert +source "$TOP/tests/common" +source "$TOP/tests/common.convert" setup_root_helper prepare_test_dev 512M check_prereq btrfs-convert for feature in '' 'extref' 'skinny-metadata' 'no-holes'; do - convert_test "$feature" "ext3 4k nodesize" 4096 mke2fs -j -b 4096 - convert_test "$feature" "ext3 8k nodesize" 8192 mke2fs -j -b 4096 - convert_test "$feature" "ext3 16k nodesize" 16384 mke2fs -j -b 4096 - convert_test "$feature" "ext3 32k nodesize" 32768 mke2fs -j -b 4096 - convert_test "$feature" "ext3 64k nodesize" 65536 mke2fs -j -b 4096 + convert_test ext3 "$feature" "ext3 4k nodesize" 4096 mke2fs -j -b 4096 + convert_test ext3 "$feature" "ext3 8k nodesize" 8192 mke2fs -j -b 4096 + convert_test ext3 "$feature" "ext3 16k nodesize" 16384 mke2fs -j -b 4096 + convert_test ext3 "$feature" "ext3 32k nodesize" 32768 mke2fs -j -b 4096 + convert_test ext3 "$feature" "ext3 64k nodesize" 65536 mke2fs -j -b 4096 done diff --git a/tests/convert-tests/003-ext4-basic/test.sh b/tests/convert-tests/003-ext4-basic/test.sh index 531c81bd..df8bec28 100755 --- a/tests/convert-tests/003-ext4-basic/test.sh +++ b/tests/convert-tests/003-ext4-basic/test.sh @@ -1,16 +1,16 @@ #!/bin/bash -source $TOP/tests/common -source $TOP/tests/common.convert +source "$TOP/tests/common" +source "$TOP/tests/common.convert" setup_root_helper prepare_test_dev 512M check_prereq btrfs-convert for feature in '' 'extref' 'skinny-metadata' 'no-holes'; do - convert_test "$feature" "ext4 4k nodesize" 4096 mke2fs -t ext4 -b 4096 - convert_test "$feature" "ext4 8k nodesize" 8192 mke2fs -t ext4 -b 4096 - convert_test "$feature" "ext4 16k nodesize" 16384 mke2fs -t ext4 -b 4096 - convert_test "$feature" "ext4 32k nodesize" 32768 mke2fs -t ext4 -b 4096 - convert_test "$feature" "ext4 64k nodesize" 65536 mke2fs -t ext4 -b 4096 + convert_test ext4 "$feature" "ext4 4k nodesize" 4096 mke2fs -t ext4 -b 4096 + convert_test ext4 "$feature" "ext4 8k nodesize" 8192 mke2fs -t ext4 -b 4096 + convert_test ext4 "$feature" "ext4 16k nodesize" 16384 mke2fs -t ext4 -b 4096 + convert_test ext4 "$feature" "ext4 32k nodesize" 32768 mke2fs -t ext4 -b 4096 + convert_test ext4 "$feature" "ext4 64k nodesize" 65536 mke2fs -t ext4 -b 4096 done diff --git a/tests/convert-tests/004-ext2-backup-superblock-ranges/test.sh b/tests/convert-tests/004-ext2-backup-superblock-ranges/test.sh index c56650b2..0ce62f78 100755 --- a/tests/convert-tests/004-ext2-backup-superblock-ranges/test.sh +++ b/tests/convert-tests/004-ext2-backup-superblock-ranges/test.sh @@ -10,7 +10,7 @@ # 4) Overlap file extents # 5) Unable to rollback -source $TOP/tests/common +source "$TOP/tests/common" check_prereq btrfs-convert check_prereq btrfs @@ -23,20 +23,20 @@ prepare_test_dev 512M # override common function function check_image() { TEST_DEV="$1" - run_check e2fsck -n -f $TEST_DEV - run_check $TOP/btrfs-convert $TEST_DEV - run_check $TOP/btrfs check $TEST_DEV - run_check $TOP/btrfs inspect-internal dump-super $TEST_DEV + run_check e2fsck -n -f "$TEST_DEV" + run_check "$TOP/btrfs-convert" "$TEST_DEV" + run_check "$TOP/btrfs" check "$TEST_DEV" + run_check "$TOP/btrfs" inspect-internal dump-super "$TEST_DEV" run_check_mount_test_dev - run_check $SUDO_HELPER e2fsck -n -f $TEST_MNT/ext2_saved/image - run_check $SUDO_HELPER umount $TEST_MNT + run_check $SUDO_HELPER e2fsck -n -f "$TEST_MNT/ext2_saved/image" + run_check $SUDO_HELPER umount "$TEST_MNT" - run_check $TOP/btrfs check $TEST_DEV - run_check $TOP/btrfs-convert -r $TEST_DEV - run_check e2fsck -n -f $TEST_DEV + run_check "$TOP/btrfs" check "$TEST_DEV" + run_check "$TOP/btrfs-convert" -r "$TEST_DEV" + run_check e2fsck -n -f "$TEST_DEV" - rm -f $TEST_DEV + rm -f "$TEST_DEV" } check_all_images diff --git a/tests/convert-tests/005-delete-all-rollback/test.sh b/tests/convert-tests/005-delete-all-rollback/test.sh index cf576e70..337413bb 100755 --- a/tests/convert-tests/005-delete-all-rollback/test.sh +++ b/tests/convert-tests/005-delete-all-rollback/test.sh @@ -2,8 +2,8 @@ # create a base image, convert to btrfs, remove all files, rollback the ext4 image # note: ext4 only -source $TOP/tests/common -source $TOP/tests/common.convert +source "$TOP/tests/common" +source "$TOP/tests/common.convert" setup_root_helper prepare_test_dev 512M @@ -26,7 +26,7 @@ do_test() { nodesize="$3" shift 3 convert_test_preamble "$features" "$msg" "$nodesize" "$@" - convert_test_prep_fs "$@" + convert_test_prep_fs ext4 "$@" populate_fs CHECKSUMTMP=$(mktemp --tmpdir btrfs-progs-convert.XXXXXXXXXX) convert_test_gen_checksums "$CHECKSUMTMP" @@ -43,16 +43,16 @@ do_test() { # ext2_saved/image must not be deleted run_mayfail $SUDO_HELPER find "$TEST_MNT"/ -mindepth 1 -path '*ext2_saved' -prune -o -exec rm -vrf "{}" \; cd "$here" - run_check $TOP/btrfs filesystem sync "$TEST_MNT" + run_check "$TOP/btrfs" filesystem sync "$TEST_MNT" run_check_umount_test_dev convert_test_post_rollback - run_check_mount_test_dev + run_check_mount_convert_dev ext4 convert_test_post_check_checksums "$CHECKSUMTMP" run_check_umount_test_dev # mount again and verify checksums - run_check_mount_test_dev + run_check_mount_convert_dev ext4 convert_test_post_check_checksums "$CHECKSUMTMP" run_check_umount_test_dev diff --git a/tests/convert-tests/006-large-hole-extent/test.sh b/tests/convert-tests/006-large-hole-extent/test.sh index d3bc093c..f63a1186 100755 --- a/tests/convert-tests/006-large-hole-extent/test.sh +++ b/tests/convert-tests/006-large-hole-extent/test.sh @@ -5,8 +5,8 @@ # Fast pinpoint regression test. No options combination nor checksum # verification -source $TOP/tests/common -source $TOP/tests/common.convert +source "$TOP/tests/common" +source "$TOP/tests/common.convert" setup_root_helper prepare_test_dev 512M @@ -14,9 +14,9 @@ check_prereq btrfs-convert default_mke2fs="mke2fs -t ext4 -b 4096" convert_test_preamble '' 'large hole extent test' 16k "$default_mke2fs" -convert_test_prep_fs $default_mke2fs +convert_test_prep_fs ext4 $default_mke2fs -run_check $SUDO_HELPER dd if=/dev/zero of=$TEST_MNT/file bs=1M \ +run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT/file" bs=1M \ count=1 seek=1024 > /dev/null 2>&1 run_check_umount_test_dev diff --git a/tests/convert-tests/007-unsupported-block-sizes/test.sh b/tests/convert-tests/007-unsupported-block-sizes/test.sh index 9ba17751..af8ec357 100755 --- a/tests/convert-tests/007-unsupported-block-sizes/test.sh +++ b/tests/convert-tests/007-unsupported-block-sizes/test.sh @@ -1,8 +1,8 @@ #!/bin/bash # Check if block sizes smaller than 4k expectedly fail to convert -source $TOP/tests/common -source $TOP/tests/common.convert +source "$TOP/tests/common" +source "$TOP/tests/common.convert" setup_root_helper prepare_test_dev 512M @@ -11,11 +11,11 @@ check_prereq btrfs-convert for bs in 1024 2048; do default_mke2fs="mke2fs -t ext4 -b $bs" convert_test_preamble '' "unsupported block size $bs" 16k "$default_mke2fs" - convert_test_prep_fs $default_mke2fs + convert_test_prep_fs ext4 $default_mke2fs - run_check $SUDO_HELPER dd if=/dev/zero of=$TEST_MNT/file bs=1M \ + run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT/file" bs=1M \ count=1 seek=1024 > /dev/null 2>&1 run_check_umount_test_dev - run_mustfail "$bs block converted" $TOP/btrfs-convert $TEST_DEV + run_mustfail "$bs block converted" "$TOP/btrfs-convert" "$TEST_DEV" done diff --git a/tests/convert-tests/008-readonly-image/test.sh b/tests/convert-tests/008-readonly-image/test.sh index 4e422378..4d5e629f 100755 --- a/tests/convert-tests/008-readonly-image/test.sh +++ b/tests/convert-tests/008-readonly-image/test.sh @@ -1,8 +1,8 @@ #!/bin/bash # Check if the converted ext2 image is readonly -source $TOP/tests/common -source $TOP/tests/common.convert +source "$TOP/tests/common" +source "$TOP/tests/common.convert" setup_root_helper prepare_test_dev 512M @@ -10,13 +10,13 @@ check_prereq btrfs-convert default_mke2fs="mke2fs -t ext4 -b 4096" convert_test_preamble '' 'readonly image test' 16k "$default_mke2fs" -convert_test_prep_fs $default_mke2fs +convert_test_prep_fs ext4 $default_mke2fs run_check_umount_test_dev convert_test_do_convert run_check_mount_test_dev # It's expected to fail -$SUDO_HELPER dd if=/dev/zero of=$TEST_MNT/ext2_save/image bs=1M count=1 \ +$SUDO_HELPER dd if=/dev/zero of="$TEST_MNT/ext2_save/image" bs=1M count=1 \ &> /dev/null if [ $? -ne 1 ]; then echo "after convert ext2_save/image is not read-only" diff --git a/tests/convert-tests/009-common-inode-flags/test.sh b/tests/convert-tests/009-common-inode-flags/test.sh index 6f26d187..f42fb681 100755 --- a/tests/convert-tests/009-common-inode-flags/test.sh +++ b/tests/convert-tests/009-common-inode-flags/test.sh @@ -1,8 +1,8 @@ #!/bin/bash # Check if btrfs-convert can copy common inode flags like SYNC/IMMUTABLE -source $TOP/tests/common -source $TOP/tests/common.convert +source "$TOP/tests/common" +source "$TOP/tests/common.convert" setup_root_helper prepare_test_dev 512M @@ -11,20 +11,20 @@ check_prereq btrfs-convert fail=0 default_mke2fs="mke2fs -t ext4 -b 4096" convert_test_preamble '' 'common inode flags test' 16k "$default_mke2fs" -convert_test_prep_fs $default_mke2fs +convert_test_prep_fs ext4 $default_mke2fs # create file with specific flags -run_check $SUDO_HELPER touch $TEST_MNT/flag_test -run_check $SUDO_HELPER chattr +aAdSi $TEST_MNT/flag_test +run_check $SUDO_HELPER touch "$TEST_MNT/flag_test" +run_check $SUDO_HELPER chattr +aAdSi "$TEST_MNT/flag_test" run_check_umount_test_dev convert_test_do_convert run_check_mount_test_dev # Log the status -run_check lsattr $TEST_MNT/flag_test +run_check lsattr "$TEST_MNT/flag_test" # Above flags should be copied to btrfs flags, and lsattr should get them -run_check_stdout lsattr $TEST_MNT/flag_test | cut -f1 -d\ | grep "[aAdiS]" -q +run_check_stdout lsattr "$TEST_MNT/flag_test" | cut -f1 -d\ | grep "[aAdiS]" -q if [ $? -ne 0 ]; then rm tmp_output _fail "no common inode flags are copied after convert" diff --git a/tests/fsck-tests.sh b/tests/fsck-tests.sh index 44cca1b8..15d26c70 100755 --- a/tests/fsck-tests.sh +++ b/tests/fsck-tests.sh @@ -23,6 +23,7 @@ rm -f "$RESULTS" check_prereq btrfs-corrupt-block check_prereq btrfs-image check_prereq btrfs +check_prereq btrfstune check_kernel_support run_one_test() { diff --git a/tests/fsck-tests/006-bad-root-items/test.sh b/tests/fsck-tests/006-bad-root-items/test.sh index 84332348..bf3ef781 100755 --- a/tests/fsck-tests/006-bad-root-items/test.sh +++ b/tests/fsck-tests/006-bad-root-items/test.sh @@ -1,15 +1,15 @@ #!/bin/bash -source $TOP/tests/common +source "$TOP/tests/common" check_prereq btrfs -echo "extracting image default_case.tar.xz" >> $RESULTS +echo "extracting image default_case.tar.xz" >> "$RESULTS" tar --no-same-owner -xJf default_case.tar.xz || \ _fail "failed to extract default_case.tar.xz" check_image test.img -echo "extracting image skinny_case.tar.xz" >> $RESULTS +echo "extracting image skinny_case.tar.xz" >> "$RESULTS" tar --no-same-owner -xJf skinny_case.tar.xz || \ _fail "failed to extract skinny_case.tar.xz" check_image test.img diff --git a/tests/fsck-tests/012-leaf-corruption/test.sh b/tests/fsck-tests/012-leaf-corruption/test.sh index a308727d..fc10a4ff 100755 --- a/tests/fsck-tests/012-leaf-corruption/test.sh +++ b/tests/fsck-tests/012-leaf-corruption/test.sh @@ -1,6 +1,6 @@ #!/bin/bash -source $TOP/tests/common +source "$TOP/tests/common" check_prereq btrfs-image @@ -37,16 +37,16 @@ leaf_no_data_ext_list=( generate_leaf_corrupt_no_data_ext() { dest=$1 - echo "generating leaf_corrupt_no_data_ext.btrfs-image" >> $RESULTS + echo "generating leaf_corrupt_no_data_ext.btrfs-image" >> "$RESULTS" tar --no-same-owner -xJf ./no_data_extent.tar.xz || \ _fail "failed to extract leaf_corrupt_no_data_ext.btrfs-image" - $TOP/btrfs-image -r test.img.btrfs-image $dest || \ + "$TOP/btrfs-image" -r test.img.btrfs-image "$dest" || \ _fail "failed to extract leaf_corrupt_no_data_ext.btrfs-image" # leaf at 4206592 and 20905984 contains no regular data # extent, clear its csum to corrupt the leaf. for x in 4206592 20905984; do - dd if=/dev/zero of=$dest bs=1 count=32 conv=notrunc seek=$x \ + dd if=/dev/zero of="$dest" bs=1 count=32 conv=notrunc seek="$x" \ 1>/dev/null 2>&1 done } @@ -60,21 +60,21 @@ check_inode() name=$5 # Check whether the inode exists - exists=$($SUDO_HELPER find $path -inum $ino) + exists=$($SUDO_HELPER find "$path" -inum "$ino") if [ -z "$exists" ]; then _fail "inode $ino not recovered correctly" fi # Check inode type - found_mode=$(printf "%o" 0x$($SUDO_HELPER stat $exists -c %f)) - if [ $found_mode -ne $mode ]; then + found_mode=$(printf "%o" 0x$($SUDO_HELPER stat "$exists" -c %f)) + if [ "$found_mode" -ne "$mode" ]; then echo "$found_mode" _fail "inode $ino modes not recovered" fi # Check inode size - found_size=$($SUDO_HELPER stat $exists -c %s) - if [ $mode -ne 41700 -a $found_size -ne $size ]; then + found_size=$($SUDO_HELPER stat "$exists" -c %s) + if [ $mode -ne 41700 -a "$found_size" -ne "$size" ]; then _fail "inode $ino size not recovered correctly" fi @@ -90,11 +90,11 @@ check_inode() check_leaf_corrupt_no_data_ext() { image=$1 - $SUDO_HELPER mount -o loop $image -o ro $TEST_MNT + $SUDO_HELPER mount -o loop -t btrfs "$image" -o ro "$TEST_MNT" i=0 while [ $i -lt ${#leaf_no_data_ext_list[@]} ]; do - check_inode $TEST_MNT/lost+found \ + check_inode "$TEST_MNT/lost+found" \ ${leaf_no_data_ext_list[i]} \ ${leaf_no_data_ext_list[i + 1]} \ ${leaf_no_data_ext_list[i + 2]} \ @@ -102,7 +102,7 @@ check_leaf_corrupt_no_data_ext() ${leaf_no_data_ext_list[i + 4]} ((i+=4)) done - $SUDO_HELPER umount $TEST_MNT + $SUDO_HELPER umount "$TEST_MNT" } setup_root_helper diff --git a/tests/fsck-tests/013-extent-tree-rebuild/test.sh b/tests/fsck-tests/013-extent-tree-rebuild/test.sh index 37bdcd9c..90fe2e83 100755 --- a/tests/fsck-tests/013-extent-tree-rebuild/test.sh +++ b/tests/fsck-tests/013-extent-tree-rebuild/test.sh @@ -1,6 +1,6 @@ #!/bin/bash -source $TOP/tests/common +source "$TOP/tests/common" check_prereq btrfs-corrupt-block check_prereq mkfs.btrfs @@ -12,32 +12,32 @@ prepare_test_dev 1G # test whether fsck can rebuild a corrupted extent tree test_extent_tree_rebuild() { - run_check $SUDO_HELPER $TOP/mkfs.btrfs -f $TEST_DEV + run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f "$TEST_DEV" run_check_mount_test_dev generate_dataset small for i in `seq 1 100`;do - run_check $SUDO_HELPER $TOP/btrfs sub snapshot $TEST_MNT \ - $TEST_MNT/snapaaaaaaa_$i + run_check $SUDO_HELPER "$TOP/btrfs" sub snapshot "$TEST_MNT" \ + "$TEST_MNT/snapaaaaaaa_$i" done run_check_umount_test_dev # get extent root bytenr - extent_root_bytenr=`$SUDO_HELPER $TOP/btrfs inspect-internal dump-tree -r $TEST_DEV | \ + extent_root_bytenr=`$SUDO_HELPER "$TOP/btrfs" inspect-internal dump-tree -r "$TEST_DEV" | \ grep extent | awk '{print $7}'` - if [ -z $extent_root_bytenr ];then + if [ -z "$extent_root_bytenr" ];then _fail "fail to get extent root bytenr" fi # corrupt extent root node block - run_check $SUDO_HELPER $TOP/btrfs-corrupt-block -l $extent_root_bytenr \ - -b 4096 $TEST_DEV + run_check $SUDO_HELPER "$TOP/btrfs-corrupt-block" -l "$extent_root_bytenr" \ + -b 4096 "$TEST_DEV" - $SUDO_HELPER $TOP/btrfs check $TEST_DEV >& /dev/null && \ + $SUDO_HELPER "$TOP/btrfs" check "$TEST_DEV" >& /dev/null && \ _fail "btrfs check should detect failure" - run_check $SUDO_HELPER $TOP/btrfs check --init-extent-tree $TEST_DEV - run_check $SUDO_HELPER $TOP/btrfs check $TEST_DEV + run_check $SUDO_HELPER "$TOP/btrfs" check --repair --init-extent-tree "$TEST_DEV" + run_check $SUDO_HELPER "$TOP/btrfs" check "$TEST_DEV" } test_extent_tree_rebuild diff --git a/tests/fsck-tests/018-leaf-crossing-stripes/test.sh b/tests/fsck-tests/018-leaf-crossing-stripes/test.sh index c453ab5c..29eb20b5 100755 --- a/tests/fsck-tests/018-leaf-crossing-stripes/test.sh +++ b/tests/fsck-tests/018-leaf-crossing-stripes/test.sh @@ -1,11 +1,11 @@ #!/bin/bash -source $TOP/tests/common +source "$TOP/tests/common" check_prereq btrfs image=$(extract_image "./default_case.raw.xz") -run_check_stdout $TOP/btrfs check "$image" 2>&1 | +run_check_stdout "$TOP/btrfs" check "$image" 2>&1 | grep -q "crossing stripe boundary" || _fail "no expected error message in the output" diff --git a/tests/fsck-tests/019-non-skinny-false-alert/test.sh b/tests/fsck-tests/019-non-skinny-false-alert/test.sh index a7f8e862..550f2947 100755 --- a/tests/fsck-tests/019-non-skinny-false-alert/test.sh +++ b/tests/fsck-tests/019-non-skinny-false-alert/test.sh @@ -11,12 +11,12 @@ # # a buggy check leads to the above messages -source $TOP/tests/common +source "$TOP/tests/common" check_prereq btrfs image=$(extract_image "./default_case.img.xz") -run_check_stdout $TOP/btrfs check "$image" 2>&1 | +run_check_stdout "$TOP/btrfs" check "$image" 2>&1 | grep -q "type mismatch with chunk" && _fail "unexpected error message in the output" diff --git a/tests/fsck-tests/020-extent-ref-cases/block_group_item_false_alert.raw.xz b/tests/fsck-tests/020-extent-ref-cases/block_group_item_false_alert.raw.xz Binary files differnew file mode 100644 index 00000000..559c3fa9 --- /dev/null +++ b/tests/fsck-tests/020-extent-ref-cases/block_group_item_false_alert.raw.xz diff --git a/tests/fsck-tests/020-extent-ref-cases/shared_data_ref_only.img b/tests/fsck-tests/020-extent-ref-cases/shared_data_ref_only.img Binary files differnew file mode 100644 index 00000000..6d2b95e4 --- /dev/null +++ b/tests/fsck-tests/020-extent-ref-cases/shared_data_ref_only.img diff --git a/tests/fsck-tests/020-extent-ref-cases/test.sh b/tests/fsck-tests/020-extent-ref-cases/test.sh index c2b6a006..1e1e4e23 100755 --- a/tests/fsck-tests/020-extent-ref-cases/test.sh +++ b/tests/fsck-tests/020-extent-ref-cases/test.sh @@ -8,16 +8,23 @@ # * shared_data_ref # * no_inline_ref (a extent item without inline ref) # * no_skinny_ref +# +# Special check for lowmem regression +# * block_group_item_false_alert +# Containing a block group and its first extent at +# the beginning of leaf. +# Which caused false alert for lowmem mode. -source $TOP/tests/common +source "$TOP/tests/common" check_prereq btrfs -for img in *.img +for img in *.img *.raw.xz do - image=$(extract_image $img) - run_check_stdout $TOP/btrfs check "$image" 2>&1 | - grep -q "Errors found in extent allocation tree or chunk allocation" && - _fail "unexpected error occurred when checking $img" + image=$(extract_image "$img") + + # Since the return value bug is already fixed, we don't need + # the old grep hack to detect bug. + run_check "$TOP/btrfs" check "$image" rm -f "$image" done diff --git a/tests/fsck-tests/021-partially-dropped-snapshot-case/test.sh b/tests/fsck-tests/021-partially-dropped-snapshot-case/test.sh index eb8d8849..44a33a63 100755 --- a/tests/fsck-tests/021-partially-dropped-snapshot-case/test.sh +++ b/tests/fsck-tests/021-partially-dropped-snapshot-case/test.sh @@ -1,14 +1,14 @@ #!/bin/bash # confirm whether btrfsck supports to check a partially dropped snapshot -source $TOP/tests/common +source "$TOP/tests/common" check_prereq btrfs for img in *.img do - image=$(extract_image $img) - run_check_stdout $TOP/btrfs check "$image" 2>&1 | + image=$(extract_image "$img") + run_check_stdout "$TOP/btrfs" check "$image" 2>&1 | grep -q "Errors found in extent allocation tree or chunk allocation" if [ $? -eq 0 ]; then rm -f "$image" diff --git a/tests/fsck-tests/022-qgroup-rescan-halfway/test.sh b/tests/fsck-tests/022-qgroup-rescan-halfway/test.sh index 1dc8f8fc..dcdc1b42 100755 --- a/tests/fsck-tests/022-qgroup-rescan-halfway/test.sh +++ b/tests/fsck-tests/022-qgroup-rescan-halfway/test.sh @@ -1,7 +1,7 @@ #!/bin/bash # check whether btrfsck can detect running qgroup rescan -source $TOP/tests/common +source "$TOP/tests/common" check_prereq btrfs @@ -9,7 +9,7 @@ check_image() { local image image=$1 - run_check_stdout $TOP/btrfs check "$image" 2>&1 | \ + run_check_stdout "$TOP/btrfs" check "$image" 2>&1 | \ grep -q "Counts for qgroup id" if [ $? -eq 0 ]; then _fail "Btrfs check doesn't detect rescan correctly" diff --git a/tests/fsck-tests/023-qgroup-stack-overflow/test.sh b/tests/fsck-tests/023-qgroup-stack-overflow/test.sh index a304eac5..ebb07f36 100755 --- a/tests/fsck-tests/023-qgroup-stack-overflow/test.sh +++ b/tests/fsck-tests/023-qgroup-stack-overflow/test.sh @@ -5,13 +5,13 @@ # Fixed by patch: # btrfs-progs: Fix stack overflow for checking qgroup on tree reloc tree -source $TOP/tests/common +source "$TOP/tests/common" check_prereq btrfs check_image() { - run_check $TOP/btrfs check "$1" + run_check "$TOP/btrfs" check "$1" } check_all_images diff --git a/tests/fsck-tests/024-clear-space-cache/test.sh b/tests/fsck-tests/024-clear-space-cache/test.sh index 2945ae87..6cf8440b 100755 --- a/tests/fsck-tests/024-clear-space-cache/test.sh +++ b/tests/fsck-tests/024-clear-space-cache/test.sh @@ -1,7 +1,7 @@ #!/bin/bash # confirm that clearing space cache works -source $TOP/tests/common +source "$TOP/tests/common" check_prereq btrfs check_prereq mkfs.btrfs @@ -9,21 +9,21 @@ check_prereq mkfs.btrfs setup_root_helper prepare_test_dev 1G -run_check $SUDO_HELPER $TOP/mkfs.btrfs -f $TEST_DEV +run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f "$TEST_DEV" run_check_mount_test_dev # Create files that takes at least 3 data chunks, while # can still be removed to create free space inside one chunk. for i in $(seq 0 6); do - run_check $SUDO_HELPER dd if=/dev/zero of=$TEST_MNT/file_${i} bs=1M \ + run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_MNT/file_${i}" bs=1M \ count=64 > /dev/null 2>&1 done sync # Remove file 1 3 5 to create holes for i in 1 3 5; do - run_check $SUDO_HELPER rm $TEST_MNT/file_${i} + run_check $SUDO_HELPER rm "$TEST_MNT/file_${i}" done sync @@ -31,17 +31,17 @@ sync run_check_umount_test_dev # Clear space cache and re-check fs -run_check $TOP/btrfs check --clear-space-cache v1 $TEST_DEV -run_check $TOP/btrfs check $TEST_DEV +run_check "$TOP/btrfs" check --clear-space-cache v1 "$TEST_DEV" +run_check "$TOP/btrfs" check "$TEST_DEV" # Manually recheck space cache and super space cache generation -run_check_stdout $TOP/btrfs inspect-internal dump-tree -t root $TEST_DEV | \ +run_check_stdout "$TOP/btrfs" inspect-internal dump-tree -t root "$TEST_DEV" | \ grep -q FREE_SPACE if [ $? -eq 0 ]; then _fail "clear space cache doesn't clear all space cache" fi -run_check_stdout $TOP/btrfs inspect-internal dump-super $TEST_DEV | +run_check_stdout $TOP/btrfs inspect-internal dump-super "$TEST_DEV" | grep -q 'cache_generation.*18446744073709551615' if [ $? -ne 0 ]; then _fail "clear space cache doesn't set cache_generation correctly" diff --git a/tests/fsck-tests/025-file-extents/test.sh b/tests/fsck-tests/025-file-extents/test.sh new file mode 100755 index 00000000..ebe8a305 --- /dev/null +++ b/tests/fsck-tests/025-file-extents/test.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Confirm btrfs check can check file extents without causing false alert + +source "$TOP/tests/common" + +check_prereq btrfs +check_prereq mkfs.btrfs +check_prereq btrfstune +check_global_prereq dd +check_global_prereq fallocate +check_global_prereq truncate + +setup_root_helper +prepare_test_dev 128M + +# Do some write into a large prealloc range +# Lowmem mode can report missing csum due to wrong csum range +test_paritical_write_into_prealloc() +{ + run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f "$TEST_DEV" + run_check_mount_test_dev + + run_check $SUDO_HELPER fallocate -l 128K "$TEST_MNT/file" + sync + run_check $SUDO_HELPER dd conv=notrunc if=/dev/zero of="$TEST_MNT/file" bs=1K count=64 + run_check_umount_test_dev + run_check "$TOP/btrfs" check "$TEST_DEV" +} + +# Inline compressed file extent +# Lowmem mode can cause silent error without any error message +# due to too restrict check on inline extent size +test_compressed_inline_extent() +{ + run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f "$TEST_DEV" + run_check_mount_test_dev -o compress=lzo,max_inline=2048 + + run_check $SUDO_HELPER dd conv=notrunc if=/dev/null of="$TEST_MNT/file" bs=1K count=1 + run_check_umount_test_dev + run_check "$TOP/btrfs" check "$TEST_DEV" +} + +# File extent hole with NO_HOLES incompat feature set. +# Lowmem mode will cause a false alert as it doesn't allow any file hole +# extents, while we can set NO_HOLES at anytime we want, it's definitely a +# false alert +test_hole_extent_with_no_holes_flag() +{ + run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f "$TEST_DEV" + run_check_mount_test_dev + + run_check $SUDO_HELPER truncate -s 16K "$TEST_MNT/tmp" + run_check_umount_test_dev + run_check $SUDO_HELPER "$TOP/btrfstune" -n "$TEST_DEV" + run_check "$TOP/btrfs" check "$TEST_DEV" +} + +test_paritical_write_into_prealloc +test_compressed_inline_extent +test_hole_extent_with_no_holes_flag diff --git a/tests/fssum.c b/tests/fssum.c new file mode 100644 index 00000000..5dde9984 --- /dev/null +++ b/tests/fssum.c @@ -0,0 +1,839 @@ +/* + * Copyright (C) 2012 STRATO AG. All rights reserved. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public + * License v2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + * You should have received a copy of the GNU General Public + * License along with this program; if not, write to the + * Free Software Foundation, Inc., 59 Temple Place - Suite 330, + * Boston, MA 021110-1307, USA. + */ + +#include "kerncompat.h" +#include <stdio.h> +#include <stdlib.h> +#include <unistd.h> +#include <string.h> +#include <fcntl.h> +#include <dirent.h> +#include <errno.h> +#include <sys/sysmacros.h> +#include <sys/stat.h> +#include <assert.h> +#include <time.h> +#include <stdint.h> +#include "tests/sha.h" + +#define CS_SIZE 32 +#define CHUNKS 128 + +#ifndef SEEK_DATA +#define SEEK_DATA 3 +#define SEEK_HOLE 4 +#endif + +/* TODO: add hardlink recognition */ +/* TODO: add xattr/acl */ + +struct excludes { + char *path; + int len; +}; + +typedef struct _sum { + SHA256Context sha; + unsigned char out[CS_SIZE]; +} sum_t; + +typedef int (*sum_file_data_t)(int fd, sum_t *dst); + +int gen_manifest = 0; +int in_manifest = 0; +char *checksum = NULL; +struct excludes *excludes; +int n_excludes = 0; +int verbose = 0; +FILE *out_fp; +FILE *in_fp; + +enum _flags { + FLAG_UID, + FLAG_GID, + FLAG_MODE, + FLAG_ATIME, + FLAG_MTIME, + FLAG_CTIME, + FLAG_DATA, + FLAG_OPEN_ERROR, + FLAG_STRUCTURE, + NUM_FLAGS +}; + +const char flchar[] = "ugoamcdes"; +char line[65536]; + +int flags[NUM_FLAGS] = {1, 1, 1, 1, 1, 0, 1, 0, 0}; + +char * +getln(char *buf, int size, FILE *fp) +{ + char *p; + int l; + + p = fgets(buf, size, fp); + if (!p) + return NULL; + + l = strlen(p); + while(l > 0 && (p[l - 1] == '\n' || p[l - 1] == '\r')) + p[--l] = 0; + + return p; +} + +void +parse_flag(int c) +{ + int i; + int is_upper = 0; + + if (c >= 'A' && c <= 'Z') { + is_upper = 1; + c += 'a' - 'A'; + } + for (i = 0; flchar[i]; ++i) { + if (flchar[i] == c) { + flags[i] = is_upper ? 0 : 1; + return; + } + } + fprintf(stderr, "unrecognized flag %c\n", c); + exit(-1); +} + +void +parse_flags(char *p) +{ + while (*p) + parse_flag(*p++); +} + +void +usage(void) +{ + fprintf(stderr, "usage: fssum <options> <path>\n"); + fprintf(stderr, " options:\n"); + fprintf(stderr, " -f : write out a full manifest file\n"); + fprintf(stderr, " -w <file> : send output to file\n"); + fprintf(stderr, " -v : verbose mode (debugging only)\n"); + fprintf(stderr, + " -r <file> : read checksum or manifest from file\n"); + fprintf(stderr, " -[ugoamcde] : specify which fields to include in checksum calculation.\n"); + fprintf(stderr, " u : include uid\n"); + fprintf(stderr, " g : include gid\n"); + fprintf(stderr, " o : include mode\n"); + fprintf(stderr, " m : include mtime\n"); + fprintf(stderr, " a : include atime\n"); + fprintf(stderr, " c : include ctime\n"); + fprintf(stderr, " d : include file data\n"); + fprintf(stderr, " e : include open errors (aborts otherwise)\n"); + fprintf(stderr, " s : include block structure (holes)\n"); + fprintf(stderr, " -[UGOAMCDES]: exclude respective field from calculation\n"); + fprintf(stderr, " -n : reset all flags\n"); + fprintf(stderr, " -N : set all flags\n"); + fprintf(stderr, " -x path : exclude path when building checksum (multiple ok)\n"); + fprintf(stderr, " -h : this help\n\n"); + fprintf(stderr, "The default field mask is ugoamCdES. If the checksum/manifest is read from a\n"); + fprintf(stderr, "file, the mask is taken from there and the values given on the command line\n"); + fprintf(stderr, "are ignored.\n"); + exit(-1); +} + +static char buf[65536]; + +void * +alloc(size_t sz) +{ + void *p = malloc(sz); + + if (!p) { + fprintf(stderr, "malloc failed\n"); + exit(-1); + } + + return p; +} + +void +sum_init(sum_t *cs) +{ + SHA256Reset(&cs->sha); +} + +void +sum_fini(sum_t *cs) +{ + SHA256Result(&cs->sha, cs->out); +} + +void +sum_add(sum_t *cs, void *buf, int size) +{ + SHA256Input(&cs->sha, buf, size); +} + +void +sum_add_sum(sum_t *dst, sum_t *src) +{ + sum_add(dst, src->out, sizeof(src->out)); +} + +void +sum_add_u64(sum_t *dst, uint64_t val) +{ + uint64_t v = cpu_to_le64(val); + sum_add(dst, &v, sizeof(v)); +} + +void +sum_add_time(sum_t *dst, time_t t) +{ + sum_add_u64(dst, t); +} + +char * +sum_to_string(sum_t *dst) +{ + int i; + char *s = alloc(CS_SIZE * 2 + 1); + + for (i = 0; i < CS_SIZE; ++i) + sprintf(s + i * 2, "%02x", dst->out[i]); + + return s; +} + +int +sum_file_data_permissive(int fd, sum_t *dst) +{ + int ret; + off_t pos; + off_t old; + int i; + uint64_t zeros = 0; + + pos = lseek(fd, 0, SEEK_CUR); + if (pos == (off_t)-1) + return errno == ENXIO ? 0 : -2; + + while (1) { + old = pos; + pos = lseek(fd, pos, SEEK_DATA); + if (pos == (off_t)-1) { + if (errno == ENXIO) { + ret = 0; + pos = lseek(fd, 0, SEEK_END); + if (pos != (off_t)-1) + zeros += pos - old; + } else { + ret = -2; + } + break; + } + ret = read(fd, buf, sizeof(buf)); + assert(ret); /* eof found by lseek */ + if (ret <= 0) + break; + if (old < pos) /* hole */ + zeros += pos - old; + for (i = 0; i < ret; ++i) { + for (old = i; buf[i] == 0 && i < ret; ++i) + ; + if (old < i) /* code like a hole */ + zeros += i - old; + if (i == ret) + break; + if (zeros) { + if (verbose >= 2) + fprintf(stderr, + "adding %llu zeros to sum\n", + (unsigned long long)zeros); + sum_add_u64(dst, 0); + sum_add_u64(dst, zeros); + zeros = 0; + } + for (old = i; buf[i] != 0 && i < ret; ++i) + ; + if (verbose >= 2) + fprintf(stderr, "adding %u non-zeros to sum\n", + i - (int)old); + sum_add(dst, buf + old, i - old); + } + pos += ret; + } + + if (zeros) { + if (verbose >= 2) + fprintf(stderr, + "adding %llu zeros to sum (finishing)\n", + (unsigned long long)zeros); + sum_add_u64(dst, 0); + sum_add_u64(dst, zeros); + } + + return ret; +} + +int +sum_file_data_strict(int fd, sum_t *dst) +{ + int ret; + off_t pos; + + pos = lseek(fd, 0, SEEK_CUR); + if (pos == (off_t)-1) + return errno == ENXIO ? 0 : -2; + + while (1) { + pos = lseek(fd, pos, SEEK_DATA); + if (pos == (off_t)-1) + return errno == ENXIO ? 0 : -2; + ret = read(fd, buf, sizeof(buf)); + assert(ret); /* eof found by lseek */ + if (ret <= 0) + return ret; + if (verbose >= 2) + fprintf(stderr, + "adding to sum at file offset %llu, %d bytes\n", + (unsigned long long)pos, ret); + sum_add_u64(dst, (uint64_t)pos); + sum_add(dst, buf, ret); + pos += ret; + } +} + +char * +escape(char *in) +{ + char *out = alloc(strlen(in) * 3 + 1); + char *src = in; + char *dst = out; + + for (; *src; ++src) { + if (*src >= 32 && *src < 127 && *src != '\\') { + *dst++ = *src; + } else { + sprintf(dst, "\\%02x", (unsigned char)*src); + dst += 3; + } + } + *dst = 0; + + return out; +} + +void +excess_file(const char *fn) +{ + printf("only in local fs: %s\n", fn); +} + +void +missing_file(const char *fn) +{ + printf("only in remote fs: %s\n", fn); +} + +int +pathcmp(const char *a, const char *b) +{ + int len_a = strlen(a); + int len_b = strlen(b); + + /* + * as the containing directory is sent after the files, it has to + * come out bigger in the comparison. + */ + if (len_a < len_b && a[len_a - 1] == '/' && strncmp(a, b, len_a) == 0) + return 1; + if (len_a > len_b && b[len_b - 1] == '/' && strncmp(a, b, len_b) == 0) + return -1; + + return strcmp(a, b); +} + +void +check_match(char *fn, char *local_m, char *remote_m, + char *local_c, char *remote_c) +{ + int match_m = !strcmp(local_m, remote_m); + int match_c = !strcmp(local_c, remote_c); + + if (match_m && !match_c) { + printf("data mismatch in %s\n", fn); + } else if (!match_m && match_c) { + printf("metadata mismatch in %s\n", fn); + } else if (!match_m && !match_c) { + printf("metadata and data mismatch in %s\n", fn); + } +} + +char *prev_fn; +char *prev_m; +char *prev_c; +void +check_manifest(char *fn, char *m, char *c, int last_call) +{ + char *rem_m; + char *rem_c; + char *l; + int cmp; + + if (prev_fn) { + if (last_call) + cmp = -1; + else + cmp = pathcmp(prev_fn, fn); + if (cmp > 0) { + excess_file(fn); + return; + } else if (cmp < 0) { + missing_file(prev_fn); + } else { + check_match(fn, m, prev_m, c, prev_c); + } + free(prev_fn); + free(prev_m); + free(prev_c); + prev_fn = NULL; + prev_m = NULL; + prev_c = NULL; + if (cmp == 0) + return; + } + while ((l = getln(line, sizeof(line), in_fp))) { + rem_c = strrchr(l, ' '); + if (!rem_c) { + if (checksum) + free(checksum); + + /* final cs */ + checksum = strdup(l); + break; + } + if (rem_c == l) { +malformed: + fprintf(stderr, "malformed input\n"); + exit(-1); + } + *rem_c++ = 0; + rem_m = strrchr(l, ' '); + if (!rem_m) + goto malformed; + *rem_m++ = 0; + + if (last_call) + cmp = -1; + else + cmp = pathcmp(l, fn); + if (cmp == 0) { + check_match(fn, m, rem_m, c, rem_c); + return; + } else if (cmp > 0) { + excess_file(fn); + prev_fn = strdup(l); + prev_m = strdup(rem_m); + prev_c = strdup(rem_c); + return; + } + missing_file(l); + } + if (!last_call) + excess_file(fn); +} + +int +namecmp(const void *aa, const void *bb) +{ + char * const *a = aa; + char * const *b = bb; + + return strcmp(*a, *b); +} + +void +sum(int dirfd, int level, sum_t *dircs, char *path_prefix, char *path_in) +{ + DIR *d; + struct dirent *de; + char **namelist = NULL; + int alloclen = 0; + int entries = 0; + int i; + int ret; + int fd; + int excl; + sum_file_data_t sum_file_data = flags[FLAG_STRUCTURE] ? + sum_file_data_strict : sum_file_data_permissive; + + d = fdopendir(dirfd); + if (!d) { + perror("opendir"); + exit(-1); + } + while((de = readdir(d))) { + if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, "..")) + continue; + if (entries == alloclen) { + alloclen += CHUNKS; + namelist = realloc(namelist, + alloclen * sizeof(*namelist)); + if (!namelist) { + fprintf(stderr, "malloc failed\n"); + exit(-1); + } + } + namelist[entries] = strdup(de->d_name); + if (!namelist[entries]) { + fprintf(stderr, "malloc failed\n"); + exit(-1); + } + ++entries; + } + + qsort(namelist, entries, sizeof(*namelist), namecmp); + for (i = 0; i < entries; ++i) { + struct stat64 st; + sum_t cs; + sum_t meta; + char *path; + + sum_init(&cs); + sum_init(&meta); + path = alloc(strlen(path_in) + strlen(namelist[i]) + 3); + sprintf(path, "%s/%s", path_in, namelist[i]); + for (excl = 0; excl < n_excludes; ++excl) { + if (strncmp(excludes[excl].path, path, + excludes[excl].len) == 0) + goto next; + } + + ret = fchdir(dirfd); + if (ret == -1) { + perror("fchdir"); + exit(-1); + } + ret = lstat64(namelist[i], &st); + if (ret) { + fprintf(stderr, "stat failed for %s/%s: %s\n", + path_prefix, path, strerror(errno)); + exit(-1); + } + sum_add_u64(&meta, level); + sum_add(&meta, namelist[i], strlen(namelist[i])); + if (!S_ISDIR(st.st_mode)) + sum_add_u64(&meta, st.st_nlink); + if (flags[FLAG_UID]) + sum_add_u64(&meta, st.st_uid); + if (flags[FLAG_GID]) + sum_add_u64(&meta, st.st_gid); + if (flags[FLAG_MODE]) + sum_add_u64(&meta, st.st_mode); + if (flags[FLAG_ATIME]) + sum_add_time(&meta, st.st_atime); + if (flags[FLAG_MTIME]) + sum_add_time(&meta, st.st_mtime); + if (flags[FLAG_CTIME]) + sum_add_time(&meta, st.st_ctime); + if (S_ISDIR(st.st_mode)) { + fd = openat(dirfd, namelist[i], 0); + if (fd == -1 && flags[FLAG_OPEN_ERROR]) { + sum_add_u64(&meta, errno); + } else if (fd == -1) { + fprintf(stderr, "open failed for %s/%s: %s\n", + path_prefix, path, strerror(errno)); + exit(-1); + } else { + sum(fd, level + 1, &cs, path_prefix, path); + close(fd); + } + } else if (S_ISREG(st.st_mode)) { + sum_add_u64(&meta, st.st_size); + if (flags[FLAG_DATA]) { + if (verbose) + fprintf(stderr, "file %s\n", + namelist[i]); + fd = openat(dirfd, namelist[i], 0); + if (fd == -1 && flags[FLAG_OPEN_ERROR]) { + sum_add_u64(&meta, errno); + } else if (fd == -1) { + fprintf(stderr, + "open failed for %s/%s: %s\n", + path_prefix, path, + strerror(errno)); + exit(-1); + } + if (fd != -1) { + ret = sum_file_data(fd, &cs); + if (ret < 0) { + fprintf(stderr, + "read failed for " + "%s/%s: %s\n", + path_prefix, path, + strerror(errno)); + exit(-1); + } + close(fd); + } + } + } else if (S_ISLNK(st.st_mode)) { + ret = readlink(namelist[i], buf, sizeof(buf)); + if (ret == -1) { + perror("readlink"); + exit(-1); + } + sum_add(&cs, buf, ret); + } else if (S_ISCHR(st.st_mode) || S_ISBLK(st.st_mode)) { + sum_add_u64(&cs, major(st.st_rdev)); + sum_add_u64(&cs, minor(st.st_rdev)); + } + sum_fini(&cs); + sum_fini(&meta); + if (gen_manifest || in_manifest) { + char *fn; + char *m; + char *c; + + if (S_ISDIR(st.st_mode)) + strcat(path, "/"); + fn = escape(path); + m = sum_to_string(&meta); + c = sum_to_string(&cs); + + if (gen_manifest) + fprintf(out_fp, "%s %s %s\n", fn, m, c); + if (in_manifest) + check_manifest(fn, m, c, 0); + free(c); + free(m); + free(fn); + } + sum_add_sum(dircs, &cs); + sum_add_sum(dircs, &meta); +next: + free(path); + free(namelist[i]); + } + + free(namelist); + closedir(d); +} + +int +main(int argc, char *argv[]) +{ + extern char *optarg; + extern int optind; + int c; + char *path; + int fd; + sum_t cs; + char *sumstring; + char flagstring[sizeof(flchar)]; + int ret = 0; + int i; + int plen; + int elen; + int n_flags = 0; + const char *allopts = "heEfuUgGoOaAmMcCdDsSnNw:r:vx:"; + + out_fp = stdout; + while ((c = getopt(argc, argv, allopts)) != EOF) { + switch(c) { + case 'f': + gen_manifest = 1; + break; + case 'u': + case 'U': + case 'g': + case 'G': + case 'o': + case 'O': + case 'a': + case 'A': + case 'm': + case 'M': + case 'c': + case 'C': + case 'd': + case 'D': + case 'e': + case 'E': + case 's': + case 'S': + ++n_flags; + parse_flag(c); + break; + case 'n': + for (i = 0; i < NUM_FLAGS; ++i) + flags[i] = 0; + break; + case 'N': + for (i = 0; i < NUM_FLAGS; ++i) + flags[i] = 1; + break; + case 'w': + out_fp = fopen(optarg, "w"); + if (!out_fp) { + fprintf(stderr, + "failed to open output file: %s\n", + strerror(errno)); + exit(-1); + } + break; + case 'r': + in_fp = fopen(optarg, "r"); + if (!in_fp) { + fprintf(stderr, + "failed to open input file: %s\n", + strerror(errno)); + exit(-1); + } + break; + case 'x': + ++n_excludes; + excludes = realloc(excludes, + sizeof(*excludes) * n_excludes); + if (!excludes) { + fprintf(stderr, + "failed to alloc exclude space\n"); + exit(-1); + } + excludes[n_excludes - 1].path = optarg; + break; + case 'v': + ++verbose; + break; + case 'h': + case '?': + usage(); + } + } + + if (optind + 1 != argc) { + fprintf(stderr, "missing path\n"); + usage(); + } + + if (in_fp) { + char *l = getln(line, sizeof(line), in_fp); + char *p; + + if (l == NULL) { + fprintf(stderr, "failed to read line from input\n"); + exit(-1); + } + if (strncmp(l, "Flags: ", 7) == 0) { + l += 7; + in_manifest = 1; + parse_flags(l); + } else if ((p = strchr(l, ':'))) { + *p++ = 0; + parse_flags(l); + + if (checksum) + free(checksum); + checksum = strdup(p); + } else { + fprintf(stderr, "invalid input file format\n"); + exit(-1); + } + if (n_flags) + fprintf(stderr, "warning: " + "command line flags ignored in -r mode\n"); + } + strcpy(flagstring, flchar); + for (i = 0; i < NUM_FLAGS; ++i) { + if (flags[i] == 0) + flagstring[i] -= 'a' - 'A'; + } + + path = argv[optind]; + plen = strlen(path); + if (path[plen - 1] == '/') { + --plen; + path[plen] = '\0'; + } + + for (i = 0; i < n_excludes; ++i) { + if (strncmp(path, excludes[i].path, plen) != 0) + fprintf(stderr, + "warning: exclude %s outside of path %s\n", + excludes[i].path, path); + else + excludes[i].path += plen; + elen = strlen(excludes[i].path); + if (excludes[i].path[elen - 1] == '/') + --elen; + excludes[i].path[elen] = '\0'; + excludes[i].len = elen; + } + + fd = open(path, O_RDONLY); + if (fd == -1) { + fprintf(stderr, "failed to open %s: %s\n", path, + strerror(errno)); + exit(-1); + } + + if (gen_manifest) + fprintf(out_fp, "Flags: %s\n", flagstring); + + sum_init(&cs); + sum(fd, 1, &cs, path, ""); + sum_fini(&cs); + + close(fd); + if (in_manifest) + check_manifest("", "", "", 1); + + if (!checksum) { + if (in_manifest) { + fprintf(stderr, "malformed input\n"); + exit(-1); + } + if (!gen_manifest) + fprintf(out_fp, "%s:", flagstring); + + sumstring = sum_to_string(&cs); + fprintf(out_fp, "%s\n", sumstring); + free(sumstring); + } else { + sumstring = sum_to_string(&cs); + if (strcmp(checksum, sumstring) == 0) { + printf("OK\n"); + ret = 0; + } else { + printf("FAIL\n"); + ret = 1; + } + + free(checksum); + free(sumstring); + } + + if (in_fp) + fclose(in_fp); + + if (out_fp != stdout) + fclose(out_fp); + + exit(ret); +} diff --git a/tests/fsck-tests/015-check-bad-memory-access/bko-97171-btrfs-image.raw.txt b/tests/fuzz-tests/images/bko-97171-btrfs-image.raw.txt index 9685ed46..9685ed46 100644 --- a/tests/fsck-tests/015-check-bad-memory-access/bko-97171-btrfs-image.raw.txt +++ b/tests/fuzz-tests/images/bko-97171-btrfs-image.raw.txt diff --git a/tests/fsck-tests/015-check-bad-memory-access/bko-97171-btrfs-image.raw.xz b/tests/fuzz-tests/images/bko-97171-btrfs-image.raw.xz Binary files differindex f3f0944d..f3f0944d 100644 --- a/tests/fsck-tests/015-check-bad-memory-access/bko-97171-btrfs-image.raw.xz +++ b/tests/fuzz-tests/images/bko-97171-btrfs-image.raw.xz diff --git a/tests/misc-tests.sh b/tests/misc-tests.sh index 1c645c9b..08988016 100755 --- a/tests/misc-tests.sh +++ b/tests/misc-tests.sh @@ -24,6 +24,9 @@ check_prereq btrfs-corrupt-block check_prereq btrfs-image check_prereq btrfstune check_prereq btrfs +check_prereq btrfs-zero-log +check_prereq btrfs-find-root +check_prereq btrfs-select-super check_kernel_support # The tests are driven by their custom script called 'test.sh' diff --git a/tests/misc-tests/009-subvolume-sync-must-wait/test.sh b/tests/misc-tests/009-subvolume-sync-must-wait/test.sh index 92c896f9..fa3f09ab 100755 --- a/tests/misc-tests/009-subvolume-sync-must-wait/test.sh +++ b/tests/misc-tests/009-subvolume-sync-must-wait/test.sh @@ -9,9 +9,9 @@ check_prereq btrfs setup_root_helper -run_check truncate -s 2G $IMAGE -run_check $TOP/mkfs.btrfs -f $IMAGE -run_check $SUDO_HELPER mount $IMAGE $TEST_MNT +prepare_test_dev +run_check $TOP/mkfs.btrfs -f "$TEST_DEV" +run_check_mount_test_dev run_check $SUDO_HELPER chmod a+rw $TEST_MNT cd $TEST_MNT @@ -49,4 +49,4 @@ run_check $SUDO_HELPER $TOP/btrfs subvolume list -d . wait cd .. -run_check $SUDO_HELPER umount $TEST_MNT +run_check_umount_test_dev diff --git a/tests/misc-tests/013-subvolume-sync-crash/test.sh b/tests/misc-tests/013-subvolume-sync-crash/test.sh index 4cb1b4e7..cd445961 100755 --- a/tests/misc-tests/013-subvolume-sync-crash/test.sh +++ b/tests/misc-tests/013-subvolume-sync-crash/test.sh @@ -10,9 +10,9 @@ check_prereq btrfs setup_root_helper -run_check truncate -s 2G $IMAGE -run_check $TOP/mkfs.btrfs -f $IMAGE -run_check $SUDO_HELPER mount $IMAGE $TEST_MNT +prepare_test_dev +run_check "$TOP/mkfs.btrfs" -f "$TEST_DEV" +run_check_mount_test_dev run_check $SUDO_HELPER chmod a+rw $TEST_MNT cd $TEST_MNT @@ -46,4 +46,4 @@ run_check $SUDO_HELPER $TOP/btrfs subvolume list -d . wait cd .. -run_check $SUDO_HELPER umount $TEST_MNT +run_check_umount_test_dev diff --git a/tests/misc-tests/014-filesystem-label/test.sh b/tests/misc-tests/014-filesystem-label/test.sh index a5e08ccc..753aa9ea 100755 --- a/tests/misc-tests/014-filesystem-label/test.sh +++ b/tests/misc-tests/014-filesystem-label/test.sh @@ -9,9 +9,9 @@ check_prereq btrfs setup_root_helper -run_check truncate -s 2G $IMAGE -run_check $TOP/mkfs.btrfs -L BTRFS-TEST-LABEL -f $IMAGE -run_check $SUDO_HELPER mount $IMAGE $TEST_MNT +prepare_test_dev +run_check "$TOP/mkfs.btrfs" -L BTRFS-TEST-LABEL -f "$TEST_DEV" +run_check_mount_test_dev run_check $SUDO_HELPER chmod a+rw $TEST_MNT cd $TEST_MNT @@ -66,4 +66,4 @@ run_check $SUDO_HELPER $TOP/btrfs filesystem label $TEST_MNT cd .. -run_check $SUDO_HELPER umount $TEST_MNT +run_check_umount_test_dev diff --git a/tests/misc-tests/016-send-clone-src/test.sh b/tests/misc-tests/016-send-clone-src/test.sh index e256eef9..479da677 100755 --- a/tests/misc-tests/016-send-clone-src/test.sh +++ b/tests/misc-tests/016-send-clone-src/test.sh @@ -9,9 +9,9 @@ check_prereq mkfs.btrfs check_prereq btrfs setup_root_helper -prepare_test_dev 1g -run_check $TOP/mkfs.btrfs -f $IMAGE +prepare_test_dev 1g +run_check "$TOP/mkfs.btrfs" -f "$TEST_DEV" run_check_mount_test_dev here=`pwd` diff --git a/tests/misc-tests/017-recv-stream-malformatted/test.sh b/tests/misc-tests/017-recv-stream-malformatted/test.sh index 884b7d42..3756be27 100755 --- a/tests/misc-tests/017-recv-stream-malformatted/test.sh +++ b/tests/misc-tests/017-recv-stream-malformatted/test.sh @@ -8,9 +8,9 @@ check_prereq mkfs.btrfs check_prereq btrfs setup_root_helper -prepare_test_dev 1g -run_check $TOP/mkfs.btrfs -f $IMAGE +prepare_test_dev 1g +run_check "$TOP/mkfs.btrfs" -f "$TEST_DEV" run_check_mount_test_dev echo -n '' | run_mayfail $SUDO_HELPER "$TOP/btrfs" receive "$TEST_MNT" && diff --git a/tests/misc-tests/018-recv-end-of-stream/test.sh b/tests/misc-tests/018-recv-end-of-stream/test.sh index d39683e9..3b8a0319 100755 --- a/tests/misc-tests/018-recv-end-of-stream/test.sh +++ b/tests/misc-tests/018-recv-end-of-stream/test.sh @@ -13,6 +13,8 @@ prepare_test_dev 1g here=`pwd` +# All helpers can exercise various options passed to 'btrfs receive' + test_full_empty_stream() { local str @@ -34,7 +36,7 @@ test_full_empty_stream() { run_check $TOP/mkfs.btrfs -f $TEST_DEV run_check_mount_test_dev - run_check $SUDO_HELPER $TOP/btrfs receive -v -f "$str" "$TEST_MNT" + run_check $SUDO_HELPER $TOP/btrfs receive "$@" -v -f "$str" "$TEST_MNT" run_check_umount_test_dev run_check rm -f -- "$str" @@ -65,7 +67,7 @@ test_full_simple_stream() { run_check $TOP/mkfs.btrfs -f $TEST_DEV run_check_mount_test_dev - run_check $SUDO_HELPER $TOP/btrfs receive -v -f "$str" "$TEST_MNT" + run_check $SUDO_HELPER $TOP/btrfs receive "$@" -v -f "$str" "$TEST_MNT" run_check_umount_test_dev run_check rm -f -- "$str" @@ -96,8 +98,8 @@ test_incr_empty_stream() { run_check $TOP/mkfs.btrfs -f $TEST_DEV run_check_mount_test_dev - run_check $SUDO_HELPER $TOP/btrfs receive -v -f "$fstr" "$TEST_MNT" - run_check $SUDO_HELPER $TOP/btrfs receive -v -f "$istr" "$TEST_MNT" + run_check $SUDO_HELPER $TOP/btrfs receive "$@" -v -f "$fstr" "$TEST_MNT" + run_check $SUDO_HELPER $TOP/btrfs receive "$@" -v -f "$istr" "$TEST_MNT" run_check_umount_test_dev run_check rm -f -- "$fstr" "$istr" @@ -136,8 +138,8 @@ test_incr_simple_stream() { run_check $TOP/mkfs.btrfs -f $TEST_DEV run_check_mount_test_dev - run_check $SUDO_HELPER $TOP/btrfs receive -v -f "$fstr" "$TEST_MNT" - run_check $SUDO_HELPER $TOP/btrfs receive -v -f "$istr" "$TEST_MNT" + run_check $SUDO_HELPER $TOP/btrfs receive "$@" -v -f "$fstr" "$TEST_MNT" + run_check $SUDO_HELPER $TOP/btrfs receive "$@" -v -f "$istr" "$TEST_MNT" run_check_umount_test_dev run_check rm -f -- "$fstr" "$istr" @@ -147,3 +149,9 @@ test_full_empty_stream test_full_simple_stream test_incr_empty_stream test_incr_simple_stream + +extra_opt=-e +test_full_empty_stream $extra_opt +test_full_simple_stream $extra_opt +test_incr_empty_stream $extra_opt +test_incr_simple_stream $extra_opt diff --git a/tests/misc-tests/019-receive-clones-on-munted-subvol/test.sh b/tests/misc-tests/019-receive-clones-on-munted-subvol/test.sh new file mode 100755 index 00000000..182b0cf9 --- /dev/null +++ b/tests/misc-tests/019-receive-clones-on-munted-subvol/test.sh @@ -0,0 +1,127 @@ +#! /bin/bash +# +# Test that an incremental send operation works when in both snapshots there are +# two directory inodes that have the same number but different generations and +# have an entry with the same name that corresponds to different inodes in each +# snapshot. + +source $TOP/tests/common + +check_prereq mkfs.btrfs +check_prereq btrfs +check_prereq fssum + +setup_root_helper +prepare_test_dev + +FSSUM_PROG="$TOP/fssum" +srcdir=./send-test-dir +rm -rf "$srcdir" +mkdir -p "$srcdir" +run_check chmod a+rw "$srcdir" + +run_check "$TOP/mkfs.btrfs" -f "$TEST_DEV" +run_check_mount_test_dev + +BLOCK_SIZE=$(stat -f -c %S "$TEST_MNT") + +run_check $SUDO_HELPER "$TOP/btrfs" subvolume create "$TEST_MNT/foo" +run_check $SUDO_HELPER "$TOP/btrfs" subvolume create "$TEST_MNT/bar" +run_check $SUDO_HELPER "$TOP/btrfs" subvolume create "$TEST_MNT/baz" +run_check $SUDO_HELPER "$TOP/btrfs" subvolume create "$TEST_MNT/snap" + +tr '\000' 'A' < /dev/null | + run_check $SUDO_HELPER dd of=$TEST_MNT/foo/file_a bs=$BLOCK_SIZE count=32 +tr '\000' 'B' < /dev/null | + run_check $SUDO_HELPER dd of=$TEST_MNT/bar/file_b bs=$BLOCK_SIZE count=32 + +run_check $SUDO_HELPER cp --reflink=always "$TEST_MNT/foo/file_a" "$TEST_MNT/baz/file_a" +run_check $SUDO_HELPER cp --reflink=always "$TEST_MNT/bar/file_b" "$TEST_MNT/baz/file_b" + +# Filesystem looks like: +# +# . +# |--- foo/ +# | |--- file_a +# |--- bar/ +# | |--- file_b +# |--- baz/ +# | |--- file_a (clone of "foo/file_a") +# | |--- file_b (clone of "bar/file_b") +# |--- snap/ +# + +# create snapshots and send streams + +run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot -r "$TEST_MNT/foo" "$TEST_MNT/snap/foo.0" +run_check $SUDO_HELPER "$TOP/btrfs" send -f "$srcdir/foo.0.snap" "$TEST_MNT/snap/foo.0" + +run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot -r "$TEST_MNT/bar" "$TEST_MNT/snap/bar.0" +run_check $SUDO_HELPER "$TOP/btrfs" send -f "$srcdir/bar.0.snap" "$TEST_MNT/snap/bar.0" + +run_check $SUDO_HELPER cp --reflink=always "$TEST_MNT/foo/file_a" "$TEST_MNT/foo/file_a.clone" +run_check $SUDO_HELPER rm -f -- "$TEST_MNT/foo/file_a" + +run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot -r "$TEST_MNT/foo" \ + "$TEST_MNT/snap/foo.1" +run_check $SUDO_HELPER "$TOP/btrfs" send -p "$TEST_MNT/snap/foo.0" -f "$srcdir/foo.1.snap" \ + "$TEST_MNT/snap/foo.1" + +run_check $SUDO_HELPER "$TOP/btrfs" subvolume snapshot -r "$TEST_MNT/baz" \ + "$TEST_MNT/snap/baz.0" +run_check $SUDO_HELPER "$TOP/btrfs" send -p "$TEST_MNT/snap/foo.1" \ + -c "$TEST_MNT/snap/bar.0" -f "$srcdir/baz.0.snap" \ + "$TEST_MNT/snap/baz.0" + +# Filesystem looks like: +# +# . +# |--- foo/ +# | |--- file_a.clone (clone of "foo/file_a") +# |--- bar/ +# | |--- file_b +# |--- baz/ +# | |--- file_a (clone of "foo/file_a") +# | |--- file_b (clone of "bar/file_b") +# |--- snap/ +# |--- bar.0/ (snapshot of "bar") +# | |--- file_b +# |--- foo.0/ (snapshot of "foo") +# | |--- file_a +# |--- foo.1/ (snapshot of "foo") +# | |--- file_a.clone +# |--- baz.0/ (snapshot of "baz") +# | |--- file_a +# | |--- file_b + +run_check $FSSUM_PROG -A -f -w "$srcdir/foo.0.fssum" "$TEST_MNT/snap/foo.0" +run_check $FSSUM_PROG -A -f -w "$srcdir/foo.1.fssum" "$TEST_MNT/snap/foo.1" +run_check $FSSUM_PROG -A -f -w "$srcdir/bar.0.fssum" "$TEST_MNT/snap/bar.0" +run_check $FSSUM_PROG -A -f -w "$srcdir/baz.0.fssum" "$TEST_MNT/snap/baz.0" + +run_check_umount_test_dev +run_check "$TOP/mkfs.btrfs" -f "$TEST_DEV" +run_check_mount_test_dev +run_check $SUDO_HELPER "$TOP/btrfs" subvolume create "$TEST_MNT/dest" +run_check_umount_test_dev +run_check_mount_test_dev -o subvol=/dest + +run_check $SUDO_HELPER "$TOP/btrfs" receive -f "$srcdir/foo.0.snap" "$TEST_MNT" +run_check $SUDO_HELPER "$TOP/btrfs" receive -f "$srcdir/bar.0.snap" "$TEST_MNT" + +# if "dest/" is not correctly stripped from the beginning of the path to +# "foo.0" in the target fs, we would get an error here because the clone source +# "foo.0/file_a" for "foo.1/file_a.clone" can't be found. +run_check $SUDO_HELPER "$TOP/btrfs" receive -f "$srcdir/foo.1.snap" "$TEST_MNT" + +# same here, but with send -c instead of -p +run_check $SUDO_HELPER "$TOP/btrfs" receive -f "$srcdir/baz.0.snap" "$TEST_MNT" + +run_check $FSSUM_PROG -r "$srcdir/foo.0.fssum" "$TEST_MNT/foo.0" +run_check $FSSUM_PROG -r "$srcdir/foo.1.fssum" "$TEST_MNT/foo.1" +run_check $FSSUM_PROG -r "$srcdir/bar.0.fssum" "$TEST_MNT/bar.0" +run_check $FSSUM_PROG -r "$srcdir/baz.0.fssum" "$TEST_MNT/baz.0" + +run_check_umount_test_dev + +rm -rf -- "$srcdir" diff --git a/tests/misc-tests/020-fix-superblock-corruption/test.sh b/tests/misc-tests/020-fix-superblock-corruption/test.sh new file mode 100755 index 00000000..77c1a5aa --- /dev/null +++ b/tests/misc-tests/020-fix-superblock-corruption/test.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Corrupt primary superblock and restore it using backup superblock. + +source "$TOP/tests/common" + +check_prereq btrfs-select-super +check_prereq btrfs + +setup_root_helper +prepare_test_dev + +FIRST_SUPERBLOCK_OFFSET=65536 + +test_superblock_restore() +{ + run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f "$TEST_DEV" + + # Corrupt superblock checksum + run_check $SUDO_HELPER dd if=/dev/zero of="$TEST_DEV" \ + seek="$FIRST_SUPERBLOCK_OFFSET" bs=1 count=4 conv=notrunc + + # Run btrfs check to detect corruption + run_mayfail "$TOP/btrfs" check "$TEST_DEV" && \ + _fail "btrfs check should detect corruption" + + # Copy backup superblock to primary + run_check "$TOP/btrfs-select-super" -s 1 "$TEST_DEV" + + # Perform btrfs check + run_check "$TOP/btrfs" check "$TEST_DEV" +} + +test_superblock_restore diff --git a/tests/misc-tests/021-image-multi-devices/test.sh b/tests/misc-tests/021-image-multi-devices/test.sh new file mode 100755 index 00000000..abf67f90 --- /dev/null +++ b/tests/misc-tests/021-image-multi-devices/test.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Test btrfs-image with multiple devices filesystem and verify that restoring +# the created image works against a single device. + +source "$TOP/tests/common" + +check_prereq btrfs-image +check_prereq mkfs.btrfs +check_prereq btrfs + +setup_root_helper + +rm -f dev1 dev2 +run_check truncate -s 2G dev1 +run_check truncate -s 2G dev2 +chmod a+w dev1 dev2 + +loop1=$(run_check_stdout $SUDO_HELPER losetup --find --show dev1) +loop2=$(run_check_stdout $SUDO_HELPER losetup --find --show dev2) + +# Create the test file system. + +run_check $SUDO_HELPER "$TOP/mkfs.btrfs" -f $loop1 $loop2 +run_check $SUDO_HELPER mount $loop1 "$TEST_MNT" +run_check $SUDO_HELPER dd bs=1M count=1 if=/dev/zero "of=$TEST_MNT/foobar" +orig_md5=$(run_check_stdout md5sum "$TEST_MNT/foobar" | cut -d ' ' -f 1) +run_check $SUDO_HELPER umount "$TEST_MNT" + +# Create the image to restore later. +run_check $SUDO_HELPER "$TOP/btrfs-image" $loop1 "$IMAGE" + +# Wipe out the filesystem from the devices, restore the image on a single +# device, check everything works and file foobar is there and with 1Mb of +# zeroes. +run_check $SUDO_HELPER wipefs -a $loop1 +run_check $SUDO_HELPER wipefs -a $loop2 + +run_check $SUDO_HELPER $TOP/btrfs-image -r "$IMAGE" $loop1 + +run_check $SUDO_HELPER mount $loop1 "$TEST_MNT" +new_md5=$(run_check_stdout md5sum "$TEST_MNT/foobar" | cut -d ' ' -f 1) +run_check $SUDO_HELPER umount "$TEST_MNT" + +# Cleanup loop devices. +run_check $SUDO_HELPER losetup -d $loop1 +run_check $SUDO_HELPER losetup -d $loop2 +rm -f dev1 dev2 + +# Compare the file digests. +[ $orig_md5 == $new_md5 ] || _fail "File digests do not match" diff --git a/tests/mkfs-tests/002-no-force-mixed-on-small-volume/test.sh b/tests/mkfs-tests/002-no-force-mixed-on-small-volume/test.sh index 855fbd18..37846234 100755 --- a/tests/mkfs-tests/002-no-force-mixed-on-small-volume/test.sh +++ b/tests/mkfs-tests/002-no-force-mixed-on-small-volume/test.sh @@ -8,6 +8,5 @@ check_prereq mkfs.btrfs setup_root_helper -run_check truncate -s 512M $IMAGE -mixed=$(run_check_stdout $TOP/mkfs.btrfs -n 64k -f $IMAGE | egrep 'Data|Metadata') +mixed=$(run_check_stdout "$TOP/mkfs.btrfs" -b 512M -n 64k -f "$TEST_DEV" | egrep 'Data|Metadata') echo "$mixed" | grep -q -v 'Data+Metadata:' || _fail "unexpected: created a mixed-bg filesystem" diff --git a/tests/mkfs-tests/003-mixed-with-wrong-nodesize/test.sh b/tests/mkfs-tests/003-mixed-with-wrong-nodesize/test.sh index 289d5ff0..074fc22e 100755 --- a/tests/mkfs-tests/003-mixed-with-wrong-nodesize/test.sh +++ b/tests/mkfs-tests/003-mixed-with-wrong-nodesize/test.sh @@ -6,7 +6,6 @@ source $TOP/tests/common check_prereq mkfs.btrfs -run_check truncate -s 512M $IMAGE -run_mayfail $TOP/mkfs.btrfs -f -M -s 4096 -n 16384 "$IMAGE" && _fail +run_mayfail "$TOP/mkfs.btrfs" -b 512M -f -M -s 4096 -n 16384 "$TEST_DEV" && _fail exit 0 diff --git a/tests/sha-private.h b/tests/sha-private.h new file mode 100644 index 00000000..6e9c4520 --- /dev/null +++ b/tests/sha-private.h @@ -0,0 +1,28 @@ +/************************ sha-private.h ************************/ +/***************** See RFC 6234 for details. *******************/ +#ifndef _SHA_PRIVATE__H +#define _SHA_PRIVATE__H +/* + * These definitions are defined in FIPS 180-3, section 4.1. + * Ch() and Maj() are defined identically in sections 4.1.1, + * 4.1.2, and 4.1.3. + * + * The definitions used in FIPS 180-3 are as follows: + */ + +#ifndef USE_MODIFIED_MACROS +#define SHA_Ch(x,y,z) (((x) & (y)) ^ ((~(x)) & (z))) +#define SHA_Maj(x,y,z) (((x) & (y)) ^ ((x) & (z)) ^ ((y) & (z))) +#else /* USE_MODIFIED_MACROS */ +/* + * The following definitions are equivalent and potentially faster. + */ + +#define SHA_Ch(x, y, z) (((x) & ((y) ^ (z))) ^ (z)) +#define SHA_Maj(x, y, z) (((x) & ((y) | (z))) | ((y) & (z))) + +#endif /* USE_MODIFIED_MACROS */ + +#define SHA_Parity(x, y, z) ((x) ^ (y) ^ (z)) + +#endif /* _SHA_PRIVATE__H */ diff --git a/tests/sha.h b/tests/sha.h new file mode 100644 index 00000000..1ffd6880 --- /dev/null +++ b/tests/sha.h @@ -0,0 +1,356 @@ +/**************************** sha.h ****************************/ +/***************** See RFC 6234 for details. *******************/ +/* + Copyright (c) 2011 IETF Trust and the persons identified as + authors of the code. All rights reserved. + + Redistribution and use in source and binary forms, with or + without modification, are permitted provided that the following + conditions are met: + + - Redistributions of source code must retain the above + copyright notice, this list of conditions and + the following disclaimer. + + - Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + - Neither the name of Internet Society, IETF or IETF Trust, nor + the names of specific contributors, may be used to endorse or + promote products derived from this software without specific + prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ +#ifndef _SHA_H_ +#define _SHA_H_ + +/* + * Description: + * This file implements the Secure Hash Algorithms + * as defined in the U.S. National Institute of Standards + * and Technology Federal Information Processing Standards + * Publication (FIPS PUB) 180-3 published in October 2008 + * and formerly defined in its predecessors, FIPS PUB 180-1 + * and FIP PUB 180-2. + * + * A combined document showing all algorithms is available at + * http://csrc.nist.gov/publications/fips/ + * fips180-3/fips180-3_final.pdf + * + * The five hashes are defined in these sizes: + * SHA-1 20 byte / 160 bit + * SHA-224 28 byte / 224 bit + * SHA-256 32 byte / 256 bit + * SHA-384 48 byte / 384 bit + * SHA-512 64 byte / 512 bit + * + * Compilation Note: + * These files may be compiled with two options: + * USE_32BIT_ONLY - use 32-bit arithmetic only, for systems + * without 64-bit integers + * + * USE_MODIFIED_MACROS - use alternate form of the SHA_Ch() + * and SHA_Maj() macros that are equivalent + * and potentially faster on many systems + * + */ + +#include <stdint.h> +/* + * If you do not have the ISO standard stdint.h header file, then you + * must typedef the following: + * name meaning + * uint64_t unsigned 64-bit integer + * uint32_t unsigned 32-bit integer + * uint8_t unsigned 8-bit integer (i.e., unsigned char) + * int_least16_t integer of >= 16 bits + * + * See stdint-example.h + */ + +#ifndef _SHA_enum_ +#define _SHA_enum_ +/* + * All SHA functions return one of these values. + */ +enum { + shaSuccess = 0, + shaNull, /* Null pointer parameter */ + shaInputTooLong, /* input data too long */ + shaStateError, /* called Input after FinalBits or Result */ + shaBadParam /* passed a bad parameter */ +}; +#endif /* _SHA_enum_ */ + +/* + * These constants hold size information for each of the SHA + * hashing operations + */ +enum { + SHA1_Message_Block_Size = 64, SHA224_Message_Block_Size = 64, + SHA256_Message_Block_Size = 64, SHA384_Message_Block_Size = 128, + SHA512_Message_Block_Size = 128, + USHA_Max_Message_Block_Size = SHA512_Message_Block_Size, + + SHA1HashSize = 20, SHA224HashSize = 28, SHA256HashSize = 32, + SHA384HashSize = 48, SHA512HashSize = 64, + USHAMaxHashSize = SHA512HashSize, + + SHA1HashSizeBits = 160, SHA224HashSizeBits = 224, + SHA256HashSizeBits = 256, SHA384HashSizeBits = 384, + SHA512HashSizeBits = 512, USHAMaxHashSizeBits = SHA512HashSizeBits +}; + +/* + * These constants are used in the USHA (Unified SHA) functions. + */ +typedef enum SHAversion { + SHA1, SHA224, SHA256, SHA384, SHA512 +} SHAversion; + +/* + * This structure will hold context information for the SHA-1 + * hashing operation. + */ +typedef struct SHA1Context { + uint32_t Intermediate_Hash[SHA1HashSize/4]; /* Message Digest */ + + uint32_t Length_High; /* Message length in bits */ + uint32_t Length_Low; /* Message length in bits */ + + int_least16_t Message_Block_Index; /* Message_Block array index */ + /* 512-bit message blocks */ + uint8_t Message_Block[SHA1_Message_Block_Size]; + + int Computed; /* Is the hash computed? */ + int Corrupted; /* Cumulative corruption code */ +} SHA1Context; + +/* + * This structure will hold context information for the SHA-256 + * hashing operation. + */ +typedef struct SHA256Context { + uint32_t Intermediate_Hash[SHA256HashSize/4]; /* Message Digest */ + + uint32_t Length_High; /* Message length in bits */ + uint32_t Length_Low; /* Message length in bits */ + + int_least16_t Message_Block_Index; /* Message_Block array index */ + /* 512-bit message blocks */ + uint8_t Message_Block[SHA256_Message_Block_Size]; + + int Computed; /* Is the hash computed? */ + int Corrupted; /* Cumulative corruption code */ +} SHA256Context; + +/* + * This structure will hold context information for the SHA-512 + * hashing operation. + */ +typedef struct SHA512Context { +#ifdef USE_32BIT_ONLY + uint32_t Intermediate_Hash[SHA512HashSize/4]; /* Message Digest */ + uint32_t Length[4]; /* Message length in bits */ +#else /* !USE_32BIT_ONLY */ + uint64_t Intermediate_Hash[SHA512HashSize/8]; /* Message Digest */ + uint64_t Length_High, Length_Low; /* Message length in bits */ +#endif /* USE_32BIT_ONLY */ + + int_least16_t Message_Block_Index; /* Message_Block array index */ + /* 1024-bit message blocks */ + uint8_t Message_Block[SHA512_Message_Block_Size]; + + int Computed; /* Is the hash computed?*/ + int Corrupted; /* Cumulative corruption code */ +} SHA512Context; + +/* + * This structure will hold context information for the SHA-224 + * hashing operation. It uses the SHA-256 structure for computation. + */ +typedef struct SHA256Context SHA224Context; + +/* + * This structure will hold context information for the SHA-384 + * hashing operation. It uses the SHA-512 structure for computation. + */ +typedef struct SHA512Context SHA384Context; + +/* + * This structure holds context information for all SHA + * hashing operations. + */ +typedef struct USHAContext { + int whichSha; /* which SHA is being used */ + union { + SHA1Context sha1Context; + SHA224Context sha224Context; SHA256Context sha256Context; + SHA384Context sha384Context; SHA512Context sha512Context; + } ctx; +} USHAContext; + +/* + * This structure will hold context information for the HMAC + * keyed-hashing operation. + */ +typedef struct HMACContext { + int whichSha; /* which SHA is being used */ + int hashSize; /* hash size of SHA being used */ + int blockSize; /* block size of SHA being used */ + USHAContext shaContext; /* SHA context */ + unsigned char k_opad[USHA_Max_Message_Block_Size]; + /* outer padding - key XORd with opad */ + int Computed; /* Is the MAC computed? */ + int Corrupted; /* Cumulative corruption code */ + +} HMACContext; + +/* + * This structure will hold context information for the HKDF + * extract-and-expand Key Derivation Functions. + */ +typedef struct HKDFContext { + int whichSha; /* which SHA is being used */ + HMACContext hmacContext; + int hashSize; /* hash size of SHA being used */ + unsigned char prk[USHAMaxHashSize]; + /* pseudo-random key - output of hkdfInput */ + int Computed; /* Is the key material computed? */ + int Corrupted; /* Cumulative corruption code */ +} HKDFContext; + +/* + * Function Prototypes + */ + +/* SHA-1 */ +extern int SHA1Reset(SHA1Context *); +extern int SHA1Input(SHA1Context *, const uint8_t *bytes, + unsigned int bytecount); +extern int SHA1FinalBits(SHA1Context *, uint8_t bits, + unsigned int bit_count); +extern int SHA1Result(SHA1Context *, + uint8_t Message_Digest[SHA1HashSize]); + +/* SHA-224 */ +extern int SHA224Reset(SHA224Context *); +extern int SHA224Input(SHA224Context *, const uint8_t *bytes, + unsigned int bytecount); +extern int SHA224FinalBits(SHA224Context *, uint8_t bits, + unsigned int bit_count); +extern int SHA224Result(SHA224Context *, + uint8_t Message_Digest[SHA224HashSize]); + +/* SHA-256 */ +extern int SHA256Reset(SHA256Context *); +extern int SHA256Input(SHA256Context *, const uint8_t *bytes, + unsigned int bytecount); +extern int SHA256FinalBits(SHA256Context *, uint8_t bits, + unsigned int bit_count); +extern int SHA256Result(SHA256Context *, + uint8_t Message_Digest[SHA256HashSize]); + +/* SHA-384 */ +extern int SHA384Reset(SHA384Context *); +extern int SHA384Input(SHA384Context *, const uint8_t *bytes, + unsigned int bytecount); +extern int SHA384FinalBits(SHA384Context *, uint8_t bits, + unsigned int bit_count); +extern int SHA384Result(SHA384Context *, + uint8_t Message_Digest[SHA384HashSize]); + +/* SHA-512 */ +extern int SHA512Reset(SHA512Context *); +extern int SHA512Input(SHA512Context *, const uint8_t *bytes, + unsigned int bytecount); +extern int SHA512FinalBits(SHA512Context *, uint8_t bits, + unsigned int bit_count); +extern int SHA512Result(SHA512Context *, + uint8_t Message_Digest[SHA512HashSize]); + +/* Unified SHA functions, chosen by whichSha */ +extern int USHAReset(USHAContext *context, SHAversion whichSha); +extern int USHAInput(USHAContext *context, + const uint8_t *bytes, unsigned int bytecount); +extern int USHAFinalBits(USHAContext *context, + uint8_t bits, unsigned int bit_count); +extern int USHAResult(USHAContext *context, + uint8_t Message_Digest[USHAMaxHashSize]); +extern int USHABlockSize(enum SHAversion whichSha); +extern int USHAHashSize(enum SHAversion whichSha); +extern int USHAHashSizeBits(enum SHAversion whichSha); +extern const char *USHAHashName(enum SHAversion whichSha); + +/* + * HMAC Keyed-Hashing for Message Authentication, RFC 2104, + * for all SHAs. + * This interface allows a fixed-length text input to be used. + */ +extern int hmac(SHAversion whichSha, /* which SHA algorithm to use */ + const unsigned char *text, /* pointer to data stream */ + int text_len, /* length of data stream */ + const unsigned char *key, /* pointer to authentication key */ + int key_len, /* length of authentication key */ + uint8_t digest[USHAMaxHashSize]); /* caller digest to fill in */ + +/* + * HMAC Keyed-Hashing for Message Authentication, RFC 2104, + * for all SHAs. + * This interface allows any length of text input to be used. + */ +extern int hmacReset(HMACContext *context, enum SHAversion whichSha, + const unsigned char *key, int key_len); +extern int hmacInput(HMACContext *context, const unsigned char *text, + int text_len); +extern int hmacFinalBits(HMACContext *context, uint8_t bits, + unsigned int bit_count); +extern int hmacResult(HMACContext *context, + uint8_t digest[USHAMaxHashSize]); + +/* + * HKDF HMAC-based Extract-and-Expand Key Derivation Function, + * RFC 5869, for all SHAs. + */ +extern int hkdf(SHAversion whichSha, const unsigned char *salt, + int salt_len, const unsigned char *ikm, int ikm_len, + const unsigned char *info, int info_len, + uint8_t okm[ ], int okm_len); +extern int hkdfExtract(SHAversion whichSha, const unsigned char *salt, + int salt_len, const unsigned char *ikm, + int ikm_len, uint8_t prk[USHAMaxHashSize]); +extern int hkdfExpand(SHAversion whichSha, const uint8_t prk[ ], + int prk_len, const unsigned char *info, + int info_len, uint8_t okm[ ], int okm_len); + +/* + * HKDF HMAC-based Extract-and-Expand Key Derivation Function, + * RFC 5869, for all SHAs. + * This interface allows any length of text input to be used. + */ +extern int hkdfReset(HKDFContext *context, enum SHAversion whichSha, + const unsigned char *salt, int salt_len); +extern int hkdfInput(HKDFContext *context, const unsigned char *ikm, + int ikm_len); +extern int hkdfFinalBits(HKDFContext *context, uint8_t ikm_bits, + unsigned int ikm_bit_count); +extern int hkdfResult(HKDFContext *context, + uint8_t prk[USHAMaxHashSize], + const unsigned char *info, int info_len, + uint8_t okm[USHAMaxHashSize], int okm_len); +#endif /* _SHA_H_ */ diff --git a/tests/sha224-256.c b/tests/sha224-256.c new file mode 100644 index 00000000..2d963e65 --- /dev/null +++ b/tests/sha224-256.c @@ -0,0 +1,601 @@ +/* +RFC 6234 SHAs, HMAC-SHAs, and HKDF May 2011 + + +Copyright Notice + + Copyright (c) 2011 IETF Trust and the persons identified as the + document authors. All rights reserved. + + This document is subject to BCP 78 and the IETF Trust's Legal + Provisions Relating to IETF Documents + (http://trustee.ietf.org/license-info) in effect on the date of + publication of this document. Please review these documents + carefully, as they describe your rights and restrictions with respect + to this document. Code Components extracted from this document must + include Simplified BSD License text as described in Section 4.e of + the Trust Legal Provisions and are provided without warranty as + described in the Simplified BSD License. +*/ + +/************************* sha224-256.c ************************/ +/***************** See RFC 6234 for details. *******************/ +/* Copyright (c) 2011 IETF Trust and the persons identified as */ +/* authors of the code. All rights reserved. */ +/* See sha.h for terms of use and redistribution. */ + +/* + * Description: + * This file implements the Secure Hash Algorithms SHA-224 and + * SHA-256 as defined in the U.S. National Institute of Standards + * and Technology Federal Information Processing Standards + * Publication (FIPS PUB) 180-3 published in October 2008 + * and formerly defined in its predecessors, FIPS PUB 180-1 + * and FIP PUB 180-2. + * + * A combined document showing all algorithms is available at + * http://csrc.nist.gov/publications/fips/ + * fips180-3/fips180-3_final.pdf + * + * The SHA-224 and SHA-256 algorithms produce 224-bit and 256-bit + * message digests for a given data stream. It should take about + * 2**n steps to find a message with the same digest as a given + * message and 2**(n/2) to find any two messages with the same + * digest, when n is the digest size in bits. Therefore, this + * algorithm can serve as a means of providing a + * "fingerprint" for a message. + * + * Portability Issues: + * SHA-224 and SHA-256 are defined in terms of 32-bit "words". + * This code uses <stdint.h> (included via "sha.h") to define 32- + * and 8-bit unsigned integer types. If your C compiler does not + * support 32-bit unsigned integers, this code is not + * appropriate. + * + * Caveats: + * SHA-224 and SHA-256 are designed to work with messages less + * than 2^64 bits long. This implementation uses SHA224/256Input() + * to hash the bits that are a multiple of the size of an 8-bit + * octet, and then optionally uses SHA224/256FinalBits() + * to hash the final few bits of the input. + */ + +#include "tests/sha.h" +#include "tests/sha-private.h" + +/* Define the SHA shift, rotate left, and rotate right macros */ +#define SHA256_SHR(bits,word) ((word) >> (bits)) +#define SHA256_ROTL(bits,word) \ + (((word) << (bits)) | ((word) >> (32-(bits)))) +#define SHA256_ROTR(bits,word) \ + (((word) >> (bits)) | ((word) << (32-(bits)))) + +/* Define the SHA SIGMA and sigma macros */ +#define SHA256_SIGMA0(word) \ + (SHA256_ROTR( 2,word) ^ SHA256_ROTR(13,word) ^ SHA256_ROTR(22,word)) +#define SHA256_SIGMA1(word) \ + (SHA256_ROTR( 6,word) ^ SHA256_ROTR(11,word) ^ SHA256_ROTR(25,word)) +#define SHA256_sigma0(word) \ + (SHA256_ROTR( 7,word) ^ SHA256_ROTR(18,word) ^ SHA256_SHR( 3,word)) +#define SHA256_sigma1(word) \ + (SHA256_ROTR(17,word) ^ SHA256_ROTR(19,word) ^ SHA256_SHR(10,word)) + +/* + * Add "length" to the length. + * Set Corrupted when overflow has occurred. + */ +static uint32_t addTemp; +#define SHA224_256AddLength(context, length) \ + (addTemp = (context)->Length_Low, (context)->Corrupted = \ + (((context)->Length_Low += (length)) < addTemp) && \ + (++(context)->Length_High == 0) ? shaInputTooLong : \ + (context)->Corrupted ) + +/* Local Function Prototypes */ +static int SHA224_256Reset(SHA256Context *context, uint32_t *H0); +static void SHA224_256ProcessMessageBlock(SHA256Context *context); +static void SHA224_256Finalize(SHA256Context *context, + uint8_t Pad_Byte); +static void SHA224_256PadMessage(SHA256Context *context, + uint8_t Pad_Byte); +static int SHA224_256ResultN(SHA256Context *context, + uint8_t Message_Digest[ ], int HashSize); + +/* Initial Hash Values: FIPS 180-3 section 5.3.2 */ +static uint32_t SHA224_H0[SHA256HashSize/4] = { + 0xC1059ED8, 0x367CD507, 0x3070DD17, 0xF70E5939, + 0xFFC00B31, 0x68581511, 0x64F98FA7, 0xBEFA4FA4 +}; + +/* Initial Hash Values: FIPS 180-3 section 5.3.3 */ +static uint32_t SHA256_H0[SHA256HashSize/4] = { + 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A, + 0x510E527F, 0x9B05688C, 0x1F83D9AB, 0x5BE0CD19 +}; + +/* + * SHA224Reset + * + * Description: + * This function will initialize the SHA224Context in preparation + * for computing a new SHA224 message digest. + * + * Parameters: + * context: [in/out] + * The context to reset. + * + * Returns: + * sha Error Code. + */ +int SHA224Reset(SHA224Context *context) +{ + return SHA224_256Reset(context, SHA224_H0); +} + +/* + * SHA224Input + * + * Description: + * This function accepts an array of octets as the next portion + * of the message. + * + * Parameters: + * context: [in/out] + * The SHA context to update. + * message_array[ ]: [in] + * An array of octets representing the next portion of + * the message. + * length: [in] + * The length of the message in message_array. + * + * Returns: + * sha Error Code. + * + */ +int SHA224Input(SHA224Context *context, const uint8_t *message_array, + unsigned int length) +{ + return SHA256Input(context, message_array, length); +} + +/* + * SHA224FinalBits + * + * Description: + * This function will add in any final bits of the message. + * + * Parameters: + * context: [in/out] + * The SHA context to update. + * message_bits: [in] + * The final bits of the message, in the upper portion of the + * byte. (Use 0b###00000 instead of 0b00000### to input the + * three bits ###.) + * length: [in] + * The number of bits in message_bits, between 1 and 7. + * + * Returns: + * sha Error Code. + */ +int SHA224FinalBits(SHA224Context *context, + uint8_t message_bits, unsigned int length) +{ + return SHA256FinalBits(context, message_bits, length); +} + +/* + * SHA224Result + * + * Description: + * This function will return the 224-bit message digest + * into the Message_Digest array provided by the caller. + * NOTE: + * The first octet of hash is stored in the element with index 0, + * the last octet of hash in the element with index 27. + * + * Parameters: + * context: [in/out] + * The context to use to calculate the SHA hash. + * Message_Digest[ ]: [out] + * Where the digest is returned. + * + * Returns: + * sha Error Code. + */ +int SHA224Result(SHA224Context *context, + uint8_t Message_Digest[SHA224HashSize]) +{ + return SHA224_256ResultN(context, Message_Digest, SHA224HashSize); +} + +/* + * SHA256Reset + * + * Description: + * This function will initialize the SHA256Context in preparation + * for computing a new SHA256 message digest. + * + * Parameters: + * context: [in/out] + * The context to reset. + * + * Returns: + * sha Error Code. + */ +int SHA256Reset(SHA256Context *context) +{ + return SHA224_256Reset(context, SHA256_H0); +} + +/* + * SHA256Input + * + * Description: + * This function accepts an array of octets as the next portion + * of the message. + * + * Parameters: + * context: [in/out] + * The SHA context to update. + * message_array[ ]: [in] + * An array of octets representing the next portion of + * the message. + * length: [in] + * The length of the message in message_array. + * + * Returns: + * sha Error Code. + */ +int SHA256Input(SHA256Context *context, const uint8_t *message_array, + unsigned int length) +{ + if (!context) return shaNull; + if (!length) return shaSuccess; + if (!message_array) return shaNull; + if (context->Computed) return context->Corrupted = shaStateError; + if (context->Corrupted) return context->Corrupted; + + while (length--) { + context->Message_Block[context->Message_Block_Index++] = + *message_array; + + if ((SHA224_256AddLength(context, 8) == shaSuccess) && + (context->Message_Block_Index == SHA256_Message_Block_Size)) + SHA224_256ProcessMessageBlock(context); + + message_array++; + } + + return context->Corrupted; + +} + +/* + * SHA256FinalBits + * + * Description: + * This function will add in any final bits of the message. + * + * Parameters: + * context: [in/out] + * The SHA context to update. + * message_bits: [in] + * The final bits of the message, in the upper portion of the + * byte. (Use 0b###00000 instead of 0b00000### to input the + * three bits ###.) + * length: [in] + * The number of bits in message_bits, between 1 and 7. + * + * Returns: + * sha Error Code. + */ +int SHA256FinalBits(SHA256Context *context, + uint8_t message_bits, unsigned int length) +{ + static uint8_t masks[8] = { + /* 0 0b00000000 */ 0x00, /* 1 0b10000000 */ 0x80, + /* 2 0b11000000 */ 0xC0, /* 3 0b11100000 */ 0xE0, + /* 4 0b11110000 */ 0xF0, /* 5 0b11111000 */ 0xF8, + /* 6 0b11111100 */ 0xFC, /* 7 0b11111110 */ 0xFE + }; + static uint8_t markbit[8] = { + /* 0 0b10000000 */ 0x80, /* 1 0b01000000 */ 0x40, + /* 2 0b00100000 */ 0x20, /* 3 0b00010000 */ 0x10, + /* 4 0b00001000 */ 0x08, /* 5 0b00000100 */ 0x04, + /* 6 0b00000010 */ 0x02, /* 7 0b00000001 */ 0x01 + }; + + if (!context) return shaNull; + if (!length) return shaSuccess; + if (context->Corrupted) return context->Corrupted; + if (context->Computed) return context->Corrupted = shaStateError; + if (length >= 8) return context->Corrupted = shaBadParam; + + SHA224_256AddLength(context, length); + SHA224_256Finalize(context, (uint8_t) + ((message_bits & masks[length]) | markbit[length])); + + return context->Corrupted; +} + +/* + * SHA256Result + * + * Description: + * This function will return the 256-bit message digest + * into the Message_Digest array provided by the caller. + * NOTE: + * The first octet of hash is stored in the element with index 0, + * the last octet of hash in the element with index 31. + * + * Parameters: + * context: [in/out] + * The context to use to calculate the SHA hash. + * Message_Digest[ ]: [out] + * Where the digest is returned. + * + * Returns: + * sha Error Code. + */ +int SHA256Result(SHA256Context *context, + uint8_t Message_Digest[SHA256HashSize]) +{ + return SHA224_256ResultN(context, Message_Digest, SHA256HashSize); +} + +/* + * SHA224_256Reset + * + * Description: + * This helper function will initialize the SHA256Context in + * preparation for computing a new SHA-224 or SHA-256 message digest. + * + * Parameters: + * context: [in/out] + * The context to reset. + * H0[ ]: [in] + * The initial hash value array to use. + * + * Returns: + * sha Error Code. + */ +static int SHA224_256Reset(SHA256Context *context, uint32_t *H0) +{ + if (!context) return shaNull; + + context->Length_High = context->Length_Low = 0; + context->Message_Block_Index = 0; + + context->Intermediate_Hash[0] = H0[0]; + context->Intermediate_Hash[1] = H0[1]; + context->Intermediate_Hash[2] = H0[2]; + context->Intermediate_Hash[3] = H0[3]; + context->Intermediate_Hash[4] = H0[4]; + context->Intermediate_Hash[5] = H0[5]; + context->Intermediate_Hash[6] = H0[6]; + context->Intermediate_Hash[7] = H0[7]; + + context->Computed = 0; + context->Corrupted = shaSuccess; + + return shaSuccess; +} + +/* + * SHA224_256ProcessMessageBlock + * + * Description: + * This helper function will process the next 512 bits of the + * message stored in the Message_Block array. + * + * Parameters: + * context: [in/out] + * The SHA context to update. + * + * Returns: + * Nothing. + * + * Comments: + * Many of the variable names in this code, especially the + * single character names, were used because those were the + * names used in the Secure Hash Standard. + */ +static void SHA224_256ProcessMessageBlock(SHA256Context *context) +{ + /* Constants defined in FIPS 180-3, section 4.2.2 */ + static const uint32_t K[64] = { + 0x428a2f98, 0x71374491, 0xb5c0fbcf, 0xe9b5dba5, 0x3956c25b, + 0x59f111f1, 0x923f82a4, 0xab1c5ed5, 0xd807aa98, 0x12835b01, + 0x243185be, 0x550c7dc3, 0x72be5d74, 0x80deb1fe, 0x9bdc06a7, + 0xc19bf174, 0xe49b69c1, 0xefbe4786, 0x0fc19dc6, 0x240ca1cc, + 0x2de92c6f, 0x4a7484aa, 0x5cb0a9dc, 0x76f988da, 0x983e5152, + 0xa831c66d, 0xb00327c8, 0xbf597fc7, 0xc6e00bf3, 0xd5a79147, + 0x06ca6351, 0x14292967, 0x27b70a85, 0x2e1b2138, 0x4d2c6dfc, + 0x53380d13, 0x650a7354, 0x766a0abb, 0x81c2c92e, 0x92722c85, + 0xa2bfe8a1, 0xa81a664b, 0xc24b8b70, 0xc76c51a3, 0xd192e819, + 0xd6990624, 0xf40e3585, 0x106aa070, 0x19a4c116, 0x1e376c08, + 0x2748774c, 0x34b0bcb5, 0x391c0cb3, 0x4ed8aa4a, 0x5b9cca4f, + 0x682e6ff3, 0x748f82ee, 0x78a5636f, 0x84c87814, 0x8cc70208, + 0x90befffa, 0xa4506ceb, 0xbef9a3f7, 0xc67178f2 + }; + int t, t4; /* Loop counter */ + uint32_t temp1, temp2; /* Temporary word value */ + uint32_t W[64]; /* Word sequence */ + uint32_t A, B, C, D, E, F, G, H; /* Word buffers */ + + /* + * Initialize the first 16 words in the array W + */ + for (t = t4 = 0; t < 16; t++, t4 += 4) + W[t] = (((uint32_t)context->Message_Block[t4]) << 24) | + (((uint32_t)context->Message_Block[t4 + 1]) << 16) | + (((uint32_t)context->Message_Block[t4 + 2]) << 8) | + (((uint32_t)context->Message_Block[t4 + 3])); + + for (t = 16; t < 64; t++) + W[t] = SHA256_sigma1(W[t-2]) + W[t-7] + + SHA256_sigma0(W[t-15]) + W[t-16]; + + A = context->Intermediate_Hash[0]; + B = context->Intermediate_Hash[1]; + C = context->Intermediate_Hash[2]; + D = context->Intermediate_Hash[3]; + E = context->Intermediate_Hash[4]; + F = context->Intermediate_Hash[5]; + G = context->Intermediate_Hash[6]; + H = context->Intermediate_Hash[7]; + + for (t = 0; t < 64; t++) { + temp1 = H + SHA256_SIGMA1(E) + SHA_Ch(E,F,G) + K[t] + W[t]; + temp2 = SHA256_SIGMA0(A) + SHA_Maj(A,B,C); + H = G; + G = F; + F = E; + E = D + temp1; + D = C; + C = B; + B = A; + A = temp1 + temp2; + } + + context->Intermediate_Hash[0] += A; + context->Intermediate_Hash[1] += B; + context->Intermediate_Hash[2] += C; + context->Intermediate_Hash[3] += D; + context->Intermediate_Hash[4] += E; + context->Intermediate_Hash[5] += F; + context->Intermediate_Hash[6] += G; + context->Intermediate_Hash[7] += H; + + context->Message_Block_Index = 0; +} + +/* + * SHA224_256Finalize + * + * Description: + * This helper function finishes off the digest calculations. + * + * Parameters: + * context: [in/out] + * The SHA context to update. + * Pad_Byte: [in] + * The last byte to add to the message block before the 0-padding + * and length. This will contain the last bits of the message + * followed by another single bit. If the message was an + * exact multiple of 8-bits long, Pad_Byte will be 0x80. + * + * Returns: + * sha Error Code. + */ +static void SHA224_256Finalize(SHA256Context *context, + uint8_t Pad_Byte) +{ + int i; + SHA224_256PadMessage(context, Pad_Byte); + /* message may be sensitive, so clear it out */ + for (i = 0; i < SHA256_Message_Block_Size; ++i) + context->Message_Block[i] = 0; + context->Length_High = 0; /* and clear length */ + context->Length_Low = 0; + context->Computed = 1; +} + +/* + * SHA224_256PadMessage + * + * Description: + * According to the standard, the message must be padded to the next + * even multiple of 512 bits. The first padding bit must be a '1'. + * The last 64 bits represent the length of the original message. + * All bits in between should be 0. This helper function will pad + * the message according to those rules by filling the + * Message_Block array accordingly. When it returns, it can be + * assumed that the message digest has been computed. + * + * Parameters: + * context: [in/out] + * The context to pad. + * Pad_Byte: [in] + * The last byte to add to the message block before the 0-padding + * and length. This will contain the last bits of the message + * followed by another single bit. If the message was an + * exact multiple of 8-bits long, Pad_Byte will be 0x80. + * + * Returns: + * Nothing. + */ +static void SHA224_256PadMessage(SHA256Context *context, + uint8_t Pad_Byte) +{ + /* + * Check to see if the current message block is too small to hold + * the initial padding bits and length. If so, we will pad the + * block, process it, and then continue padding into a second + * block. + */ + if (context->Message_Block_Index >= (SHA256_Message_Block_Size-8)) { + context->Message_Block[context->Message_Block_Index++] = Pad_Byte; + while (context->Message_Block_Index < SHA256_Message_Block_Size) + context->Message_Block[context->Message_Block_Index++] = 0; + SHA224_256ProcessMessageBlock(context); + } else + context->Message_Block[context->Message_Block_Index++] = Pad_Byte; + + while (context->Message_Block_Index < (SHA256_Message_Block_Size-8)) + context->Message_Block[context->Message_Block_Index++] = 0; + + /* + * Store the message length as the last 8 octets + */ + context->Message_Block[56] = (uint8_t)(context->Length_High >> 24); + context->Message_Block[57] = (uint8_t)(context->Length_High >> 16); + context->Message_Block[58] = (uint8_t)(context->Length_High >> 8); + context->Message_Block[59] = (uint8_t)(context->Length_High); + context->Message_Block[60] = (uint8_t)(context->Length_Low >> 24); + context->Message_Block[61] = (uint8_t)(context->Length_Low >> 16); + context->Message_Block[62] = (uint8_t)(context->Length_Low >> 8); + context->Message_Block[63] = (uint8_t)(context->Length_Low); + + SHA224_256ProcessMessageBlock(context); +} + +/* + * SHA224_256ResultN + * + * Description: + * This helper function will return the 224-bit or 256-bit message + * digest into the Message_Digest array provided by the caller. + * NOTE: + * The first octet of hash is stored in the element with index 0, + * the last octet of hash in the element with index 27/31. + * + * Parameters: + * context: [in/out] + * The context to use to calculate the SHA hash. + * Message_Digest[ ]: [out] + * Where the digest is returned. + * HashSize: [in] + * The size of the hash, either 28 or 32. + * + * Returns: + * sha Error Code. + */ +static int SHA224_256ResultN(SHA256Context *context, + uint8_t Message_Digest[ ], int HashSize) +{ + int i; + + if (!context) return shaNull; + if (!Message_Digest) return shaNull; + if (context->Corrupted) return context->Corrupted; + + if (!context->Computed) + SHA224_256Finalize(context, 0x80); + + for (i = 0; i < HashSize; ++i) + Message_Digest[i] = (uint8_t) + (context->Intermediate_Hash[i>>2] >> 8 * ( 3 - ( i & 0x03 ) )); + + return shaSuccess; +} |