diff options
author | Johan Fleury <jfleury@arcaik.net> | 2018-12-06 08:08:20 -0500 |
---|---|---|
committer | Johan Fleury <jfleury@arcaik.net> | 2018-12-06 08:08:20 -0500 |
commit | 08a5c60f8832ef214b2b4cd56d88cf4658253dae (patch) | |
tree | fb9c22adba70cb08d29862838fb2b2119c39f946 | |
parent | f68f0de9d6fa3185ac612a307b179c754fde232c (diff) |
New upstream version 1.2.11
-rw-r--r-- | .drone.yml | 11 | ||||
-rw-r--r-- | NEWS | 17 | ||||
-rw-r--r-- | README.md | 21 | ||||
-rw-r--r-- | borgmatic/borg/create.py | 4 | ||||
-rw-r--r-- | borgmatic/commands/borgmatic.py | 50 | ||||
-rw-r--r-- | borgmatic/commands/generate_config.py | 2 | ||||
-rw-r--r-- | borgmatic/config/checks.py | 9 | ||||
-rw-r--r-- | borgmatic/config/schema.yaml | 17 | ||||
-rw-r--r-- | borgmatic/config/validate.py | 13 | ||||
-rwxr-xr-x | scripts/black | 7 | ||||
-rwxr-xr-x | scripts/find-unsupported-borg-options | 1 | ||||
-rwxr-xr-x | scripts/release | 25 | ||||
-rw-r--r-- | setup.py | 3 | ||||
-rw-r--r-- | test_requirements.txt | 20 | ||||
-rw-r--r-- | tests/end-to-end/test_borgmatic.py | 24 | ||||
-rw-r--r-- | tests/unit/borg/test_create.py | 19 | ||||
-rw-r--r-- | tests/unit/commands/test_borgmatic.py | 2 | ||||
-rw-r--r-- | tests/unit/config/test_checks.py | 23 | ||||
-rw-r--r-- | tests/unit/config/test_validate.py | 23 | ||||
-rw-r--r-- | tox.ini | 3 |
20 files changed, 256 insertions, 38 deletions
@@ -1,9 +1,18 @@ pipeline: build: - image: python:3.7.0-alpine3.8 + image: python:${PYTHON_VERSION}-alpine${ALPINE_VERSION} pull: true commands: - pip install tox - tox - apk add --no-cache borgbackup - tox -e end-to-end + +matrix: + ALPINE_VERSION: + - 3.7 + - 3.8 + PYTHON_VERSION: + - 3.5 + - 3.6 + - 3.7 @@ -1,3 +1,20 @@ +1.2.11 + * #108: Support for Borg create --progress via borgmatic command-line flag. + +1.2.10 + * #105: Support for Borg --chunker-params create option via "chunker_params" in borgmatic's storage + section. + +1.2.9 + * #102: Fix for syntax error that occurred in Python 3.5 and below. + * Make automated tests support running in Python 3.5. + +1.2.8 + * #73: Enable consistency checks for only certain repositories via "check_repositories" option in + borgmatic's consistency configuration. Handy for large repositories that take forever to check. + * Include link to issue tracker within various command output. + * Run continuous integration tests on a matrix of Python and Borg versions. + 1.2.7 * #98: Support for Borg --keep-secondly prune option. * Use Black code formatter and Flake8 code checker as part of running automated tests. @@ -56,13 +56,17 @@ href="https://asciinema.org/a/203761" target="_blank">screencast</a>. ## Installation -To get up and running, follow the [Borg Quick +To get up and running, first [install +Borg](https://borgbackup.readthedocs.io/en/latest/installation.html), at +least version 1.1. Then, follow the [Borg Quick Start](https://borgbackup.readthedocs.org/en/latest/quickstart.html) to create -a repository on a local or remote host. Note that if you plan to run borgmatic -on a schedule with cron, and you encrypt your Borg repository with a -passphrase instead of a key file, you'll either need to set the borgmatic -`encryption_passphrase` configuration variable or set the `BORG_PASSPHRASE` -environment variable. See the [repository encryption +a repository on a local or remote host. + +Note that if you plan to run borgmatic on a schedule with cron, and you +encrypt your Borg repository with a passphrase instead of a key file, you'll +either need to set the borgmatic `encryption_passphrase` configuration +variable or set the `BORG_PASSPHRASE` environment variable. See the +[repository encryption section](https://borgbackup.readthedocs.io/en/latest/quickstart.html#repository-encryption) of the Quick Start for more info. @@ -441,9 +445,6 @@ cd borgmatic tox ``` -Note that while running borgmatic itself only requires Python 3+, running -borgmatic's tests require Python 3.6+. - If when running tests, you get an error from the [Black](https://black.readthedocs.io/en/stable/) code formatter about files that would be reformatted, you can ask Black to format them for you via the @@ -478,7 +479,7 @@ consistency: ``` This error can be caused by an ssh timeout, which you can rectify by adding -the following to the ~/.ssh/config file on the client: +the following to the `~/.ssh/config` file on the client: ```text Host * diff --git a/borgmatic/borg/create.py b/borgmatic/borg/create.py index 51504b2..eaf2e6e 100644 --- a/borgmatic/borg/create.py +++ b/borgmatic/borg/create.py @@ -104,6 +104,7 @@ def create_archive( storage_config, local_path='borg', remote_path=None, + progress=False, json=False, ): ''' @@ -115,6 +116,7 @@ def create_archive( pattern_file = _write_pattern_file(location_config.get('patterns')) exclude_file = _write_pattern_file(_expand_directories(location_config.get('exclude_patterns'))) checkpoint_interval = storage_config.get('checkpoint_interval', None) + chunker_params = storage_config.get('chunker_params', None) compression = storage_config.get('compression', None) remote_rate_limit = storage_config.get('remote_rate_limit', None) umask = storage_config.get('umask', None) @@ -135,6 +137,7 @@ def create_archive( + _make_pattern_flags(location_config, pattern_file.name if pattern_file else None) + _make_exclude_flags(location_config, exclude_file.name if exclude_file else None) + (('--checkpoint-interval', str(checkpoint_interval)) if checkpoint_interval else ()) + + (('--chunker-params', chunker_params) if chunker_params else ()) + (('--compression', compression) if compression else ()) + (('--remote-ratelimit', str(remote_rate_limit)) if remote_rate_limit else ()) + (('--one-file-system',) if location_config.get('one_file_system') else ()) @@ -149,6 +152,7 @@ def create_archive( + (('--stats',) if not dry_run and logger.isEnabledFor(logging.INFO) else ()) + (('--debug', '--show-rc') if logger.isEnabledFor(logging.DEBUG) else ()) + (('--dry-run',) if dry_run else ()) + + (('--progress',) if progress else ()) + (('--json',) if json else ()) ) diff --git a/borgmatic/commands/borgmatic.py b/borgmatic/commands/borgmatic.py index 27bbed0..d49c029 100644 --- a/borgmatic/commands/borgmatic.py +++ b/borgmatic/commands/borgmatic.py @@ -13,7 +13,7 @@ from borgmatic.borg import ( info as borg_info, ) from borgmatic.commands import hook -from borgmatic.config import collect, convert, validate +from borgmatic.config import checks, collect, convert, validate from borgmatic.signals import configure_signals from borgmatic.verbosity import verbosity_to_log_level @@ -79,6 +79,13 @@ def parse_arguments(*arguments): help='Display summary information on archives', ) parser.add_argument( + '--progress', + dest='progress', + default=False, + action='store_true', + help='Display progress with --create option for each file as it is backed up', + ) + parser.add_argument( '--json', dest='json', default=False, @@ -103,6 +110,9 @@ def parse_arguments(*arguments): args = parser.parse_args(arguments) + if args.progress and not args.create: + raise ValueError('The --progress option can only be used with the --create option') + if args.json and not (args.create or args.list or args.info): raise ValueError( 'The --json option can only be used with the --create, --list, or --info options' @@ -144,7 +154,15 @@ def run_configuration(config_filename, args): # pragma: no cover if args.create: hook.execute_hook(hooks.get('before_backup'), config_filename, 'pre-backup') - _run_commands(args, consistency, local_path, location, remote_path, retention, storage) + _run_commands( + args=args, + consistency=consistency, + local_path=local_path, + location=location, + remote_path=remote_path, + retention=retention, + storage=storage, + ) if args.create: hook.execute_hook(hooks.get('after_backup'), config_filename, 'post-backup') @@ -153,25 +171,26 @@ def run_configuration(config_filename, args): # pragma: no cover raise -def _run_commands(args, consistency, local_path, location, remote_path, retention, storage): +def _run_commands(*, args, consistency, local_path, location, remote_path, retention, storage): json_results = [] for unexpanded_repository in location['repositories']: _run_commands_on_repository( - args, - consistency, - json_results, - local_path, - location, - remote_path, - retention, - storage, - unexpanded_repository, + args=args, + consistency=consistency, + json_results=json_results, + local_path=local_path, + location=location, + remote_path=remote_path, + retention=retention, + storage=storage, + unexpanded_repository=unexpanded_repository, ) if args.json: sys.stdout.write(json.dumps(json_results)) def _run_commands_on_repository( + *, args, consistency, json_results, @@ -180,7 +199,7 @@ def _run_commands_on_repository( remote_path, retention, storage, - unexpanded_repository, + unexpanded_repository ): # pragma: no cover repository = os.path.expanduser(unexpanded_repository) dry_run_label = ' (dry run; not making any changes)' if args.dry_run else '' @@ -203,8 +222,9 @@ def _run_commands_on_repository( storage, local_path=local_path, remote_path=remote_path, + progress=args.progress, ) - if args.check: + if args.check and checks.repository_enabled_for_checks(repository, consistency): logger.info('{}: Running consistency checks'.format(repository)) borg_check.check_archives( repository, storage, consistency, local_path=local_path, remote_path=remote_path @@ -248,4 +268,6 @@ def main(): # pragma: no cover run_configuration(config_filename, args) except (ValueError, OSError, CalledProcessError) as error: print(error, file=sys.stderr) + print(file=sys.stderr) + print('Need some help? https://torsion.org/borgmatic/#issues', file=sys.stderr) sys.exit(1) diff --git a/borgmatic/commands/generate_config.py b/borgmatic/commands/generate_config.py index c552f19..d91a19b 100644 --- a/borgmatic/commands/generate_config.py +++ b/borgmatic/commands/generate_config.py @@ -38,6 +38,8 @@ def main(): # pragma: no cover print() print('Please edit the file to suit your needs. The values are just representative.') print('All fields are optional except where indicated.') + print() + print('If you ever need help: https://torsion.org/borgmatic/#issues') except (ValueError, OSError) as error: print(error, file=sys.stderr) sys.exit(1) diff --git a/borgmatic/config/checks.py b/borgmatic/config/checks.py new file mode 100644 index 0000000..13361ea --- /dev/null +++ b/borgmatic/config/checks.py @@ -0,0 +1,9 @@ +def repository_enabled_for_checks(repository, consistency): + ''' + Given a repository name and a consistency configuration dict, return whether the repository + is enabled to have consistency checks run. + ''' + if not consistency.get('check_repositories'): + return True + + return repository in consistency['check_repositories'] diff --git a/borgmatic/config/schema.yaml b/borgmatic/config/schema.yaml index f3ca6a0..a5b49d3 100644 --- a/borgmatic/config/schema.yaml +++ b/borgmatic/config/schema.yaml @@ -138,6 +138,13 @@ map: https://borgbackup.readthedocs.io/en/stable/faq.html#if-a-backup-stops-mid-way-does-the-already-backed-up-data-stay-there for details. Defaults to checkpoints every 1800 seconds (30 minutes). example: 1800 + chunker_params: + type: scalar + desc: | + Specify the parameters passed to then chunker (CHUNK_MIN_EXP, CHUNK_MAX_EXP, + HASH_MASK_BITS, HASH_WINDOW_SIZE). See https://borgbackup.readthedocs.io/en/stable/internals.html + for details. + example: 19,23,21,4095 compression: type: scalar desc: | @@ -236,6 +243,16 @@ map: example: - repository - archives + check_repositories: + seq: + - type: scalar + desc: | + Paths to a subset of the repositories in the location section on which to run + consistency checks. Handy in case some of your repositories are very large, and + so running consistency checks on them would take too long. Defaults to running + consistency checks on all repositories configured in the location section. + example: + - user@backupserver:sourcehostname.borg check_last: type: int desc: Restrict the number of checked archives to the last n. Applies only to the diff --git a/borgmatic/config/validate.py b/borgmatic/config/validate.py index a5133f4..5c0b32b 100644 --- a/borgmatic/config/validate.py +++ b/borgmatic/config/validate.py @@ -51,6 +51,19 @@ def apply_logical_validation(config_filename, parsed_configuration): ('If you provide an archive_name_format, you must also specify a retention prefix.',), ) + location_repositories = parsed_configuration.get('location', {}).get('repositories') + check_repositories = parsed_configuration.get('consistency', {}).get('check_repositories', []) + for repository in check_repositories: + if repository not in location_repositories: + raise Validation_error( + config_filename, + ( + 'Unknown repository in the consistency section\'s check_repositories: {}'.format( + repository + ), + ), + ) + consistency_prefix = parsed_configuration.get('consistency', {}).get('prefix') if archive_name_format and not consistency_prefix: logger.warning( diff --git a/scripts/black b/scripts/black new file mode 100755 index 0000000..9320501 --- /dev/null +++ b/scripts/black @@ -0,0 +1,7 @@ +#!/usr/bin/env sh + +if which black; then + black --skip-string-normalization --line-length 100 --check . +else + echo "Skipping black due to not being installed." +fi diff --git a/scripts/find-unsupported-borg-options b/scripts/find-unsupported-borg-options index bd5f2db..8b98142 100755 --- a/scripts/find-unsupported-borg-options +++ b/scripts/find-unsupported-borg-options @@ -40,6 +40,7 @@ for sub_command in prune create check list info; do | grep -v '^--list$' \ | grep -v '^--nobsdflags$' \ | grep -v '^--pattern$' \ + | grep -v '^--progress$' \ | grep -v '^--read-special$' \ | grep -v '^--repository-only$' \ | grep -v '^--show-rc$' \ diff --git a/scripts/release b/scripts/release index d109d97..0477b6b 100755 --- a/scripts/release +++ b/scripts/release @@ -2,13 +2,38 @@ set -e +projects_token=${1:-} +github_token=${2:-} + +if [[ -z $github_token ]]; then + echo "Usage: $0 [projects-token] [github-token]" + exit 1 +fi +if [[ ! -f NEWS ]]; then + echo "Missing NEWS file. Try running from root of repository." + exit 1 +fi + version=$(head --lines=1 NEWS) git tag $version git push origin $version git push github $version +# Build borgmatic and publish to pypi. rm -fr dist python3 setup.py bdist_wheel python3 setup.py sdist twine upload -r pypi dist/borgmatic-*.tar.gz twine upload -r pypi dist/borgmatic-*-py3-none-any.whl + +# Set release changelogs on projects.evoworx.org and GitHub. +release_changelog="$(cat NEWS | sed '/^$/q' | grep -v '^\S')" +escaped_release_changelog="$(echo "$release_changelog" | sed -z 's/\n/\\n/g' | sed -z 's/\"/\\"/g')" +curl --silent --request POST \ + "https://projects.torsion.org/api/v1/repos/witten/borgmatic/releases?access_token=$projects_token" \ + --header "Accept: application/json" \ + --header "Content-Type: application/json" \ + --data "{\"body\": \"$escaped_release_changelog\", \"draft\": false, \"name\": \"borgmatic $version\", \"prerelease\": false, \"tag_name\": \"$version\"}" + +github-release create --token="$github_token" --owner=witten --repo=borgmatic --tag="$version" \ + --name="borgmatic $version" --body="$release_changelog" @@ -1,7 +1,7 @@ from setuptools import setup, find_packages -VERSION = '1.2.7' +VERSION = '1.2.11' setup( @@ -30,6 +30,5 @@ setup( }, obsoletes=['atticmatic'], install_requires=('pykwalify>=1.6.0,<14.06', 'ruamel.yaml>0.15.0,<0.16.0', 'setuptools'), - tests_require=('flexmock', 'pytest'), include_package_data=True, ) diff --git a/test_requirements.txt b/test_requirements.txt index 4c6af77..4ea06ce 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,7 +1,23 @@ -black==18.9b0 +appdirs==1.4.3 +atomicwrites==1.2.1 +attrs==18.2.0 +black==18.9b0; python_version >= '3.6' +Click==7.0 +coverage==4.5.1 +docopt==0.6.2 flake8==3.5.0 flexmock==0.10.2 +mccabe==0.6.1 +more-itertools==4.3.0 +pluggy==0.7.1 +py==1.6.0 +pycodestyle==2.3.1 +pyflakes==2.0.0 pykwalify==1.7.0 -pytest==3.8.1 +pytest==3.8.2 pytest-cov==2.6.0 +python-dateutil==2.7.3 +PyYAML==3.13 ruamel.yaml>0.15.0,<0.16.0 +six==1.11.0 +toml==0.10.0 diff --git a/tests/end-to-end/test_borgmatic.py b/tests/end-to-end/test_borgmatic.py index ca3f2ad..a0314aa 100644 --- a/tests/end-to-end/test_borgmatic.py +++ b/tests/end-to-end/test_borgmatic.py @@ -11,12 +11,14 @@ def generate_configuration(config_path, repository_path): Generate borgmatic configuration into a file at the config path, and update the defaults so as to work for testing (including injecting the given repository path). ''' - subprocess.check_call(f'generate-borgmatic-config --destination {config_path}'.split(' ')) + subprocess.check_call( + 'generate-borgmatic-config --destination {}'.format(config_path).split(' ') + ) config = ( open(config_path) .read() .replace('user@backupserver:sourcehostname.borg', repository_path) - .replace('- /home', f'- {config_path}') + .replace('- /home', '- {}'.format(config_path)) .replace('- /etc', '') .replace('- /var/log/syslog*', '') ) @@ -32,7 +34,7 @@ def test_borgmatic_command(): try: subprocess.check_call( - f'borg init --encryption repokey {repository_path}'.split(' '), + 'borg init --encryption repokey {}'.format(repository_path).split(' '), env={'BORG_PASSPHRASE': '', **os.environ}, ) @@ -40,14 +42,22 @@ def test_borgmatic_command(): generate_configuration(config_path, repository_path) # Run borgmatic to generate a backup archive, and then list it to make sure it exists. - subprocess.check_call(f'borgmatic --config {config_path}'.split(' ')) + subprocess.check_call('borgmatic --config {}'.format(config_path).split(' ')) output = subprocess.check_output( - f'borgmatic --config {config_path} --list --json'.split(' '), - encoding=sys.stdout.encoding, - ) + 'borgmatic --config {} --list --json'.format(config_path).split(' ') + ).decode(sys.stdout.encoding) parsed_output = json.loads(output) assert len(parsed_output) == 1 assert len(parsed_output[0]['archives']) == 1 + + # Also exercise the info flag. + output = subprocess.check_output( + 'borgmatic --config {} --info --json'.format(config_path).split(' ') + ).decode(sys.stdout.encoding) + parsed_output = json.loads(output) + + assert len(parsed_output) == 1 + assert 'repository' in parsed_output[0] finally: shutil.rmtree(temporary_directory) diff --git a/tests/unit/borg/test_create.py b/tests/unit/borg/test_create.py index 8b739b1..2b3db1f 100644 --- a/tests/unit/borg/test_create.py +++ b/tests/unit/borg/test_create.py @@ -390,6 +390,25 @@ def test_create_archive_with_checkpoint_interval_calls_borg_with_checkpoint_inte ) +def test_create_archive_with_chunker_params_calls_borg_with_chunker_params_parameters(): + flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) + flexmock(module).should_receive('_write_pattern_file').and_return(None) + flexmock(module).should_receive('_make_pattern_flags').and_return(()) + flexmock(module).should_receive('_make_exclude_flags').and_return(()) + insert_subprocess_mock(CREATE_COMMAND + ('--chunker-params', '1,2,3,4')) + + module.create_archive( + dry_run=False, + repository='repo', + location_config={ + 'source_directories': ['foo', 'bar'], + 'repositories': ['repo'], + 'exclude_patterns': None, + }, + storage_config={'chunker_params': '1,2,3,4'}, + ) + + def test_create_archive_with_compression_calls_borg_with_compression_parameters(): flexmock(module).should_receive('_expand_directories').and_return(('foo', 'bar')).and_return(()) flexmock(module).should_receive('_write_pattern_file').and_return(None) diff --git a/tests/unit/commands/test_borgmatic.py b/tests/unit/commands/test_borgmatic.py index 20af02d..943f9aa 100644 --- a/tests/unit/commands/test_borgmatic.py +++ b/tests/unit/commands/test_borgmatic.py @@ -6,7 +6,7 @@ from flexmock import flexmock from borgmatic.commands import borgmatic -def test__run_commands_handles_multiple_json_outputs_in_array(): +def test_run_commands_handles_multiple_json_outputs_in_array(): ( flexmock(borgmatic) .should_receive('_run_commands_on_repository') diff --git a/tests/unit/config/test_checks.py b/tests/unit/config/test_checks.py new file mode 100644 index 0000000..df6df1e --- /dev/null +++ b/tests/unit/config/test_checks.py @@ -0,0 +1,23 @@ +from borgmatic.config import checks as module + + +def test_repository_enabled_for_checks_defaults_to_enabled_for_all_repositories(): + enabled = module.repository_enabled_for_checks('repo.borg', consistency={}) + + assert enabled + + +def test_repository_enabled_for_checks_is_enabled_for_specified_repositories(): + enabled = module.repository_enabled_for_checks( + 'repo.borg', consistency={'check_repositories': ['repo.borg', 'other.borg']} + ) + + assert enabled + + +def test_repository_enabled_for_checks_is_disabled_for_other_repositories(): + enabled = module.repository_enabled_for_checks( + 'repo.borg', consistency={'check_repositories': ['other.borg']} + ) + + assert not enabled diff --git a/tests/unit/config/test_validate.py b/tests/unit/config/test_validate.py index eb98b14..a542e21 100644 --- a/tests/unit/config/test_validate.py +++ b/tests/unit/config/test_validate.py @@ -51,6 +51,29 @@ def test_apply_logical_validation_warns_if_archive_name_format_present_without_c ) +def test_apply_locical_validation_raises_if_unknown_repository_in_check_repositories(): + with pytest.raises(module.Validation_error): + module.apply_logical_validation( + 'config.yaml', + { + 'location': {'repositories': ['repo.borg', 'other.borg']}, + 'retention': {'keep_secondly': 1000}, + 'consistency': {'check_repositories': ['repo.borg', 'unknown.borg']}, + }, + ) + + +def test_apply_locical_validation_does_not_raise_if_known_repository_in_check_repositories(): + module.apply_logical_validation( + 'config.yaml', + { + 'location': {'repositories': ['repo.borg', 'other.borg']}, + 'retention': {'keep_secondly': 1000}, + 'consistency': {'check_repositories': ['repo.borg']}, + }, + ) + + def test_apply_logical_validation_does_not_raise_or_warn_if_archive_name_format_and_prefix_present(): logger = flexmock(module.logger) logger.should_receive('warning').never() @@ -5,10 +5,11 @@ skipsdist = True [testenv] usedevelop = True deps = -rtest_requirements.txt +whitelist_externals = sh commands = py.test --cov-report term-missing:skip-covered --cov=borgmatic --ignore=tests/end-to-end \ tests [] - black --skip-string-normalization --line-length 100 --check . + sh scripts/black flake8 . [testenv:black] |