diff options
Diffstat (limited to 'yadm')
| -rwxr-xr-x | yadm | 696 |
1 files changed, 462 insertions, 234 deletions
@@ -1,6 +1,6 @@ #!/bin/sh # yadm - Yet Another Dotfiles Manager -# Copyright (C) 2015-2020 Tim Byrne +# Copyright (C) 2015-2021 Tim Byrne # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -20,30 +20,38 @@ if [ -z "$BASH_VERSION" ]; then [ "$YADM_TEST" != 1 ] && exec bash "$0" "$@" fi -VERSION=2.4.0 +VERSION=3.0.2 YADM_WORK="$HOME" YADM_DIR= +YADM_DATA= + YADM_LEGACY_DIR="${HOME}/.yadm" +YADM_LEGACY_ARCHIVE="files.gpg" # these are the default paths relative to YADM_DIR -YADM_REPO="repo.git" YADM_CONFIG="config" YADM_ENCRYPT="encrypt" -YADM_ARCHIVE="files.gpg" YADM_BOOTSTRAP="bootstrap" YADM_HOOKS="hooks" YADM_ALT="alt" +# these are the default paths relative to YADM_DATA +YADM_REPO="repo.git" +YADM_ARCHIVE="archive" + HOOK_COMMAND="" FULL_COMMAND="" GPG_PROGRAM="gpg" +OPENSSL_PROGRAM="openssl" GIT_PROGRAM="git" AWK_PROGRAM=("gawk" "awk") GIT_CRYPT_PROGRAM="git-crypt" +TRANSCRYPT_PROGRAM="transcrypt" J2CLI_PROGRAM="j2" ENVTPL_PROGRAM="envtpl" +ESH_PROGRAM="esh" LSB_RELEASE_PROGRAM="lsb_release" OS_RELEASE="/etc/os-release" @@ -55,6 +63,9 @@ ENCRYPT_INCLUDE_FILES="unparsed" LEGACY_WARNING_ISSUED=0 INVALID_ALT=() +GPG_OPTS=() +OPENSSL_OPTS=() + # flag causing path translations with cygpath USE_CYGPATH=0 @@ -81,18 +92,20 @@ function main() { done FULL_COMMAND="${_fc[*]}" - # create the YADM_DIR if it doesn't exist yet - [ -d "$YADM_DIR" ] || mkdir -p "$YADM_DIR" + # create the YADM_DIR & YADM_DATA if they doesn't exist yet + [ -d "$YADM_DIR" ] || mkdir -p "$YADM_DIR" + [ -d "$YADM_DATA" ] || mkdir -p "$YADM_DATA" # parse command line arguments local retval=0 - internal_commands="^(alt|bootstrap|clean|clone|config|decrypt|encrypt|enter|git-crypt|help|init|introspect|list|perms|upgrade|version)$" + internal_commands="^(alt|bootstrap|clean|clone|config|decrypt|encrypt|enter|git-crypt|help|--help|init|introspect|list|perms|transcrypt|upgrade|version|--version)$" if [ -z "$*" ] ; then # no argumnts will result in help() help elif [[ "$1" =~ $internal_commands ]] ; then # for internal commands, process all of the arguments - YADM_COMMAND="${1/-/_}" + YADM_COMMAND="${1//-/_}" + YADM_COMMAND="${YADM_COMMAND/__/}" YADM_ARGS=() shift @@ -109,7 +122,7 @@ function main() { -d) # used by all commands DEBUG="YES" ;; - -f) # used by init() and clone() + -f) # used by init(), clone() and upgrade() FORCE="YES" ;; -l) # used by decrypt() @@ -160,7 +173,7 @@ function score_file() { conditions="${src#*##}" if [ "${tgt#$YADM_ALT/}" != "${tgt}" ]; then - tgt="${YADM_WORK}/${tgt#$YADM_ALT/}" + tgt="${YADM_BASE}/${tgt#$YADM_ALT/}" fi score=0 @@ -169,6 +182,10 @@ function score_file() { label=${field%%.*} value=${field#*.} [ "$field" = "$label" ] && value="" # when .value is omitted + # extension isn't a condition and doesn't affect the score + if [[ "$label" =~ ^(e|extension)$ ]]; then + continue + fi score=$((score + 1000)) # default condition if [[ "$label" =~ ^(default)$ ]]; then @@ -222,7 +239,9 @@ function score_file() { return 0 # unsupported values else - INVALID_ALT+=("$src") + if [[ "${src##*/}" =~ .\#\#. ]]; then + INVALID_ALT+=("$src") + fi score=0 return fi @@ -249,11 +268,28 @@ function record_score() { done # if we don't find an existing index, create one by appending to the array if [ "$index" -eq -1 ]; then - alt_targets+=("$tgt") - # set index to the last index (newly created one) - for index in "${!alt_targets[@]}"; do :; done - # and set its initial score to zero - alt_scores[$index]=0 + # $YADM_CONFIG must be processed first, in case other templates lookup yadm configurations + if [ "$tgt" = "$YADM_CONFIG" ]; then + alt_targets=("$tgt" "${alt_targets[@]}") + alt_sources=("$src" "${alt_sources[@]}") + alt_scores=(0 "${alt_scores[@]}") + index=0 + # increase the index of any existing alt_template_cmds + new_cmds=() + for cmd_index in "${!alt_template_cmds[@]}"; do + new_cmds[$((cmd_index+1))]="${alt_template_cmds[$cmd_index]}" + done + alt_template_cmds=() + for cmd_index in "${!new_cmds[@]}"; do + alt_template_cmds[$cmd_index]="${new_cmds[$cmd_index]}" + done + else + alt_targets+=("$tgt") + # set index to the last index (newly created one) + for index in "${!alt_targets[@]}"; do :; done + # and set its initial score to zero + alt_scores[$index]=0 + fi fi # record nothing if a template command is registered for this file @@ -298,6 +334,8 @@ function choose_template_cmd() { if [ "$kind" = "default" ] || [ "$kind" = "" ] && awk_available; then echo "template_default" + elif [ "$kind" = "esh" ] && esh_available; then + echo "template_esh" elif [ "$kind" = "j2cli" ] || [ "$kind" = "j2" ] && j2cli_available; then echo "template_j2cli" elif [ "$kind" = "envtpl" ] || [ "$kind" = "j2" ] && envtpl_available; then @@ -327,13 +365,18 @@ BEGIN { c["user"] = user c["distro"] = distro c["source"] = source - vld = conditions() ifs = "^{%" blank "*if" els = "^{%" blank "*else" blank "*%}$" end = "^{%" blank "*endif" blank "*%}$" skp = "^{%" blank "*(if|else|endif)" + vld = conditions() + inc_start = "^{%" blank "*include" blank "+\"?" + inc_end = "\"?" blank "*%}$" + inc = inc_start ".+" inc_end prt = 1 + err = 0 } +END { exit err } { replace_vars() } # variable replacements $0 ~ vld, $0 ~ end { if ($0 ~ vld || $0 ~ end) prt=1; @@ -345,14 +388,32 @@ $0 ~ vld, $0 ~ end { if ($0 ~ els || $0 ~ end) prt=1; if ($0 ~ skp) next; } -{ if (prt) print } +{ if (!prt) next } +$0 ~ inc { + file = $0 + sub(inc_start, "", file) + sub(inc_end, "", file) + sub(/^[^\/].*$/, source_dir "/&", file) + + while ((res = getline <file) > 0) { + replace_vars() + print + } + if (res < 0) { + printf "%s:%d: error: could not read '%s'\n", FILENAME, NR, file | "cat 1>&2" + err = 1 + } + close(file) + next +} +{ print } function replace_vars() { for (label in c) { gsub(("{{" blank "*yadm\\." label blank "*}}"), c[label]) } } function conditions() { - pattern = "^{%" blank "*if" blank "*(" + pattern = ifs blank "*(" for (label in c) { value = c[label] gsub(/[\\.^$(){}\[\]|*+?]/, "\\\\&", value) @@ -371,9 +432,14 @@ EOF -v user="$local_user" \ -v distro="$local_distro" \ -v source="$input" \ + -v source_dir="$(dirname "$input")" \ "$awk_pgm" \ - "$input" > "$temp_file" - [ -f "$temp_file" ] && mv -f "$temp_file" "$output" + "$input" > "$temp_file" || rm -f "$temp_file" + + if [ -f "$temp_file" ] ; then + copy_perms "$input" "$temp_file" + mv -f "$temp_file" "$output" + fi } function template_j2cli() { @@ -388,7 +454,11 @@ function template_j2cli() { YADM_DISTRO="$local_distro" \ YADM_SOURCE="$input" \ "$J2CLI_PROGRAM" "$input" -o "$temp_file" - [ -f "$temp_file" ] && mv -f "$temp_file" "$output" + + if [ -f "$temp_file" ] ; then + copy_perms "$input" "$temp_file" + mv -f "$temp_file" "$output" + fi } function template_envtpl() { @@ -403,7 +473,30 @@ function template_envtpl() { YADM_DISTRO="$local_distro" \ YADM_SOURCE="$input" \ "$ENVTPL_PROGRAM" --keep-template "$input" -o "$temp_file" - [ -f "$temp_file" ] && mv -f "$temp_file" "$output" + + if [ -f "$temp_file" ] ; then + copy_perms "$input" "$temp_file" + mv -f "$temp_file" "$output" + fi +} + +function template_esh() { + input="$1" + output="$2" + temp_file="${output}.$$.$RANDOM" + + "$ESH_PROGRAM" -o "$temp_file" "$input" \ + YADM_CLASS="$local_class" \ + YADM_OS="$local_system" \ + YADM_HOSTNAME="$local_host" \ + YADM_USER="$local_user" \ + YADM_DISTRO="$local_distro" \ + YADM_SOURCE="$input" + + if [ -f "$temp_file" ] ; then + copy_perms "$input" "$temp_file" + mv -f "$temp_file" "$output" + fi } # ****** yadm Commands ****** @@ -429,57 +522,44 @@ function alt() { local do_copy=0 [ "$(config --bool yadm.alt-copy)" == "true" ] && do_copy=1 - # deprecated yadm.cygwin-copy option (to be removed) - [ "$(config --bool yadm.cygwin-copy)" == "true" ] && do_copy=1 - cd_work "Alternates" || return # determine all tracked files - local tracked_files - tracked_files=() + local tracked_files=() local IFS=$'\n' for tracked_file in $("$GIT_PROGRAM" ls-files | LC_ALL=C sort); do tracked_files+=("$tracked_file") done # generate data for removing stale links - local possible_alts - possible_alts=() + local possible_alts=() local IFS=$'\n' for possible_alt in "${tracked_files[@]}" "${ENCRYPT_INCLUDE_FILES[@]}"; do if [[ $possible_alt =~ .\#\#. ]]; then base_alt="${possible_alt%%##*}" - yadm_alt="${YADM_WORK}/${base_alt}" + yadm_alt="${YADM_BASE}/${base_alt}" if [ "${yadm_alt#$YADM_ALT/}" != "${yadm_alt}" ]; then base_alt="${yadm_alt#$YADM_ALT/}" fi - possible_alts+=("$YADM_WORK/${base_alt}") + possible_alts+=("$YADM_BASE/${base_alt}") fi done - local alt_linked - alt_linked=() - - if [ "$YADM_COMPATIBILITY" = "1" ]; then - alt_past_linking - else - alt_future_linking - fi + local alt_linked=() + alt_linking remove_stale_links - report_invalid_alts } function report_invalid_alts() { - [ "$YADM_COMPATIBILITY" = "1" ] && return [ "$LEGACY_WARNING_ISSUED" = "1" ] && return [ "${#INVALID_ALT[@]}" = "0" ] && return local path_list for invalid in "${INVALID_ALT[@]}"; do path_list="$path_list * $invalid"$'\n' done - cat <<EOF + cat <<EOF >&2 **WARNING** Invalid alternates have been detected. @@ -546,19 +626,15 @@ function set_local_alt_values() { } -function alt_future_linking() { +function alt_linking() { - local alt_scores - local alt_targets - local alt_sources - local alt_template_cmds - alt_scores=() - alt_targets=() - alt_sources=() - alt_template_cmds=() + local alt_scores=() + local alt_targets=() + local alt_sources=() + local alt_template_cmds=() for alt_path in $(for tracked in "${tracked_files[@]}"; do printf "%s\n" "$tracked" "${tracked%/*}"; done | LC_ALL=C sort -u) "${ENCRYPT_INCLUDE_FILES[@]}"; do - alt_path="$YADM_WORK/$alt_path" + alt_path="$YADM_BASE/$alt_path" if [[ "$alt_path" =~ .\#\#. ]]; then if [ -e "$alt_path" ] ; then score_file "$alt_path" @@ -597,85 +673,15 @@ function alt_future_linking() { } -function alt_past_linking() { - - if [ -z "$local_class" ] ; then - match_class="%" - else - match_class="$local_class" - fi - match_class="(%|$match_class)" - match_system="(%|$local_system)" - match_host="(%|$local_host)" - match_user="(%|$local_user)" - - # regex for matching "<file>##CLASS.SYSTEM.HOSTNAME.USER" - match1="^(.+)##(()|$match_system|$match_system\.$match_host|$match_system\.$match_host\.$match_user)$" - match2="^(.+)##($match_class|$match_class\.$match_system|$match_class\.$match_system\.$match_host|$match_class\.$match_system\.$match_host\.$match_user)$" - - # loop over all "tracked" files - # for every file which matches the above regex, create a symlink - for match in $match1 $match2; do - last_linked='' - local IFS=$'\n' - # the alt_paths looped over here are a unique sorted list of both files and their immediate parent directory - for alt_path in $(for tracked in "${tracked_files[@]}"; do printf "%s\n" "$tracked" "${tracked%/*}"; done | LC_ALL=C sort -u) "${ENCRYPT_INCLUDE_FILES[@]}"; do - alt_path="$YADM_WORK/$alt_path" - if [ -e "$alt_path" ] ; then - if [[ $alt_path =~ $match ]] ; then - if [ "$alt_path" != "$last_linked" ] ; then - new_link="${BASH_REMATCH[1]}" - debug "Linking $alt_path to $new_link" - [ -n "$loud" ] && echo "Linking $alt_path to $new_link" - if [ "$do_copy" -eq 1 ]; then - if [ -L "$new_link" ]; then - rm -f "$new_link" - fi - cp -f "$alt_path" "$new_link" - else - ln_relative "$alt_path" "$new_link" - fi - last_linked="$alt_path" - fi - fi - fi - done - done - - # loop over all "tracked" files - # for every file which is a *##yadm.j2 create a real file - local match="^(.+)##yadm\\.j2$" - for tracked_file in "${tracked_files[@]}" "${ENCRYPT_INCLUDE_FILES[@]}"; do - tracked_file="$YADM_WORK/$tracked_file" - if [ -e "$tracked_file" ] ; then - if [[ $tracked_file =~ $match ]] ; then - real_file="${BASH_REMATCH[1]}" - if envtpl_available; then - debug "Creating $real_file from template $tracked_file" - [ -n "$loud" ] && echo "Creating $real_file from template $tracked_file" - temp_file="${real_file}.$$.$RANDOM" - YADM_CLASS="$local_class" \ - YADM_OS="$local_system" \ - YADM_HOSTNAME="$local_host" \ - YADM_USER="$local_user" \ - YADM_DISTRO="$local_distro" \ - "$ENVTPL_PROGRAM" --keep-template "$tracked_file" -o "$temp_file" - [ -f "$temp_file" ] && mv -f "$temp_file" "$real_file" - else - debug "envtpl not available, not creating $real_file from template $tracked_file" - [ -n "$loud" ] && echo "envtpl not available, not creating $real_file from template $tracked_file" - fi - fi - fi - done - -} - function ln_relative() { local full_source full_target target_dir - full_source="$1" - full_target="$2" - target_dir="${full_target%/*}" + local full_source="$1" + local full_target="$2" + local target_dir="${full_target%/*}" + if [ "$target_dir" == "" ]; then + target_dir="/" + fi + local rel_source rel_source=$(relative_path "$target_dir" "$full_source") ln -nfs "$rel_source" "$full_target" alt_linked+=("$rel_source") @@ -699,13 +705,23 @@ function clean() { } +function _default_remote_branch() { + local ls_remote + ls_remote=$("$GIT_PROGRAM" ls-remote -q --symref "$1" 2>/dev/null) + match="^ref:[[:blank:]]+refs/heads/([^[:blank:]]+)" + if [[ "$ls_remote" =~ $match ]] ; then + echo "${BASH_REMATCH[1]}" + else + echo master + fi +} + function clone() { DO_BOOTSTRAP=1 - local branch - branch="master" + local branch= - clone_args=() + local repo_url= while [[ $# -gt 0 ]] ; do key="$1" case $key in @@ -722,22 +738,29 @@ function clone() { --no-bootstrap) # prevent bootstrap, without prompt DO_BOOTSTRAP=3 ;; - *) # main arguments are kept intact - clone_args+=("$1") + *) # use first found argument as the URL + [ -z "$repo_url" ] && repo_url="$1" ;; esac shift done + [ -z "$repo_url" ] && error_out "No repository provided" + + [ -z "$branch" ] && branch=$(_default_remote_branch "$repo_url") + [ -n "$DEBUG" ] && display_private_perms "initial" + # shellcheck disable=SC2119 # clone will begin with a bare repo - local empty= - init $empty + init + + # configure local HEAD with the correct branch + printf 'ref: refs/heads/%s\n' "$branch" > "${YADM_REPO}/HEAD" # add the specified remote, and configure the repo to track origin/$branch debug "Adding remote to new repo" - "$GIT_PROGRAM" remote add origin "${clone_args[@]}" + "$GIT_PROGRAM" remote add origin "$repo_url" debug "Configuring new repo to track origin/${branch}" "$GIT_PROGRAM" config "branch.${branch}.remote" origin "$GIT_PROGRAM" config "branch.${branch}.merge" "refs/heads/${branch}" @@ -747,13 +770,13 @@ function clone() { "$GIT_PROGRAM" fetch origin || { debug "Removing repo after failed clone" rm -rf "$YADM_REPO" - error_out "Unable to fetch origin ${clone_args[0]}" + error_out "Unable to fetch origin $repo_url" } debug "Verifying '${branch}' is a valid branch to merge" [ -f "${YADM_REPO}/refs/remotes/origin/${branch}" ] || { debug "Removing repo after failed clone" rm -rf "$YADM_REPO" - error_out "Clone failed, 'origin/${branch}' does not exist in ${clone_args[0]}" + error_out "Clone failed, 'origin/${branch}' does not exist in $repo_url" } if [ "$YADM_WORK" = "$HOME" ]; then @@ -848,6 +871,8 @@ EOF CHANGES_POSSIBLE=1 else + # make sure parent folder of config file exists + assert_parent "$YADM_CONFIG" # operate on the yadm configuration file "$GIT_PROGRAM" config --file="$(mixed_path "$YADM_CONFIG")" "$@" @@ -855,9 +880,98 @@ EOF } +function _set_gpg_options() { + gpg_key="$(config yadm.gpg-recipient)" + if [ "$gpg_key" = "ASK" ]; then + GPG_OPTS=("--no-default-recipient" "-e") + elif [ "$gpg_key" != "" ]; then + GPG_OPTS=("-e" "-r $gpg_key") + else + GPG_OPTS=("-c") + fi +} + +function _get_openssl_ciphername() { + OPENSSL_CIPHERNAME="$(config yadm.openssl-ciphername)" + if [ -z "$OPENSSL_CIPHERNAME" ]; then + OPENSSL_CIPHERNAME="aes-256-cbc" + fi + echo "$OPENSSL_CIPHERNAME" +} + +function _set_openssl_options() { + cipher_name="$(_get_openssl_ciphername)" + OPENSSL_OPTS=("-${cipher_name}" -salt) + if [ "$(config --bool yadm.openssl-old)" == "true" ]; then + OPENSSL_OPTS+=(-md md5) + else + OPENSSL_OPTS+=(-pbkdf2 -iter 100000 -md sha512) + fi +} + +function _get_cipher() { + output_archive="$1" + yadm_cipher="$(config yadm.cipher)" + if [ -z "$yadm_cipher" ]; then + yadm_cipher="gpg" + fi +} + +function _decrypt_from() { + + local output_archive + local yadm_cipher + _get_cipher "$1" + + case "$yadm_cipher" in + gpg) + require_gpg + $GPG_PROGRAM -d "$output_archive" + ;; + + openssl) + require_openssl + _set_openssl_options + $OPENSSL_PROGRAM enc -d "${OPENSSL_OPTS[@]}" -in "$output_archive" + ;; + + *) + error_out "Unknown cipher '$yadm_cipher'" + ;; + + esac + +} + +function _encrypt_to() { + + local output_archive + local yadm_cipher + _get_cipher "$1" + + case "$yadm_cipher" in + gpg) + require_gpg + _set_gpg_options + $GPG_PROGRAM --yes "${GPG_OPTS[@]}" --output "$output_archive" + ;; + + openssl) + require_openssl + _set_openssl_options + $OPENSSL_PROGRAM enc -e "${OPENSSL_OPTS[@]}" -out "$output_archive" + ;; + + *) + error_out "Unknown cipher '$yadm_cipher'" + ;; + + esac + +} + function decrypt() { - require_gpg require_archive [ -f "$YADM_ENCRYPT" ] && exclude_encrypted @@ -869,7 +983,7 @@ function decrypt() { fi # decrypt the archive - if ($GPG_PROGRAM -d "$YADM_ARCHIVE" || echo 1) | tar v${tar_option}f - -C "$YADM_WORK"; then + if (_decrypt_from "$YADM_ARCHIVE" || echo 1) | tar v${tar_option}f - -C "$YADM_WORK"; then [ ! "$DO_LIST" = "YES" ] && echo "All files decrypted." else error_out "Unable to extract encrypted files." @@ -881,33 +995,19 @@ function decrypt() { function encrypt() { - require_gpg require_encrypt exclude_encrypted parse_encrypt cd_work "Encryption" || return - # Build gpg options for gpg - GPG_KEY="$(config yadm.gpg-recipient)" - if [ "$GPG_KEY" = "ASK" ]; then - GPG_OPTS=("--no-default-recipient" "-e") - elif [ "$GPG_KEY" != "" ]; then - GPG_OPTS=("-e") - for key in $GPG_KEY; do - GPG_OPTS+=("-r $key") - done - else - GPG_OPTS=("-c") - fi - # report which files will be encrypted echo "Encrypting the following files:" printf '%s\n' "${ENCRYPT_INCLUDE_FILES[@]}" echo # encrypt all files which match the globs - if tar -f - -c "${ENCRYPT_INCLUDE_FILES[@]}" | $GPG_PROGRAM --yes "${GPG_OPTS[@]}" --output "$YADM_ARCHIVE"; then + if tar -f - -c "${ENCRYPT_INCLUDE_FILES[@]}" | _encrypt_to "$YADM_ARCHIVE"; then echo "Wrote new file: $YADM_ARCHIVE" else error_out "Unable to write $YADM_ARCHIVE" @@ -934,18 +1034,27 @@ function git_crypt() { enter "${GIT_CRYPT_PROGRAM} $*" } +function transcrypt() { + require_transcrypt + enter "${TRANSCRYPT_PROGRAM} $*" +} + function enter() { command="$*" require_shell require_repo - shell_opts="" - shell_path="" + local -a shell_opts + local shell_path="" if [[ "$SHELL" =~ bash$ ]]; then - shell_opts="--norc" + shell_opts=("--norc") shell_path="\w" elif [[ "$SHELL" =~ [cz]sh$ ]]; then - shell_opts="-f" + shell_opts=("-f") + if [[ "$SHELL" =~ zsh$ && "$TERM" = "dumb" ]]; then + # Disable ZLE for tramp + shell_opts+=("--no-zle") + fi shell_path="%~" fi @@ -960,7 +1069,7 @@ function enter() { [ "${#shell_cmd[@]}" -eq 0 ] && echo "Entering yadm repo" yadm_prompt="yadm shell ($YADM_REPO) $shell_path > " - PROMPT="$yadm_prompt" PS1="$yadm_prompt" "$SHELL" $shell_opts "${shell_cmd[@]}" + PROMPT="$yadm_prompt" PS1="$yadm_prompt" "$SHELL" "${shell_opts[@]}" "${shell_cmd[@]}" return_code="$?" if [ "${#shell_cmd[@]}" -eq 0 ]; then @@ -1023,12 +1132,14 @@ Commands: yadm perms - Fix perms for private files yadm enter [COMMAND] - Run sub-shell with GIT variables set yadm git-crypt [OPTIONS] - Run git-crypt commands for the yadm repo + yadm transcrypt [OPTIONS] - Run transcrypt commands for the yadm repo Files: - \$HOME/.config/yadm/config - yadm's configuration file - \$HOME/.config/yadm/repo.git - yadm's Git repository - \$HOME/.config/yadm/encrypt - List of globs used for encrypt/decrypt - \$HOME/.config/yadm/files.gpg - Encrypted data stored here + \$HOME/.config/yadm/config - yadm's configuration file + \$HOME/.config/yadm/encrypt - List of globs to encrypt/decrypt + \$HOME/.config/yadm/bootstrap - Script run via: yadm bootstrap + \$HOME/.local/share/yadm/repo.git - yadm's Git repository + \$HOME/.local/share/yadm/archive - Encrypted data stored here Use "man yadm" for complete documentation. EOF @@ -1037,6 +1148,7 @@ EOF } +# shellcheck disable=SC2120 function init() { # safety check, don't attempt to init when the repo is already present @@ -1076,13 +1188,14 @@ config decrypt encrypt enter -gitconfig git-crypt +gitconfig help init introspect list perms +transcrypt upgrade version EOF @@ -1099,10 +1212,14 @@ yadm.auto-alt yadm.auto-exclude yadm.auto-perms yadm.auto-private-dirs +yadm.cipher yadm.git-program yadm.gpg-perms yadm.gpg-program yadm.gpg-recipient +yadm.openssl-ciphername +yadm.openssl-old +yadm.openssl-program yadm.ssh-perms EOF } @@ -1116,6 +1233,7 @@ function introspect_switches() { --yadm-archive --yadm-bootstrap --yadm-config +--yadm-data --yadm-dir --yadm-encrypt --yadm-repo @@ -1177,40 +1295,82 @@ function perms() { function upgrade() { - local actions_performed - actions_performed=0 - local repo_updates - repo_updates=0 + local actions_performed=0 + local -a submodules + local repo_updates=0 - [ "$YADM_COMPATIBILITY" = "1" ] && \ - error_out "Unable to upgrade. YADM_COMPATIBILITY is set to '1'." + [[ -n "${YADM_OVERRIDE_REPO}${YADM_OVERRIDE_ARCHIVE}" || "$YADM_DATA" = "$YADM_DIR" ]] && \ + error_out "Unable to upgrade. Paths have been overridden with command line options" - [ "$YADM_DIR" = "$YADM_LEGACY_DIR" ] && \ - error_out "Unable to upgrade. yadm dir has been resolved as '$YADM_LEGACY_DIR'." + # choose a legacy repo, the version 2 location will be favored + local LEGACY_REPO= + [ -d "$YADM_LEGACY_DIR/repo.git" ] && LEGACY_REPO="$YADM_LEGACY_DIR/repo.git" + [ -d "$YADM_DIR/repo.git" ] && LEGACY_REPO="$YADM_DIR/repo.git" # handle legacy repo - if [ -d "$YADM_LEGACY_DIR/repo.git" ]; then + if [ -d "$LEGACY_REPO" ]; then + # choose # legacy repo detected, it must be moved to YADM_REPO if [ -e "$YADM_REPO" ]; then error_out "Unable to upgrade. '$YADM_REPO' already exists. Refusing to overwrite it." else actions_performed=1 - echo "Moving $YADM_LEGACY_DIR/repo.git to $YADM_REPO" + echo "Moving $LEGACY_REPO to $YADM_REPO" + + export GIT_DIR="$LEGACY_REPO" + + # Must absorb git dirs, otherwise deinit below will fail for modules that have + # been cloned first and then added as a submodule. + "$GIT_PROGRAM" submodule absorbgitdirs + + local submodule_status + submodule_status=$("$GIT_PROGRAM" -C "$YADM_WORK" submodule status) + while read -r sha submodule rest; do + [ "$submodule" == "" ] && continue + if [[ "$sha" = -* ]]; then + continue + fi + "$GIT_PROGRAM" -C "$YADM_WORK" submodule deinit ${FORCE:+-f} -- "$submodule" || { + for other in "${submodules[@]}"; do + "$GIT_PROGRAM" -C "$YADM_WORK" submodule update --init --recursive -- "$other" + done + error_out "Unable to upgrade. Could not deinit submodule $submodule" + } + submodules+=("$submodule") + done <<< "$submodule_status" + assert_parent "$YADM_REPO" - mv "$YADM_LEGACY_DIR/repo.git" "$YADM_REPO" + mv "$LEGACY_REPO" "$YADM_REPO" fi fi - - # handle other legacy paths GIT_DIR="$YADM_REPO" export GIT_DIR + + # choose a legacy archive, the version 2 location will be favored + local LEGACY_ARCHIVE= + [ -e "$YADM_LEGACY_DIR/$YADM_LEGACY_ARCHIVE" ] && LEGACY_ARCHIVE="$YADM_LEGACY_DIR/$YADM_LEGACY_ARCHIVE" + [ -e "$YADM_DIR/$YADM_LEGACY_ARCHIVE" ] && LEGACY_ARCHIVE="$YADM_DIR/$YADM_LEGACY_ARCHIVE" + + # handle legacy archive + if [ -e "$LEGACY_ARCHIVE" ]; then + actions_performed=1 + echo "Moving $LEGACY_ARCHIVE to $YADM_ARCHIVE" + assert_parent "$YADM_ARCHIVE" + # test to see if path is "tracked" in repo, if so 'git mv' must be used + if "$GIT_PROGRAM" ls-files --error-unmatch "$LEGACY_ARCHIVE" &> /dev/null; then + "$GIT_PROGRAM" mv "$LEGACY_ARCHIVE" "$YADM_ARCHIVE" && repo_updates=1 + else + mv -i "$LEGACY_ARCHIVE" "$YADM_ARCHIVE" + fi + fi + + # handle any remaining version 1 paths for legacy_path in \ "$YADM_LEGACY_DIR/config" \ "$YADM_LEGACY_DIR/encrypt" \ - "$YADM_LEGACY_DIR/files.gpg" \ "$YADM_LEGACY_DIR/bootstrap" \ "$YADM_LEGACY_DIR"/hooks/{pre,post}_* \ - ; \ + ; do if [ -e "$legacy_path" ]; then new_filename=${legacy_path#$YADM_LEGACY_DIR/} @@ -1228,19 +1388,15 @@ function upgrade() { done # handle submodules, which need to be reinitialized - if [ "$actions_performed" -ne 0 ]; then - cd_work "Upgrade submodules" - if "$GIT_PROGRAM" ls-files --error-unmatch .gitmodules &> /dev/null; then - "$GIT_PROGRAM" submodule deinit -f . - "$GIT_PROGRAM" submodule update --init --recursive - fi - fi + for submodule in "${submodules[@]}"; do + "$GIT_PROGRAM" -C "$YADM_WORK" submodule update --init --recursive -- "$submodule" + done [ "$actions_performed" -eq 0 ] && \ echo "No legacy paths found. Upgrade is not necessary" [ "$repo_updates" -eq 1 ] && \ - echo "Some files tracked by yadm have been renamed. This changes should probably be commited now." + echo "Some files tracked by yadm have been renamed. These changes should probably be commited now." exit 0 @@ -1310,7 +1466,8 @@ function is_valid_branch_name() { # * "~", "^", ":", "\", space # * end with a "/" # * end with ".lock" - [[ "$1" =~ (\/\.|\.\.|[~^:\\ ]|\/$|\.lock$) ]] && return 1 + pattern='(\/\.|\.\.|[~^:\\ ]|\/$|\.lock$)' + [[ "$1" =~ $pattern ]] && return 1 return 0 } @@ -1344,6 +1501,13 @@ function process_global_args() { YADM_DIR="$2" shift ;; + --yadm-data) # override the standard YADM_DATA + if [[ ! "$2" =~ ^/ ]] ; then + error_out "You must specify a fully qualified yadm data directory" + fi + YADM_DATA="$2" + shift + ;; --yadm-repo) # override the standard YADM_REPO if [[ ! "$2" =~ ^/ ]] ; then error_out "You must specify a fully qualified repo path" @@ -1388,23 +1552,25 @@ function process_global_args() { } -function set_yadm_dir() { - - # only resolve YADM_DIR if it hasn't been provided already - [ -n "$YADM_DIR" ] && return +function set_yadm_dirs() { - # compatibility with major version 1 ignores XDG_CONFIG_HOME - if [ "$YADM_COMPATIBILITY" = "1" ]; then - YADM_DIR="$YADM_LEGACY_DIR" - return + # only resolve YADM_DATA if it hasn't been provided already + if [ -z "$YADM_DATA" ]; then + local base_yadm_data="$XDG_DATA_HOME" + if [[ ! "$base_yadm_data" =~ ^/ ]] ; then + base_yadm_data="${HOME}/.local/share" + fi + YADM_DATA="${base_yadm_data}/yadm" fi - local base_yadm_dir - base_yadm_dir="$XDG_CONFIG_HOME" - if [[ ! "$base_yadm_dir" =~ ^/ ]] ; then - base_yadm_dir="${HOME}/.config" + # only resolve YADM_DIR if it hasn't been provided already + if [ -z "$YADM_DIR" ]; then + local base_yadm_dir="$XDG_CONFIG_HOME" + if [[ ! "$base_yadm_dir" =~ ^/ ]] ; then + base_yadm_dir="${HOME}/.config" + fi + YADM_DIR="${base_yadm_dir}/yadm" fi - YADM_DIR="${base_yadm_dir}/yadm" issue_legacy_path_warning @@ -1418,21 +1584,22 @@ function issue_legacy_path_warning() { # no warnings if YADM_DIR is resolved as the leacy path [ "$YADM_DIR" = "$YADM_LEGACY_DIR" ] && return - # no warnings if the legacy directory doesn't exist - [ ! -d "$YADM_LEGACY_DIR" ] && return + # no warnings if overrides have been provided + [[ -n "${YADM_OVERRIDE_REPO}${YADM_OVERRIDE_ARCHIVE}" || "$YADM_DATA" = "$YADM_DIR" ]] && return # test for legacy paths - local legacy_found - legacy_found=() + local legacy_found=() # this is ordered by importance for legacy_path in \ + "$YADM_DIR/$YADM_REPO" \ + "$YADM_DIR/$YADM_LEGACY_ARCHIVE" \ "$YADM_LEGACY_DIR/$YADM_REPO" \ + "$YADM_LEGACY_DIR/$YADM_BOOTSTRAP" \ "$YADM_LEGACY_DIR/$YADM_CONFIG" \ "$YADM_LEGACY_DIR/$YADM_ENCRYPT" \ - "$YADM_LEGACY_DIR/$YADM_ARCHIVE" \ - "$YADM_LEGACY_DIR/$YADM_BOOTSTRAP" \ "$YADM_LEGACY_DIR/$YADM_HOOKS"/{pre,post}_* \ - ; \ + "$YADM_LEGACY_DIR/$YADM_LEGACY_ARCHIVE" \ + ; do [ -e "$legacy_path" ] && legacy_found+=("$legacy_path") done @@ -1444,26 +1611,25 @@ function issue_legacy_path_warning() { path_list="$path_list * $legacy_path"$'\n' done - cat <<EOF + cat <<EOF >&2 **WARNING** - Legacy configuration paths have been detected. + Legacy paths have been detected. - Beginning with version 2.0.0, yadm uses the XDG Base Directory Specification - to find its configurations. Read more about this change here: + With version 3.0.0, yadm uses the XDG Base Directory Specification + to find its configurations and data. Read more about these changes here: + https://yadm.io/docs/upgrade_from_2 https://yadm.io/docs/upgrade_from_1 - In your environment, the configuration directory has been resolved to: + In your environment, the data directory has been resolved to: - $YADM_DIR + $YADM_DATA To remove this warning do one of the following: - * Run "yadm upgrade" to move the yadm data to the new directory. (RECOMMENDED) - * Manually move yadm configurations to the directory listed above. - * Specify your preferred yadm directory with -Y each execution. - * Define an environment variable "YADM_COMPATIBILITY=1" to run in version 1 - compatibility mode. (DEPRECATED) + * Run "yadm upgrade" to move the yadm data to the new paths. (RECOMMENDED) + * Manually move yadm data to new default paths and reinit any submodules. + * Specify your preferred paths with --yadm-data and --yadm-archive each execution. Legacy paths detected: ${path_list} @@ -1476,15 +1642,17 @@ LEGACY_WARNING_ISSUED=1 function configure_paths() { - # change all paths to be relative to YADM_DIR - YADM_REPO="$YADM_DIR/$YADM_REPO" + # change paths to be relative to YADM_DIR YADM_CONFIG="$YADM_DIR/$YADM_CONFIG" YADM_ENCRYPT="$YADM_DIR/$YADM_ENCRYPT" - YADM_ARCHIVE="$YADM_DIR/$YADM_ARCHIVE" YADM_BOOTSTRAP="$YADM_DIR/$YADM_BOOTSTRAP" YADM_HOOKS="$YADM_DIR/$YADM_HOOKS" YADM_ALT="$YADM_DIR/$YADM_ALT" + # change paths to be relative to YADM_DATA + YADM_REPO="$YADM_DATA/$YADM_REPO" + YADM_ARCHIVE="$YADM_DATA/$YADM_ARCHIVE" + # independent overrides for paths if [ -n "$YADM_OVERRIDE_REPO" ]; then YADM_REPO="$YADM_OVERRIDE_REPO" @@ -1513,6 +1681,14 @@ function configure_paths() { [ -n "$work" ] && YADM_WORK="$work" fi + # YADM_BASE is used for manipulating the base worktree path for much of the + # alternate file processing + if [ "$YADM_WORK" == "/" ]; then + YADM_BASE="" + else + YADM_BASE="$YADM_WORK" + fi + } function configure_repo() { @@ -1572,7 +1748,7 @@ function debug() { function error_out() { - echo_e "ERROR: $*" + echo_e "ERROR: $*" >&2 exit_with_hook 1 } @@ -1590,12 +1766,14 @@ function invoke_hook() { exit_status="$2" hook_command="${YADM_HOOKS}/${mode}_$HOOK_COMMAND" - if [ -x "$hook_command" ] ; then + if [ -x "$hook_command" ] || \ + { [[ $OPERATING_SYSTEM == MINGW* ]] && [ -f "$hook_command" ] ;} ; then debug "Invoking hook: $hook_command" # expose some internal data to all hooks YADM_HOOK_COMMAND=$HOOK_COMMAND YADM_HOOK_DIR=$YADM_DIR + YADM_HOOK_DATA=$YADM_DATA YADM_HOOK_EXIT=$exit_status YADM_HOOK_FULL_COMMAND=$FULL_COMMAND YADM_HOOK_REPO=$YADM_REPO @@ -1607,6 +1785,7 @@ function invoke_hook() { export YADM_HOOK_COMMAND export YADM_HOOK_DIR + export YADM_HOOK_DATA export YADM_HOOK_EXIT export YADM_HOOK_FULL_COMMAND export YADM_HOOK_REPO @@ -1660,7 +1839,9 @@ function assert_private_dirs() { function assert_parent() { basedir=${1%/*} - [ -e "$basedir" ] || mkdir -p "$basedir" + if [ -n "$basedir" ]; then + [ -e "$basedir" ] || mkdir -p "$basedir" + fi } function display_private_perms() { @@ -1705,7 +1886,7 @@ function parse_encrypt() { if [ -f "$YADM_ENCRYPT" ] ; then # parse both included/excluded while IFS='' read -r line || [ -n "$line" ]; do - if [[ ! $line =~ ^# && ! $line =~ ^[[:space:]]*$ ]] ; then + if [[ ! $line =~ ^# && ! $line =~ ^[[:blank:]]*$ ]] ; then local IFS=$'\n' for pattern in $line; do if [[ "$pattern" =~ $exclude_pattern ]]; then @@ -1862,6 +2043,34 @@ function join_string { printf "%s" "${*:2}" } +function get_mode { + local filename="$1" + local mode + + # most *nixes + mode=$(stat -c '%a' "$filename" 2>/dev/null) + if [ -z "$mode" ] ; then + # BSD-style + mode=$(stat -f '%p' "$filename" 2>/dev/null) + mode=${mode: -4} + fi + + # only accept results if they are octal + if [[ ! $mode =~ ^[0-7]+$ ]] ; then + mode="" + fi + + echo "$mode" +} + +function copy_perms { + local source="$1" + local dest="$2" + mode=$(get_mode "$source") + [ -n "$mode" ] && chmod "$mode" "$dest" + return 0 +} + # ****** Prerequisites Functions ****** function require_archive() { @@ -1874,8 +2083,7 @@ function require_git() { local alt_git alt_git="$(config yadm.git-program)" - local more_info - more_info="" + local more_info="" if [ "$alt_git" != "" ] ; then GIT_PROGRAM="$alt_git" @@ -1888,8 +2096,7 @@ function require_gpg() { local alt_gpg alt_gpg="$(config yadm.gpg-program)" - local more_info - more_info="" + local more_info="" if [ "$alt_gpg" != "" ] ; then GPG_PROGRAM="$alt_gpg" @@ -1898,6 +2105,19 @@ function require_gpg() { command -v "$GPG_PROGRAM" &> /dev/null || error_out "This functionality requires GPG to be installed, but the command '$GPG_PROGRAM' cannot be located.$more_info" } +function require_openssl() { + local alt_openssl + alt_openssl="$(config yadm.openssl-program)" + + local more_info="" + + if [ "$alt_openssl" != "" ] ; then + OPENSSL_PROGRAM="$alt_openssl" + more_info="\nThis command has been set via the yadm.openssl-program configuration." + fi + command -v "$OPENSSL_PROGRAM" &> /dev/null || + error_out "This functionality requires OpenSSL to be installed, but the command '$OPENSSL_PROGRAM' cannot be located.$more_info" +} function require_repo() { [ -d "$YADM_REPO" ] || error_out "Git repo does not exist. did you forget to run 'init' or 'clone'?" } @@ -1908,6 +2128,10 @@ function require_git_crypt() { command -v "$GIT_CRYPT_PROGRAM" &> /dev/null || error_out "This functionality requires git-crypt to be installed, but the command '$GIT_CRYPT_PROGRAM' cannot be located." } +function require_transcrypt() { + command -v "$TRANSCRYPT_PROGRAM" &> /dev/null || + error_out "This functionality requires transcrypt to be installed, but the command '$TRANSCRYPT_PROGRAM' cannot be located." +} function bootstrap_available() { [ -f "$YADM_BOOTSTRAP" ] && [ -x "$YADM_BOOTSTRAP" ] && return return 1 @@ -1924,6 +2148,10 @@ function envtpl_available() { command -v "$ENVTPL_PROGRAM" &> /dev/null && return return 1 } +function esh_available() { + command -v "$ESH_PROGRAM" &> /dev/null && return + return 1 +} function readlink_available() { command -v "readlink" &> /dev/null && return return 1 @@ -1969,7 +2197,7 @@ if [ "$YADM_TEST" != 1 ] ; then process_global_args "$@" set_operating_system set_awk - set_yadm_dir + set_yadm_dirs configure_paths main "${MAIN_ARGS[@]}" fi |