Merge tag '10.20240129'

tagging package git-annex version 10.20240129

435 files changed, 10658 insertions, 1363 deletions

diff --git a/Annex/AutoMerge.hs b/Annex/AutoMerge.hs
index 77afe521c9..bb43d0593b 100644
--- a/Annex/AutoMerge.hs
+++ b/Annex/AutoMerge.hs
@@ -242,7 +242,7 @@ resolveMerge' unstagedmap (Just us) them inoverlay u = do
 		stageSymlink dest' =<< hashSymlink l
 
 	replacewithsymlink dest link = replaceWorkTreeFile dest $
-		makeGitLink link . toRawFilePath
+		makeGitLink link
 
 	makepointer key dest destmode = do
 		unless inoverlay $ 
@@ -267,10 +267,10 @@ resolveMerge' unstagedmap (Just us) them inoverlay u = do
 			Nothing -> noop
 			Just sha -> replaceWorkTreeFile item $ \tmp -> do
 				c <- catObject sha
-				liftIO $ L.writeFile tmp c
+				liftIO $ L.writeFile (decodeBS tmp) c
 				when isexecutable $
 					liftIO $ void $ tryIO $ 
-						modifyFileMode (toRawFilePath tmp) $
+						modifyFileMode tmp $
 							addModes executeModes
 
 		-- Update the work tree to reflect the graft.
diff --git a/Annex/Branch.hs b/Annex/Branch.hs
index dce16667d7..8f927a78b4 100644
--- a/Annex/Branch.hs
+++ b/Annex/Branch.hs
@@ -1,6 +1,6 @@
 {- management of the git-annex branch
  -
- - Copyright 2011-2022 Joey Hess <id@joeyh.name>
+ - Copyright 2011-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -14,6 +14,7 @@ module Annex.Branch (
 	hasSibling,
 	siblingBranches,
 	create,
+	getBranch,
 	UpdateMade(..),
 	update,
 	forceUpdate,
@@ -36,11 +37,11 @@ module Annex.Branch (
 	withIndex,
 	precache,
 	overBranchFileContents,
+	updatedFromTree,
 ) where
 
 import qualified Data.ByteString as B
 import qualified Data.ByteString.Lazy as L
-import qualified Data.ByteString.Char8 as B8
 import qualified Data.Set as S
 import qualified Data.Map as M
 import Data.Function
@@ -49,6 +50,7 @@ import Data.ByteString.Builder
 import Control.Concurrent (threadDelay)
 import Control.Concurrent.MVar
 import qualified System.FilePath.ByteString as P
+import System.PosixCompat.Files (isRegularFile)
 
 import Annex.Common hiding (append)
 import Types.BranchState
@@ -119,7 +121,7 @@ siblingBranches = inRepo $ Git.Ref.matchingUniq [name]
 create :: Annex ()
 create = void getBranch
 
-{- Returns the ref of the branch, creating it first if necessary. -}
+{- Returns the sha of the branch, creating it first if necessary. -}
 getBranch :: Annex Git.Ref
 getBranch = maybe (hasOrigin >>= go >>= use) return =<< branchsha
   where
@@ -134,7 +136,7 @@ getBranch = maybe (hasOrigin >>= go >>= use) return =<< branchsha
 			<$> branchsha
 	go False = withIndex' True $ do
 		-- Create the index file. This is not necessary,
-		-- except to avoid a bug in git that causes
+		-- except to avoid a bug in git 2.37 that causes
 		-- git write-tree to segfault when the index file does not
 		-- exist.
 		inRepo $ flip Git.UpdateIndex.streamUpdateIndex []
@@ -595,21 +597,24 @@ files = do
 		then return Nothing
 		else do
 			(bfs, cleanup) <- branchFiles
+			jfs <- journalledFiles
+			pjfs <- journalledFilesPrivate
 			-- ++ forces the content of the first list to be
 			-- buffered in memory, so use journalledFiles,
 			-- which should be much smaller most of the time.
 			-- branchFiles will stream as the list is consumed.
-			l <- (++) <$> journalledFiles <*> pure bfs
+			let l = jfs ++ pjfs ++ bfs
 			return (Just (l, cleanup))
 
-{- Lists all files currently in the journal. There may be duplicates in
- - the list when using a private journal. -}
+{- Lists all files currently in the journal, but not files in the private
+ - journal. -}
 journalledFiles :: Annex [RawFilePath]
-journalledFiles = ifM privateUUIDsKnown
-	( (++)
-		<$> getJournalledFilesStale gitAnnexPrivateJournalDir
-		<*> getJournalledFilesStale gitAnnexJournalDir
-	, getJournalledFilesStale gitAnnexJournalDir
+journalledFiles = getJournalledFilesStale gitAnnexJournalDir
+
+journalledFilesPrivate :: Annex [RawFilePath]
+journalledFilesPrivate = ifM privateUUIDsKnown
+	( getJournalledFilesStale gitAnnexPrivateJournalDir
+	, return []
 	)
 
 {- Files in the branch, not including any from journalled changes,
@@ -726,13 +731,14 @@ stageJournal jl commitindex = withIndex $ withOtherTmp $ \tmpdir -> do
 	genstream dir h jh jlogh streamer = readDirectory jh >>= \case
 		Nothing -> return ()
 		Just file -> do
-			unless (dirCruft file) $ do
-				let path = dir P.</> toRawFilePath file
+			let path = dir P.</> toRawFilePath file
+			unless (dirCruft file) $ whenM (isfile path) $ do
 				sha <- Git.HashObject.hashFile h path
 				hPutStrLn jlogh file
 				streamer $ Git.UpdateIndex.updateIndexLine
 					sha TreeFile (asTopFilePath $ fileJournal $ toRawFilePath file)
 			genstream dir h jh jlogh streamer
+	isfile file = isRegularFile <$> R.getFileStatus file
 	-- Clean up the staged files, as listed in the temp log file.
 	-- The temp file is used to avoid needing to buffer all the
 	-- filenames in memory.
@@ -883,7 +889,7 @@ ignoreRefs rs = do
 
 getIgnoredRefs :: Annex (S.Set Git.Sha)
 getIgnoredRefs = 
-	S.fromList . mapMaybe Git.Sha.extractSha . B8.lines <$> content
+	S.fromList . mapMaybe Git.Sha.extractSha . fileLines' <$> content
   where
 	content = do
 		f <- fromRawFilePath <$> fromRepo gitAnnexIgnoredRefs
@@ -906,7 +912,7 @@ getMergedRefs' :: Annex [(Git.Sha, Git.Branch)]
 getMergedRefs' = do
 	f <- fromRawFilePath <$> fromRepo gitAnnexMergedRefs
 	s <- liftIO $ catchDefaultIO mempty $ B.readFile f
-	return $ map parse $ B8.lines s
+	return $ map parse $ fileLines' s
   where
 	parse l = 
 		let (s, b) = separate' (== (fromIntegral (ord '\t'))) l
@@ -915,10 +921,14 @@ getMergedRefs' = do
 {- Grafts a treeish into the branch at the specified location,
  - and then removes it. This ensures that the treeish won't get garbage
  - collected, and will always be available as long as the git-annex branch
- - is available. -}
-rememberTreeish :: Git.Ref -> TopFilePath -> Annex ()
-rememberTreeish treeish graftpoint = lockJournal $ rememberTreeishLocked treeish graftpoint
-rememberTreeishLocked :: Git.Ref -> TopFilePath -> JournalLocked -> Annex ()
+ - is available.
+ -
+ - Returns the sha of the git commit made to the git-annex branch.
+ -}
+rememberTreeish :: Git.Ref -> TopFilePath -> Annex Git.Sha
+rememberTreeish treeish graftpoint = lockJournal $
+	rememberTreeishLocked treeish graftpoint
+rememberTreeishLocked :: Git.Ref -> TopFilePath -> JournalLocked -> Annex Git.Sha
 rememberTreeishLocked treeish graftpoint jl = do
 	branchref <- getBranch
 	updateIndex jl branchref
@@ -935,6 +945,7 @@ rememberTreeishLocked treeish graftpoint jl = do
 	-- and the index was updated to that above, so it's safe to
 	-- say that the index contains c'.
 	setIndexSha c'
+	return c'
 
 {- Runs an action on the content of selected files from the branch.
  - This is much faster than reading the content of each file in turn,
@@ -989,8 +1000,11 @@ overBranchFileContents' select go st = do
 			-- This can cause the action to be run a
 			-- second time with a file it already ran on.
 			| otherwise -> liftIO (tryTakeMVar buf) >>= \case
-				Nothing -> drain buf =<< journalledFiles
-				Just fs -> drain buf fs
+				Nothing -> do
+					jfs <- journalledFiles
+					pjfs <- journalledFilesPrivate
+					drain buf jfs pjfs
+				Just (jfs, pjfs) -> drain buf jfs pjfs
 	catObjectStreamLsTree l (select' . getTopFilePath . Git.LsTree.file) g go'
 		`finally` liftIO (void cleanup)
   where
@@ -1004,9 +1018,9 @@ overBranchFileContents' select go st = do
 			PossiblyStaleJournalledContent journalledcontent ->
 				Just (fromMaybe mempty branchcontent <> journalledcontent)
 				
-	drain buf fs = case getnext fs of
-		Just (v, f, fs') -> do
-			liftIO $ putMVar buf fs'
+	drain buf fs pfs = case getnext fs pfs of
+		Just (v, f, fs', pfs') -> do
+			liftIO $ putMVar buf (fs', pfs')
 			content <- getJournalFileStale (GetPrivate True) f >>= \case
 				NoJournalledContent -> return Nothing
 				JournalledContent journalledcontent ->
@@ -1019,11 +1033,22 @@ overBranchFileContents' select go st = do
 					return (Just (content <> journalledcontent))
 			return (Just (v, f, content))
 		Nothing -> do
-			liftIO $ putMVar buf []
+			liftIO $ putMVar buf ([], [])
 			return Nothing
 	
-	getnext [] = Nothing
-	getnext (f:fs) = case select f of
-		Nothing -> getnext fs
-		Just v -> Just (v, f, fs)
-
+	getnext [] [] = Nothing
+	getnext (f:fs) pfs = case select f of
+		Nothing -> getnext fs pfs
+		Just v -> Just (v, f, fs, pfs)
+	getnext [] (pf:pfs) = case select pf of
+		Nothing -> getnext [] pfs
+		Just v -> Just (v, pf, [], pfs)
+
+{- Check if the git-annex branch has been updated from the oldtree.
+ - If so, returns the tuple of the old and new trees. -}
+updatedFromTree :: Git.Sha -> Annex (Maybe (Git.Sha, Git.Sha))
+updatedFromTree oldtree =
+	inRepo (Git.Ref.tree fullname) >>= \case
+		Just currtree | currtree /= oldtree ->
+			return $ Just (oldtree, currtree)
+		_ -> return Nothing
diff --git a/Annex/Branch/Transitions.hs b/Annex/Branch/Transitions.hs
index 1cbe62120e..d7f45cb067 100644
--- a/Annex/Branch/Transitions.hs
+++ b/Annex/Branch/Transitions.hs
@@ -18,6 +18,7 @@ import qualified Logs.Presence.Pure as Presence
 import qualified Logs.Chunk.Pure as Chunk
 import qualified Logs.MetaData.Pure as MetaData
 import qualified Logs.Remote.Pure as Remote
+import Logs.MapLog
 import Types.TrustLevel
 import Types.UUID
 import Types.MetaData
@@ -53,7 +54,7 @@ dropDead trustmap remoteconfigmap gc f content
 	| f == trustLog = PreserveFile
 	| f == remoteLog = ChangeFile $
 		Remote.buildRemoteConfigLog $
-			M.mapWithKey minimizesameasdead $
+			mapLogWithKey minimizesameasdead $
 				filterMapLog (notdead trustmap) id $
 					Remote.parseRemoteConfigLog content
 	| otherwise = filterBranch (notdead trustmap') gc f content
@@ -95,8 +96,8 @@ filterBranch wantuuid gc f content = case getLogVariety gc f of
 	Just OtherLog -> PreserveFile
 	Nothing -> PreserveFile
 
-filterMapLog :: (UUID -> Bool) -> (k -> UUID) -> M.Map k v -> M.Map k v
-filterMapLog wantuuid getuuid = M.filterWithKey $ \k _v -> wantuuid (getuuid k)
+filterMapLog :: (UUID -> Bool) -> (k -> UUID) -> MapLog k v -> MapLog k v
+filterMapLog wantuuid getuuid = filterMapLogWith (\k _v -> wantuuid (getuuid k))
 
 filterLocationLog :: (UUID -> Bool) -> [Presence.LogLine] -> [Presence.LogLine]
 filterLocationLog wantuuid = filter $
diff --git a/Annex/Content.hs b/Annex/Content.hs
index b212fcc77a..995eb6ed15 100644
--- a/Annex/Content.hs
+++ b/Annex/Content.hs
@@ -1,6 +1,6 @@
 {- git-annex file content managing
  -
- - Copyright 2010-2022 Joey Hess <id@joeyh.name>
+ - Copyright 2010-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -66,6 +66,7 @@ module Annex.Content (
 	getKeyStatus,
 	getKeyFileStatus,
 	cleanObjectDirs,
+	contentSize,
 ) where
 
 import System.IO.Unsafe (unsafeInterleaveIO)
@@ -278,9 +279,10 @@ lockContentUsing contentlocker key fallback a = withContentLockFile key $ \mlock
 {- Runs an action, passing it the temp file to get,
  - and if the action succeeds, verifies the file matches
  - the key and moves the file into the annex as a key's content. -}
-getViaTmp :: RetrievalSecurityPolicy -> VerifyConfig -> Key -> AssociatedFile -> (RawFilePath -> Annex (Bool, Verification)) -> Annex Bool
-getViaTmp rsp v key af action = checkDiskSpaceToGet key False $
-	getViaTmpFromDisk rsp v key af action
+getViaTmp :: RetrievalSecurityPolicy -> VerifyConfig -> Key -> AssociatedFile -> Maybe FileSize -> (RawFilePath -> Annex (Bool, Verification)) -> Annex Bool
+getViaTmp rsp v key af sz action =
+	checkDiskSpaceToGet key sz False $
+		getViaTmpFromDisk rsp v key af action
 
 {- Like getViaTmp, but does not check that there is enough disk space
  - for the incoming key. For use when the key content is already on disk
@@ -348,14 +350,14 @@ verificationOfContentFailed tmpfile = do
  -
  - Wen there's enough free space, runs the download action.
  -}
-checkDiskSpaceToGet :: Key -> a -> Annex a -> Annex a
-checkDiskSpaceToGet key unabletoget getkey = do
+checkDiskSpaceToGet :: Key -> Maybe FileSize -> a -> Annex a -> Annex a
+checkDiskSpaceToGet key sz unabletoget getkey = do
 	tmp <- fromRepo (gitAnnexTmpObjectLocation key)
 	e <- liftIO $ doesFileExist (fromRawFilePath tmp)
 	alreadythere <- liftIO $ if e
 		then getFileSize tmp
 		else return 0
-	ifM (checkDiskSpace Nothing key alreadythere True)
+	ifM (checkDiskSpace sz Nothing key alreadythere True)
 		( do
 			-- The tmp file may not have been left writable
 			when e $ thawContent tmp
@@ -449,7 +451,7 @@ checkSecureHashes' :: Key -> Annex Bool
 checkSecureHashes' key = checkSecureHashes key >>= \case
 	Nothing -> return True
 	Just msg -> do
-		warning $ UnquotedString $ msg ++ "to annex objects"
+		warning $ UnquotedString $ msg ++ " to annex objects"
 		return False
 
 data LinkAnnexResult = LinkAnnexOk | LinkAnnexFailed | LinkAnnexNoop
@@ -477,7 +479,7 @@ linkToAnnex key src srcic = ifM (checkSecureHashes' key)
 linkFromAnnex :: Key -> RawFilePath -> Maybe FileMode -> Annex LinkAnnexResult
 linkFromAnnex key dest destmode =
 	replaceFile' (const noop) (fromRawFilePath dest) (== LinkAnnexOk) $ \tmp ->
-		linkFromAnnex' key (toRawFilePath tmp) destmode
+		linkFromAnnex' key tmp destmode
 
 {- This is only safe to use when dest is not a worktree file. -}
 linkFromAnnex' :: Key -> RawFilePath -> Maybe FileMode -> Annex LinkAnnexResult
@@ -541,17 +543,18 @@ unlinkAnnex key = do
 		secureErase obj
 		liftIO $ removeWhenExistsWith R.removeLink obj
 
-{- Runs an action to transfer an object's content.
+{- Runs an action to transfer an object's content. The action is also
+ - passed the size of the object.
  -
  - In some cases, it's possible for the file to change as it's being sent.
  - If this happens, runs the rollback action and throws an exception.
  - The rollback action should remove the data that was transferred.
  -}
-sendAnnex :: Key -> Annex () -> (FilePath -> Annex a) -> Annex a
+sendAnnex :: Key -> Annex () -> (FilePath -> FileSize -> Annex a) -> Annex a
 sendAnnex key rollback sendobject = go =<< prepSendAnnex' key
   where
-	go (Just (f, check)) = do
-		r <- sendobject f
+	go (Just (f, sz, check)) = do
+		r <- sendobject f sz
 		check >>= \case
 			Nothing -> return r
 			Just err -> do
@@ -570,9 +573,13 @@ sendAnnex key rollback sendobject = go =<< prepSendAnnex' key
  - Annex monad of the remote that is receiving the object, rather than
  - the sender. So it cannot rely on Annex state.
  -}
-prepSendAnnex :: Key -> Annex (Maybe (FilePath, Annex Bool))
+prepSendAnnex :: Key -> Annex (Maybe (FilePath, FileSize, Annex Bool))
 prepSendAnnex key = withObjectLoc key $ \f -> do
-	let retval c = return $ Just (fromRawFilePath f, sameInodeCache f c)
+	let retval c cs = return $ Just 
+		(fromRawFilePath f
+		, inodeCacheFileSize c
+		, sameInodeCache f cs
+		)
 	cache <- Database.Keys.getInodeCaches key
 	if null cache
 		-- Since no inode cache is in the database, this
@@ -580,7 +587,7 @@ prepSendAnnex key = withObjectLoc key $ \f -> do
 		-- change while the transfer is in progress, so
 		-- generate an inode cache for the starting
 		-- content.
-		then maybe (return Nothing) (retval . (:[]))
+		then maybe (return Nothing) (\fc -> retval fc [fc])
 			=<< withTSDelta (liftIO . genInodeCache f)
 		-- Verify that the object is not modified. Usually this
 		-- only has to check the inode cache, but if the cache
@@ -588,19 +595,19 @@ prepSendAnnex key = withObjectLoc key $ \f -> do
 		-- content.
 		else withTSDelta (liftIO . genInodeCache f) >>= \case
 			Just fc -> ifM (isUnmodified' key f fc cache)
-				( retval (fc:cache)
+				( retval fc (fc:cache)
 				, return Nothing
 				)
 			Nothing -> return Nothing
 
-prepSendAnnex' :: Key -> Annex (Maybe (FilePath, Annex (Maybe String)))
+prepSendAnnex' :: Key -> Annex (Maybe (FilePath, FileSize, Annex (Maybe String)))
 prepSendAnnex' key = prepSendAnnex key >>= \case
-	Just (f, checksuccess) -> 
+	Just (f, sz, checksuccess) -> 
 		let checksuccess' = ifM checksuccess
 			( return Nothing
 			, return (Just "content changed while it was being sent")
 			)
-		in return (Just (f, checksuccess'))
+		in return (Just (f, sz, checksuccess'))
 	Nothing -> return Nothing
 
 cleanObjectLoc :: Key -> Annex () -> Annex ()
@@ -744,9 +751,9 @@ downloadUrl :: Bool -> Key -> MeterUpdate -> Maybe IncrementalVerifier -> [Url.U
 downloadUrl listfailedurls k p iv urls file uo = 
 	-- Poll the file to handle configurations where an external
 	-- download command is used.
-	meteredFile file (Just p) k (go urls [])
+	meteredFile (toRawFilePath file) (Just p) k (go urls [])
   where
-	go (u:us) errs = Url.download' p iv u file uo >>= \case
+	go (u:us) errs p' = Url.download' p' iv u file uo >>= \case
 		Right () -> return True
 		Left err -> do
 			-- If the incremental verifier was fed anything
@@ -758,9 +765,9 @@ downloadUrl listfailedurls k p iv urls file uo =
 					Just n | n > 0 -> unableIncrementalVerifier iv'
 					_ -> noop
 				Nothing -> noop
-			go us ((u, err) : errs)
-	go [] [] = return False
-	go [] errs@((_, err):_) = do
+			go us ((u, err) : errs) p'
+	go [] [] _ = return False
+	go [] errs@((_, err):_) _ = do
 		if listfailedurls
 			then warning $ UnquotedString $
 				unlines $ flip map errs $ \(u, err') ->
@@ -916,3 +923,25 @@ getKeyFileStatus key file = do
 				)
 		_ -> return s
 
+{- Gets the size of the content of a key when it is present.
+ - Useful when the key does not have keySize set. 
+ -
+ - When the object file appears possibly modified with annex.thin set, does
+ - not do an expensive verification that the content is good, just returns
+ - Nothing.
+ -}
+contentSize :: Key -> Annex (Maybe FileSize)
+contentSize key = catchDefaultIO Nothing $
+	withObjectLoc key $ \loc ->
+		withTSDelta (liftIO . genInodeCache loc) >>= \case
+			Just ic -> ifM (unmodified ic)
+				( return (Just (inodeCacheFileSize ic))
+				, return Nothing
+				)
+			Nothing -> return Nothing
+  where
+	unmodified ic =
+		ifM (annexThin <$> Annex.getGitConfig)
+			( isUnmodifiedCheap' key ic
+			, return True
+			)
diff --git a/Annex/Content/LowLevel.hs b/Annex/Content/LowLevel.hs
index ec8284bedb..9d732f6a6e 100644
--- a/Annex/Content/LowLevel.hs
+++ b/Annex/Content/LowLevel.hs
@@ -1,6 +1,6 @@
 {- git-annex low-level content functions
  -
- - Copyright 2010-2018 Joey Hess <id@joeyh.name>
+ - Copyright 2010-2024 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -102,13 +102,13 @@ preserveGitMode _ _ = return True
  - to be downloaded from the free space. This way, we avoid overcommitting
  - when doing concurrent downloads.
  -}
-checkDiskSpace :: Maybe RawFilePath -> Key -> Integer -> Bool -> Annex Bool
-checkDiskSpace destdir key = checkDiskSpace' (fromMaybe 1 (fromKey keySize key)) destdir key
+checkDiskSpace :: Maybe FileSize -> Maybe RawFilePath -> Key -> Integer -> Bool -> Annex Bool
+checkDiskSpace msz destdir key = checkDiskSpace' sz destdir key
+  where
+	sz = fromMaybe 1 (fromKey keySize key <|> msz)
 
-{- Allows specifying the size of the key, if it's known, which is useful
- - as not all keys know their size. -}
-checkDiskSpace' :: Integer -> Maybe RawFilePath -> Key -> Integer -> Bool -> Annex Bool
-checkDiskSpace' need destdir key alreadythere samefilesystem = ifM (Annex.getRead Annex.force)
+checkDiskSpace' :: FileSize -> Maybe RawFilePath -> Key -> Integer -> Bool -> Annex Bool
+checkDiskSpace' sz destdir key alreadythere samefilesystem = ifM (Annex.getRead Annex.force)
 	( return True
 	, do
 		-- We can't get inprogress and free at the same
@@ -123,7 +123,7 @@ checkDiskSpace' need destdir key alreadythere samefilesystem = ifM (Annex.getRea
 		dir >>= liftIO . getDiskFree . fromRawFilePath >>= \case
 			Just have -> do
 				reserve <- annexDiskReserve <$> Annex.getGitConfig
-				let delta = need + reserve - have - alreadythere + inprogress
+				let delta = sz + reserve - have - alreadythere + inprogress
 				let ok = delta <= 0
 				unless ok $
 					warning $ UnquotedString $ 
diff --git a/Annex/Content/PointerFile.hs b/Annex/Content/PointerFile.hs
index 7fc4be5327..c2acc9ab93 100644
--- a/Annex/Content/PointerFile.hs
+++ b/Annex/Content/PointerFile.hs
@@ -38,11 +38,10 @@ populatePointerFile restage k obj f = go =<< liftIO (isPointerFile f)
 		destmode <- liftIO $ catchMaybeIO $ fileMode <$> R.getFileStatus f
 		liftIO $ removeWhenExistsWith R.removeLink f
 		(ic, populated) <- replaceWorkTreeFile f' $ \tmp -> do
-			let tmp' = toRawFilePath tmp
-			ok <- linkOrCopy k obj tmp' destmode >>= \case
-				Just _ -> thawContent tmp' >> return True
-				Nothing -> liftIO (writePointerFile tmp' k destmode) >> return False
-			ic <- withTSDelta (liftIO . genInodeCache tmp')
+			ok <- linkOrCopy k obj tmp destmode >>= \case
+				Just _ -> thawContent tmp >> return True
+				Nothing -> liftIO (writePointerFile tmp k destmode) >> return False
+			ic <- withTSDelta (liftIO . genInodeCache tmp)
 			return (ic, ok)
 		maybe noop (restagePointerFile restage f) ic
 		if populated
@@ -60,14 +59,13 @@ depopulatePointerFile key file = do
 	secureErase file
 	liftIO $ removeWhenExistsWith R.removeLink file
 	ic <- replaceWorkTreeFile (fromRawFilePath file) $ \tmp -> do
-		let tmp' = toRawFilePath tmp
-		liftIO $ writePointerFile tmp' key mode
+		liftIO $ writePointerFile tmp key mode
 #if ! defined(mingw32_HOST_OS)
 		-- Don't advance mtime; this avoids unnecessary re-smudging
 		-- by git in some cases.
 		liftIO $ maybe noop
-			(\t -> touch tmp' t False)
+			(\t -> touch tmp t False)
 			(fmap Posix.modificationTimeHiRes st)
 #endif
-		withTSDelta (liftIO . genInodeCache tmp')
+		withTSDelta (liftIO . genInodeCache tmp)
 	maybe noop (restagePointerFile (Restage True) file) ic
diff --git a/Annex/Content/Presence.hs b/Annex/Content/Presence.hs
index d3aea87151..2eb0016ddd 100644
--- a/Annex/Content/Presence.hs
+++ b/Annex/Content/Presence.hs
@@ -18,6 +18,7 @@ module Annex.Content.Presence (
 	isUnmodified,
 	isUnmodified',
 	isUnmodifiedCheap,
+	isUnmodifiedCheap',
 	withContentLockFile,
 	contentLockFile,
 ) where
@@ -206,7 +207,7 @@ isUnmodified' = isUnmodifiedLowLevel Database.Keys.addInodeCaches
  - within a small time window (eg 1 second).
  -}
 isUnmodifiedCheap :: Key -> RawFilePath -> Annex Bool
-isUnmodifiedCheap key f = maybe (return False) (isUnmodifiedCheap' key) 
+isUnmodifiedCheap key f = maybe (pure False) (isUnmodifiedCheap' key) 
 	=<< withTSDelta (liftIO . genInodeCache f)
 
 isUnmodifiedCheap' :: Key -> InodeCache -> Annex Bool
diff --git a/Annex/CopyFile.hs b/Annex/CopyFile.hs
index 0be9debd5f..55c7d908e2 100644
--- a/Annex/CopyFile.hs
+++ b/Annex/CopyFile.hs
@@ -57,7 +57,7 @@ tryCopyCoW (CopyCoWTried copycowtried) src dest meterupdate =
 			)
 		)
   where
-	docopycow = watchFileSize dest meterupdate $
+	docopycow = watchFileSize dest' meterupdate $ const $
 		copyCoW CopyTimeStamps src dest
 	
 	dest' = toRawFilePath dest
diff --git a/Annex/FileMatcher.hs b/Annex/FileMatcher.hs
index 6c049dfce4..e48931f360 100644
--- a/Annex/FileMatcher.hs
+++ b/Annex/FileMatcher.hs
@@ -16,7 +16,6 @@ module Annex.FileMatcher (
 	matchAll,
 	PreferredContentData(..),
 	preferredContentTokens,
-	preferredContentKeylessTokens,
 	preferredContentParser,
 	ParseToken,
 	parsedToMatcher,
@@ -139,19 +138,15 @@ tokenizeMatcher = filter (not . null) . concatMap splitparens . words
   where
 	splitparens = segmentDelim (`elem` "()")
 
-commonKeylessTokens :: LimitBy -> [ParseToken (MatchFiles Annex)]
-commonKeylessTokens lb =
+commonTokens :: LimitBy -> [ParseToken (MatchFiles Annex)]
+commonTokens lb =
 	[ SimpleToken "anything" (simply limitAnything)
 	, SimpleToken "nothing" (simply limitNothing)
 	, ValueToken "include" (usev limitInclude)
 	, ValueToken "exclude" (usev limitExclude)
 	, ValueToken "largerthan" (usev $ limitSize lb "largerthan" (>))
 	, ValueToken "smallerthan" (usev $ limitSize lb "smallerthan" (<))
-	]
-
-commonKeyedTokens :: [ParseToken (MatchFiles Annex)]
-commonKeyedTokens = 
-	[ SimpleToken "unused" (simply limitUnused)
+	, SimpleToken "unused" (simply limitUnused)
 	]
 
 data PreferredContentData = PCD
@@ -162,25 +157,12 @@ data PreferredContentData = PCD
 	, repoUUID :: Maybe UUID
 	}
 
--- Tokens of preferred content expressions that do not need a Key to be
--- known. 
---
--- When importing from a special remote, this is used to match
--- some preferred content expressions before the content is downloaded,
--- so the Key is not known.
-preferredContentKeylessTokens :: PreferredContentData -> [ParseToken (MatchFiles Annex)]
-preferredContentKeylessTokens pcd =
+preferredContentTokens :: PreferredContentData -> [ParseToken (MatchFiles Annex)]
+preferredContentTokens pcd =
 	[ SimpleToken "standard" (call "standard" $ matchStandard pcd)
 	, SimpleToken "groupwanted" (call "groupwanted" $ matchGroupWanted pcd)
 	, SimpleToken "inpreferreddir" (simply $ limitInDir preferreddir "inpreferreddir")
-	] ++ commonKeylessTokens LimitAnnexFiles
-  where
-	preferreddir = maybe "public" fromProposedAccepted $
-		M.lookup preferreddirField =<< (`M.lookup` configMap pcd) =<< repoUUID pcd
-
-preferredContentKeyedTokens :: PreferredContentData -> [ParseToken (MatchFiles Annex)]
-preferredContentKeyedTokens pcd =
-	[ SimpleToken "present" (simply $ limitPresent $ repoUUID pcd)
+	, SimpleToken "present" (simply $ limitPresent $ repoUUID pcd)
 	, SimpleToken "securehash" (simply limitSecureHash)
 	, ValueToken "copies" (usev limitCopies)
 	, ValueToken "lackingcopies" (usev $ limitLackingCopies "lackingcopies" False)
@@ -189,13 +171,10 @@ preferredContentKeyedTokens pcd =
 	, ValueToken "metadata" (usev limitMetaData)
 	, ValueToken "inallgroup" (usev $ limitInAllGroup $ getGroupMap pcd)
 	, ValueToken "onlyingroup" (usev $ limitOnlyInGroup $ getGroupMap pcd)
-	] ++ commonKeyedTokens
-
-preferredContentTokens :: PreferredContentData -> [ParseToken (MatchFiles Annex)]
-preferredContentTokens pcd = concat
-	[ preferredContentKeylessTokens pcd
-	, preferredContentKeyedTokens pcd
-	]
+	] ++ commonTokens LimitAnnexFiles
+  where
+	preferreddir = maybe "public" fromProposedAccepted $
+		M.lookup preferreddirField =<< (`M.lookup` configMap pcd) =<< repoUUID pcd
 
 preferredContentParser :: [ParseToken (MatchFiles Annex)] -> String -> [ParseResult (MatchFiles Annex)]
 preferredContentParser tokens = map (parseToken tokens) . tokenizeMatcher
@@ -210,8 +189,7 @@ mkMatchExpressionParser = do
 		const $ Left $ "\""++n++"\" not supported; not built with MagicMime support"
 #endif
 	let parse = parseToken $
-		commonKeyedTokens ++
-		commonKeylessTokens LimitDiskFiles ++
+		commonTokens LimitDiskFiles ++
 #ifdef WITH_MAGICMIME
 		[ mimer "mimetype" $
 			matchMagic "mimetype" getMagicMimeType providedMimeType userProvidedMimeType
diff --git a/Annex/Import.hs b/Annex/Import.hs
index 1dbb50d4aa..eaf41f4f79 100644
--- a/Annex/Import.hs
+++ b/Annex/Import.hs
@@ -1,6 +1,6 @@
 {- git-annex import from remotes
  -
- - Copyright 2019-2023 Joey Hess <id@joeyh.name>
+ - Copyright 2019-2024 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -223,20 +223,27 @@ buildImportCommit' remote importcommitconfig mtrackingcommit imported@(History t
 		-- nothing new needs to be committed.
 		-- (This is unlikely to happen.)
 		| sametodepth h' = return Nothing
-		| otherwise = do
-			importedcommit <- case getRemoteTrackingBranchImportHistory h of
-				Nothing -> mkcommitsunconnected imported
-				Just oldimported@(History oldhc _)
-					| importeddepth == 1 ->
-						mkcommitconnected imported oldimported
-					| otherwise -> do
-						let oldimportedtrees = mapHistory historyCommitTree oldimported
-						mknewcommits oldhc oldimportedtrees imported
-			ti' <- addBackExportExcluded remote ti
-			Just <$> makeRemoteTrackingBranchMergeCommit'
-				trackingcommit importedcommit ti'
+		-- If the imported tree is unchanged,
+		-- nothing new needs to be committed.
+		| otherwise = getLastImportedTree remote >>= \case
+			Just (LastImportedTree lasttree)
+				| lasttree == ti -> return Nothing
+			_ -> gencommit trackingcommit h
 	  where
 		h'@(History t s) = mapHistory historyCommitTree h
+	
+	gencommit trackingcommit h = do
+		importedcommit <- case getRemoteTrackingBranchImportHistory h of
+			Nothing -> mkcommitsunconnected imported
+			Just oldimported@(History oldhc _)
+				| importeddepth == 1 ->
+					mkcommitconnected imported oldimported
+				| otherwise -> do
+					let oldimportedtrees = mapHistory historyCommitTree oldimported
+					mknewcommits oldhc oldimportedtrees imported
+		ti' <- addBackExportExcluded remote ti
+		Just <$> makeRemoteTrackingBranchMergeCommit'
+			trackingcommit importedcommit ti'
 
 	importeddepth = historyDepth imported
 
@@ -791,7 +798,6 @@ importKeys remote importtreeconfig importcontent thirdpartypopulated importablec
 					, providedMimeEncoding = Nothing
 					, providedLinkType = Nothing
 					}
-				let bwlimit = remoteAnnexBwLimit (Remote.gitconfig remote)
 				islargefile <- checkMatcher' matcher mi mempty
 				metered Nothing sz bwlimit $ const $ if islargefile
 					then doimportlarge importkey cidmap loc cid sz f
@@ -834,7 +840,7 @@ importKeys remote importtreeconfig importcontent thirdpartypopulated importablec
 				when ok $
 					logStatus k InfoPresent
 				return (Just (k, ok))
-			checkDiskSpaceToGet k Nothing $
+			checkDiskSpaceToGet k Nothing Nothing $
 				notifyTransfer Download af $
 					download' (Remote.uuid remote) k af Nothing stdRetry $ \p' ->
 						withTmp k $ downloader p'
@@ -853,7 +859,7 @@ importKeys remote importtreeconfig importcontent thirdpartypopulated importablec
 					recordcidkey cidmap cid k
 					return sha
 				Nothing -> error "internal"
-		checkDiskSpaceToGet tmpkey Nothing $
+		checkDiskSpaceToGet tmpkey Nothing Nothing $
 			withTmp tmpkey $ \tmpfile ->
 				tryNonAsync (downloader tmpfile) >>= \case
 					Right sha -> return $ Just (loc, Left sha)
@@ -888,8 +894,7 @@ importKeys remote importtreeconfig importcontent thirdpartypopulated importablec
 			Left e -> do
 				warning (UnquotedString (show e))
 				return Nothing
-		let bwlimit = remoteAnnexBwLimit (Remote.gitconfig remote)
-		checkDiskSpaceToGet tmpkey Nothing $
+		checkDiskSpaceToGet tmpkey Nothing Nothing $
 			notifyTransfer Download af $
 				download' (Remote.uuid remote) tmpkey af Nothing stdRetry $ \p ->
 					withTmp tmpkey $ \tmpfile ->
@@ -917,6 +922,9 @@ importKeys remote importtreeconfig importcontent thirdpartypopulated importablec
 				else gitShaKey <$> hashFile tmpfile
 	
 	ia = Remote.importActions remote
+				
+	bwlimit = remoteAnnexBwLimitDownload (Remote.gitconfig remote)
+			<|> remoteAnnexBwLimit (Remote.gitconfig remote)
 
 	locworktreefile loc = fromRepo $ fromTopFilePath $ asTopFilePath $
 		case importtreeconfig of
@@ -990,20 +998,22 @@ addBackExportExcluded remote importtree =
  -
  - Only keyless tokens are supported, because the keys are not known
  - until an imported file is downloaded, which is too late to bother
- - excluding it from an import.
+ - excluding it from an import. So prunes any tokens in the preferred
+ - content expression that need keys.
  -}
 makeImportMatcher :: Remote -> Annex (Either String (FileMatcher Annex))
-makeImportMatcher r = load preferredContentKeylessTokens >>= \case
+makeImportMatcher r = load preferredContentTokens >>= \case
 	Nothing -> return $ Right (matchAll, matcherdesc)
 	Just (Right v) -> return $ Right (v, matcherdesc)
-	Just (Left err) -> load preferredContentTokens >>= \case
-		Just (Left err') -> return $ Left err'
-		_ -> return $ Left $
-			"The preferred content expression contains terms that cannot be checked when importing: " ++ err
+	Just (Left err) -> return $ Left err
   where
-	load t = M.lookup (Remote.uuid r) . fst <$> preferredRequiredMapsLoad' t
+	load t = M.lookup (Remote.uuid r) . fst
+		<$> preferredRequiredMapsLoad' pruneImportMatcher t
 	matcherdesc = MatcherDesc "preferred content"
 
+pruneImportMatcher :: Utility.Matcher.Matcher (MatchFiles a) -> Utility.Matcher.Matcher (MatchFiles a)
+pruneImportMatcher = Utility.Matcher.pruneMatcher matchNeedsKey
+
 {- Gets the ImportableContents from the remote.
  -
  - Filters out any paths that include a ".git" component, because git does
diff --git a/Annex/Ingest.hs b/Annex/Ingest.hs
index 26a5e388eb..1ef4d346b9 100644
--- a/Annex/Ingest.hs
+++ b/Annex/Ingest.hs
@@ -19,6 +19,7 @@ module Annex.Ingest (
 	finishIngestUnlocked,
 	cleanOldKeys,
 	addSymlink,
+	genSymlink,
 	makeLink,
 	addUnlocked,
 	CheckGitIgnore(..),
@@ -38,6 +39,7 @@ import Annex.MetaData
 import Annex.CurrentBranch
 import Annex.CheckIgnore
 import Logs.Location
+import qualified Git
 import qualified Annex
 import qualified Database.Keys
 import Config
@@ -306,7 +308,7 @@ restoreFile file key e = do
 makeLink :: RawFilePath -> Key -> Maybe InodeCache -> Annex LinkTarget
 makeLink file key mcache = flip catchNonAsync (restoreFile file key) $ do
 	l <- calcRepo $ gitAnnexLink file key
-	replaceWorkTreeFile file' $ makeAnnexLink l . toRawFilePath
+	replaceWorkTreeFile file' $ makeAnnexLink l
 
 	-- touch symlink to have same time as the original file,
 	-- as provided in the InodeCache
@@ -320,9 +322,12 @@ makeLink file key mcache = flip catchNonAsync (restoreFile file key) $ do
 
 {- Creates the symlink to the annexed content, and stages it in git. -}
 addSymlink :: RawFilePath -> Key -> Maybe InodeCache -> Annex ()
-addSymlink file key mcache = do
+addSymlink file key mcache = stageSymlink file =<< genSymlink file key mcache
+
+genSymlink :: RawFilePath -> Key -> Maybe InodeCache -> Annex Git.Sha
+genSymlink file key mcache = do
 	linktarget <- makeLink file key mcache
-	stageSymlink file =<< hashSymlink linktarget
+	hashSymlink linktarget
 
 {- Parameters to pass to git add, forcing addition of ignored files.
  -
diff --git a/Annex/Locations.hs b/Annex/Locations.hs
index 1f793321a6..6e4f0faa2a 100644
--- a/Annex/Locations.hs
+++ b/Annex/Locations.hs
@@ -54,6 +54,10 @@ module Annex.Locations (
 	gitAnnexRestageLock,
 	gitAnnexAdjustedBranchUpdateLog,
 	gitAnnexAdjustedBranchUpdateLock,
+	gitAnnexMigrateLog,
+	gitAnnexMigrateLock,
+	gitAnnexMigrationsLog,
+	gitAnnexMigrationsLock,
 	gitAnnexMoveLog,
 	gitAnnexMoveLock,
 	gitAnnexExportDir,
@@ -65,6 +69,8 @@ module Annex.Locations (
 	gitAnnexImportLog,
 	gitAnnexContentIdentifierDbDir,
 	gitAnnexContentIdentifierLock,
+	gitAnnexImportFeedDbDir,
+	gitAnnexImportFeedDbLock,
 	gitAnnexScheduleState,
 	gitAnnexTransferDir,
 	gitAnnexCredsDir,
@@ -405,6 +411,20 @@ gitAnnexAdjustedBranchUpdateLog r = gitAnnexDir r P.</> "adjust.log"
 gitAnnexAdjustedBranchUpdateLock :: Git.Repo -> RawFilePath
 gitAnnexAdjustedBranchUpdateLock r = gitAnnexDir r P.</> "adjust.lck"
 
+{- .git/annex/migrate.log is used to log migrations before committing them. -}
+gitAnnexMigrateLog :: Git.Repo -> RawFilePath
+gitAnnexMigrateLog r = gitAnnexDir r P.</> "migrate.log"
+
+gitAnnexMigrateLock :: Git.Repo -> RawFilePath
+gitAnnexMigrateLock r = gitAnnexDir r P.</> "migrate.lck"
+
+{- .git/annex/migrations.log is used to log committed migrations. -}
+gitAnnexMigrationsLog :: Git.Repo -> RawFilePath
+gitAnnexMigrationsLog r = gitAnnexDir r P.</> "migrations.log"
+
+gitAnnexMigrationsLock :: Git.Repo -> RawFilePath
+gitAnnexMigrationsLock r = gitAnnexDir r P.</> "migrations.lck"
+
 {- .git/annex/move.log is used to log moves that are in progress,
  - to better support resuming an interrupted move. -}
 gitAnnexMoveLog :: Git.Repo -> RawFilePath
@@ -460,6 +480,15 @@ gitAnnexImportLog :: UUID -> Git.Repo -> GitConfig -> RawFilePath
 gitAnnexImportLog u r c = 
 	gitAnnexImportDir r c P.</> fromUUID u P.</> "log"
 
+{- Directory containing database used by importfeed. -}
+gitAnnexImportFeedDbDir :: Git.Repo -> GitConfig -> RawFilePath
+gitAnnexImportFeedDbDir r c =
+	fromMaybe (gitAnnexDir r) (annexDbDir c) P.</> "importfeed"
+
+{- Lock file for writing to the importfeed database. -}
+gitAnnexImportFeedDbLock :: Git.Repo -> GitConfig -> RawFilePath
+gitAnnexImportFeedDbLock r c = gitAnnexImportFeedDbDir r c <> ".lck"
+
 {- .git/annex/schedulestate is used to store information about when
  - scheduled jobs were last run. -}
 gitAnnexScheduleState :: Git.Repo -> RawFilePath
diff --git a/Annex/MetaData/StandardFields.hs b/Annex/MetaData/StandardFields.hs
index 9cc3ed4c49..061133b41c 100644
--- a/Annex/MetaData/StandardFields.hs
+++ b/Annex/MetaData/StandardFields.hs
@@ -15,7 +15,8 @@ module Annex.MetaData.StandardFields (
 	isDateMetaField,
 	lastChangedField,
 	mkLastChangedField,
-	isLastChangedField
+	isLastChangedField,
+	itemIdField
 ) where
 
 import Types.MetaData
@@ -61,3 +62,6 @@ lastchanged = "lastchanged"
 
 lastchangedSuffix :: T.Text
 lastchangedSuffix = "-lastchanged"
+
+itemIdField :: MetaField
+itemIdField = mkMetaFieldUnchecked "itemid"
diff --git a/Annex/NumCopies.hs b/Annex/NumCopies.hs
index a3c6f92dcd..69b1943d87 100644
--- a/Annex/NumCopies.hs
+++ b/Annex/NumCopies.hs
@@ -282,9 +282,11 @@ notEnoughCopies key neednum needmin have skip bad nolocmsg lockunsupported = do
 	showNote "unsafe"
 	if length have < fromNumCopies neednum
 		then showLongNote $ UnquotedString $
-			"Could only verify the existence of " ++
-			show (length have) ++ " out of " ++ show (fromNumCopies neednum) ++ 
-			" necessary " ++ pluralCopies (fromNumCopies neednum)
+			if fromNumCopies neednum == 1
+				then "Could not verify the existence of the 1 necessary copy."
+				else "Could only verify the existence of " ++
+					show (length have) ++ " out of " ++ show (fromNumCopies neednum) ++ 
+					" necessary " ++ pluralCopies (fromNumCopies neednum) ++ "."
 		else do
 			showLongNote $ UnquotedString $ "Unable to lock down " ++ show (fromMinCopies needmin) ++ 
 				" " ++ pluralCopies (fromMinCopies needmin) ++ 
@@ -317,7 +319,7 @@ pluralCopies _ = "copies"
 verifiableCopies :: Key -> [UUID] -> Annex ([UnVerifiedCopy], [VerifiedCopy])
 verifiableCopies key exclude = do
 	locs <- Remote.keyLocations key
-	(remotes, trusteduuids) <- Remote.remoteLocations locs
+	(remotes, trusteduuids) <- Remote.remoteLocations (Remote.IncludeIgnored False) locs
 		=<< trustGet Trusted
 	untrusteduuids <- trustGet UnTrusted
 	let exclude' = exclude ++ untrusteduuids
diff --git a/Annex/ReplaceFile.hs b/Annex/ReplaceFile.hs
index 9f671cb9d6..21735eba14 100644
--- a/Annex/ReplaceFile.hs
+++ b/Annex/ReplaceFile.hs
@@ -26,17 +26,17 @@ import Utility.Path.Max
 #endif
 
 {- replaceFile on a file located inside the gitAnnexDir. -}
-replaceGitAnnexDirFile :: FilePath -> (FilePath -> Annex a) -> Annex a
+replaceGitAnnexDirFile :: FilePath -> (RawFilePath -> Annex a) -> Annex a
 replaceGitAnnexDirFile = replaceFile createAnnexDirectory
 
 {- replaceFile on a file located inside the .git directory. -}
-replaceGitDirFile :: FilePath -> (FilePath -> Annex a) -> Annex a
+replaceGitDirFile :: FilePath -> (RawFilePath -> Annex a) -> Annex a
 replaceGitDirFile = replaceFile $ \dir -> do
 	top <- fromRepo localGitDir
 	liftIO $ createDirectoryUnder [top] dir
 
 {- replaceFile on a worktree file. -}
-replaceWorkTreeFile :: FilePath -> (FilePath -> Annex a) -> Annex a
+replaceWorkTreeFile :: FilePath -> (RawFilePath -> Annex a) -> Annex a
 replaceWorkTreeFile = replaceFile createWorkTreeDirectory
 
 {- Replaces a possibly already existing file with a new version, 
@@ -54,10 +54,10 @@ replaceWorkTreeFile = replaceFile createWorkTreeDirectory
  - The createdirectory action is only run when moving the file into place
  - fails, and can create any parent directory structure needed.
  -}
-replaceFile :: (RawFilePath -> Annex ()) -> FilePath -> (FilePath -> Annex a) -> Annex a
+replaceFile :: (RawFilePath -> Annex ()) -> FilePath -> (RawFilePath -> Annex a) -> Annex a
 replaceFile createdirectory file action = replaceFile' createdirectory file (const True) action
 
-replaceFile' :: (RawFilePath -> Annex ()) -> FilePath -> (a -> Bool) -> (FilePath -> Annex a) -> Annex a
+replaceFile' :: (RawFilePath -> Annex ()) -> FilePath -> (a -> Bool) -> (RawFilePath -> Annex a) -> Annex a
 replaceFile' createdirectory file checkres action = withOtherTmp $ \othertmpdir -> do
 	let othertmpdir' = fromRawFilePath othertmpdir
 #ifndef mingw32_HOST_OS
@@ -72,10 +72,10 @@ replaceFile' createdirectory file checkres action = withOtherTmp $ \othertmpdir
 	let basetmp = "t"
 #endif
 	withTmpDirIn othertmpdir' basetmp $ \tmpdir -> do
-		let tmpfile = tmpdir </> basetmp
+		let tmpfile = toRawFilePath (tmpdir </> basetmp)
 		r <- action tmpfile
 		when (checkres r) $
-			replaceFileFrom (toRawFilePath tmpfile) (toRawFilePath file) createdirectory
+			replaceFileFrom tmpfile (toRawFilePath file) createdirectory
 		return r
 
 replaceFileFrom :: RawFilePath -> RawFilePath -> (RawFilePath -> Annex ()) -> Annex ()
diff --git a/Annex/StallDetection.hs b/Annex/StallDetection.hs
index 9c095b4c86..9b885c2ecf 100644
--- a/Annex/StallDetection.hs
+++ b/Annex/StallDetection.hs
@@ -1,14 +1,20 @@
 {- Stall detection for transfers.
  -
- - Copyright 2020-2021 Joey Hess <id@joeyh.name>
+ - Copyright 2020-2024 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
 
-module Annex.StallDetection (detectStalls, StallDetection) where
+module Annex.StallDetection (
+	getStallDetection,
+	detectStalls,
+	StallDetection,
+) where
 
 import Annex.Common
 import Types.StallDetection
+import Types.Direction
+import Types.Remote (gitconfig)
 import Utility.Metered
 import Utility.HumanTime
 import Utility.DataUnits
@@ -16,41 +22,71 @@ import Utility.ThreadScheduler
 
 import Control.Concurrent.STM
 import Control.Monad.IO.Class (MonadIO)
+import Data.Time.Clock
+
+getStallDetection :: Direction -> Remote -> Maybe StallDetection
+getStallDetection Download r = 
+	remoteAnnexStallDetectionDownload (gitconfig r)
+		<|> remoteAnnexStallDetection (gitconfig r)
+getStallDetection Upload r =
+	remoteAnnexStallDetectionUpload (gitconfig r)
+		<|> remoteAnnexStallDetection (gitconfig r)
 
 {- This may be safely canceled (with eg uninterruptibleCancel),
  - as long as the passed action can be safely canceled. -}
 detectStalls :: (Monad m, MonadIO m) => Maybe StallDetection -> TVar (Maybe BytesProcessed) -> m () -> m ()
 detectStalls Nothing _ _ = noop
 detectStalls (Just StallDetectionDisabled) _ _ = noop
-detectStalls (Just (StallDetection (BwRate minsz duration))) metervar onstall =
-	detectStalls' minsz duration metervar onstall Nothing
+detectStalls (Just (StallDetection bwrate@(BwRate _minsz duration))) metervar onstall = do
+	-- If the progress is being updated, but less frequently than
+	-- the specified duration, a stall would be incorrectly detected.
+	--
+	-- For example, consider the case of a remote that does
+	-- not support progress updates, but is chunked with a large chunk
+	-- size. In that case, progress is only updated after each chunk.
+	--
+	-- So, wait for the first update, and see how long it takes.
+	-- When it's longer than the duration (or close to it), 
+	-- upscale the duration and minsz accordingly.
+	starttime <- liftIO getCurrentTime
+	v <- waitforfirstupdate =<< readMeterVar metervar
+	endtime <- liftIO getCurrentTime
+	let timepassed = floor (endtime `diffUTCTime` starttime)
+	let BwRate scaledminsz scaledduration = upscale bwrate timepassed
+	detectStalls' scaledminsz scaledduration metervar onstall v
+  where
+	minwaitsecs = Seconds $
+		min 60 (fromIntegral (durationSeconds duration))
+	waitforfirstupdate startval = do
+		liftIO $ threadDelaySeconds minwaitsecs
+		v <- readMeterVar metervar
+		if v > startval
+			then return v
+			else waitforfirstupdate startval
 detectStalls (Just ProbeStallDetection) metervar onstall = do
 	-- Only do stall detection once the progress is confirmed to be
 	-- consistently updating. After the first update, it needs to
 	-- advance twice within 30 seconds. With that established,
 	-- if no data at all is sent for a 60 second period, it's
 	-- assumed to be a stall.
-	v <- getval >>= waitforfirstupdate
+	v <- readMeterVar metervar >>= waitforfirstupdate
 	ontimelyadvance v $ \v' -> ontimelyadvance v' $
 		detectStalls' 1 duration metervar onstall
   where
-	getval = liftIO $ atomically $ fmap fromBytesProcessed
-		<$> readTVar metervar
-	
 	duration = Duration 60
 
 	delay = Seconds (fromIntegral (durationSeconds duration) `div` 2)
 	
 	waitforfirstupdate startval = do
 		liftIO $ threadDelaySeconds delay
-		v <- getval
+		v <- readMeterVar metervar
 		if v > startval
 			then return v
 			else waitforfirstupdate startval
 
 	ontimelyadvance v cont = do
 		liftIO $ threadDelaySeconds delay
-		v' <- getval
+		v' <- readMeterVar metervar
 		when (v' > v) $
 			cont v'
 
@@ -65,8 +101,7 @@ detectStalls'
 detectStalls' minsz duration metervar onstall st = do
 	liftIO $ threadDelaySeconds delay
 	-- Get whatever progress value was reported most recently, if any.
-	v <- liftIO $ atomically $ fmap fromBytesProcessed
-		<$> readTVar metervar
+	v <- readMeterVar metervar
 	let cont = detectStalls' minsz duration metervar onstall v
 	case (st, v) of
 		(Nothing, _) -> cont
@@ -81,3 +116,39 @@ detectStalls' minsz duration metervar onstall st = do
 			| otherwise -> cont
   where
 	delay = Seconds (fromIntegral (durationSeconds duration))
+
+readMeterVar
+	:: MonadIO m
+	=> TVar (Maybe BytesProcessed)
+	-> m (Maybe ByteSize)
+readMeterVar metervar = liftIO $ atomically $ 
+	fmap fromBytesProcessed <$> readTVar metervar
+
+-- Scale up the minsz and duration to match the observed time that passed
+-- between progress updates. This allows for some variation in the transfer
+-- rate causing later progress updates to happen less frequently.
+upscale :: BwRate -> Integer -> BwRate
+upscale input@(BwRate minsz duration) timepassedsecs
+	| timepassedsecs > dsecs `div` allowedvariation = BwRate 
+		(ceiling (fromIntegral minsz * scale))
+		(Duration (ceiling (fromIntegral dsecs * scale)))
+	| otherwise = input
+  where
+	scale = max (1 :: Double) $
+		(fromIntegral timepassedsecs / fromIntegral (max dsecs 1))
+		* fromIntegral allowedvariation
+	
+	dsecs = durationSeconds duration
+
+	-- Setting this too low will make normal bandwidth variations be
+	-- considered to be stalls, while setting it too high will make
+	-- stalls not be detected for much longer than the expected
+	-- duration.
+	--
+	-- For example, a BwRate of 20MB/1m, when the first progress
+	-- update takes 10m to arrive, is scaled to 600MB/30m. That 30m
+	-- is a reasonable since only 3 chunks get sent in that amount of
+	-- time at that rate. If allowedvariation = 10, that would
+	-- be 2000MB/100m, which seems much too long to wait to detect a
+	-- stall.
+	allowedvariation = 3
diff --git a/Annex/TaggedPush.hs b/Annex/TaggedPush.hs
index cde24cdb74..d728678e9a 100644
--- a/Annex/TaggedPush.hs
+++ b/Annex/TaggedPush.hs
@@ -38,14 +38,14 @@ toTaggedBranch :: UUID -> Maybe String -> Git.Branch -> Git.Ref
 toTaggedBranch u info b = Git.Ref $ S.intercalate "/" $ catMaybes
 	[ Just "refs/synced"
 	, Just $ fromUUID u
-	, toB64' . encodeBS <$> info
+	, toB64 . encodeBS <$> info
 	, Just $ Git.fromRef' $ Git.Ref.base b
 	]
 
-fromTaggedBranch :: Git.Ref -> Maybe (UUID, Maybe String)
+fromTaggedBranch :: Git.Ref -> Maybe (UUID, Maybe S.ByteString)
 fromTaggedBranch b = case splitc '/' $ Git.fromRef b of
 	("refs":"synced":u:info:_base) ->
-		Just (toUUID u, fromB64Maybe info)
+		Just (toUUID u, fromB64Maybe (encodeBS info))
 	("refs":"synced":u:_base) ->
 		Just (toUUID u, Nothing)
 	_ -> Nothing
diff --git a/Annex/Transfer.hs b/Annex/Transfer.hs
index 879082d712..d31863f2b8 100644
--- a/Annex/Transfer.hs
+++ b/Annex/Transfer.hs
@@ -56,7 +56,7 @@ import Data.Ord
 -- Upload, supporting canceling detected stalls.
 upload :: Remote -> Key -> AssociatedFile -> RetryDecider -> NotifyWitness -> Annex Bool
 upload r key f d witness = 
-	case remoteAnnexStallDetection (Remote.gitconfig r) of
+	case getStallDetection Upload r of
 		Nothing -> go (Just ProbeStallDetection)
 		Just StallDetectionDisabled -> go Nothing
 		Just sd -> runTransferrer sd r key f d Upload witness
@@ -75,12 +75,12 @@ alwaysUpload u key f sd d a _witness = guardHaveUUID u $
 -- Download, supporting canceling detected stalls.
 download :: Remote -> Key -> AssociatedFile -> RetryDecider -> NotifyWitness -> Annex Bool
 download r key f d witness = 
-	case remoteAnnexStallDetection (Remote.gitconfig r) of
+	case getStallDetection Download r of
 		Nothing -> go (Just ProbeStallDetection)
 		Just StallDetectionDisabled -> go Nothing
 		Just sd -> runTransferrer sd r key f d Download witness
   where
-	go sd = getViaTmp (Remote.retrievalSecurityPolicy r) vc key f $ \dest ->
+	go sd = getViaTmp (Remote.retrievalSecurityPolicy r) vc key f Nothing $ \dest ->
 		download' (Remote.uuid r) key f sd d (go' dest) witness
 	go' dest p = verifiedAction $
 		Remote.retrieveKeyFile r key f (fromRawFilePath dest) p vc
diff --git a/Annex/UUID.hs b/Annex/UUID.hs
index 872fb9091a..8986f2d7a0 100644
--- a/Annex/UUID.hs
+++ b/Annex/UUID.hs
@@ -41,6 +41,7 @@ import Config
 import qualified Data.UUID as U
 import qualified Data.UUID.V4 as U4
 import qualified Data.UUID.V5 as U5
+import qualified Data.ByteString as S
 import Data.String
 
 configkeyUUID :: ConfigKey
@@ -53,13 +54,13 @@ genUUID = toUUID <$> U4.nextRandom
 {- Generates a UUID from a given string, using a namespace.
  - Given the same namespace, the same string will always result
  - in the same UUID. -}
-genUUIDInNameSpace :: U.UUID -> String -> UUID
-genUUIDInNameSpace namespace = toUUID . U5.generateNamed namespace . s2w8
+genUUIDInNameSpace :: U.UUID -> S.ByteString -> UUID
+genUUIDInNameSpace namespace = toUUID . U5.generateNamed namespace . S.unpack
 
 {- Namespace used for UUIDs derived from git-remote-gcrypt ids. -}
 gCryptNameSpace :: U.UUID
 gCryptNameSpace = U5.generateNamed U5.namespaceURL $
-	s2w8 "http://git-annex.branchable.com/design/gcrypt/" 
+	S.unpack "http://git-annex.branchable.com/design/gcrypt/" 
 
 {- Get current repository's UUID. -}
 getUUID :: Annex UUID
diff --git a/Annex/VectorClock/Utility.hs b/Annex/VectorClock/Utility.hs
index 1396dd54cb..76b74d9cd5 100644
--- a/Annex/VectorClock/Utility.hs
+++ b/Annex/VectorClock/Utility.hs
@@ -20,4 +20,14 @@ startVectorClock = go =<< getEnv "GIT_ANNEX_VECTOR_CLOCK"
 	go (Just s) = case parsePOSIXTime s of
 		Just t -> return (pure (CandidateVectorClock t))
 		Nothing -> timebased
-	timebased = return (CandidateVectorClock <$> getPOSIXTime)
+	-- Avoid using fractional seconds in the CandidateVectorClock.
+	-- This reduces the size of the packed git-annex branch by up
+	-- to 8%. 
+	--
+	-- Due to the use of vector clocks, high resolution timestamps are
+	-- not necessary to make clear which information is most recent when
+	-- eg, a value is changed repeatedly in the same second. In such a
+	-- case, the vector clock will be advanced to the next second after
+	-- the last modification.
+	timebased = return $
+		CandidateVectorClock . truncateResolution 0 <$> getPOSIXTime
diff --git a/Assistant.hs b/Assistant.hs
index 3bfaaaa7f9..2e50a79ff1 100644
--- a/Assistant.hs
+++ b/Assistant.hs
@@ -59,7 +59,7 @@ import System.Environment (getArgs)
 #endif
 import qualified Utility.Debug as Debug
 
-import Network.Socket (HostName)
+import Network.Socket (HostName, PortNumber)
 
 stopDaemon :: Annex ()
 stopDaemon = liftIO . Utility.Daemon.stopDaemon . fromRawFilePath
@@ -70,8 +70,8 @@ stopDaemon = liftIO . Utility.Daemon.stopDaemon . fromRawFilePath
  -
  - startbrowser is passed the url and html shim file, as well as the original
  - stdout and stderr descriptors. -}
-startDaemon :: Bool -> Bool -> Maybe Duration -> Maybe String -> Maybe HostName ->  Maybe (Maybe Handle -> Maybe Handle -> String -> FilePath -> IO ()) -> Annex ()
-startDaemon assistant foreground startdelay cannotrun listenhost startbrowser = do
+startDaemon :: Bool -> Bool -> Maybe Duration -> Maybe String -> Maybe HostName -> Maybe PortNumber ->  Maybe (Maybe Handle -> Maybe Handle -> String -> FilePath -> IO ()) -> Annex ()
+startDaemon assistant foreground startdelay cannotrun listenhost listenport startbrowser = do
 	Annex.changeState $ \s -> s { Annex.daemon = True }
 	enableInteractiveBranchAccess
 	pidfile <- fromRepo gitAnnexPidFile
@@ -141,7 +141,7 @@ startDaemon assistant foreground startdelay cannotrun listenhost startbrowser =
 #endif
 		urlrenderer <- liftIO newUrlRenderer
 #ifdef WITH_WEBAPP
-		let webappthread = [ assist $ webAppThread d urlrenderer False cannotrun Nothing listenhost webappwaiter ]
+		let webappthread = [ assist $ webAppThread d urlrenderer False cannotrun Nothing listenhost listenport webappwaiter ]
 #else
 		let webappthread = []
 #endif
diff --git a/Assistant/MakeRepo.hs b/Assistant/MakeRepo.hs
index ca3be78a5f..d85eb45775 100644
--- a/Assistant/MakeRepo.hs
+++ b/Assistant/MakeRepo.hs
@@ -95,4 +95,4 @@ initRepo' desc mgroup = unlessM isInitialized $ do
 {- Checks if a git repo exists at a location. -}
 probeRepoExists :: FilePath -> IO Bool
 probeRepoExists dir = isJust <$>
-	catchDefaultIO Nothing (Git.Construct.checkForRepo dir)
+	catchDefaultIO Nothing (Git.Construct.checkForRepo (encodeBS dir))
diff --git a/Assistant/Threads/Watcher.hs b/Assistant/Threads/Watcher.hs
index 1f5ebf80a0..2df29ce76c 100644
--- a/Assistant/Threads/Watcher.hs
+++ b/Assistant/Threads/Watcher.hs
@@ -293,7 +293,7 @@ onAddSymlink' linktarget mk file filestatus = go mk
 			then ensurestaged (Just link) =<< getDaemonStatus
 			else do
 				liftAnnex $ replaceWorkTreeFile file $
-					makeAnnexLink link . toRawFilePath
+					makeAnnexLink link
 				addLink file link (Just key)
 	-- other symlink, not git-annex
 	go Nothing = ensurestaged linktarget =<< getDaemonStatus
diff --git a/Assistant/Threads/WebApp.hs b/Assistant/Threads/WebApp.hs
index f8b3a2b41e..3fdd12d05f 100644
--- a/Assistant/Threads/WebApp.hs
+++ b/Assistant/Threads/WebApp.hs
@@ -45,7 +45,7 @@ import Git
 import qualified Annex
 
 import Yesod
-import Network.Socket (SockAddr, HostName)
+import Network.Socket (SockAddr, HostName, PortNumber)
 import Data.Text (pack, unpack)
 import qualified Network.Wai.Handler.WarpTLS as TLS
 import Network.Wai.Middleware.RequestLogger
@@ -61,12 +61,16 @@ webAppThread
 	-> Maybe String
 	-> Maybe (IO Url)
 	-> Maybe HostName
+	-> Maybe PortNumber
 	-> Maybe (Url -> FilePath -> IO ())
 	-> NamedThread
-webAppThread assistantdata urlrenderer noannex cannotrun postfirstrun listenhost onstartup = thread $ liftIO $ do
+webAppThread assistantdata urlrenderer noannex cannotrun postfirstrun listenhost listenport onstartup = thread $ liftIO $ do
 	listenhost' <- if isJust listenhost
 		then pure listenhost
 		else getAnnex $ annexListen <$> Annex.getGitConfig
+	listenport' <- if isJust listenport
+		then pure listenport
+		else getAnnex $ annexPort <$> Annex.getGitConfig
 	tlssettings <- getAnnex getTlsSettings
 	webapp <- WebApp
 		<$> pure assistantdata
@@ -84,7 +88,7 @@ webAppThread assistantdata urlrenderer noannex cannotrun postfirstrun listenhost
 		( return $ logStdout app
 		, return app
 		)
-	runWebApp tlssettings listenhost' app' $ \addr -> if noannex
+	runWebApp tlssettings listenhost' listenport' app' $ \addr -> if noannex
 		then withTmpFile "webapp.html" $ \tmpfile h -> do
 			hClose h
 			go tlssettings addr webapp tmpfile Nothing
diff --git a/Assistant/TransferQueue.hs b/Assistant/TransferQueue.hs
index d2d245b7b1..571899bb6d 100644
--- a/Assistant/TransferQueue.hs
+++ b/Assistant/TransferQueue.hs
@@ -93,7 +93,7 @@ queueTransfersMatching matching reason schedule k f direction
 				filter (\r -> not (inset s r || Remote.readonly r))
 					(syncDataRemotes st)
 	  where
-		locs = S.fromList . map Remote.uuid <$> Remote.keyPossibilities k
+		locs = S.fromList . map Remote.uuid <$> Remote.keyPossibilities (Remote.IncludeIgnored False) k
 		inset s r = S.member (Remote.uuid r) s
 	gentransfer r = Transfer
 		{ transferDirection = direction
diff --git a/Assistant/TransferSlots.hs b/Assistant/TransferSlots.hs
index bf14118f64..c16871f468 100644
--- a/Assistant/TransferSlots.hs
+++ b/Assistant/TransferSlots.hs
@@ -33,6 +33,7 @@ import qualified Remote
 import qualified Types.Remote as Remote
 import Annex.Content
 import Annex.Wanted
+import Annex.StallDetection
 import Utility.Batch
 import Types.NumCopies
 
@@ -126,8 +127,7 @@ genTransfer t info = case transferRemote info of
 				qp <- liftAnnex $ coreQuotePath <$> Annex.getGitConfig
 				debug [ "Transferring:" , describeTransfer qp t info ]
 				notifyTransfer
-				let sd = remoteAnnexStallDetection
-					(Remote.gitconfig remote)
+				let sd = getStallDetection (transferDirection t) remote
 				return $ Just (t, info, go remote sd)
 			, do
 				qp <- liftAnnex $ coreQuotePath <$> Annex.getGitConfig
diff --git a/Assistant/WebApp/Configurators/Ssh.hs b/Assistant/WebApp/Configurators/Ssh.hs
index 86065b8fc4..04ac8ceb1d 100644
--- a/Assistant/WebApp/Configurators/Ssh.hs
+++ b/Assistant/WebApp/Configurators/Ssh.hs
@@ -319,7 +319,7 @@ testServer sshinput@(SshInput { inputHostname = Just hn }) = do
 		finduuid (k, v)
 			| k == "annex.uuid" = Just $ toUUID v
 			| k == fromConfigKey GCrypt.coreGCryptId =
-				Just $ genUUIDInNameSpace gCryptNameSpace v
+				Just $ genUUIDInNameSpace gCryptNameSpace (encodeBS v)
 			| otherwise = Nothing
 	
 	checkcommand c = "if which " ++ c ++ "; then " ++ report c ++ "; fi"
diff --git a/Assistant/WebApp/Gpg.hs b/Assistant/WebApp/Gpg.hs
index 3723d03907..79c36a2e0e 100644
--- a/Assistant/WebApp/Gpg.hs
+++ b/Assistant/WebApp/Gpg.hs
@@ -54,7 +54,7 @@ withNewSecretKey :: (KeyId -> Handler Html) -> Handler Html
 withNewSecretKey use = do
 	cmd <- liftAnnex $ gpgCmd <$> Annex.getGitConfig
 	userid <- liftIO $ newUserId cmd
-	liftIO $ genSecretKey cmd RSA "" userid maxRecommendedKeySize
+	liftIO $ genSecretKey cmd "" userid
 	results <- M.keys . M.filter (== userid) <$> liftIO (secretKeys cmd)
 	case results of
 		[] -> giveup "Failed to generate gpg key!"
diff --git a/Author.hs b/Author.hs
new file mode 100644
index 0000000000..281e8def9f
--- /dev/null
+++ b/Author.hs
@@ -0,0 +1,35 @@
+{- authorship made explicit in the code
+ -
+ - Copyright 2023 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU AGPL version 3 or higher.
+ -}
+
+{-# LANGUAGE FlexibleInstances, RankNTypes #-}
+{-# OPTIONS_GHC -fno-warn-tabs #-}
+
+module Author where
+
+data Author = JoeyHess
+
+-- This allows writing eg:
+--
+-- copyright = author JoeyHess 1999 :: Copyright
+type Copyright = forall t. Authored t => t
+
+class Authored t where
+	author:: Author -> Int -> t
+
+instance Authored Bool where
+	author _ year = year >= 2010
+	{-# INLINE author #-}
+
+instance Authored (a -> a) where
+	author by year f
+		| author by year = f
+		| otherwise = author by (pred year) f
+	{-# INLINE author #-}
+
+instance Monad m => Authored (a -> m a) where
+	author by year = pure . author by year
+	{-# INLINE author #-}
diff --git a/CHANGELOG b/CHANGELOG
index e7113df01c..9441bdbce7 100644
--- a/CHANGELOG
+++ b/CHANGELOG
@@ -1,3 +1,89 @@
+git-annex (10.20240129) upstream; urgency=medium
+
+  * info: Added "annex sizes of repositories" table to the overall display.
+  * import: Sped up import from special remotes.
+  * import: Added --message/-m option.
+  * Support using commands that implement the Stateless OpenPGP command line
+    interface, as an alternative to gpg.
+    Currently only supported for encryption=shared special remotes,
+    when annex.shared-sop-command is configured.
+  * test: Test a specified Stateless OpenPGP command when
+    run with eg --test-git-config annex.shared-sop-command=sqop
+  * Improve disk free space checking when transferring unsized keys to
+    local git remotes.
+  * Added configs annex.stalldetection-download, annex.stalldetection-upload,
+    annex.bwlimit-download, annex.bwlimit-upload,
+    and similar per-remote configs.
+  * Improve annex.stalldetection to handle remotes that update progress
+    less frequently than the configured time period.
+  * external: Monitor file size when getting content from external
+    special remotes and use that to update the progress meter,
+    in case the external special remote program does not report progress.
+  * Added --expected-present file matching option.
+  * webapp: Added --port option, and annex.port config.
+  * assistant: When generating a gpg secret key, avoid hardcoding the
+    key algorithm and size.
+
+ -- Joey Hess <id@joeyh.name>  Mon, 29 Jan 2024 13:12:00 -0400
+
+git-annex (10.20231227) upstream; urgency=medium
+
+  * migrate: Support distributed migrations by recording each migration,
+    and adding a --update option that updates the local repository
+    incrementally, hard linking annex objects to their new keys.
+  * pull, sync: When operating on content, automatically handle
+    distributed migrations.
+  * Added annex.syncmigrations config that can be set to false to prevent
+    pull and sync from migrating object content.
+  * migrate: Added --apply option that (re)applies all recorded 
+    distributed migrations to the objects in repository.
+  * migrate: Support adding size to URL keys that were added with
+    --relaxed, by running eg: git-annex migrate --backend=URL foo
+  * When importing from a special remote, support preferred content
+    expressions that use terms that match on keys (eg "present", "copies=1").
+    Such terms are ignored when importing, since the key is not known yet.
+    Before, such expressions caused the import to fail.
+  * Support git-annex copy/move --from-anywhere --to remote.
+  * Make git-annex get/copy/move --from foo override configuration of 
+    remote.foo.annex-ignore, as documented.
+  * Lower precision of timestamps in git-annex branch, which can reduce the
+    size of the branch by up to 8%.
+  * sync: Fix locking problems during merge when annex.pidlock is set.
+  * Avoid a problem with temp file names ending in "." on certian
+    filesystems that have problems with such filenames.
+  * sync, push: Avoid trying to send individual files to special remotes
+    configured with importtree=yes exporttree=no, which would always fail.
+  * Fix a crash opening sqlite databases when run in a non-unicode locale.
+    (Needs persistent-sqlite 2.13.3.)
+
+ -- Joey Hess <id@joeyh.name>  Wed, 27 Dec 2023 19:27:37 -0400
+
+git-annex (10.20231129) upstream; urgency=medium
+
+  * Fix bug in git-annex copy --from --to that skipped files that were
+    locally present.
+  * Make git-annex copy --from --to --fast actually fast.
+  * Fix crash of enableremote when the special remote has embedcreds=yes.
+  * Ignore directories and other unusual files in .git/annex/journal/
+  * info: Added calculation of combined annex size of all repositories.
+  * log: Added options --sizesof, --sizes and --totalsizes that
+    display how the size of repositories changed over time.
+  * log: Added options --interval, --bytes, --received, and --gnuplot
+    to tune the output of the above added options.
+  * findkeys: Support --largerthan and --smallerthan.
+  * importfeed: Use caching database to avoid needing to list urls
+    on every run, and avoid using too much memory.
+  * Improve memory use of --all when using annex.private.
+  * lookupkey: Sped up --batch.
+  * Windows: Consistently avoid ending standard output lines with CR.
+    This matches the behavior of git on Windows.
+  * Windows: Fix CRLF handling in some log files.
+  * Windows: When git-annex init is installing hook scripts, it will
+    avoid ending lines with CR for portability. Existing hook scripts
+    that do have CR line endings will not be changed.
+
+ -- Joey Hess <id@joeyh.name>  Wed, 29 Nov 2023 15:59:20 -0400
+
 git-annex (10.20230926) upstream; urgency=medium
 
   * Fix more breakage caused by git's fix for CVE-2022-24765, this time
diff --git a/COPYRIGHT b/COPYRIGHT
index 039bb805f6..afc2fe99e7 100644
--- a/COPYRIGHT
+++ b/COPYRIGHT
@@ -2,7 +2,7 @@ Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
 Source: native package
 
 Files: *
-Copyright: © 2010-2023 Joey Hess <id@joeyh.name>
+Copyright: © 2010-2024 Joey Hess <id@joeyh.name>
 License: AGPL-3+
 
 Files: Assistant/WebApp.hs Assistant/WebApp/* templates/* static/*
@@ -68,6 +68,11 @@ Files: Annex/DirHashes.hs
 Copyright: © 2010-2017 Joey Hess <id@joeyh.name>
 License: GPL-3+
 
+Files: Database/RawFilePath.hs
+Copyright: © 2012 Michael Snoyman, http://www.yesodweb.com/
+           © 2023 Joey Hess <id@joeyh.name>
+License: Expat
+
 Files: doc/tips/automatically_adding_metadata/pre-commit-annex 
 Copyright: 2014 Joey Hess <id@joeyh.name>
            2016 Klaus Ethgen <Klaus@Ethgen.ch>
diff --git a/CmdLine/Batch.hs b/CmdLine/Batch.hs
index 323f1d9f6e..2a7924ab2b 100644
--- a/CmdLine/Batch.hs
+++ b/CmdLine/Batch.hs
@@ -164,9 +164,8 @@ batchFilesKeys fmt a = do
 	matcher <- getMatcher
 	go $ \si v -> case v of
 		Right f -> 
-			let f' = toRawFilePath f
-			in ifM (matcher $ MatchingFile $ FileInfo f' f' Nothing)
-				( a (si, Right f')
+			ifM (matcher $ MatchingFile $ FileInfo f f Nothing)
+				( a (si, Right f)
 				, return Nothing
 				)
 		Left k -> a (si, Left k)
@@ -177,7 +176,7 @@ batchFilesKeys fmt a = do
 		-- because in non-batch mode, that is done when
 		-- CmdLine.Seek uses git ls-files.
 		BatchFormat _ (BatchKeys False) -> 
-			Right . Right . fromRawFilePath 
+			Right . Right
 				<$$> liftIO . relPathCwdToFile . toRawFilePath
 		BatchFormat _ (BatchKeys True) -> \i ->
 			pure $ case deserializeKey i of
@@ -199,7 +198,7 @@ batchAnnexed fmt seeker keyaction = do
 			Right f -> lookupKeyStaged f >>= \case
 				Nothing -> return Nothing
 				Just k -> checkpresent k $
-					startAction seeker si f k
+					startAction seeker Nothing si f k
 			Left k -> ifM (matcher (MatchingInfo (mkinfo k)))
 				( checkpresent k $
 					keyaction (si, k, mkActionItem k)
diff --git a/CmdLine/GitAnnex/Options.hs b/CmdLine/GitAnnex/Options.hs
index 92649fae34..694b60d240 100644
--- a/CmdLine/GitAnnex/Options.hs
+++ b/CmdLine/GitAnnex/Options.hs
@@ -162,40 +162,55 @@ parseToOption = strOption
 	<> completeRemotes
 	)
 
+parseFromAnywhereOption :: Parser Bool
+parseFromAnywhereOption = switch
+	( long "from-anywhere"
+	<> help "from any remote"
+	)
+
 parseRemoteOption :: Parser RemoteName
 parseRemoteOption = strOption
 	( long "remote" <> metavar paramRemote
 	<> completeRemotes
 	)
 
--- | From or to a remote, or both, or a special --to=here
+-- | --from or --to a remote, or both, or a special --to=here,
+-- or --from-anywhere --to remote.
 data FromToHereOptions 
 	= FromOrToRemote FromToOptions
 	| ToHere
 	| FromRemoteToRemote (DeferredParse Remote) (DeferredParse Remote)
+	| FromAnywhereToRemote (DeferredParse Remote)
 
 parseFromToHereOptions :: Parser (Maybe FromToHereOptions)
 parseFromToHereOptions = go
 	<$> optional parseFromOption
 	<*> optional parseToOption
+	<*> parseFromAnywhereOption
   where
-	go (Just from) (Just to) = Just $ FromRemoteToRemote
+	go _ (Just to) True = Just $ FromAnywhereToRemote
+		(mkParseRemoteOption to)
+	go (Just from) (Just to) _ = Just $ FromRemoteToRemote
 		(mkParseRemoteOption from)
 		(mkParseRemoteOption to)
-	go (Just from) Nothing = Just $ FromOrToRemote
+	go (Just from) Nothing _ = Just $ FromOrToRemote
 		(FromRemote $ mkParseRemoteOption from)
-	go Nothing (Just to) = Just $ case to of
+	go Nothing (Just to) _ = Just $ case to of
 		"here" -> ToHere
 		"." -> ToHere
 		_ -> FromOrToRemote $ ToRemote $ mkParseRemoteOption to
-	go Nothing Nothing = Nothing
+	go Nothing Nothing _ = Nothing
 
 instance DeferredParseClass FromToHereOptions where
-	finishParse (FromOrToRemote v) = FromOrToRemote <$> finishParse v
+	finishParse (FromOrToRemote v) = 
+		FromOrToRemote <$> finishParse v
 	finishParse ToHere = pure ToHere
-	finishParse (FromRemoteToRemote v1 v2) = FromRemoteToRemote
-		<$> finishParse v1
-		<*> finishParse v2
+	finishParse (FromRemoteToRemote v1 v2) = 
+		FromRemoteToRemote
+			<$> finishParse v1
+			<*> finishParse v2
+	finishParse (FromAnywhereToRemote v) = 
+		FromAnywhereToRemote <$> finishParse v
 
 -- Options for acting on keys, rather than work tree files.
 data KeyOptions
@@ -266,6 +281,7 @@ annexedMatchingOptions = concat
 keyMatchingOptions :: [AnnexOption]
 keyMatchingOptions = concat
 	[ keyMatchingOptions'
+	, sizeMatchingOptions Limit.LimitAnnexFiles
 	, anythingNothingOptions
 	, combiningOptions 
 	, timeLimitOption 
@@ -277,7 +293,7 @@ keyMatchingOptions' :: [AnnexOption]
 keyMatchingOptions' = 
 	[ annexOption (setAnnexState . Limit.addIn) $ strOption
 		( long "in" <> short 'i' <> metavar paramRemote
-		<> help "match files present in a remote"
+		<> help "match files present in a repository"
 		<> hidden
 		<> completeRemotes
 		)
@@ -370,6 +386,11 @@ keyMatchingOptions' =
 		<> help "match files that are locked"
 		<> hidden
 		)
+	, annexFlag (setAnnexState Limit.addExpectedPresent)
+		( long "expected-present"
+		<> help "match files expected to be present"
+		<> hidden
+		)
 	]
 
 -- Options to match files which may not yet be annexed.
@@ -398,7 +419,11 @@ fileMatchingOptions' lb =
 		<> help "limit to files whose content is the same as another file matching the glob pattern"
 		<> hidden
 		)
-	, annexOption (setAnnexState . Limit.addLargerThan lb) $ strOption
+	] ++ sizeMatchingOptions lb
+
+sizeMatchingOptions :: Limit.LimitBy -> [AnnexOption]
+sizeMatchingOptions lb =
+	[ annexOption (setAnnexState . Limit.addLargerThan lb) $ strOption
 		( long "largerthan" <> metavar paramSize
 		<> help "match files larger than a size"
 		<> hidden
diff --git a/CmdLine/Seek.hs b/CmdLine/Seek.hs
index 18be0a44f7..620ff81da3 100644
--- a/CmdLine/Seek.hs
+++ b/CmdLine/Seek.hs
@@ -4,7 +4,7 @@
  - the values a user passes to a command, and prepare actions operating
  - on them.
  -
- - Copyright 2010-2022 Joey Hess <id@joeyh.name>
+ - Copyright 2010-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -58,11 +58,14 @@ import System.PosixCompat.Files (isDirectory, isSymbolicLink, deviceID, fileID)
 import qualified System.FilePath.ByteString as P
 
 data AnnexedFileSeeker = AnnexedFileSeeker
-	{ startAction :: SeekInput -> RawFilePath -> Key -> CommandStart
+	{ startAction :: Maybe KeySha -> SeekInput -> RawFilePath -> Key -> CommandStart
 	, checkContentPresent :: Maybe Bool
 	, usesLocationLog :: Bool
 	}
 
+-- The Sha that was read to get the Key.
+newtype KeySha = KeySha Git.Sha
+
 withFilesInGitAnnex :: WarnUnmatchWhen -> AnnexedFileSeeker -> WorkTreeItems -> CommandSeek
 withFilesInGitAnnex ww a l = seekFilteredKeys a $
 	seekHelper fst3 ww LsFiles.inRepoDetails l
@@ -276,24 +279,14 @@ withKeyOptions' ko auto mkkeyaction fallbackaction worktreeitems = do
 	-- those. This significantly speeds up typical operations
 	-- that need to look at the location log for each key.
 	runallkeys = do
-		checktimelimit <- mkCheckTimeLimit
 		keyaction <- mkkeyaction
-		config <- Annex.getGitConfig
-		
-		let getk = locationLogFileKey config
+		checktimelimit <- mkCheckTimeLimit
 		let discard reader = reader >>= \case
 			Nothing -> noop
 			Just _ -> discard reader
-		let go reader = reader >>= \case
-			Just (k, f, content) -> checktimelimit (discard reader) $ do
-				maybe noop (Annex.Branch.precache f) content
-				unlessM (checkDead k) $
-					keyaction Nothing (SeekInput [], k, mkActionItem k)
-				go reader
-			Nothing -> return ()
-		Annex.Branch.overBranchFileContents getk go >>= \case
-			Just r -> return r
-			Nothing -> giveup "This repository is read-only, and there are unmerged git-annex branches, which prevents operating on all keys. (Set annex.merge-annex-branches to false to ignore the unmerged git-annex branches.)"
+		overLocationLogs' () 
+			(\reader cont -> checktimelimit (discard reader) cont) 
+			(\k _ () -> keyaction Nothing (SeekInput [], k, mkActionItem k))
 
 	runkeyaction getks = do
 		keyaction <- mkkeyaction
@@ -385,9 +378,9 @@ seekFilteredKeys seeker listfs = do
 	propagateLsFilesError cleanup
   where
 	finisher mi oreader checktimelimit = liftIO oreader >>= \case
-		Just ((si, f), content) -> checktimelimit (liftIO discard) $ do
-			keyaction f mi content $ 
-				commandAction . startAction seeker si f
+		Just ((si, f, keysha), content) -> checktimelimit (liftIO discard) $ do
+			keyaction f mi content $
+				commandAction . startAction seeker keysha si f
 			finisher mi oreader checktimelimit
 		Nothing -> return ()
 	  where
@@ -396,12 +389,12 @@ seekFilteredKeys seeker listfs = do
 			Just _ -> discard
 
 	precachefinisher mi lreader checktimelimit = liftIO lreader >>= \case
-		Just ((logf, (si, f), k), logcontent) -> checktimelimit (liftIO discard) $ do
+		Just ((logf, (si, f, keysha), k), logcontent) -> checktimelimit (liftIO discard) $ do
 			maybe noop (Annex.Branch.precache logf) logcontent
 			checkMatcherWhen mi
 				(matcherNeedsLocationLog mi && not (matcherNeedsFileName mi))
 				(MatchingFile $ FileInfo f f (Just k))
-				(commandAction $ startAction seeker si f k)
+				(commandAction $ startAction seeker keysha si f k)
 			precachefinisher mi lreader checktimelimit
 		Nothing -> return ()
 	  where
@@ -410,11 +403,11 @@ seekFilteredKeys seeker listfs = do
 			Just _ -> discard
 	
 	precacher mi config oreader lfeeder lcloser = liftIO oreader >>= \case
-		Just ((si, f), content) -> do
+		Just ((si, f, keysha), content) -> do
 			keyaction f mi content $ \k -> 
 				let logf = locationLogFile config k
 				    ref = Git.Ref.branchFileRef Annex.Branch.fullname logf
-				in liftIO $ lfeeder ((logf, (si, f), k), ref)
+				in liftIO $ lfeeder ((logf, (si, f, keysha), k), ref)
 			precacher mi config oreader lfeeder lcloser
 		Nothing -> liftIO $ void lcloser
 	
@@ -425,7 +418,7 @@ seekFilteredKeys seeker listfs = do
 		(not ((matcherNeedsKey mi || matcherNeedsLocationLog mi) 
 			&& not (matcherNeedsFileName mi)))
 		(MatchingFile $ FileInfo f f Nothing)
-		(liftIO $ ofeeder ((si, f), sha))
+		(liftIO $ ofeeder ((si, f, Just (KeySha sha)), sha))
 
 	keyaction f mi content a = 
 		case parseLinkTargetOrPointerLazy =<< content of
diff --git a/Command/AddUrl.hs b/Command/AddUrl.hs
index 842a854831..d98c1f95c0 100644
--- a/Command/AddUrl.hs
+++ b/Command/AddUrl.hs
@@ -439,7 +439,7 @@ downloadWith canadd addunlockedmatcher downloader dummykey u url file =
  - the returned location. -}
 downloadWith' :: (FilePath -> MeterUpdate -> Annex Bool) -> Key -> UUID -> URLString -> RawFilePath -> Annex (Maybe (RawFilePath, Backend))
 downloadWith' downloader dummykey u url file =
-	checkDiskSpaceToGet dummykey Nothing $ do
+	checkDiskSpaceToGet dummykey Nothing Nothing $ do
 		backend <- chooseBackend file
 		tmp <- fromRepo $ gitAnnexTmpObjectLocation dummykey
 		let t = (Transfer.Transfer Transfer.Download u (fromKey id dummykey))
diff --git a/Command/CheckPresentKey.hs b/Command/CheckPresentKey.hs
index f3b4ef921c..efefc71aeb 100644
--- a/Command/CheckPresentKey.hs
+++ b/Command/CheckPresentKey.hs
@@ -51,7 +51,7 @@ check :: String -> Maybe Remote -> Annex Result
 check ks mr = case mr of
 	Just r -> go Nothing [r]
 	Nothing -> do
-		mostlikely <- Remote.keyPossibilities k
+		mostlikely <- Remote.keyPossibilities (Remote.IncludeIgnored False) k
 		otherremotes <- flip Remote.remotesWithoutUUID 
 			(map Remote.uuid mostlikely)
 			<$> remoteList
diff --git a/Command/Copy.hs b/Command/Copy.hs
index 0034bb26cb..8fe19e2ff9 100644
--- a/Command/Copy.hs
+++ b/Command/Copy.hs
@@ -63,12 +63,13 @@ seek' o fto = startConcurrency (Command.Move.stages fto) $ do
 	ww = WarnUnmatchLsFiles "copy"
 	
 	seeker = AnnexedFileSeeker
-		{ startAction = start o fto
+		{ startAction = const $ start o fto
 		, checkContentPresent = case fto of
 			FromOrToRemote (FromRemote _) -> Just False
 			FromOrToRemote (ToRemote _) -> Just True
 			ToHere -> Just False
-			FromRemoteToRemote _ _ -> Just False
+			FromRemoteToRemote _ _ -> Nothing
+			FromAnywhereToRemote _ -> Nothing
 		, usesLocationLog = True
 		}
 	keyaction = Command.Move.startKey fto Command.Move.RemoveNever
@@ -84,12 +85,13 @@ start o fto si file key = stopUnless shouldCopy $
 		| autoMode o = want <||> numCopiesCheck file key (<)
 		| otherwise = return True
 	want = case fto of
-		FromOrToRemote (ToRemote dest) ->
-			(Remote.uuid <$> getParsed dest) >>= checkwantsend
+		FromOrToRemote (ToRemote dest) -> checkwantsend dest
 		FromOrToRemote (FromRemote _) -> checkwantget
 		ToHere -> checkwantget
-		FromRemoteToRemote _ dest ->
-			(Remote.uuid <$> getParsed dest) >>= checkwantsend
-			
-	checkwantsend = wantGetBy False (Just key) (AssociatedFile (Just file))
+		FromRemoteToRemote _ dest -> checkwantsend dest
+		FromAnywhereToRemote dest -> checkwantsend dest
+
+	checkwantsend dest = 
+		(Remote.uuid <$> getParsed dest) >>=
+			wantGetBy False (Just key) (AssociatedFile (Just file))
 	checkwantget = wantGet False (Just key) (AssociatedFile (Just file))
diff --git a/Command/Drop.hs b/Command/Drop.hs
index 0de97c72e0..2fcb5d6ebb 100644
--- a/Command/Drop.hs
+++ b/Command/Drop.hs
@@ -60,7 +60,7 @@ seek o = startConcurrency commandStages $ do
 				then pure Nothing
 				else pure (Just remote)				
 	let seeker = AnnexedFileSeeker
-		{ startAction = start o from
+		{ startAction = const $ start o from
 		, checkContentPresent = case from of
 			Nothing -> Just True
 			Just _ -> Nothing
diff --git a/Command/Expire.hs b/Command/Expire.hs
index 1971f84562..7cbf0a8230 100644
--- a/Command/Expire.hs
+++ b/Command/Expire.hs
@@ -73,7 +73,7 @@ start (Expire expire) noact actlog descs u =
 					trustSet u DeadTrusted
 				next $ return True
   where
-	lastact = changed <$> M.lookup u actlog
+	lastact = changed <$> M.lookup u (fromMapLog actlog)
 	whenactive = case lastact of
 		Just (VectorClock c) -> do
 			d <- liftIO $ durationSince $ posixSecondsToUTCTime c
diff --git a/Command/Export.hs b/Command/Export.hs
index 779d300ff9..bca9c9ffed 100644
--- a/Command/Export.hs
+++ b/Command/Export.hs
@@ -304,7 +304,7 @@ performExport r db ek af contentsha loc allfilledvar = do
 				alwaysUpload (uuid r) ek af Nothing stdRetry $ \pm -> do
 					let rollback = void $
 						performUnexport r db [ek] loc
-					sendAnnex ek rollback $ \f ->
+					sendAnnex ek rollback $ \f _sz ->
 						Remote.action $
 							storer f ek loc pm
 			, do
diff --git a/Command/FilterBranch.hs b/Command/FilterBranch.hs
index 0e1b0a7514..10f03cccda 100644
--- a/Command/FilterBranch.hs
+++ b/Command/FilterBranch.hs
@@ -157,7 +157,7 @@ seek o = withOtherTmp $ \tmpdir -> do
 					=<< Annex.Branch.get f
 			next (return True)
 		let seeker = AnnexedFileSeeker
-			{ startAction = \_ _ k -> addkeyinfo k
+			{ startAction = \_ _ _ k -> addkeyinfo k
 			, checkContentPresent = Nothing
 			, usesLocationLog = True
 			}
diff --git a/Command/Find.hs b/Command/Find.hs
index d5971e3875..3a1fabe5e2 100644
--- a/Command/Find.hs
+++ b/Command/Find.hs
@@ -63,7 +63,7 @@ seek o = do
 		checkNotBareRepo
 	isterminal <- liftIO $ checkIsTerminal stdout
 	seeker <- contentPresentUnlessLimited $ AnnexedFileSeeker
-		{ startAction = start o isterminal
+		{ startAction = const (start o isterminal)
 		, checkContentPresent = Nothing
 		, usesLocationLog = False
 		}
diff --git a/Command/FindKeys.hs b/Command/FindKeys.hs
index e24075dacb..f105d7f8b9 100644
--- a/Command/FindKeys.hs
+++ b/Command/FindKeys.hs
@@ -33,7 +33,7 @@ seek o = do
 		, usesLocationLog = False
 		-- startAction is not actually used since this
 		-- is not used to seek files
-		, startAction = \_ _ key -> start' o isterminal key
+		, startAction = \_ _ _ key -> start' o isterminal key
 		}
 	withKeyOptions (Just WantAllKeys) False seeker
 		(commandAction . start o isterminal)
diff --git a/Command/Fix.hs b/Command/Fix.hs
index 28c8cfa9be..862853a861 100644
--- a/Command/Fix.hs
+++ b/Command/Fix.hs
@@ -37,7 +37,7 @@ seek ps = unlessM crippledFileSystem $
   where
 	ww = WarnUnmatchLsFiles "fix"
 	seeker = AnnexedFileSeeker
-		{ startAction = start FixAll
+		{ startAction = const $ start FixAll
 		, checkContentPresent = Nothing
 		, usesLocationLog = False
 		}
@@ -73,12 +73,11 @@ start fixwhat si file key = do
 breakHardLink :: RawFilePath -> Key -> RawFilePath -> CommandPerform
 breakHardLink file key obj = do
 	replaceWorkTreeFile (fromRawFilePath file) $ \tmp -> do
-		let tmp' = toRawFilePath tmp
 		mode <- liftIO $ catchMaybeIO $ fileMode <$> R.getFileStatus file
-		unlessM (checkedCopyFile key obj tmp' mode) $
+		unlessM (checkedCopyFile key obj tmp mode) $
 			giveup "unable to break hard link"
-		thawContent tmp'
-		Database.Keys.storeInodeCaches key [tmp']
+		thawContent tmp
+		Database.Keys.storeInodeCaches key [tmp]
 		modifyContentDir obj $ freezeContent obj
 	next $ return True
 
@@ -86,7 +85,7 @@ makeHardLink :: RawFilePath -> Key -> CommandPerform
 makeHardLink file key = do
 	replaceWorkTreeFile (fromRawFilePath file) $ \tmp -> do
 		mode <- liftIO $ catchMaybeIO $ fileMode <$> R.getFileStatus file
-		linkFromAnnex' key (toRawFilePath tmp) mode >>= \case
+		linkFromAnnex' key tmp mode >>= \case
 			LinkAnnexFailed -> giveup "unable to make hard link"
 			_ -> noop
 	next $ return True
@@ -99,10 +98,9 @@ fixSymlink file link = do
 		<$> R.getSymbolicLinkStatus file
 #endif
 	replaceWorkTreeFile (fromRawFilePath file) $ \tmpfile -> do
-		let tmpfile' = toRawFilePath tmpfile
-		liftIO $ R.createSymbolicLink link tmpfile'
+		liftIO $ R.createSymbolicLink link tmpfile
 #if ! defined(mingw32_HOST_OS)
-		liftIO $ maybe noop (\t -> touch tmpfile' t False) mtime
+		liftIO $ maybe noop (\t -> touch tmpfile t False) mtime
 #endif
 	stageSymlink file =<< hashSymlink link
 	next $ return True
diff --git a/Command/Fsck.hs b/Command/Fsck.hs
index b25e49b73e..0a8ac45d9c 100644
--- a/Command/Fsck.hs
+++ b/Command/Fsck.hs
@@ -102,7 +102,7 @@ seek o = startConcurrency commandStages $ do
 	checkDeadRepo u
 	i <- prepIncremental u (incrementalOpt o)
 	let seeker = AnnexedFileSeeker
-		{ startAction = start from i
+		{ startAction = const $ start from i
 		, checkContentPresent = Nothing
 		, usesLocationLog = True
 		}
@@ -195,7 +195,7 @@ performRemote key afile backend numcopies remote =
 		let cleanup = liftIO $ catchIO (R.removeLink tmp) (const noop)
 		cleanup
 		cleanup `after` a tmp
-	getfile tmp = ifM (checkDiskSpace (Just (P.takeDirectory tmp)) key 0 True)
+	getfile tmp = ifM (checkDiskSpace Nothing (Just (P.takeDirectory tmp)) key 0 True)
 		( ifM (getcheap tmp)
 			( return (Just (Right UnVerified))
 			, ifM (Annex.getRead Annex.fast)
@@ -417,16 +417,15 @@ verifyWorkTree key file = do
 		Just k | k == key -> whenM (inAnnex key) $ do
 			showNote "fixing worktree content"
 			replaceWorkTreeFile (fromRawFilePath file) $ \tmp -> do
-				let tmp' = toRawFilePath tmp
 				mode <- liftIO $ catchMaybeIO $ fileMode <$> R.getFileStatus file
 				ifM (annexThin <$> Annex.getGitConfig)
-					( void $ linkFromAnnex' key tmp' mode
+					( void $ linkFromAnnex' key tmp mode
 					, do
 						obj <- calcRepo (gitAnnexLocation key)
-						void $ checkedCopyFile key obj tmp' mode
-						thawContent tmp'
+						void $ checkedCopyFile key obj tmp mode
+						thawContent tmp
 					)
-				Database.Keys.storeInodeCaches key [tmp']
+				Database.Keys.storeInodeCaches key [tmp]
 		_ -> return ()
 	return True
 
diff --git a/Command/GCryptSetup.hs b/Command/GCryptSetup.hs
index fe2f06ad9b..ae17702cce 100644
--- a/Command/GCryptSetup.hs
+++ b/Command/GCryptSetup.hs
@@ -29,7 +29,7 @@ start gcryptid = starting "gcryptsetup" (ActionItemOther Nothing) (SeekInput [gc
 	
 	g <- gitRepo
 	gu <- Remote.GCrypt.getGCryptUUID True g
-	let newgu = genUUIDInNameSpace gCryptNameSpace gcryptid
+	let newgu = genUUIDInNameSpace gCryptNameSpace (encodeBS gcryptid)
 	if isNothing gu || gu == Just newgu
 		then if Git.repoIsLocalBare g
 			then do
diff --git a/Command/Get.hs b/Command/Get.hs
index 7d3d4a2ef1..c8f76568ed 100644
--- a/Command/Get.hs
+++ b/Command/Get.hs
@@ -41,7 +41,7 @@ seek :: GetOptions -> CommandSeek
 seek o = startConcurrency transferStages $ do
 	from <- maybe (pure Nothing) (Just <$$> getParsed) (getFrom o)
 	let seeker = AnnexedFileSeeker
-		{ startAction = start o from
+		{ startAction = const $ start o from
 		, checkContentPresent = Just False
 		, usesLocationLog = True
 		}
@@ -87,7 +87,8 @@ perform key afile = stopUnless (getKey key afile) $
 {- Try to find a copy of the file in one of the remotes,
  - and copy it to here. -}
 getKey :: Key -> AssociatedFile -> Annex Bool
-getKey key afile = getKey' key afile =<< Remote.keyPossibilities key
+getKey key afile = getKey' key afile
+	=<< Remote.keyPossibilities (Remote.IncludeIgnored False) key
 
 getKey' :: Key -> AssociatedFile -> [Remote] -> Annex Bool
 getKey' key afile = dispatch
diff --git a/Command/Import.hs b/Command/Import.hs
index b3cf4d4ad1..a37064eefc 100644
--- a/Command/Import.hs
+++ b/Command/Import.hs
@@ -70,6 +70,7 @@ data ImportOptions
 		, importToSubDir :: Maybe FilePath
 		, importContent :: Bool
 		, checkGitIgnoreOption :: CheckGitIgnore
+		, messageOption :: Maybe String
 		}
 
 optParser :: CmdParamsDesc -> Parser ImportOptions
@@ -81,7 +82,11 @@ optParser desc = do
 		)
 	dupmode <- fromMaybe Default <$> optional duplicateModeParser
 	ic <- Command.Add.checkGitIgnoreSwitch
-	return $ case mfromremote of
+	message <- optional (strOption
+		( long "message" <> short 'm' <> metavar "MSG"
+		<> help "commit message"
+		))
+	pure $ case mfromremote of
 		Nothing -> LocalImportOptions ps dupmode ic
 		Just r -> case ps of
 			[bs] -> 
@@ -91,6 +96,7 @@ optParser desc = do
 					(if null subdir then Nothing else Just subdir)
 					content
 					ic
+					message
 			_ -> giveup "expected BRANCH[:SUBDIR]"
 
 data DuplicateMode = Default | Duplicate | DeDuplicate | CleanDuplicates | SkipDuplicates | ReinjectDuplicates
@@ -141,7 +147,9 @@ seek o@(RemoteImportOptions {}) = startConcurrency commandStages $ do
 		(pure Nothing)
 		(Just <$$> inRepo . toTopFilePath . toRawFilePath)
 		(importToSubDir o)
-	seekRemote r (importToBranch o) subdir (importContent o) (checkGitIgnoreOption o)
+	seekRemote r (importToBranch o) subdir (importContent o) 
+		(checkGitIgnoreOption o)
+		(messageOption o)
 
 startLocal :: ImportOptions -> AddUnlockedMatcher -> GetFileMatcher -> DuplicateMode -> (RawFilePath, RawFilePath) -> CommandStart
 startLocal o addunlockedmatcher largematcher mode (srcfile, destfile) =
@@ -314,8 +322,8 @@ verifyExisting key destfile (yes, no) = do
 	verifyEnoughCopiesToDrop [] key Nothing needcopies mincopies [] preverified tocheck
 		(const yes) no
 
-seekRemote :: Remote -> Branch -> Maybe TopFilePath -> Bool -> CheckGitIgnore -> CommandSeek
-seekRemote remote branch msubdir importcontent ci = do
+seekRemote :: Remote -> Branch -> Maybe TopFilePath -> Bool -> CheckGitIgnore -> Maybe String -> CommandSeek
+seekRemote remote branch msubdir importcontent ci mimportmessage = do
 	importtreeconfig <- case msubdir of
 		Nothing -> return ImportTree
 		Just subdir ->
@@ -345,7 +353,9 @@ seekRemote remote branch msubdir importcontent ci = do
 				includeCommandAction $ 
 					commitimport imported
   where
-	importmessage = "import from " ++ Remote.name remote
+	importmessage = fromMaybe 
+		("import from " ++ Remote.name remote)
+		mimportmessage
 
 	tb = mkRemoteTrackingBranch remote branch
 
diff --git a/Command/ImportFeed.hs b/Command/ImportFeed.hs
index 3122c8f601..98a328bff0 100644
--- a/Command/ImportFeed.hs
+++ b/Command/ImportFeed.hs
@@ -48,9 +48,8 @@ import Logs.MetaData
 import Annex.MetaData
 import Annex.FileMatcher
 import Annex.UntrustedFilePath
-import qualified Annex.Branch
-import Logs
 import qualified Utility.RawFilePath as R
+import qualified Database.ImportFeed as Db
 
 cmd :: Command
 cmd = notBareRepo $ withAnnexOptions os $
@@ -202,53 +201,25 @@ data DownloadLocation = Enclosure URLString | MediaLink URLString
 type ItemId = String
 
 data Cache = Cache
-	{ knownurls :: S.Set URLString
-	, knownitems :: S.Set ItemId
+	{ dbhandle :: Maybe Db.ImportFeedDbHandle 
 	, template :: Utility.Format.Format
 	}
 
 getCache :: Maybe String -> Annex Cache
 getCache opttemplate = ifM (Annex.getRead Annex.force)
-	( ret S.empty S.empty
+	( ret Nothing
 	, do
 		j <- jsonOutputEnabled
 		unless j $
 			showStartMessage (StartMessage "importfeed" (ActionItemOther (Just "gathering known urls")) (SeekInput []))
-		(us, is) <- knownItems
+		h <- Db.openDb
 		unless j
 			showEndOk
-		ret (S.fromList us) (S.fromList is)
+		ret (Just h)
 	)
   where
 	tmpl = Utility.Format.gen $ fromMaybe defaultTemplate opttemplate
-	ret us is = return $ Cache us is tmpl
-
-{- Scan all url logs and metadata logs in the branch and find urls
- - and ItemIds that are already known. -}
-knownItems :: Annex ([URLString], [ItemId])
-knownItems = Annex.Branch.overBranchFileContents select (go [] []) >>= \case
-		Just r -> return r
-		Nothing -> giveup "This repository is read-only."
-  where
-	select f
-		| isUrlLog f = Just ()
-		| isMetaDataLog f = Just ()
-		| otherwise = Nothing
-
-	go uc ic reader = reader >>= \case
-		Just ((), f, Just content)
-			| isUrlLog f -> case parseUrlLog content of
-				[] -> go uc ic reader
-				us -> go (us++uc) ic reader
-			| isMetaDataLog f ->
-				let s = currentMetaDataValues itemIdField $
-					parseCurrentMetaData content
-				in if S.null s
-					then go uc ic reader
-					else go uc (map (decodeBS . fromMetaValue) (S.toList s)++ic) reader
-			| otherwise -> go uc ic reader
-		Just ((), _, Nothing) -> go uc ic reader
-		Nothing -> return (uc, ic)
+	ret h = return $ Cache h tmpl
 
 findDownloads :: URLString -> Feed -> [ToDownload]
 findDownloads u f = catMaybes $ map mk (feedItems f)
@@ -285,14 +256,20 @@ startDownload addunlockedmatcher opts cache cv todownload = case location todown
 				, downloadmedia linkurl mediaurl mediakey
 				)
   where
-	forced = Annex.getRead Annex.force
-
 	{- Avoids downloading any items that are already known to be
-	 - associated with a file in the annex, unless forced. -}
-	checkknown url a
-		| knownitemid || S.member url (knownurls cache)
-			= ifM forced (a, nothingtodo)
-		| otherwise = a
+	 - associated with a file in the annex. -}
+	checkknown url a = case dbhandle cache of
+		Just db -> ifM (liftIO $ Db.isKnownUrl db url)
+			( nothingtodo
+			, case getItemId (item todownload) of
+				Just (_, itemid) ->
+					ifM (liftIO $ Db.isKnownItemId db (fromFeedText itemid))
+						( nothingtodo
+						, a
+						)
+				_ -> a
+			)
+		Nothing -> a
 
 	nothingtodo = recordsuccess >> stop
 
@@ -302,11 +279,6 @@ startDownload addunlockedmatcher opts cache cv todownload = case location todown
 	startdownloadenclosure url = checkknown url $ startUrlDownload cv todownload url $
 		downloadEnclosure addunlockedmatcher opts cache cv todownload url 
 
-	knownitemid = case getItemId (item todownload) of
-		Just (_, itemid) ->
-			S.member (decodeBS $ fromFeedText itemid) (knownitems cache)
-		_ -> False
-
 	downloadmedia linkurl mediaurl mediakey
 		| rawOption (downloadOptions opts) = startdownloadlink
 		| otherwise = ifM (youtubeDlSupported linkurl)
@@ -555,9 +527,6 @@ extractFields i = map (uncurry extractField)
 	feedauthor = decodeBS . fromFeedText <$> getFeedAuthor (feed i)
 	itemauthor = decodeBS . fromFeedText <$> getItemAuthor (item i)
 
-itemIdField :: MetaField
-itemIdField = mkMetaFieldUnchecked "itemid"
-
 extractField :: String -> [Maybe String] -> (String, String)
 extractField k [] = (k, noneValue)
 extractField k (Just v:_)
diff --git a/Command/Info.hs b/Command/Info.hs
index f487f8db19..2c989a91be 100644
--- a/Command/Info.hs
+++ b/Command/Info.hs
@@ -89,12 +89,13 @@ data StatInfo = StatInfo
 	{ presentData :: Maybe KeyInfo
 	, referencedData :: Maybe KeyInfo
 	, repoData :: M.Map UUID KeyInfo
+	, allRepoData :: Maybe KeyInfo
 	, numCopiesStats :: Maybe NumCopiesStats
 	, infoOptions :: InfoOptions
 	}
 
 emptyStatInfo :: InfoOptions -> StatInfo
-emptyStatInfo = StatInfo Nothing Nothing M.empty Nothing
+emptyStatInfo = StatInfo Nothing Nothing M.empty Nothing Nothing
 
 -- a state monad for running Stats in
 type StatState = StateT StatInfo Annex
@@ -281,8 +282,10 @@ global_slow_stats =
 	, local_annex_size
 	, known_annex_files True
 	, known_annex_size True
-	, bloom_info
+	, total_annex_size
+	, reposizes_stats_global
 	, backend_usage
+	, bloom_info
 	]
 
 tree_fast_stats :: Bool -> [FilePath -> Stat]
@@ -296,7 +299,7 @@ tree_fast_stats isworktree =
 tree_slow_stats :: [FilePath -> Stat]
 tree_slow_stats =
 	[ const numcopies_stats
-	, const reposizes_stats
+	, const reposizes_stats_tree
 	, const reposizes_total
 	]
 
@@ -370,6 +373,10 @@ countRepoList :: Int -> String -> String
 countRepoList _ [] = "0"
 countRepoList n s = show n ++ "\n" ++ beginning s
 
+dispRepoList :: String -> String
+dispRepoList [] = ""
+dispRepoList s = "\n" ++ beginning s
+
 dir_name :: FilePath -> Stat
 dir_name dir = simpleStat "directory" $ pure dir
 
@@ -435,6 +442,12 @@ known_annex_size :: Bool -> Stat
 known_annex_size isworktree = 
 	simpleStat ("size of annexed files in " ++ treeDesc isworktree) $
 		showSizeKeys =<< cachedReferencedData
+
+total_annex_size :: Stat
+total_annex_size = 
+	simpleStat "combined annex size of all repositories" $
+		showSizeKeys . fromMaybe mempty . allRepoData
+		=<< cachedAllRepoData
   
 treeDesc :: Bool -> String
 treeDesc True = "working tree"
@@ -538,21 +551,29 @@ numcopies_stats = stat "numcopies stats" $ json fmt $
 		. map (\(variance, count) -> "numcopies " ++ variance ++ ": " ++ show count)
 		. V.toList
 
-reposizes_stats :: Stat
-reposizes_stats = stat desc $ nojson $ do
+reposizes_stats_tree :: Stat
+reposizes_stats_tree = reposizes_stats True "repositories containing these files"
+	=<< cachedRepoData
+
+reposizes_stats_global :: Stat
+reposizes_stats_global = reposizes_stats False "annex sizes of repositories" 
+	. repoData =<< cachedAllRepoData
+
+reposizes_stats :: Bool -> String -> M.Map UUID KeyInfo -> Stat
+reposizes_stats count desc m = stat desc $ nojson $ do
 	sizer <- mkSizer
-	l <- map (\(u, kd) -> (u, sizer storageUnits True (sizeKeys kd)))
-		. sortBy (flip (comparing (sizeKeys . snd)))
-		. M.toList
-		<$> cachedRepoData
+	let l = map (\(u, kd) -> (u, sizer storageUnits True (sizeKeys kd))) $
+		sortBy (flip (comparing (sizeKeys . snd))) $
+		M.toList m
 	let maxlen = maximum (map (length . snd) l)
 	descm <- lift Remote.uuidDescriptions
 	-- This also handles json display.
 	s <- lift $ Remote.prettyPrintUUIDsWith (Just "size") desc descm (Just . show) $
 		map (\(u, sz) -> (u, Just $ mkdisp sz maxlen)) l
-	return $ countRepoList (length l) s
+	return $ if count
+		then countRepoList (length l) s
+		else dispRepoList s
   where
-	desc = "repositories containing these files"
 	mkdisp sz maxlen = DualDisp
 		{ dispNormal = lpad maxlen sz
 		, dispJson = sz
@@ -612,11 +633,32 @@ cachedReferencedData = do
 			put s { referencedData = Just v }
 			return v
 
--- currently only available for directory info
+cachedAllRepoData :: StatState StatInfo
+cachedAllRepoData = do
+	s <- get
+	case allRepoData s of
+		Just _ -> return s
+		Nothing -> do
+			matcher <- lift getKeyOnlyMatcher
+			!(d, rd) <- lift $ overLocationLogs (emptyKeyInfo, mempty) $ \k locs (d, rd) -> do
+				ifM (matchOnKey matcher k)
+					( do
+						alivelocs <- snd
+							<$> trustPartition DeadTrusted locs
+						let !d' = addKeyCopies (genericLength alivelocs) k d
+						let !rd' = foldl' (flip (accumrepodata k)) rd alivelocs
+						return (d', rd')
+					, return (d, rd)
+					)
+			let s' = s { allRepoData = Just d, repoData = rd }
+			put s'
+			return s'
+  where
+	accumrepodata k = M.alter (Just . addKey k . fromMaybe emptyKeyInfo)
+
 cachedNumCopiesStats :: StatState (Maybe NumCopiesStats)
 cachedNumCopiesStats = numCopiesStats <$> get
 
--- currently only available for directory info
 cachedRepoData :: StatState (M.Map UUID KeyInfo)
 cachedRepoData = repoData <$> get
 
@@ -627,7 +669,13 @@ getDirStatInfo o dir = do
 	(presentdata, referenceddata, numcopiesstats, repodata) <-
 		Command.Unused.withKeysFilesReferencedIn dir initial
 			(update matcher fast)
-	return $ StatInfo (Just presentdata) (Just referenceddata) repodata (Just numcopiesstats) o
+	return $ StatInfo
+		(Just presentdata)
+		(Just referenceddata)
+		repodata
+		Nothing
+		(Just numcopiesstats)
+		o
   where
 	initial = (emptyKeyInfo, emptyKeyInfo, emptyNumCopiesStats, M.empty)
 	update matcher fast key file vs@(presentdata, referenceddata, numcopiesstats, repodata) =
@@ -663,7 +711,7 @@ getTreeStatInfo o r = do
 	(presentdata, referenceddata, repodata) <- go fast matcher ls initial
 	ifM (liftIO cleanup)
 		( return $ Just $
-			StatInfo (Just presentdata) (Just referenceddata) repodata Nothing o
+			StatInfo (Just presentdata) (Just referenceddata) repodata Nothing Nothing o
 		, return Nothing
 		)
   where
@@ -695,16 +743,19 @@ emptyNumCopiesStats :: NumCopiesStats
 emptyNumCopiesStats = NumCopiesStats M.empty
 
 addKey :: Key -> KeyInfo -> KeyInfo
-addKey key (KeyInfo count size unknownsize backends) =
+addKey = addKeyCopies 1
+
+addKeyCopies :: Integer -> Key -> KeyInfo -> KeyInfo
+addKeyCopies numcopies key (KeyInfo count size unknownsize backends) =
 	KeyInfo count' size' unknownsize' backends'
   where
 	{- All calculations strict to avoid thunks when repeatedly
 	 - applied to many keys. -}
 	!count' = count + 1
 	!backends' = M.insertWith (+) (fromKey keyVariety key) 1 backends
-	!size' = maybe size (+ size) ks
+	!size' = maybe size (\sz -> sz * numcopies + size) ks
 	!unknownsize' = maybe (unknownsize + 1) (const unknownsize) ks
-	ks = fromKey keySize key
+	!ks = fromKey keySize key
 
 updateRepoData :: Key -> [UUID] -> M.Map UUID KeyInfo -> M.Map UUID KeyInfo
 updateRepoData key locs m = m'
diff --git a/Command/Inprogress.hs b/Command/Inprogress.hs
index 8ab920242f..7b5f1482ea 100644
--- a/Command/Inprogress.hs
+++ b/Command/Inprogress.hs
@@ -42,7 +42,7 @@ seek o = do
 		_ -> do
 			let s = S.fromList ts
 			let seeker = AnnexedFileSeeker
-				{ startAction = start isterminal s
+				{ startAction = const $ start isterminal s
 				, checkContentPresent = Nothing
 				, usesLocationLog = False
 				}
diff --git a/Command/List.hs b/Command/List.hs
index b14c55d707..46185e6092 100644
--- a/Command/List.hs
+++ b/Command/List.hs
@@ -50,7 +50,7 @@ seek o = do
 	list <- getList o
 	printHeader list
 	let seeker = AnnexedFileSeeker
-		{ startAction = start list
+		{ startAction = const $ start list
 		, checkContentPresent = Nothing
 		, usesLocationLog = True
 		}
diff --git a/Command/Lock.hs b/Command/Lock.hs
index d547a07f93..cfd0846cd5 100644
--- a/Command/Lock.hs
+++ b/Command/Lock.hs
@@ -34,7 +34,7 @@ seek ps = withFilesInGitAnnex ww seeker =<< workTreeItems ww ps
   where
 	ww = WarnUnmatchLsFiles "lock"
 	seeker = AnnexedFileSeeker
-		{ startAction = start
+		{ startAction = const start
 		, checkContentPresent = Nothing
 		, usesLocationLog = False
 		}
@@ -79,7 +79,7 @@ perform file key = do
 		mfc <- withTSDelta (liftIO . genInodeCache file)
 		unlessM (sameInodeCache obj (maybeToList mfc)) $ do
 			modifyContentDir obj $ replaceGitAnnexDirFile (fromRawFilePath obj) $ \tmp -> do
-				unlessM (checkedCopyFile key obj (toRawFilePath tmp) Nothing) $
+				unlessM (checkedCopyFile key obj tmp Nothing) $
 					giveup "unable to lock file"
 			Database.Keys.storeInodeCaches key [obj]
 
diff --git a/Command/Log.hs b/Command/Log.hs
index c6a53451d4..149b099dd5 100644
--- a/Command/Log.hs
+++ b/Command/Log.hs
@@ -1,11 +1,11 @@
 {- git-annex command
  -
- - Copyright 2012-2021 Joey Hess <id@joeyh.name>
+ - Copyright 2012-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
 
-{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE OverloadedStrings, BangPatterns #-}
 
 module Command.Log where
 
@@ -16,23 +16,24 @@ import Data.Time.Clock.POSIX
 import Data.Time
 import qualified Data.ByteString.Char8 as B8
 import qualified System.FilePath.ByteString as P
+import Control.Concurrent.Async
 
 import Command
 import Logs
 import Logs.Location
+import Logs.UUID
+import qualified Logs.Presence.Pure as PLog
+import Logs.Trust.Pure (parseTrustLog)
+import Logs.UUIDBased (simpleMap)
+import qualified Annex
 import qualified Annex.Branch
-import qualified Git
-import Git.Command
 import qualified Remote
-import qualified Annex
-
-data RefChange = RefChange 
-	{ changetime :: POSIXTime
-	, oldref :: Git.Ref
-	, newref :: Git.Ref
-	, changekey :: Key
-	}
-	deriving (Show)
+import qualified Git
+import Git.Log
+import Git.CatFile
+import Types.TrustLevel
+import Utility.DataUnits
+import Utility.HumanTime
 
 data LogChange = Added | Removed
 
@@ -46,7 +47,14 @@ cmd = withAnnexOptions [jsonOptions, annexedMatchingOptions] $
 data LogOptions = LogOptions
 	{ logFiles :: CmdParams
 	, allOption :: Bool
+	, sizesOfOption :: Maybe (DeferredParse UUID)
+	, sizesOption :: Bool
+	, totalSizesOption :: Bool
+	, intervalOption :: Maybe Duration
+	, receivedOption :: Bool
+	, gnuplotOption :: Bool
 	, rawDateOption :: Bool
+	, bytesOption :: Bool
 	, gourceOption :: Bool
 	, passthruOptions :: [CommandParam]
 	}
@@ -59,11 +67,41 @@ optParser desc = LogOptions
 		<> short 'A'
 		<> help "display location log changes to all files"
 		)
+	<*> optional ((parseUUIDOption <$> strOption
+		( long "sizesof"
+		<> metavar (paramRemote `paramOr` paramDesc `paramOr` paramUUID)
+		<> help "display history of sizes of this repository"
+		<> completeRemotes
+		)))
+	<*> switch
+		( long "sizes"
+		<> help "display history of sizes of all repositories"
+		)
+	<*> switch
+		( long "totalsizes"
+		<> help "display history of total sizes of all repositories"
+		)
+	<*> optional (option (eitherReader parseDuration)
+		( long "interval" <> metavar paramTime
+		<> help "minimum time between displays of changed size"
+		))
+	<*> switch
+		( long "received"
+		<> help "display received data per interval rather than repository sizes"
+		)
+	<*> switch
+		( long "gnuplot"
+		<> help "graph the history"
+		)
 	<*> switch
 		( long "raw-date"
 		<> help "display seconds from unix epoch"
 		)
 	<*> switch
+		( long "bytes"
+		<> help "display sizes in bytes"
+		)
+	<*> switch
 		( long "gource"
 		<> help "format output for gource"
 		)
@@ -86,12 +124,21 @@ optParser desc = LogOptions
 
 seek :: LogOptions -> CommandSeek
 seek o = ifM (null <$> Annex.Branch.getUnmergedRefs)
-	( do
+	( maybe (pure Nothing) (Just <$$> getParsed) (sizesOfOption o) >>= \case
+		Just u -> sizeHistoryInfo (Just u) o
+		Nothing -> if sizesOption o || totalSizesOption o
+			then sizeHistoryInfo Nothing o
+			else go	
+	, giveup "This repository is read-only, and there are unmerged git-annex branches, which prevents displaying location log changes. (Set annex.merge-annex-branches to false to ignore the unmerged git-annex branches.)"
+	)
+  where
+	ww = WarnUnmatchLsFiles "log"
+	go = do
 		m <- Remote.uuidDescriptions
 		zone <- liftIO getCurrentTimeZone
 		outputter <- mkOutputter m zone o <$> jsonOutputEnabled
 		let seeker = AnnexedFileSeeker
-			{ startAction = start o outputter
+			{ startAction = const $ start o outputter
 			, checkContentPresent = Nothing
 			-- the way this uses the location log would not be
 			-- helped by precaching the current value
@@ -102,10 +149,6 @@ seek o = ifM (null <$> Annex.Branch.getUnmergedRefs)
 				=<< workTreeItems ww fs
 			([], True) -> commandAction (startAll o outputter)
 			(_, True) -> giveup "Cannot specify both files and --all"
-	, giveup "This repository is read-only, and there are unmerged git-annex branches, which prevents displaying location log changes. (Set annex.merge-annex-branches to false to ignore the unmerged git-annex branches.)"
-	)
-  where
-	ww = WarnUnmatchLsFiles "log"
 
 start :: LogOptions -> (ActionItem -> SeekInput -> Outputter) -> SeekInput -> RawFilePath -> Key -> CommandStart
 start o outputter si file key = do
@@ -117,12 +160,12 @@ start o outputter si file key = do
 
 startAll :: LogOptions -> (ActionItem -> SeekInput -> Outputter) -> CommandStart
 startAll o outputter = do
-	(changes, cleanup) <- getAllLog (passthruOptions o)
+	(changes, cleanup) <- getGitLogAnnex [] (passthruOptions o)
 	showLog (\ai -> outputter ai (SeekInput [])) changes
 	void $ liftIO cleanup
 	stop
 
-{- Displays changes made. Only works when all the RefChanges are for the
+{- Displays changes made. Only works when all the LoggedFileChanges are for the
  - same key. The method is to compare each value with the value
  - after it in the list, which is the old version of the value.
  -
@@ -136,7 +179,7 @@ startAll o outputter = do
  - This also generates subtly better output when the git-annex branch
  - got diverged.
  -}
-showLogIncremental :: Outputter -> [RefChange] -> Annex ()
+showLogIncremental :: Outputter -> [LoggedFileChange Key] -> Annex ()
 showLogIncremental outputter ps = do
 	sets <- mapM (getset newref) ps
 	previous <- maybe (return genesis) (getset oldref) (lastMaybe ps)
@@ -153,9 +196,9 @@ showLogIncremental outputter ps = do
 {- Displays changes made. Streams, and can display changes affecting
  - different keys, but does twice as much reading of logged values
  - as showLogIncremental. -}
-showLog :: (ActionItem -> Outputter) -> [RefChange] -> Annex ()
+showLog :: (ActionItem -> Outputter) -> [LoggedFileChange Key] -> Annex ()
 showLog outputter cs = forM_ cs $ \c -> do
-	let ai = mkActionItem (changekey c)
+	let ai = mkActionItem (changed c)
 	new <- S.fromList <$> loggedLocationsRef (newref c)
 	old <- S.fromList <$> loggedLocationsRef (oldref c)
 	sequence_ $ compareChanges (outputter ai)
@@ -166,7 +209,7 @@ mkOutputter m zone o jsonenabled ai si
 	| jsonenabled = jsonOutput m ai si
 	| rawDateOption o = normalOutput lookupdescription ai rawTimeStamp
 	| gourceOption o = gourceOutput lookupdescription ai 
-	| otherwise = normalOutput lookupdescription ai (showTimeStamp zone)
+	| otherwise = normalOutput lookupdescription ai (showTimeStamp zone rfc822DateFormat)
   where
 	lookupdescription u = maybe (fromUUID u) (fromUUIDDesc) (M.lookup u m)
 
@@ -236,85 +279,251 @@ compareChanges format changes = concatMap diff changes
  - once the location log file is gone avoids it checking all the way back
  - to commit 0 to see if it used to exist, so generally speeds things up a
  - *lot* for newish files. -}
-getKeyLog :: Key -> [CommandParam] -> Annex ([RefChange], IO Bool)
+getKeyLog :: Key -> [CommandParam] -> Annex ([LoggedFileChange Key], IO Bool)
 getKeyLog key os = do
 	top <- fromRepo Git.repoPath
 	p <- liftIO $ relPathCwdToFile top
 	config <- Annex.getGitConfig
 	let logfile = p P.</> locationLogFile config key
-	getGitLog [fromRawFilePath logfile] (Param "--remove-empty" : os)
-
-{- Streams the git log for all git-annex branch changes. -}
-getAllLog :: [CommandParam] -> Annex ([RefChange], IO Bool)
-getAllLog = getGitLog []
+	getGitLogAnnex [fromRawFilePath logfile] (Param "--remove-empty" : os)
 
-getGitLog :: [FilePath] -> [CommandParam] -> Annex ([RefChange], IO Bool)
-getGitLog fs os = do
+getGitLogAnnex :: [FilePath] -> [CommandParam] -> Annex ([LoggedFileChange Key], IO Bool)
+getGitLogAnnex fs os = do
 	config <- Annex.getGitConfig
-	(ls, cleanup) <- inRepo $ pipeNullSplit $
-		[ Param "log"
-		, Param "-z"
-		, Param "--pretty=format:%ct"
-		, Param "--raw"
-		, Param "--no-abbrev"
-		] ++ os ++
-		[ Param $ Git.fromRef Annex.Branch.fullname
-		, Param "--"
-		] ++ map Param fs
-	return (parseGitRawLog config (map decodeBL ls), cleanup)
-
--- Parses chunked git log --raw output, which looks something like:
---
--- [ "timestamp\n:changeline"
--- , "logfile"
--- , ""
--- , "timestamp\n:changeline"
--- , "logfile"
--- , ":changeline"
--- , "logfile"
--- , ""
--- ]
---
--- The timestamp is not included before all changelines, so
--- keep track of the most recently seen timestamp.
-parseGitRawLog :: GitConfig -> [String] -> [RefChange]
-parseGitRawLog config = parse epoch
-  where
-	epoch = toEnum 0 :: POSIXTime
-	parse oldts ([]:rest) = parse oldts rest
-	parse oldts (c1:c2:rest) = case mrc of
-		Just rc -> rc : parse ts rest
-		Nothing -> parse ts (c2:rest)
-	  where
-		(ts, cl) = case separate (== '\n') c1 of
-			(cl', []) -> (oldts, cl')
-			(tss, cl') -> (parseTimeStamp tss, cl')
-	  	mrc = do
-			(old, new) <- parseRawChangeLine cl
-			key <- locationLogFileKey config (toRawFilePath c2)
-			return $ RefChange
-				{ changetime = ts
-				, oldref = old
-				, newref = new
-				, changekey = key
-				}	
-	parse _ _ = []
-
--- Parses something like "100644 100644 oldsha newsha M"
-parseRawChangeLine :: String -> Maybe (Git.Ref, Git.Ref)
-parseRawChangeLine = go . words
-  where
-	go (_:_:oldsha:newsha:_) = 
-		Just (Git.Ref (encodeBS oldsha), Git.Ref (encodeBS newsha))
-	go _ = Nothing
+	let fileselector = \_sha f ->
+		locationLogFileKey config (toRawFilePath f)
+	inRepo $ getGitLog Annex.Branch.fullname Nothing fs os fileselector
 
-parseTimeStamp :: String -> POSIXTime
-parseTimeStamp = utcTimeToPOSIXSeconds . fromMaybe (giveup "bad timestamp") .
-	parseTimeM True defaultTimeLocale "%s"
-
-showTimeStamp :: TimeZone -> POSIXTime -> String
-showTimeStamp zone = formatTime defaultTimeLocale rfc822DateFormat 
+showTimeStamp :: TimeZone -> String -> POSIXTime -> String
+showTimeStamp zone format = formatTime defaultTimeLocale format
 	. utcToZonedTime zone . posixSecondsToUTCTime
 
 rawTimeStamp :: POSIXTime -> String
 rawTimeStamp t = filter (/= 's') (show t)
+
+sizeHistoryInfo :: (Maybe UUID) -> LogOptions -> Annex ()
+sizeHistoryInfo mu o = do
+	uuidmap <- getuuidmap
+	zone <- liftIO getCurrentTimeZone
+	dispst <- displaystart uuidmap zone
+	(l, cleanup) <- getlog
+	g <- Annex.gitRepo
+	liftIO $ catObjectStream g $ \feeder closer reader -> do
+		tid <- async $ do
+			forM_ l $ \c -> 
+				feeder ((changed c, changetime c), newref c)
+			closer
+		go reader mempty mempty mempty uuidmap dispst
+		wait tid
+	void $ liftIO cleanup
+  where
+	-- Go through the log of the git-annex branch in reverse,
+	-- and in date order, and pick out changes to location log files
+	-- and to the trust log.
+	getlog = do
+		config <- Annex.getGitConfig
+		let fileselector = \_sha f -> let f' = toRawFilePath f in
+			case locationLogFileKey config f' of
+				Just k -> Just (Right k)
+				Nothing
+					| f' == trustLog -> Just (Left ())
+					| otherwise -> Nothing
+		inRepo $ getGitLog Annex.Branch.fullname Nothing []
+			[ Param "--date-order"
+			, Param "--reverse"
+			]
+			fileselector
+
+	go reader sizemap locmap trustlog uuidmap dispst = reader >>= \case
+		Just ((Right k, t), Just logcontent) -> do
+			let !newlog = parselocationlog logcontent uuidmap
+			let !(sizemap', locmap') = case M.lookup k locmap of
+				Nothing -> addnew k sizemap locmap newlog
+				Just v -> update k sizemap locmap v newlog
+			dispst' <- displaysizes dispst trustlog uuidmap sizemap' t
+			go reader sizemap' locmap' trustlog uuidmap dispst'
+		Just ((Left (), t), Just logcontent) -> do
+			let !trustlog' = trustlog <> parseTrustLog logcontent
+			dispst' <- displaysizes dispst trustlog' uuidmap sizemap t
+			go reader sizemap locmap trustlog' uuidmap dispst'
+		Just (_, Nothing) -> 
+			go reader sizemap locmap trustlog uuidmap dispst
+		Nothing -> 
+			displayend dispst
+
+	-- Known uuids are stored in this map, and when uuids are stored in the
+	-- state, it's a value from this map. This avoids storing multiple
+	-- copies of the same uuid in memory.
+	getuuidmap = do
+		(us, ds) <- unzip . M.toList <$> uuidDescMap
+		return $ M.fromList (zip us (zip us ds))
+	
+	-- Parses a location log file, and replaces the logged uuid
+	-- with one from the uuidmap.
+	parselocationlog logcontent uuidmap = 
+		map replaceuuid $ PLog.parseLog logcontent
+	  where
+		replaceuuid ll = 
+			let !u = toUUID $ PLog.fromLogInfo $ PLog.info ll
+			    !ushared = maybe u fst $ M.lookup u uuidmap
+			in ll { PLog.info = PLog.LogInfo (fromUUID ushared) }
+
+	presentlocs = map (toUUID . PLog.fromLogInfo . PLog.info)
+		. PLog.filterPresent
+	
+	-- Since the git log is being traversed in date order, commits
+	-- from different branches can appear one after the other, and so
+	-- the newlog is not necessarily the complete state known at that
+	-- time across all git-annex repositories.
+	--
+	-- This combines the new location log with what has been
+	-- accumulated so far, which is equivilant to merging together
+	-- all git-annex branches at that point in time.
+	update k sizemap locmap (oldlog, oldlocs) newlog = 
+		( updatesize (updatesize sizemap sz (S.toList addedlocs))
+			(negate sz) (S.toList removedlocs)
+		, M.insert k (combinedlog, combinedlocs) locmap
+		)
+	  where
+		sz = ksz k
+		combinedlog = PLog.compactLog (oldlog ++ newlog)
+		combinedlocs = S.fromList (presentlocs combinedlog)
+		addedlocs = S.difference combinedlocs oldlocs
+		removedlocs
+			| receivedOption o = S.empty
+			| otherwise = S.difference oldlocs combinedlocs
+	
+	addnew k sizemap locmap newlog = 
+		( updatesize sizemap (ksz k) locs
+		, M.insert k (newlog, S.fromList locs) locmap
+		)
+	  where
+		locs = presentlocs newlog
+	
+	ksz k = fromMaybe 0 (fromKey keySize k)
+	
+	updatesize sizemap _ [] = sizemap
+	updatesize sizemap sz (l:ls) =
+		updatesize (M.insertWith (+) l sz sizemap) sz ls
+
+	epoch = toEnum 0
+
+	displaystart uuidmap zone
+		| gnuplotOption o = do
+			file <- (</>)
+				<$> fromRepo (fromRawFilePath . gitAnnexDir)
+				<*> pure "gnuplot"
+			liftIO $ putStrLn $ "Generating gnuplot script in " ++ file
+			h <- liftIO $ openFile file WriteMode
+			liftIO $ mapM_ (hPutStrLn h)
+				[ "set datafile separator ','"
+				, "set timefmt \"%Y-%m-%dT%H:%M:%S\""
+				, "set xdata time"
+				, "set xtics out"
+				, "set ytics format '%s%c'"
+				, "set tics front"
+				, "set key spacing 1 font \",8\""
+				]
+			unless (sizesOption o) $
+				liftIO $ hPutStrLn h "set key off"
+			liftIO $ hPutStrLn h "$data << EOD"
+			liftIO $ hPutStrLn h $ if sizesOption o
+				then uuidmapheader
+				else csvheader ["value"]
+			let endaction = do
+				mapM_ (hPutStrLn h)
+					[ "EOD"
+					, ""
+					, "plot for [i=2:" ++ show ncols ++ ":1] \\"
+					, "  \"$data\" using 1:(sum [col=i:" ++ show ncols ++ "] column(col)) \\"
+					, "  title columnheader(i) \\"
+					, if receivedOption o
+						then "  with boxes"
+						else "  with filledcurves x1"
+					]
+				hFlush h
+				putStrLn $ "Running gnuplot..."
+				void $ liftIO $ boolSystem "gnuplot"
+					[Param "-p", File file]
+			return (dispst h endaction)
+		| sizesOption o = do
+			liftIO $ putStrLn uuidmapheader
+			return (dispst stdout noop)
+		| otherwise = return (dispst stdout noop)
+	  where
+		dispst fileh endaction = 
+			(zone, False, epoch, Nothing, mempty, fileh, endaction)
+		ncols
+			| sizesOption o = 1 + length (M.elems uuidmap)
+			| otherwise = 2
+		uuidmapheader = csvheader $
+			map (fromUUIDDesc . snd) (M.elems uuidmap)
+
+	displaysizes (zone, displayedyet, prevt, prevoutput, prevsizemap, h, endaction) trustlog uuidmap sizemap t
+		| t - prevt >= dt && changedoutput = do
+			displayts zone t output h
+			return (zone, True, t, Just output, sizemap', h, endaction)
+		| t < prevt = return (zone, displayedyet, t, Just output, prevsizemap, h, endaction)
+		| otherwise = return (zone, displayedyet, prevt, prevoutput, prevsizemap, h, endaction)
+	  where
+		output = intercalate "," (map showsize sizes)
+		us = case mu of
+			Just u -> [u]
+			Nothing -> M.keys uuidmap
+		sizes
+			| totalSizesOption o = [sum (M.elems sizedisplaymap)]
+			| otherwise = map (\u -> fromMaybe 0 (M.lookup u sizedisplaymap)) us
+		dt = maybe 1 durationToPOSIXTime (intervalOption o)
+
+		changedoutput
+			| receivedOption o = 
+				any (/= 0) sizes 
+					|| prevoutput /= Just output
+			| otherwise = 
+				(displayedyet || any (/= 0) sizes)
+					&& (prevoutput /= Just output)
+
+		sizedisplaymap
+			| receivedOption o = 
+				M.unionWith posminus sizemap' prevsizemap
+			| otherwise = sizemap'
+
+		posminus a b = max 0 (a - b)
+
+		-- A verison of sizemap where uuids that are currently dead
+		-- have 0 size.
+		sizemap' = M.mapWithKey zerodead sizemap
+		zerodead u v = case M.lookup u (simpleMap trustlog) of
+			Just DeadTrusted -> 0
+			_ -> v
+
+	displayts zone t output h = do
+		hPutStrLn h (ts ++ "," ++ output)
+		hFlush h
+	  where
+		ts = if rawDateOption o && not (gnuplotOption o)
+			then rawTimeStamp t
+			else showTimeStamp zone "%Y-%m-%dT%H:%M:%S" t
+
+	displayend dispst@(_, _, _, _, _, _, endaction) = do
+		displayendsizes dispst
+		endaction
+
+	displayendsizes (zone, _, _, Just output, _, h, _) = do
+		now <- getPOSIXTime
+		displayts zone now output h
+	displayendsizes _ = return ()
+
+	showsize n
+		| bytesOption o || gnuplotOption o = show n
+		| otherwise = roughSize storageUnits True n
+	
+	csvquote s
+		| ',' `elem` s || '"' `elem` s = 
+			'"' : concatMap escquote s ++ ['"']
+		| otherwise = s
+	  where
+		escquote '"' = "\"\""
+		escquote c = [c]
+	
+	csvheader l = intercalate "," ("date" : map csvquote l)
diff --git a/Command/LookupKey.hs b/Command/LookupKey.hs
index 8e5d28b4eb..f191aa1e8b 100644
--- a/Command/LookupKey.hs
+++ b/Command/LookupKey.hs
@@ -50,11 +50,13 @@ display Nothing = return False
 -- To support absolute filenames, pass through git ls-files.
 -- But, this plumbing command does not recurse through directories.
 seekSingleGitFile :: FilePath -> Annex (Maybe RawFilePath)
-seekSingleGitFile file = do
-	(l, cleanup) <- inRepo (Git.LsFiles.inRepo [] [toRawFilePath file])
-	r <- case l of
-		(f:[]) | takeFileName (fromRawFilePath f) == takeFileName file ->
-			return (Just f)
-		_ -> return Nothing
-	void $ liftIO cleanup
-	return r
+seekSingleGitFile file
+	| isRelative file = return (Just (toRawFilePath file))
+	| otherwise = do
+		(l, cleanup) <- inRepo (Git.LsFiles.inRepo [] [toRawFilePath file])
+		r <- case l of
+			(f:[]) | takeFileName (fromRawFilePath f) == takeFileName file ->
+				return (Just f)
+			_ -> return Nothing
+		void $ liftIO cleanup
+		return r
diff --git a/Command/MetaData.hs b/Command/MetaData.hs
index e07e5e99f2..e0a16c9249 100644
--- a/Command/MetaData.hs
+++ b/Command/MetaData.hs
@@ -77,7 +77,7 @@ seek o = case batchOption o of
 		c <- currentVectorClock
 		let ww = WarnUnmatchLsFiles "metadata"
 		let seeker = AnnexedFileSeeker
-			{ startAction = start c o
+			{ startAction = const $ start c o
 			, checkContentPresent = Nothing
 			, usesLocationLog = False
 			}
diff --git a/Command/Migrate.hs b/Command/Migrate.hs
index 67cf69c2a8..c07377aae2 100644
--- a/Command/Migrate.hs
+++ b/Command/Migrate.hs
@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2011 Joey Hess <id@joeyh.name>
+ - Copyright 2011-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -15,9 +15,15 @@ import Annex.Content
 import qualified Command.ReKey
 import qualified Command.Fsck
 import qualified Annex
+import Logs.Migrate
 import Logs.MetaData
 import Logs.Web
+import Logs.Location
 import Utility.Metered
+import qualified Database.Keys
+import Git.FilePath
+import Annex.Link
+import Annex.UUID
 
 cmd :: Command
 cmd = withAnnexOptions [backendOption, annexedMatchingOptions, jsonOptions] $
@@ -27,6 +33,8 @@ cmd = withAnnexOptions [backendOption, annexedMatchingOptions, jsonOptions] $
 
 data MigrateOptions = MigrateOptions
 	{ migrateThese :: CmdParams
+	, updateOption :: Bool
+	, applyOption :: Bool
 	, removeSize :: Bool
 	}
 
@@ -34,12 +42,27 @@ optParser :: CmdParamsDesc -> Parser MigrateOptions
 optParser desc = MigrateOptions
 	<$> cmdParams desc
 	<*> switch
+		( long "update"
+		<> help "incrementally apply migrations performed elsewhere"
+		)
+	<*> switch
+		( long "apply"
+		<> help "(re)apply migrations performed elsewhere"
+		)
+	<*> switch
 		( long "remove-size"
 		<> help "remove size field from keys"
 		)
 
 seek :: MigrateOptions -> CommandSeek
-seek o = withFilesInGitAnnex ww seeker =<< workTreeItems ww (migrateThese o)
+seek o
+	| updateOption o || applyOption o = do
+		unless (null (migrateThese o)) $
+			error "Cannot combine --update or --apply with files to migrate."
+		seekDistributedMigrations (not (applyOption o))
+	| otherwise = do
+		withFilesInGitAnnex ww seeker =<< workTreeItems ww (migrateThese o)
+		commitMigration
   where
 	ww = WarnUnmatchLsFiles "migrate"
 	seeker = AnnexedFileSeeker
@@ -48,8 +71,16 @@ seek o = withFilesInGitAnnex ww seeker =<< workTreeItems ww (migrateThese o)
 		, usesLocationLog = False
 		}
 
-start :: MigrateOptions -> SeekInput -> RawFilePath -> Key -> CommandStart
-start o si file key = do
+seekDistributedMigrations :: Bool -> CommandSeek
+seekDistributedMigrations incremental =
+	streamNewDistributedMigrations incremental $ \oldkey newkey ->
+		-- Not using commandAction because this is not necessarily
+		-- concurrency safe, and also is unlikely to be sped up
+		-- by multiple jobs.
+		void $ includeCommandAction $ update oldkey newkey
+
+start :: MigrateOptions -> Maybe KeySha -> SeekInput -> RawFilePath -> Key -> CommandStart
+start o ksha si file key = do
 	forced <- Annex.getRead Annex.force
 	v <- Backend.getBackend (fromRawFilePath file) key
 	case v of
@@ -57,26 +88,26 @@ start o si file key = do
 		Just oldbackend -> do
 			exists <- inAnnex key
 			newbackend <- chooseBackend file
-			if (newbackend /= oldbackend || upgradableKey oldbackend key || forced) && exists
+			if (newbackend /= oldbackend || upgradableKey oldbackend || forced) && exists
 				then go False oldbackend newbackend
-				else if removeSize o && exists
-					then go True oldbackend oldbackend
+				else if cantweaksize newbackend oldbackend && exists
+					then go True oldbackend newbackend
 					else stop
   where
-	go onlyremovesize oldbackend newbackend =
+	go onlytweaksize oldbackend newbackend = do
+		keyrec <- case ksha of
+			Just (KeySha s) -> pure (MigrationRecord s)
+			Nothing -> error "internal"
 		starting "migrate" (mkActionItem (key, file)) si $
-			perform onlyremovesize o file key oldbackend newbackend
+			perform onlytweaksize o file key keyrec oldbackend newbackend
 
-{- Checks if a key is upgradable to a newer representation.
- - 
- - Reasons for migration:
- -  - Ideally, all keys have file size metadata. Old keys may not.
- -  - Something has changed in the backend, such as a bug fix.
- -}
-upgradableKey :: Backend -> Key -> Bool
-upgradableKey backend key = isNothing (fromKey keySize key) || backendupgradable
-  where
-	backendupgradable = maybe False (\a -> a key) (canUpgradeKey backend)
+	cantweaksize newbackend oldbackend
+		| removeSize o = isJust (fromKey keySize key)
+		| newbackend /= oldbackend = False
+		| isNothing (fromKey keySize key) = True
+		| otherwise = False
+
+	upgradableKey oldbackend = maybe False (\a -> a key) (canUpgradeKey oldbackend)
 
 {- Store the old backend's key in the new backend
  - The old backend's key is not dropped from it, because there may
@@ -87,14 +118,14 @@ upgradableKey backend key = isNothing (fromKey keySize key) || backendupgradable
  - data cannot get corrupted after the fsck but before the new key is
  - generated.
  -}
-perform :: Bool -> MigrateOptions -> RawFilePath -> Key -> Backend -> Backend -> CommandPerform
-perform onlyremovesize o file oldkey oldbackend newbackend = go =<< genkey (fastMigrate oldbackend)
+perform :: Bool -> MigrateOptions -> RawFilePath -> Key -> MigrationRecord -> Backend -> Backend -> CommandPerform
+perform onlytweaksize o file oldkey oldkeyrec oldbackend newbackend = go =<< genkey (fastMigrate oldbackend)
   where
 	go Nothing = stop
 	go (Just (newkey, knowngoodcontent))
-		| knowngoodcontent = finish (removesize newkey)
+		| knowngoodcontent = finish =<< tweaksize newkey
 		| otherwise = stopUnless checkcontent $
-			finish (removesize newkey)
+			finish =<< tweaksize newkey
 	checkcontent = Command.Fsck.checkBackend oldbackend oldkey KeyPresent afile
 	finish newkey = ifM (Command.ReKey.linkKey file oldkey newkey)
 		( do
@@ -104,10 +135,11 @@ perform onlyremovesize o file oldkey oldbackend newbackend = go =<< genkey (fast
 			urls <- getUrls oldkey
 			forM_ urls $ \url ->
 				setUrlPresent newkey url
-			next $ Command.ReKey.cleanup file newkey
+			next $ Command.ReKey.cleanup file newkey $
+				logMigration oldkeyrec
 		, giveup "failed creating link from old to new key"
 		)
-	genkey _ | onlyremovesize = return $ Just (oldkey, False)
+	genkey _ | onlytweaksize = return $ Just (oldkey, False)
 	genkey Nothing = do
 		content <- calcRepo $ gitAnnexLocation oldkey
 		let source = KeySource
@@ -120,7 +152,56 @@ perform onlyremovesize o file oldkey oldbackend newbackend = go =<< genkey (fast
 	genkey (Just fm) = fm oldkey newbackend afile >>= \case
 		Just newkey -> return (Just (newkey, True))
 		Nothing -> genkey Nothing
-	removesize k
-		| removeSize o = alterKey k $ \kd -> kd { keySize = Nothing } 
-		| otherwise = k
+	tweaksize k
+		| removeSize o = pure (removesize k)
+		| onlytweaksize = addsize k
+		| otherwise = pure k
+	removesize k = alterKey k $ \kd -> kd { keySize = Nothing }
+	addsize k
+		| fromKey keySize k == Nothing = 
+			contentSize k >>= return . \case
+				Just sz -> alterKey k $ \kd -> kd { keySize = Just sz }
+				Nothing -> k
+		| otherwise = return k
 	afile = AssociatedFile (Just file)
+
+update :: Key -> Key -> CommandStart
+update oldkey newkey =
+	stopUnless (allowed <&&> available <&&> wanted) $ do
+		ai <- findworktreefile >>= return . \case
+			Just f -> ActionItemAssociatedFile (AssociatedFile (Just f)) newkey
+			Nothing -> ActionItemKey newkey
+		starting "migrate" ai (SeekInput []) $
+			ifM (Command.ReKey.linkKey' v oldkey newkey)
+				( do
+					logStatus newkey InfoPresent
+					next $ return True
+				, next $ return False
+				)
+  where
+	available = (not <$> inAnnex newkey) <&&> inAnnex oldkey
+
+	-- annex.securehashesonly will block adding keys with insecure
+	-- hashes, this check is only to avoid doing extra work and
+	-- displaying a message when it fails.
+	allowed = isNothing <$> checkSecureHashes newkey
+
+	-- If the new key was previous present in this repository, but got
+	-- dropped, assume the user still doesn't want it there.
+	wanted = loggedPreviousLocations newkey >>= \case
+		[] -> pure True
+		us -> do
+			u <- getUUID
+			pure (u `notElem` us)
+
+	findworktreefile = do
+		fs <- Database.Keys.getAssociatedFiles newkey
+		g <- Annex.gitRepo
+		firstM (\f -> (== Just newkey) <$> isAnnexLink f) $
+			map (\f -> simplifyPath (fromTopFilePath f g)) fs
+	
+	-- Always verify the content agains the newkey, even if
+	-- annex.verify is unset. This is done to prent bad migration
+	-- information maliciously injected into the git-annex branch
+	-- from populating files with the wrong content.
+	v = AlwaysVerify
diff --git a/Command/Mirror.hs b/Command/Mirror.hs
index 8ec97e467f..d32a06790e 100644
--- a/Command/Mirror.hs
+++ b/Command/Mirror.hs
@@ -52,7 +52,7 @@ seek o = startConcurrency stages $
 		ToRemote _ -> commandStages
 	ww = WarnUnmatchLsFiles "mirror"
 	seeker = AnnexedFileSeeker
-		{ startAction = start o
+		{ startAction = const $ start o
 		, checkContentPresent = Nothing
 		, usesLocationLog = True
 		}
diff --git a/Command/Move.hs b/Command/Move.hs
index 77f7d6d3f1..056a7ca738 100644
--- a/Command/Move.hs
+++ b/Command/Move.hs
@@ -75,12 +75,13 @@ seek' o fto = startConcurrency (stages fto) $ do
 			batchAnnexed fmt seeker keyaction
   where
 	seeker = AnnexedFileSeeker
-		{ startAction = start fto (removeWhen o)
+		{ startAction = const $ start fto (removeWhen o)
 		, checkContentPresent = case fto of
 			FromOrToRemote (FromRemote _) -> Nothing
 			FromOrToRemote (ToRemote _) -> Just True
 			ToHere -> Nothing
 			FromRemoteToRemote _ _ -> Nothing
+			FromAnywhereToRemote _ -> Nothing
 		, usesLocationLog = True
 		}
 	keyaction = startKey fto (removeWhen o)
@@ -91,6 +92,7 @@ stages (FromOrToRemote (FromRemote _)) = transferStages
 stages (FromOrToRemote (ToRemote _)) = commandStages
 stages ToHere = transferStages
 stages (FromRemoteToRemote _ _) = transferStages
+stages (FromAnywhereToRemote _) = transferStages
 
 start :: FromToHereOptions -> RemoveWhen -> SeekInput -> RawFilePath -> Key -> CommandStart
 start fromto removewhen si f k = start' fromto removewhen afile si k ai
@@ -118,6 +120,9 @@ start' fromto removewhen afile si key ai =
 			src' <- getParsed src
 			dest' <- getParsed dest
 			fromToStart removewhen afile key ai si src' dest'
+		FromAnywhereToRemote dest -> do
+			dest' <- getParsed dest
+			fromAnywhereToStart removewhen afile key ai si dest'
 
 describeMoveAction :: RemoveWhen -> String
 describeMoveAction RemoveNever = "copy"
@@ -146,7 +151,7 @@ toStart' dest removewhen afile key ai si = do
 
 expectedPresent :: Remote -> Key -> Annex Bool
 expectedPresent dest key = do
-	remotes <- Remote.keyPossibilities key
+	remotes <- Remote.keyPossibilities (Remote.IncludeIgnored True) key
 	return $ dest `elem` remotes
 
 toPerform :: Remote -> RemoveWhen -> Key -> AssociatedFile -> Bool -> Either String Bool -> CommandPerform
@@ -249,7 +254,7 @@ fromOk src key
   where
 	checklog = do
 		u <- getUUID
-		remotes <- Remote.keyPossibilities key
+		remotes <- Remote.keyPossibilities (Remote.IncludeIgnored True) key
 		return $ u /= Remote.uuid src && elem src remotes
 
 fromPerform :: Remote -> RemoveWhen -> Key -> AssociatedFile -> CommandPerform
@@ -326,7 +331,7 @@ fromDrop src destuuid deststartedwithcopy key afile adjusttocheck =
 toHereStart :: RemoveWhen -> AssociatedFile -> Key -> ActionItem -> SeekInput -> CommandStart
 toHereStart removewhen afile key ai si = 
 	startingNoMessage (OnlyActionOn key ai) $ do
-		rs <- Remote.keyPossibilities key
+		rs <- Remote.keyPossibilities (Remote.IncludeIgnored False) key
 		forM_ rs $ \r ->
 			includeCommandAction $
 				starting (describeMoveAction removewhen) ai si $
@@ -334,18 +339,48 @@ toHereStart removewhen afile key ai si =
 		next $ return True
 
 fromToStart :: RemoveWhen -> AssociatedFile -> Key -> ActionItem -> SeekInput -> Remote -> Remote -> CommandStart
-fromToStart removewhen afile key ai si src dest = do
-	if Remote.uuid src == Remote.uuid dest
-		then stop
-		else do
-			u <- getUUID
-			if u == Remote.uuid src
-				then toStart removewhen afile key ai si dest
-				else if u == Remote.uuid dest
-					then fromStart removewhen afile key ai si src
-					else stopUnless (fromOk src key) $
-						starting (describeMoveAction removewhen) (OnlyActionOn key ai) si $
-							fromToPerform src dest removewhen key afile
+fromToStart removewhen afile key ai si src dest = 
+	stopUnless somethingtodo $ do
+		u <- getUUID
+		if u == Remote.uuid src
+			then toStart removewhen afile key ai si dest
+			else if u == Remote.uuid dest
+				then fromStart removewhen afile key ai si src
+				else stopUnless (fromOk src key) $
+					starting (describeMoveAction removewhen) (OnlyActionOn key ai) si $
+						fromToPerform src dest removewhen key afile
+  where
+	somethingtodo
+		| Remote.uuid src == Remote.uuid dest = return False
+		| otherwise = do
+			fast <- Annex.getRead Annex.fast
+			if fast && removewhen == RemoveNever
+				then not <$> expectedPresent dest key
+				else return True
+
+fromAnywhereToStart :: RemoveWhen -> AssociatedFile -> Key -> ActionItem -> SeekInput -> Remote -> CommandStart
+fromAnywhereToStart removewhen afile key ai si dest =
+	stopUnless somethingtodo $ do
+		u <- getUUID
+		if u == Remote.uuid dest
+			then toHereStart removewhen afile key ai si
+			else startingNoMessage (OnlyActionOn key ai) $ do
+				rs <- filter (/= dest) 
+					<$> Remote.keyPossibilities (Remote.IncludeIgnored False) key
+				forM_ rs $ \r ->
+					includeCommandAction $
+						starting (describeMoveAction removewhen) ai si $
+							fromToPerform r dest removewhen key afile
+				whenM (inAnnex key) $
+					void $ includeCommandAction $
+						toStart removewhen afile key ai si dest 
+				next $ return True
+  where
+	somethingtodo = do
+		fast <- Annex.getRead Annex.fast
+		if fast && removewhen == RemoveNever
+			then not <$> expectedPresent dest key
+			else return True
 
 {- When there is a local copy, transfer it to the dest, and drop from the src.
  -
diff --git a/Command/ReKey.hs b/Command/ReKey.hs
index 5958f48ac5..6698ad1656 100644
--- a/Command/ReKey.hs
+++ b/Command/ReKey.hs
@@ -1,6 +1,6 @@
 {- git-annex command
  -
- - Copyright 2012-2016 Joey Hess <id@joeyh.name>
+ - Copyright 2012-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -19,6 +19,7 @@ import Annex.ReplaceFile
 import Logs.Location
 import Annex.InodeSentinal
 import Annex.WorkTree
+import Logs.Migrate
 import Utility.InodeCache
 import qualified Utility.RawFilePath as R
 
@@ -88,20 +89,13 @@ perform file oldkey newkey = do
 			giveup $ decodeBS $ quote qp $ QuotedPath file
 				<> " is not available (use --force to override)"
 		)
-	next $ cleanup file newkey
+	next $ cleanup file newkey $ const noop
 
 {- Make a hard link to the old key content (when supported),
  - to avoid wasting disk space. -}
 linkKey :: RawFilePath -> Key -> Key -> Annex Bool
 linkKey file oldkey newkey = ifM (isJust <$> isAnnexLink file)
- 	{- If the object file is already hardlinked to elsewhere, a hard
-	 - link won't be made by getViaTmpFromDisk, but a copy instead.
-	 - This avoids hard linking to content linked to an
-	 - unlocked file, which would leave the new key unlocked
-	 - and vulnerable to corruption. -}
-	( getViaTmpFromDisk RetrievalAllKeysSecure DefaultVerify newkey (AssociatedFile Nothing) $ \tmp -> unVerified $ do
-		oldobj <- calcRepo (gitAnnexLocation oldkey)
-		isJust <$> linkOrCopy' (return True) newkey oldobj tmp Nothing
+	( linkKey' DefaultVerify oldkey newkey
 	, do
 	 	{- The file being rekeyed is itself an unlocked file; if
 		 - it's hard linked to the old key, that link must be broken. -}
@@ -111,10 +105,9 @@ linkKey file oldkey newkey = ifM (isJust <$> isAnnexLink file)
 			when (linkCount st > 1) $ do
 				freezeContent oldobj
 				replaceWorkTreeFile (fromRawFilePath file) $ \tmp -> do
-					let tmp' = toRawFilePath tmp
-					unlessM (checkedCopyFile oldkey oldobj tmp' Nothing) $
+					unlessM (checkedCopyFile oldkey oldobj tmp Nothing) $
 						giveup "can't lock old key"
-					thawContent tmp'
+					thawContent tmp
 		ic <- withTSDelta (liftIO . genInodeCache file)
 		case v of
 			Left e -> do
@@ -128,18 +121,34 @@ linkKey file oldkey newkey = ifM (isJust <$> isAnnexLink file)
 					LinkAnnexNoop -> True
 	)
 
-cleanup :: RawFilePath -> Key -> CommandCleanup
-cleanup file newkey = do
-	ifM (isJust <$> isAnnexLink file)
+ {- If the object file is already hardlinked to elsewhere, a hard
+ - link won't be made by getViaTmpFromDisk, but a copy instead.
+ - This avoids hard linking to content linked to an
+ - unlocked file, which would leave the new key unlocked
+ - and vulnerable to corruption. -}
+linkKey' :: VerifyConfig -> Key -> Key -> Annex Bool
+linkKey' v oldkey newkey =
+	getViaTmpFromDisk RetrievalAllKeysSecure v newkey (AssociatedFile Nothing) $ \tmp -> unVerified $ do
+		oldobj <- calcRepo (gitAnnexLocation oldkey)
+		isJust <$> linkOrCopy' (return True) newkey oldobj tmp Nothing
+
+cleanup :: RawFilePath -> Key -> (MigrationRecord -> Annex ()) -> CommandCleanup
+cleanup file newkey a = do
+	newkeyrec <- ifM (isJust <$> isAnnexLink file)
 		( do
 			-- Update symlink to use the new key.
-			addSymlink file newkey Nothing
+			sha <- genSymlink file newkey Nothing
+			stageSymlink file sha
+			return (MigrationRecord sha)
 		, do
 			mode <- liftIO $ catchMaybeIO $ fileMode <$> R.getFileStatus file
 			liftIO $ whenM (isJust <$> isPointerFile file) $
 				writePointerFile file newkey mode
-			stagePointerFile file mode =<< hashPointerFile newkey
+			sha <- hashPointerFile newkey
+			stagePointerFile file mode sha
+			return (MigrationRecord sha)
 		)
 	whenM (inAnnex newkey) $
 		logStatus newkey InfoPresent
+	a newkeyrec
 	return True
diff --git a/Command/RecvKey.hs b/Command/RecvKey.hs
index 11bd80f761..8d58ae29bf 100644
--- a/Command/RecvKey.hs
+++ b/Command/RecvKey.hs
@@ -28,7 +28,7 @@ start :: (SeekInput, Key) -> CommandStart
 start (_, key) = fieldTransfer Download key $ \_p -> do
 	-- This matches the retrievalSecurityPolicy of Remote.Git
 	let rsp = RetrievalAllKeysSecure
-	ifM (getViaTmp rsp DefaultVerify key (AssociatedFile Nothing) go)
+	ifM (getViaTmp rsp DefaultVerify key (AssociatedFile Nothing) Nothing go)
 		( do
 			logStatus key InfoPresent
 			_ <- quiesce True
diff --git a/Command/Reinject.hs b/Command/Reinject.hs
index 4fba771b1e..7385eaedd5 100644
--- a/Command/Reinject.hs
+++ b/Command/Reinject.hs
@@ -128,7 +128,7 @@ perform src key = do
 		, giveup "failed"
 		)
   where
-	move = checkDiskSpaceToGet key False $
+	move = checkDiskSpaceToGet key Nothing False $
 		moveAnnex key (AssociatedFile Nothing) src
 
 cleanup :: Key -> CommandCleanup
diff --git a/Command/SendKey.hs b/Command/SendKey.hs
index bcc24c72af..7bd71ad287 100644
--- a/Command/SendKey.hs
+++ b/Command/SendKey.hs
@@ -32,7 +32,8 @@ start (_, key) = do
 		<$> getField "RsyncOptions"
 	ifM (inAnnex key)
 		( fieldTransfer Upload key $ \_p ->
-			sendAnnex key rollback $ liftIO . rsyncServerSend (map Param opts)
+			sendAnnex key rollback $ \f _sz -> 
+				liftIO $ rsyncServerSend (map Param opts) f
 		, do
 			warning "requested key is not present"
 			liftIO exitFailure
diff --git a/Command/SetKey.hs b/Command/SetKey.hs
index 263278189f..a59fc740fe 100644
--- a/Command/SetKey.hs
+++ b/Command/SetKey.hs
@@ -36,7 +36,7 @@ perform file key = do
 	-- the file might be on a different filesystem, so moveFile is used
 	-- rather than simply calling moveAnnex; disk space is also
 	-- checked this way.
-	ok <- getViaTmp RetrievalAllKeysSecure DefaultVerify key (AssociatedFile Nothing) $ \dest -> unVerified $
+	ok <- getViaTmp RetrievalAllKeysSecure DefaultVerify key (AssociatedFile Nothing) Nothing $ \dest -> unVerified $
 		if dest /= file
 			then liftIO $ catchBoolIO $ do
 				moveFile file dest
diff --git a/Command/Sync.hs b/Command/Sync.hs
index fcdc807f1f..c4cbb0ccd1 100644
--- a/Command/Sync.hs
+++ b/Command/Sync.hs
@@ -58,6 +58,7 @@ import Command.Get (getKey')
 import qualified Command.Move
 import qualified Command.Export
 import qualified Command.Import
+import qualified Command.Migrate
 import Annex.Drop
 import Annex.UUID
 import Logs.UUID
@@ -76,6 +77,7 @@ import Annex.TaggedPush
 import Annex.CurrentBranch
 import Annex.Import
 import Annex.CheckIgnore
+import Annex.PidLock
 import Types.FileMatcher
 import Types.GitConfig
 import Types.Availability
@@ -293,6 +295,10 @@ seek' o = startConcurrency transferStages $ do
 			
 			content <- shouldSyncContent o
 
+			when content $
+				whenM (annexSyncMigrations <$> Annex.getGitConfig) $
+					Command.Migrate.seekDistributedMigrations True
+
 			forM_ (filter isImport contentremotes) $
 				withbranch . importRemote content o
 			forM_ (filter isThirdPartyPopulated contentremotes) $
@@ -352,15 +358,16 @@ mergeConfig mergeunrelated = do
 		]
 
 merge :: CurrBranch -> [Git.Merge.MergeConfig] -> SyncOptions -> Git.Branch.CommitMode -> [Git.Branch] -> Annex Bool
-merge currbranch mergeconfig o commitmode tomergel = do
-	canresolvemerge <- if resolveMergeOverride o
-		then getGitConfigVal annexResolveMerge
-		else return False
-	and <$> case currbranch of
-		(Just b, Just adj) -> forM tomergel $ \tomerge ->
-			mergeToAdjustedBranch tomerge (b, adj) mergeconfig canresolvemerge commitmode
-		(b, _) -> forM tomergel $ \tomerge ->
-			autoMergeFrom tomerge b mergeconfig commitmode canresolvemerge
+merge currbranch mergeconfig o commitmode tomergel = 
+	runsGitAnnexChildProcessViaGit $ do
+		canresolvemerge <- if resolveMergeOverride o
+			then getGitConfigVal annexResolveMerge
+			else return False
+		and <$> case currbranch of
+			(Just b, Just adj) -> forM tomergel $ \tomerge ->
+				mergeToAdjustedBranch tomerge (b, adj) mergeconfig canresolvemerge commitmode
+			(b, _) -> forM tomergel $ \tomerge ->
+				autoMergeFrom tomerge b mergeconfig commitmode canresolvemerge
 
 syncBranch :: Git.Branch -> Git.Branch
 syncBranch = Git.Ref.underBase "refs/heads/synced" . origBranch
@@ -584,7 +591,7 @@ importRemote importcontent o remote currbranch
 			let (branch, subdir) = splitRemoteAnnexTrackingBranchSubdir b
 			if canImportKeys remote importcontent
 				then do
-					Command.Import.seekRemote remote branch subdir importcontent (CheckGitIgnore True)
+					Command.Import.seekRemote remote branch subdir importcontent (CheckGitIgnore True) Nothing
 					-- Importing generates a branch
 					-- that is not initially connected
 					-- to the current branch, so allow
@@ -840,7 +847,7 @@ seekSyncContent o rs currbranch = do
   where
 	seekworktree mvar l bloomfeeder = do
 		let seeker = AnnexedFileSeeker
-			{ startAction = gofile bloomfeeder mvar
+			{ startAction = const $ gofile bloomfeeder mvar
 			, checkContentPresent = Nothing
 			, usesLocationLog = True
 			}
@@ -897,7 +904,7 @@ seekSyncContent o rs currbranch = do
 syncFile :: SyncOptions -> Either (Maybe (Bloom Key)) (Key -> Annex ()) -> [Remote] -> AssociatedFile -> Key -> Annex Bool
 syncFile o ebloom rs af k = do
 	inhere <- inAnnex k
-	locs <- map Remote.uuid <$> Remote.keyPossibilities k
+	locs <- map Remote.uuid <$> Remote.keyPossibilities (Remote.IncludeIgnored False) k
 	let (have, lack) = partition (\r -> Remote.uuid r `elem` locs) rs
 
 	got <- anyM id =<< handleget have inhere
@@ -948,7 +955,7 @@ syncFile o ebloom rs af k = do
 	wantput r
 		| pushOption o == False && operationMode o /= SatisfyMode = return False
 		| Remote.readonly r || remoteAnnexReadOnly (Remote.gitconfig r) = return False
-		| isExport r = return False
+		| isExport r || isImport r = return False
 		| isThirdPartyPopulated r = return False
 		| otherwise = wantGetBy True (Just k) af (Remote.uuid r)
 	handleput lack inhere
diff --git a/Command/TestRemote.hs b/Command/TestRemote.hs
index cd7f2a522f..9f8c5f1df9 100644
--- a/Command/TestRemote.hs
+++ b/Command/TestRemote.hs
@@ -296,7 +296,7 @@ test runannex mkr mkk =
 		Just b -> case Types.Backend.verifyKeyContent b of
 			Nothing -> return True
 			Just verifier -> verifier k (serializeKey' k)
-	get r k = logStatusAfter k $ getViaTmp (Remote.retrievalSecurityPolicy r) (RemoteVerify r) k (AssociatedFile Nothing) $ \dest ->
+	get r k = logStatusAfter k $ getViaTmp (Remote.retrievalSecurityPolicy r) (RemoteVerify r) k (AssociatedFile Nothing) Nothing $ \dest ->
 		tryNonAsync (Remote.retrieveKeyFile r k (AssociatedFile Nothing) (fromRawFilePath dest) nullMeterUpdate (RemoteVerify r)) >>= \case
 			Right v -> return (True, v)
 			Left _ -> return (False, UnVerified)
@@ -370,13 +370,13 @@ testUnavailable runannex mkr mkk =
 	, check (`notElem` [Right True, Right False]) "checkPresent" $ \r k ->
 		Remote.checkPresent r k
 	, check (== Right False) "retrieveKeyFile" $ \r k ->
-		logStatusAfter k $ getViaTmp (Remote.retrievalSecurityPolicy r) (RemoteVerify r) k (AssociatedFile Nothing) $ \dest ->
+		logStatusAfter k $ getViaTmp (Remote.retrievalSecurityPolicy r) (RemoteVerify r) k (AssociatedFile Nothing) Nothing $ \dest ->
 			tryNonAsync (Remote.retrieveKeyFile r k (AssociatedFile Nothing) (fromRawFilePath dest) nullMeterUpdate (RemoteVerify r)) >>= \case
 				Right v -> return (True, v)
 				Left _ -> return (False, UnVerified)
 	, check (== Right False) "retrieveKeyFileCheap" $ \r k -> case Remote.retrieveKeyFileCheap r of
 		Nothing -> return False
-		Just a -> logStatusAfter k $ getViaTmp (Remote.retrievalSecurityPolicy r) (RemoteVerify r) k (AssociatedFile Nothing) $ \dest -> 
+		Just a -> logStatusAfter k $ getViaTmp (Remote.retrievalSecurityPolicy r) (RemoteVerify r) k (AssociatedFile Nothing) Nothing $ \dest -> 
 			unVerified $ isRight
 				<$> tryNonAsync (a k (AssociatedFile Nothing) (fromRawFilePath dest))
 	]
diff --git a/Command/TransferKey.hs b/Command/TransferKey.hs
index 78820c0db5..e5564ce989 100644
--- a/Command/TransferKey.hs
+++ b/Command/TransferKey.hs
@@ -63,7 +63,7 @@ toPerform key file remote = go Upload file $
 fromPerform :: Key -> AssociatedFile -> Remote -> CommandPerform
 fromPerform key file remote = go Upload file $
 	download' (uuid remote) key file Nothing stdRetry $ \p ->
-		logStatusAfter key $ getViaTmp (retrievalSecurityPolicy remote) vc key file $ \t ->
+		logStatusAfter key $ getViaTmp (retrievalSecurityPolicy remote) vc key file Nothing $ \t ->
 			tryNonAsync (Remote.retrieveKeyFile remote key file (fromRawFilePath t) p vc) >>= \case
 				Right v -> return (True, v)	
 				Left e -> do
diff --git a/Command/TransferKeys.hs b/Command/TransferKeys.hs
index 3d28a23894..fe7a71fb51 100644
--- a/Command/TransferKeys.hs
+++ b/Command/TransferKeys.hs
@@ -50,7 +50,7 @@ start = do
 						return True
 		| otherwise = notifyTransfer direction file $
 			download' (Remote.uuid remote) key file Nothing stdRetry $ \p ->
-				logStatusAfter key $ getViaTmp (Remote.retrievalSecurityPolicy remote) (RemoteVerify remote) key file $ \t -> do
+				logStatusAfter key $ getViaTmp (Remote.retrievalSecurityPolicy remote) (RemoteVerify remote) key file Nothing $ \t -> do
 					r <- tryNonAsync (Remote.retrieveKeyFile remote key file (fromRawFilePath t) p (RemoteVerify remote)) >>= \case
 						Left e -> do
 							warning (UnquotedString (show e))
diff --git a/Command/Transferrer.hs b/Command/Transferrer.hs
index 83b364fb85..f48567eb00 100644
--- a/Command/Transferrer.hs
+++ b/Command/Transferrer.hs
@@ -55,7 +55,7 @@ start = do
 		-- so caller is responsible for doing notification
 		-- and for retrying, and updating location log,
 		-- and stall canceling.
-		let go p = getViaTmp (Remote.retrievalSecurityPolicy remote) (RemoteVerify remote) key file $ \t -> do
+		let go p = getViaTmp (Remote.retrievalSecurityPolicy remote) (RemoteVerify remote) key file Nothing $ \t -> do
 			Remote.verifiedAction (Remote.retrieveKeyFile remote key file (fromRawFilePath t) p (RemoteVerify remote))
 		in download' (Remote.uuid remote) key file Nothing noRetry go 
 			noNotification
@@ -72,7 +72,7 @@ start = do
 	runner (AssistantDownloadRequest _ key (TransferAssociatedFile file)) remote =
 		notifyTransfer Download file $
 			download' (Remote.uuid remote) key file Nothing stdRetry $ \p ->
-				logStatusAfter key $ getViaTmp (Remote.retrievalSecurityPolicy remote) (RemoteVerify remote) key file $ \t -> do
+				logStatusAfter key $ getViaTmp (Remote.retrievalSecurityPolicy remote) (RemoteVerify remote) key file Nothing $ \t -> do
 					r <- tryNonAsync (Remote.retrieveKeyFile remote key file (fromRawFilePath t) p (RemoteVerify remote)) >>= \case
 						Left e -> do
 							warning (UnquotedString (show e))
diff --git a/Command/Unannex.hs b/Command/Unannex.hs
index c6cb70793a..8eeae06d28 100644
--- a/Command/Unannex.hs
+++ b/Command/Unannex.hs
@@ -34,7 +34,7 @@ seek ps = withFilesInGitAnnex ww (seeker False) =<< workTreeItems ww ps
 
 seeker :: Bool -> AnnexedFileSeeker
 seeker fast = AnnexedFileSeeker
-	{ startAction = start fast
+	{ startAction = const $ start fast
 	, checkContentPresent = Just True
 	, usesLocationLog = False
 	}
diff --git a/Command/Unlock.hs b/Command/Unlock.hs
index c0c79a7a6a..c8faa7532f 100644
--- a/Command/Unlock.hs
+++ b/Command/Unlock.hs
@@ -35,7 +35,7 @@ seek ps = withFilesInGitAnnex ww seeker =<< workTreeItems ww ps
   where
 	ww = WarnUnmatchLsFiles "unlock"
 	seeker = AnnexedFileSeeker
-		{ startAction = start
+		{ startAction = const start
 		, checkContentPresent = Nothing
 		, usesLocationLog = False
 		}
@@ -54,14 +54,14 @@ perform dest key = do
 	destic <- replaceWorkTreeFile (fromRawFilePath dest) $ \tmp -> do
 		ifM (inAnnex key)
 			( do
-				r <- linkFromAnnex' key (toRawFilePath tmp) destmode
+				r <- linkFromAnnex' key tmp destmode
 				case r of
 					LinkAnnexOk -> return ()
 					LinkAnnexNoop -> return ()
 					LinkAnnexFailed -> giveup "unlock failed"
-			, liftIO $ writePointerFile (toRawFilePath tmp) key destmode
+			, liftIO $ writePointerFile tmp key destmode
 			)
-		withTSDelta (liftIO . genInodeCache (toRawFilePath tmp))
+		withTSDelta (liftIO . genInodeCache tmp)
 	next $ cleanup dest destic key destmode
 
 cleanup :: RawFilePath -> Maybe InodeCache -> Key -> Maybe FileMode -> CommandCleanup
diff --git a/Command/Watch.hs b/Command/Watch.hs
index 340559d2b6..45289f269e 100644
--- a/Command/Watch.hs
+++ b/Command/Watch.hs
@@ -24,5 +24,12 @@ start :: Bool -> DaemonOptions -> Maybe Duration -> CommandStart
 start assistant o startdelay = do
 	if stopDaemonOption o
 		then stopDaemon
-		else startDaemon assistant (foregroundDaemonOption o) startdelay Nothing Nothing Nothing -- does not return
+		else startDaemon assistant 
+			(foregroundDaemonOption o)
+			startdelay
+			Nothing 
+			Nothing
+			Nothing
+			Nothing
+			-- does not return
 	stop
diff --git a/Command/WebApp.hs b/Command/WebApp.hs
index ac0f9d3419..2958784eb7 100644
--- a/Command/WebApp.hs
+++ b/Command/WebApp.hs
@@ -36,6 +36,7 @@ import Utility.Android
 
 import Control.Concurrent
 import Control.Concurrent.STM
+import Network.Socket (PortNumber)
 
 cmd :: Command
 cmd = noCommit $ dontCheck repoExists $ notBareRepo $
@@ -45,6 +46,7 @@ cmd = noCommit $ dontCheck repoExists $ notBareRepo $
 
 data WebAppOptions = WebAppOptions
 	{ listenAddress :: Maybe String
+	, listenPort :: Maybe PortNumber
 	}
 
 optParser :: CmdParamsDesc -> Parser WebAppOptions
@@ -53,6 +55,10 @@ optParser _ = WebAppOptions
 		( long "listen" <> metavar paramAddress
 		<> help "accept connections to this address"
 		))
+	<*> optional (option auto
+		( long "port" <> metavar paramNumber
+		<> help "specify port to listen on"
+		))
 
 seek :: WebAppOptions -> CommandSeek
 seek = commandAction . start
@@ -77,9 +83,12 @@ start' allowauto o = do
 		listenAddress' <- if isJust (listenAddress o)
 			then pure (listenAddress o)
 			else annexListen <$> Annex.getGitConfig
+		listenPort' <- if isJust (listenPort o)
+			then pure (listenPort o)
+			else annexPort <$> Annex.getGitConfig
 		ifM (checkpid <&&> checkshim (fromRawFilePath f))
-			( if isJust (listenAddress o)
-				then giveup "The assistant is already running, so --listen cannot be used."
+			( if isJust (listenAddress o) || isJust (listenPort o)
+				then giveup "The assistant is already running, so --listen and --port cannot be used."
 				else do
 					url <- liftIO . readFile . fromRawFilePath
 						=<< fromRepo gitAnnexUrlFile
@@ -87,7 +96,7 @@ start' allowauto o = do
 						then putStrLn url
 						else liftIO $ openBrowser browser (fromRawFilePath f) url Nothing Nothing
 			, do
-				startDaemon True True Nothing cannotrun listenAddress' $ Just $ 
+				startDaemon True True Nothing cannotrun listenAddress' listenPort' $ Just $ 
 					\origout origerr url htmlshim ->
 						if isJust listenAddress'
 							then maybe noop (`hPutStrLn` url) origout
@@ -168,6 +177,7 @@ firstRun o = do
 			webAppThread d urlrenderer True Nothing
 				(callback signaler)
 				(listenAddress o)
+				(listenPort o)
 				(callback mainthread)
 		waitNamedThreads
   where
@@ -189,8 +199,8 @@ firstRun o = do
 			_wait <- takeMVar v
 			state <- Annex.new =<< Git.CurrentRepo.get
 			Annex.eval state $
-				startDaemon True True Nothing Nothing (listenAddress o) $ Just $
-					sendurlback v
+				startDaemon True True Nothing Nothing (listenAddress o) (listenPort o) 
+					(Just $ sendurlback v)
 	sendurlback v _origout _origerr url _htmlshim = putMVar v url
 
 openBrowser :: Maybe FilePath -> FilePath -> String -> Maybe Handle -> Maybe Handle -> IO ()
diff --git a/Command/WhereUsed.hs b/Command/WhereUsed.hs
index 5f01c9c2a6..2119c02a66 100644
--- a/Command/WhereUsed.hs
+++ b/Command/WhereUsed.hs
@@ -49,7 +49,7 @@ seek o = withKeyOptions (Just (keyOptions o)) False dummyfileseeker
 	(commandAction . start o) dummyfilecommandseek (WorkTreeItems [])
   where
 	dummyfileseeker = AnnexedFileSeeker
-		{ startAction = \_ _ _ -> return Nothing
+		{ startAction = \_ _ _ _ -> return Nothing
 		, checkContentPresent = Nothing
 		, usesLocationLog = False
 		}
diff --git a/Command/Whereis.hs b/Command/Whereis.hs
index c8ca119ed6..7c7edd1f4b 100644
--- a/Command/Whereis.hs
+++ b/Command/Whereis.hs
@@ -51,7 +51,7 @@ seek :: WhereisOptions -> CommandSeek
 seek o = do
 	m <- remoteMap id
 	let seeker = AnnexedFileSeeker
-		{ startAction = start o m
+		{ startAction = const $ start o m
 		, checkContentPresent = Nothing
 		, usesLocationLog = True
 		}
diff --git a/Creds.hs b/Creds.hs
index cfc6c3dc83..e429d796cf 100644
--- a/Creds.hs
+++ b/Creds.hs
@@ -100,10 +100,10 @@ setRemoteCredPair' pc encsetup gc storage mcreds = case mcreds of
 		cmd <- gpgCmd <$> Annex.getGitConfig
 		s <- liftIO $ encrypt cmd (pc, gc) cipher
 			(feedBytes $ L.pack $ encodeCredPair creds)
-			(readBytesStrictly $ return . S.unpack)
-		storeconfig' key (Accepted (toB64 s))
+			(readBytesStrictly return)
+		storeconfig' key (Accepted (decodeBS (toB64 s)))
 	storeconfig creds key Nothing =
-		storeconfig' key (Accepted (toB64 $ encodeCredPair creds))
+		storeconfig' key (Accepted (decodeBS $ toB64 $ encodeBS $ encodeCredPair creds))
 	
 	storeconfig' key val = return $ pc
 		{ parsedRemoteConfigMap = M.insert key (RemoteConfigValue val) (parsedRemoteConfigMap pc)
@@ -129,13 +129,13 @@ getRemoteCredPair c gc storage = maybe fromcache (return . Just) =<< fromenv
 		case (getval, mcipher) of
 			(Nothing, _) -> return Nothing
 			(Just enccreds, Just (cipher, storablecipher)) ->
-				fromenccreds enccreds cipher storablecipher
+				fromenccreds (encodeBS enccreds) cipher storablecipher
 			(Just bcreds, Nothing) ->
-				fromcreds $ fromB64 bcreds
+				fromcreds $ decodeBS $ fromB64 $ encodeBS bcreds
 	fromenccreds enccreds cipher storablecipher = do
 		cmd <- gpgCmd <$> Annex.getGitConfig
 		mcreds <- liftIO $ catchMaybeIO $ decrypt cmd (c, gc) cipher
-			(feedBytes $ L.pack $ fromB64 enccreds)
+			(feedBytes $ L.fromStrict $ fromB64 enccreds)
 			(readBytesStrictly $ return . S.unpack)
 		case mcreds of
 			Just creds -> fromcreds creds
@@ -146,7 +146,7 @@ getRemoteCredPair c gc storage = maybe fromcache (return . Just) =<< fromenv
 				case storablecipher of
 					SharedCipher {} -> showLongNote "gpg error above was caused by an old git-annex bug in credentials storage. Working around it.."
 					_ -> giveup "*** Insecure credentials storage detected for this remote! See https://git-annex.branchable.com/upgrades/insecure_embedded_creds/"
-				fromcreds $ fromB64 enccreds
+				fromcreds $ decodeBS $ fromB64 enccreds
 	fromcreds creds = case decodeCredPair creds of
 		Just credpair -> do
 			writeCacheCredPair credpair storage
diff --git a/Crypto.hs b/Crypto.hs
index 7f134a3cbc..3fa6d781d1 100644
--- a/Crypto.hs
+++ b/Crypto.hs
@@ -1,9 +1,8 @@
 {- git-annex crypto
  -
- - Currently using gpg; could later be modified to support different
- - crypto backends if necessary.
+ - Currently using gpg by default, or optionally stateless OpenPGP.
  -
- - Copyright 2011-2022 Joey Hess <id@joeyh.name>
+ - Copyright 2011-2024 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -32,55 +31,60 @@ module Crypto (
 	readBytesStrictly,
 	encrypt,
 	decrypt,
-	LensGpgEncParams(..),
+	LensEncParams(..),
 
 	prop_HmacSha1WithCipher_sane
 ) where
 
 import qualified Data.ByteString as S
 import qualified Data.ByteString.Lazy as L
-import Data.ByteString.UTF8 (fromString)
 import Control.Monad.IO.Class
+import qualified Data.ByteString.Short as S (toShort)
 
 import Annex.Common
 import qualified Utility.Gpg as Gpg
+import qualified Utility.StatelessOpenPGP as SOP
 import Types.Crypto
 import Types.Remote
 import Types.Key
 import Annex.SpecialRemote.Config
-import qualified Data.ByteString.Short as S (toShort)
+import Utility.Tmp.Dir
+
+{- The number of bytes of entropy used to generate a Cipher.
+ -
+ - Since a Cipher is base-64 encoded, the actual size of a Cipher
+ - is larger than this. 512 bytes of date base-64 encodes to 684
+ - characters.
+ -}
+cipherSize :: Int
+cipherSize = 512
 
 {- The beginning of a Cipher is used for MAC'ing; the remainder is used
- - as the GPG symmetric encryption passphrase when using the hybrid
- - scheme. Note that the cipher itself is base-64 encoded, hence the
- - string is longer than 'cipherSize': 683 characters, padded to 684.
+ - as the symmetric encryption passphrase.
  -
- - The 256 first characters that feed the MAC represent at best 192
- - bytes of entropy.  However that's more than enough for both the
- - default MAC algorithm, namely HMAC-SHA1, and the "strongest"
+ - Due to the base-64 encoding of the Cipher, the beginning 265 characters
+ - represent at best 192 bytes of entropy. However that's more than enough
+ - for both the default MAC algorithm, namely HMAC-SHA1, and the "strongest"
  - currently supported, namely HMAC-SHA512, which respectively need
  - (ideally) 64 and 128 bytes of entropy.
  -
- - The remaining characters (320 bytes of entropy) is enough for GnuPG's
- - symmetric cipher; unlike weaker public key crypto, the key does not
- - need to be too large.
+ - The remaining characters (320 bytes of entropy) is enough for
+ - the symmetric encryption passphrase; unlike weaker public key crypto,
+ - that does not need to be too large.
  -}
 cipherBeginning :: Int
 cipherBeginning = 256
 
-cipherSize :: Int
-cipherSize = 512
-
-cipherPassphrase :: Cipher -> String
-cipherPassphrase (Cipher c) = drop cipherBeginning c
+cipherPassphrase :: Cipher -> S.ByteString
+cipherPassphrase (Cipher c) = S.drop cipherBeginning c
 cipherPassphrase (MacOnlyCipher _) = giveup "MAC-only cipher"
 
-cipherMac :: Cipher -> String
-cipherMac (Cipher c) = take cipherBeginning c
+cipherMac :: Cipher -> S.ByteString
+cipherMac (Cipher c) = S.take cipherBeginning c
 cipherMac (MacOnlyCipher c) = c
 
 {- Creates a new Cipher, encrypted to the specified key id. -}
-genEncryptedCipher :: LensGpgEncParams c => Gpg.GpgCmd -> c -> Gpg.KeyId -> EncryptedCipherVariant -> Bool -> IO StorableCipher
+genEncryptedCipher :: LensEncParams c => Gpg.GpgCmd -> c -> Gpg.KeyId -> EncryptedCipherVariant -> Bool -> IO StorableCipher
 genEncryptedCipher cmd c keyid variant highQuality = do
 	ks <- Gpg.findPubKeys cmd keyid
 	random <- Gpg.genRandom cmd highQuality size
@@ -106,7 +110,7 @@ genSharedPubKeyCipher cmd keyid highQuality = do
 {- Updates an existing Cipher, making changes to its keyids.
  -
  - When the Cipher is encrypted, re-encrypts it. -}
-updateCipherKeyIds :: LensGpgEncParams encparams => Gpg.GpgCmd -> encparams -> [(Bool, Gpg.KeyId)] -> StorableCipher -> IO StorableCipher
+updateCipherKeyIds :: LensEncParams encparams => Gpg.GpgCmd -> encparams -> [(Bool, Gpg.KeyId)] -> StorableCipher -> IO StorableCipher
 updateCipherKeyIds _ _ _ SharedCipher{} = giveup "Cannot update shared cipher"
 updateCipherKeyIds _ _ [] c = return c
 updateCipherKeyIds cmd encparams changes encipher@(EncryptedCipher _ variant ks) = do
@@ -130,7 +134,7 @@ updateCipherKeyIds' cmd changes (KeyIds ks) = do
 	listKeyIds = concat <$$> mapM (keyIds <$$> Gpg.findPubKeys cmd)
 
 {- Encrypts a Cipher to the specified KeyIds. -}
-encryptCipher :: LensGpgEncParams c => Gpg.GpgCmd -> c -> Cipher -> EncryptedCipherVariant -> KeyIds -> IO StorableCipher
+encryptCipher :: LensEncParams c => Gpg.GpgCmd -> c -> Cipher -> EncryptedCipherVariant -> KeyIds -> IO StorableCipher
 encryptCipher cmd c cip variant (KeyIds ks) = do
 	-- gpg complains about duplicate recipient keyids
 	let ks' = nub $ sort ks
@@ -147,10 +151,10 @@ encryptCipher cmd c cip variant (KeyIds ks) = do
 		MacOnlyCipher x -> x
 
 {- Decrypting an EncryptedCipher is expensive; the Cipher should be cached. -}
-decryptCipher :: LensGpgEncParams c => Gpg.GpgCmd -> c -> StorableCipher -> IO Cipher
+decryptCipher :: LensEncParams c => Gpg.GpgCmd -> c -> StorableCipher -> IO Cipher
 decryptCipher cmd c cip = decryptCipher' cmd Nothing c cip
 
-decryptCipher' :: LensGpgEncParams c => Gpg.GpgCmd -> Maybe [(String, String)] -> c -> StorableCipher -> IO Cipher
+decryptCipher' :: LensEncParams c => Gpg.GpgCmd -> Maybe [(String, String)] -> c -> StorableCipher -> IO Cipher
 decryptCipher' _ _ _ (SharedCipher t) = return $ Cipher t
 decryptCipher' _ _ _ (SharedPubKeyCipher t _) = return $ MacOnlyCipher t
 decryptCipher' cmd environ c (EncryptedCipher t variant _) =
@@ -168,7 +172,7 @@ type EncKey = Key -> Key
  - on content. It does need to be repeatable. -}
 encryptKey :: Mac -> Cipher -> EncKey
 encryptKey mac c k = mkKey $ \d -> d
-	{ keyName = S.toShort $ encodeBS $ macWithCipher mac c (serializeKey k)
+	{ keyName = S.toShort $ encodeBS $ macWithCipher mac c (serializeKey' k)
 	, keyVariety = OtherKey $
 		encryptedBackendNamePrefix <> encodeBS (showMac mac)
 	}
@@ -196,19 +200,25 @@ readBytes a h = liftIO (L.hGetContents h) >>= a
 readBytesStrictly :: (MonadIO m) => (S.ByteString -> m a) -> Reader m a
 readBytesStrictly a h = liftIO (S.hGetContents h) >>= a
 
-
 {- Runs a Feeder action, that generates content that is symmetrically
  - encrypted with the Cipher (unless it is empty, in which case
- - public-key encryption is used) using the given gpg options, and then
- - read by the Reader action. 
+ - public-key encryption is used), and then read by the Reader action. 
  -
- - Note that the Reader must fully consume gpg's input before returning.
+ - Note that the Reader must fully consume all input before returning.
  -}
-encrypt :: (MonadIO m, MonadMask m, LensGpgEncParams c) => Gpg.GpgCmd -> c -> Cipher -> Feeder -> Reader m a -> m a
-encrypt cmd c cipher = case cipher of
-	Cipher{} -> Gpg.feedRead cmd (params ++ Gpg.stdEncryptionParams True) $
-			cipherPassphrase cipher
-	MacOnlyCipher{} -> Gpg.feedRead' cmd $ params ++ Gpg.stdEncryptionParams False
+encrypt :: (MonadIO m, MonadMask m, LensEncParams c) => Gpg.GpgCmd -> c -> Cipher -> Feeder -> Reader m a -> m a
+encrypt gpgcmd c cipher feeder reader = case cipher of
+	Cipher{} -> 
+		let passphrase = cipherPassphrase cipher
+		in case statelessOpenPGPCommand c of
+			Just sopcmd -> withTmpDir "sop" $ \d ->
+				SOP.encryptSymmetric sopcmd passphrase
+					(SOP.EmptyDirectory d)
+					(statelessOpenPGPProfile c)
+					(SOP.Armoring False)
+					feeder reader
+			Nothing -> Gpg.feedRead gpgcmd (params ++ Gpg.stdEncryptionParams True) passphrase feeder reader
+	MacOnlyCipher{} -> Gpg.feedRead' gpgcmd (params ++ Gpg.stdEncryptionParams False) feeder reader
   where
 	params = getGpgEncParams c
 
@@ -216,19 +226,26 @@ encrypt cmd c cipher = case cipher of
  - Cipher (or using a private key if the Cipher is empty), and read by the
  - Reader action.
  -
- - Note that the Reader must fully consume gpg's input before returning.
+ - Note that the Reader must fully consume all input before returning.
  - -}
-decrypt :: (MonadIO m, MonadMask m, LensGpgEncParams c) => Gpg.GpgCmd -> c -> Cipher -> Feeder -> Reader m a -> m a
-decrypt cmd c cipher = case cipher of
-	Cipher{} -> Gpg.feedRead cmd params $ cipherPassphrase cipher
-	MacOnlyCipher{} -> Gpg.feedRead' cmd params
+decrypt :: (MonadIO m, MonadMask m, LensEncParams c) => Gpg.GpgCmd -> c -> Cipher -> Feeder -> Reader m a -> m a
+decrypt cmd c cipher feeder reader = case cipher of
+	Cipher{} -> 
+		let passphrase = cipherPassphrase cipher
+		in case statelessOpenPGPCommand c of
+			Just sopcmd -> withTmpDir "sop" $ \d ->
+				SOP.decryptSymmetric sopcmd passphrase
+					(SOP.EmptyDirectory d)
+					feeder reader
+			Nothing -> Gpg.feedRead cmd params passphrase feeder reader
+	MacOnlyCipher{} -> Gpg.feedRead' cmd params feeder reader
   where
 	params = Param "--decrypt" : getGpgDecParams c
 
-macWithCipher :: Mac -> Cipher -> String -> String
+macWithCipher :: Mac -> Cipher -> S.ByteString -> String
 macWithCipher mac c = macWithCipher' mac (cipherMac c)
-macWithCipher' :: Mac -> String -> String -> String
-macWithCipher' mac c s = calcMac mac (fromString c) (fromString s)
+macWithCipher' :: Mac -> S.ByteString -> S.ByteString -> String
+macWithCipher' mac c s = calcMac mac c s
 
 {- Ensure that macWithCipher' returns the same thing forevermore. -}
 prop_HmacSha1WithCipher_sane :: Bool
@@ -236,19 +253,26 @@ prop_HmacSha1WithCipher_sane = known_good == macWithCipher' HmacSha1 "foo" "bar"
   where
 	known_good = "46b4ec586117154dacd49d664e5d63fdc88efb51"
 
-class LensGpgEncParams a where
-	{- Base parameters for encrypting. Does not include specification
+class LensEncParams a where
+	{- Base gpg parameters for encrypting. Does not include specification
 	 - of recipient keys. -}
 	getGpgEncParamsBase :: a -> [CommandParam]
-	{- Parameters for encrypting. When the remote is configured to use
+	{- Gpg parameters for encrypting. When the remote is configured to use
 	 - public-key encryption, includes specification of recipient keys. -}
 	getGpgEncParams :: a -> [CommandParam]
-	{- Parameters for decrypting. -}
+	{- Gpg parameters for decrypting. -}
 	getGpgDecParams :: a -> [CommandParam]
+	{- Set when stateless OpenPGP should be used rather than gpg.
+	 - It is currently only used for SharedEncryption and not the other
+	 - schemes which use public keys. -}
+	statelessOpenPGPCommand :: a -> Maybe SOP.SOPCmd
+	{- When using stateless OpenPGP, this may be set to a profile
+	 - which should be used instead of the default. -}
+	statelessOpenPGPProfile :: a -> Maybe SOP.SOPProfile
 
 {- Extract the GnuPG options from a pair of a Remote Config and a Remote
  - Git Config. -}
-instance LensGpgEncParams (ParsedRemoteConfig, RemoteGitConfig) where
+instance LensEncParams (ParsedRemoteConfig, RemoteGitConfig) where
 	getGpgEncParamsBase (_c,gc) = map Param (remoteAnnexGnupgOptions gc)
 	getGpgEncParams (c,gc) = getGpgEncParamsBase (c,gc) ++
  		{- When the remote is configured to use public-key encryption,
@@ -262,9 +286,21 @@ instance LensGpgEncParams (ParsedRemoteConfig, RemoteGitConfig) where
 					getRemoteConfigValue pubkeysField c
 			_ -> []
 	getGpgDecParams (_c,gc) = map Param (remoteAnnexGnupgDecryptOptions gc)
+	statelessOpenPGPCommand (c,gc) = case remoteAnnexSharedSOPCommand gc of
+		Nothing -> Nothing
+		Just sopcmd ->
+			{- So far stateless OpenPGP is only supported
+			 - for SharedEncryption, not other encryption
+			 - methods that involve public keys. -}
+			case getRemoteConfigValue encryptionField c of
+				Just SharedEncryption -> Just sopcmd
+				_ -> Nothing
+	statelessOpenPGPProfile (_c,gc) = remoteAnnexSharedSOPProfile gc
 
 {- Extract the GnuPG options from a Remote. -}
-instance LensGpgEncParams (RemoteA a) where
+instance LensEncParams (RemoteA a) where
 	getGpgEncParamsBase r = getGpgEncParamsBase (config r, gitconfig r)
 	getGpgEncParams r = getGpgEncParams (config r, gitconfig r)
 	getGpgDecParams r = getGpgDecParams (config r, gitconfig r)
+	statelessOpenPGPCommand r = statelessOpenPGPCommand (config r, gitconfig r)
+	statelessOpenPGPProfile r  = statelessOpenPGPProfile (config r, gitconfig r)
diff --git a/Database/ContentIdentifier.hs b/Database/ContentIdentifier.hs
index e847c00cce..bbf67dcfb1 100644
--- a/Database/ContentIdentifier.hs
+++ b/Database/ContentIdentifier.hs
@@ -46,7 +46,6 @@ import Types.RemoteState
 import Git.Types
 import Git.Sha
 import Git.FilePath
-import qualified Git.Ref
 import qualified Git.DiffTree as DiffTree
 import Logs
 import qualified Logs.ContentIdentifier as Log
@@ -54,9 +53,14 @@ import qualified Utility.RawFilePath as R
 
 import Database.Persist.Sql hiding (Key)
 import Database.Persist.TH
-import Database.Persist.Sqlite (runSqlite)
 import qualified System.FilePath.ByteString as P
+
+#if MIN_VERSION_persistent_sqlite(2,13,3)
+import Database.RawFilePath
+#else
+import Database.Persist.Sqlite (runSqlite)
 import qualified Data.Text as T
+#endif
 
 data ContentIdentifierHandle = ContentIdentifierHandle H.DbQueue Bool
 
@@ -103,8 +107,13 @@ openDb = do
 			runMigrationSilent migrateContentIdentifier
 		-- Migrate from old versions of database, which had buggy
 		-- and suboptimal uniqueness constraints.
+#if MIN_VERSION_persistent_sqlite(2,13,3)
+		else liftIO $ runSqlite' db $ void $
+			runMigrationSilent migrateContentIdentifier
+#else
 		else liftIO $ runSqlite (T.pack (fromRawFilePath db)) $ void $
 			runMigrationSilent migrateContentIdentifier
+#endif
 	h <- liftIO $ H.openDbQueue db "content_identifiers"
 	return $ ContentIdentifierHandle h isnew
 
@@ -162,10 +171,7 @@ getAnnexBranchTree (ContentIdentifierHandle h _) = H.queryDbQueue h $ do
 needsUpdateFromLog :: ContentIdentifierHandle -> Annex (Maybe (Sha, Sha))
 needsUpdateFromLog db = do
 	oldtree <- liftIO $ getAnnexBranchTree db
-	inRepo (Git.Ref.tree Annex.Branch.fullname) >>= \case
-		Just currtree | currtree /= oldtree ->
-			return $ Just (oldtree, currtree)
-		_ -> return Nothing
+	Annex.Branch.updatedFromTree oldtree
 
 {- The database should be locked for write when calling this. -}
 updateFromLog :: ContentIdentifierHandle -> (Sha, Sha) -> Annex ContentIdentifierHandle
diff --git a/Database/Export.hs b/Database/Export.hs
index b9d9879da0..2f0da8b231 100644
--- a/Database/Export.hs
+++ b/Database/Export.hs
@@ -73,18 +73,18 @@ share [mkPersist sqlSettings, mkMigrate "migrateExport"] [persistLowerCase|
 -- Files that have been exported to the remote and are present on it.
 Exported
   key Key
-  file SFilePath
+  file SByteString
   ExportedIndex key file
 -- Directories that exist on the remote, and the files that are in them.
 ExportedDirectory
-  subdir SFilePath
-  file SFilePath
+  subdir SByteString
+  file SByteString
   ExportedDirectoryIndex subdir file
 -- The content of the tree that has been exported to the remote.
 -- Not all of these files are necessarily present on the remote yet.
 ExportTree
   key Key
-  file SFilePath
+  file SByteString
   ExportTreeKeyFileIndex key file
   ExportTreeFileKeyIndex file key
 -- The tree stored in ExportTree
@@ -139,26 +139,26 @@ addExportedLocation :: ExportHandle -> Key -> ExportLocation -> IO ()
 addExportedLocation h k el = queueDb h $ do
 	void $ insertUniqueFast $ Exported k ef
 	let edirs = map
-		(\ed -> ExportedDirectory (SFilePath (fromExportDirectory ed)) ef)
+		(\ed -> ExportedDirectory (SByteString (fromExportDirectory ed)) ef)
 		(exportDirectories el)
 	putMany edirs
   where
-	ef = SFilePath (fromExportLocation el)
+	ef = SByteString (fromExportLocation el)
 
 removeExportedLocation :: ExportHandle -> Key -> ExportLocation -> IO ()
 removeExportedLocation h k el = queueDb h $ do
 	deleteWhere [ExportedKey ==. k, ExportedFile ==. ef]
-	let subdirs = map (SFilePath . fromExportDirectory)
+	let subdirs = map (SByteString . fromExportDirectory)
 		(exportDirectories el)
 	deleteWhere [ExportedDirectoryFile ==. ef, ExportedDirectorySubdir <-. subdirs]
   where
-	ef = SFilePath (fromExportLocation el)
+	ef = SByteString (fromExportLocation el)
 
 {- Note that this does not see recently queued changes. -}
 getExportedLocation :: ExportHandle -> Key -> IO [ExportLocation]
 getExportedLocation (ExportHandle h _) k = H.queryDbQueue h $ do
 	l <- selectList [ExportedKey ==. k] []
-	return $ map (mkExportLocation . (\(SFilePath f) -> f) . exportedFile . entityVal) l
+	return $ map (mkExportLocation . (\(SByteString f) -> f) . exportedFile . entityVal) l
 
 {- Note that this does not see recently queued changes. -}
 isExportDirectoryEmpty :: ExportHandle -> ExportDirectory -> IO Bool
@@ -166,13 +166,13 @@ isExportDirectoryEmpty (ExportHandle h _) d = H.queryDbQueue h $ do
 	l <- selectList [ExportedDirectorySubdir ==. ed] []
 	return $ null l
   where
-	ed = SFilePath $ fromExportDirectory d
+	ed = SByteString $ fromExportDirectory d
 
 {- Get locations in the export that might contain a key. -}
 getExportTree :: ExportHandle -> Key -> IO [ExportLocation]
 getExportTree (ExportHandle h _) k = H.queryDbQueue h $ do
 	l <- selectList [ExportTreeKey ==. k] []
-	return $ map (mkExportLocation . (\(SFilePath f) -> f) . exportTreeFile . entityVal) l
+	return $ map (mkExportLocation . (\(SByteString f) -> f) . exportTreeFile . entityVal) l
 
 {- Get keys that might be currently exported to a location.
  -
@@ -183,19 +183,19 @@ getExportTreeKey (ExportHandle h _) el = H.queryDbQueue h $ do
 	map (exportTreeKey . entityVal) 
 		<$> selectList [ExportTreeFile ==. ef] []
   where
-	ef = SFilePath (fromExportLocation el)
+	ef = SByteString (fromExportLocation el)
 
 addExportTree :: ExportHandle -> Key -> ExportLocation -> IO ()
 addExportTree h k loc = queueDb h $
 	void $ insertUniqueFast $ ExportTree k ef
   where
-	ef = SFilePath (fromExportLocation loc)
+	ef = SByteString (fromExportLocation loc)
 
 removeExportTree :: ExportHandle -> Key -> ExportLocation -> IO ()
 removeExportTree h k loc = queueDb h $
 	deleteWhere [ExportTreeKey ==. k, ExportTreeFile ==. ef]
   where
-	ef = SFilePath (fromExportLocation loc)
+	ef = SByteString (fromExportLocation loc)
 
 -- An action that is passed the old and new values that were exported,
 -- and updates state.
diff --git a/Database/Handle.hs b/Database/Handle.hs
index cf17fd3d3e..a7e65e54cb 100644
--- a/Database/Handle.hs
+++ b/Database/Handle.hs
@@ -193,11 +193,13 @@ runSqliteRobustly tablename db a = do
 				| otherwise -> rethrow $ errmsg "after successful open" ex
 	
 	opensettle retries ic = do
-		conn <- Sqlite.open tdb
+#if MIN_VERSION_persistent_sqlite(2,13,3)
+		conn <- Sqlite.open' db
+#else
+		conn <- Sqlite.open (T.pack (fromRawFilePath db))
+#endif
 		settle conn retries ic
 
-	tdb = T.pack (fromRawFilePath db)
-
 	settle conn retries ic = do
 		r <- try $ do
 			stmt <- Sqlite.prepare conn nullselect
diff --git a/Database/ImportFeed.hs b/Database/ImportFeed.hs
new file mode 100644
index 0000000000..2d44b0b9ea
--- /dev/null
+++ b/Database/ImportFeed.hs
@@ -0,0 +1,211 @@
+{- Sqlite database of known urls, and another of known itemids,
+ - for use by git-annex importfeed.
+ -
+ - Copyright 2023 Joey Hess <id@joeyh.name>
+ -:
+ - Licensed under the GNU AGPL version 3 or higher.
+ -}
+
+{-# LANGUAGE CPP #-}
+{-# LANGUAGE QuasiQuotes, TypeFamilies, TypeOperators, TemplateHaskell #-}
+{-# LANGUAGE OverloadedStrings, GADTs, FlexibleContexts, EmptyDataDecls #-}
+{-# LANGUAGE MultiParamTypeClasses, GeneralizedNewtypeDeriving #-}
+{-# LANGUAGE RankNTypes #-}
+{-# LANGUAGE DataKinds, FlexibleInstances #-}
+{-# LANGUAGE UndecidableInstances #-}
+#if MIN_VERSION_persistent_template(2,8,0)
+{-# LANGUAGE DerivingStrategies #-}
+{-# LANGUAGE StandaloneDeriving #-}
+#endif
+
+module Database.ImportFeed (
+	ImportFeedDbHandle,
+	openDb,
+	closeDb,
+	isKnownUrl,
+	isKnownItemId,
+) where
+
+import Database.Types
+import qualified Database.Queue as H
+import Database.Init
+import Database.Utility
+import Annex.Locations
+import Annex.Common hiding (delete)
+import qualified Annex.Branch
+import Git.Types
+import Git.Sha
+import Git.FilePath
+import qualified Git.DiffTree as DiffTree
+import Logs
+import Logs.Web
+import Logs.MetaData
+import Types.MetaData
+import Annex.MetaData.StandardFields
+import Annex.LockFile
+import qualified Utility.RawFilePath as R
+
+import Database.Persist.Sql hiding (Key)
+import Database.Persist.TH
+import qualified System.FilePath.ByteString as P
+import qualified Data.ByteString as B
+import qualified Data.Set as S
+
+data ImportFeedDbHandle = ImportFeedDbHandle H.DbQueue
+
+-- Note on indexes: ContentIndentifiersKeyRemoteCidIndex etc are really
+-- uniqueness constraints, which cause sqlite to automatically add indexes.
+-- So when adding indexes, have to take care to only add ones that work as
+-- uniqueness constraints. (Unfortunately persistent does not support indexes
+-- that are not uniqueness constraints; 
+-- https://github.com/yesodweb/persistent/issues/109)
+share [mkPersist sqlSettings, mkMigrate "migrateImportFeed"] [persistLowerCase|
+KnownUrls
+  url SByteString
+  UniqueUrl url
+KnownItemIds
+  itemid SByteString
+  UniqueItemId itemid
+-- The last git-annex branch tree sha that was used to update
+-- KnownUrls and KnownItemIds
+AnnexBranch
+  tree SSha
+  UniqueTree tree
+|]
+
+{- Opens the database, creating it if it doesn't exist yet.
+ - Updates the database from the git-annex branch. -}
+openDb :: Annex ImportFeedDbHandle
+openDb = do
+	dbdir <- calcRepo' gitAnnexImportFeedDbDir
+	let db = dbdir P.</> "db"
+	isnew <- liftIO $ not <$> R.doesPathExist db
+	when isnew $
+		initDb db $ void $ 
+			runMigrationSilent migrateImportFeed
+	dbh <- liftIO $ H.openDbQueue db "known_urls"
+	let h = ImportFeedDbHandle dbh
+	needsUpdateFromLog h >>= \case
+		Nothing -> return ()
+		Just v -> do
+			lck <- calcRepo' gitAnnexImportFeedDbLock
+                	withExclusiveLock lck $
+				updateFromLog h v
+	return h
+
+closeDb :: ImportFeedDbHandle -> Annex ()
+closeDb (ImportFeedDbHandle h) = liftIO $ H.closeDbQueue h
+
+isKnownUrl :: ImportFeedDbHandle -> URLString -> IO Bool
+isKnownUrl (ImportFeedDbHandle h) u = 
+	H.queryDbQueue h $ do
+		l <- selectList
+			[ KnownUrlsUrl ==. SByteString (encodeBS u)
+			] []
+		return $ not (null l)
+
+isKnownItemId :: ImportFeedDbHandle -> B.ByteString -> IO Bool
+isKnownItemId (ImportFeedDbHandle h) i = 
+	H.queryDbQueue h $ do
+		l <- selectList
+			[ KnownItemIdsItemid ==. SByteString i
+			] []
+		return $ not (null l)
+
+recordKnownUrl :: ImportFeedDbHandle -> URLByteString -> IO ()
+recordKnownUrl h u = queueDb h $
+	void $ insertUniqueFast $ KnownUrls $ SByteString u
+
+recordKnownItemId :: ImportFeedDbHandle -> SByteString -> IO ()
+recordKnownItemId h i = queueDb h $
+	void $ insertUniqueFast $ KnownItemIds i
+
+recordAnnexBranchTree :: ImportFeedDbHandle -> Sha -> IO ()
+recordAnnexBranchTree h s = queueDb h $ do
+	deleteWhere ([] :: [Filter AnnexBranch])
+	void $ insertUniqueFast $ AnnexBranch $ toSSha s
+
+getAnnexBranchTree :: ImportFeedDbHandle -> IO Sha
+getAnnexBranchTree (ImportFeedDbHandle h) = H.queryDbQueue h $ do
+	l <- selectList ([] :: [Filter AnnexBranch]) []
+	case l of
+		(s:[]) -> return $ fromSSha $ annexBranchTree $ entityVal s
+		_ -> return emptyTree
+
+queueDb :: ImportFeedDbHandle -> SqlPersistM () -> IO ()
+queueDb (ImportFeedDbHandle h) = H.queueDb h checkcommit
+  where
+        -- commit queue after 10000 changes
+        checkcommit sz _lastcommittime
+                | sz > 10000 = return True
+                | otherwise = return False
+
+{- Check if the git-annex branch has been updated and the database needs
+ - to be updated with any new information from it. -}
+needsUpdateFromLog :: ImportFeedDbHandle -> Annex (Maybe (Sha, Sha))
+needsUpdateFromLog db = do
+	oldtree <- liftIO $ getAnnexBranchTree db
+	Annex.Branch.updatedFromTree oldtree
+
+{- The database should be locked for write when calling this. -}
+updateFromLog :: ImportFeedDbHandle -> (Sha, Sha) -> Annex ()
+updateFromLog db@(ImportFeedDbHandle h) (oldtree, currtree)
+	| oldtree == emptyTree = do
+		scanbranch
+		out
+	| otherwise = do
+		scandiff
+		out
+  where
+  	out = liftIO $ do
+		recordAnnexBranchTree db currtree
+		H.flushDbQueue h
+	
+	knownitemids s = liftIO $ forM_ (S.toList s) $
+		recordKnownItemId db . SByteString . fromMetaValue
+
+	knownurls us = liftIO $ forM_ us $
+		recordKnownUrl db
+		
+	scandiff = do
+		(l, cleanup) <- inRepo $
+			DiffTree.diffTreeRecursive oldtree currtree
+		mapM_ godiff l
+		void $ liftIO $ cleanup
+	
+	godiff ti = do
+		let f = getTopFilePath (DiffTree.file ti)
+		case extLogFileKey urlLogExt f of
+			Just k -> do
+				knownurls =<< getUrls' k
+			Nothing -> case extLogFileKey metaDataLogExt f of
+				Just k -> do
+					m <- getCurrentMetaData k
+					knownitemids (currentMetaDataValues itemIdField m)
+				Nothing -> return ()
+
+	-- When initially populating the database, this 
+	-- is faster than diffing from the empty tree
+	-- and looking up every log file.
+	scanbranch = Annex.Branch.overBranchFileContents toscan goscan >>= \case
+		Just () -> return ()
+		Nothing -> scandiff
+	
+	toscan f
+		| isUrlLog f = Just ()
+		| isMetaDataLog f = Just ()
+		| otherwise = Nothing
+	
+	goscan reader = reader >>= \case
+		Just ((), f, Just content)
+			| isUrlLog f -> do
+				knownurls (parseUrlLog content)
+				goscan reader
+			| isMetaDataLog f -> do
+				knownitemids $
+					currentMetaDataValues itemIdField $
+						parseCurrentMetaData content
+				goscan reader
+			| otherwise -> goscan reader
+		Just ((), _, Nothing) -> goscan reader
+		Nothing -> return ()
diff --git a/Database/Init.hs b/Database/Init.hs
index ac33fdae03..6f7ba09faf 100644
--- a/Database/Init.hs
+++ b/Database/Init.hs
@@ -1,11 +1,11 @@
 {- Persistent sqlite database initialization
  -
- - Copyright 2015-2020 Joey Hess <id@joeyh.name>
+ - Copyright 2015-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
 
-{-# LANGUAGE OverloadedStrings #-}
+{-# LANGUAGE OverloadedStrings, CPP #-}
 
 module Database.Init where
 
@@ -13,6 +13,9 @@ import Annex.Common
 import Annex.Perms
 import Utility.FileMode
 import qualified Utility.RawFilePath as R
+#if MIN_VERSION_persistent_sqlite(2,13,3)
+import Database.RawFilePath
+#endif
 
 import Database.Persist.Sqlite
 import Lens.Micro
@@ -32,9 +35,13 @@ initDb db migration = do
 	let dbdir = P.takeDirectory db
 	let tmpdbdir = dbdir <> ".tmp"
 	let tmpdb = tmpdbdir P.</> "db"
-	let tdb = T.pack (fromRawFilePath tmpdb)
+	let tmpdb' = T.pack (fromRawFilePath tmpdb)
 	createAnnexDirectory tmpdbdir
-	liftIO $ runSqliteInfo (enableWAL tdb) migration
+#if MIN_VERSION_persistent_sqlite(2,13,3)
+	liftIO $ runSqliteInfo' tmpdb (enableWAL tmpdb') migration
+#else
+	liftIO $ runSqliteInfo (enableWAL tmpdb') migration
+#endif
 	setAnnexDirPerm tmpdbdir
 	-- Work around sqlite bug that prevents it from honoring
 	-- less restrictive umasks.
diff --git a/Database/Keys/SQL.hs b/Database/Keys/SQL.hs
index 7fea5cf2bb..2e40e39db3 100644
--- a/Database/Keys/SQL.hs
+++ b/Database/Keys/SQL.hs
@@ -46,7 +46,7 @@ import Data.Maybe
 share [mkPersist sqlSettings, mkMigrate "migrateKeysDb"] [persistLowerCase|
 Associated
   key Key
-  file SFilePath
+  file SByteString
   KeyFileIndex key file
   FileKeyIndex file key
 Content
@@ -87,7 +87,7 @@ addAssociatedFile k f = queueDb $
 		(Associated k af)
 		[AssociatedFile =. af, AssociatedKey =. k]
   where
-	af = SFilePath (getTopFilePath f)
+	af = SByteString (getTopFilePath f)
 
 -- Faster than addAssociatedFile, but only safe to use when the file
 -- was not associated with a different key before, as it does not delete
@@ -96,14 +96,14 @@ newAssociatedFile :: Key -> TopFilePath -> WriteHandle -> IO ()
 newAssociatedFile k f = queueDb $
 	insert_ $ Associated k af
   where
-	af = SFilePath (getTopFilePath f)
+	af = SByteString (getTopFilePath f)
 
 {- Note that the files returned were once associated with the key, but
  - some of them may not be any longer. -}
 getAssociatedFiles :: Key -> ReadHandle -> IO [TopFilePath]
 getAssociatedFiles k = readDb $ do
 	l <- selectList [AssociatedKey ==. k] []
-	return $ map (asTopFilePath . (\(SFilePath f) -> f) . associatedFile . entityVal) l
+	return $ map (asTopFilePath . (\(SByteString f) -> f) . associatedFile . entityVal) l
 
 {- Gets any keys that are on record as having a particular associated file.
  - (Should be one or none.) -}
@@ -112,13 +112,13 @@ getAssociatedKey f = readDb $ do
 	l <- selectList [AssociatedFile ==. af] []
 	return $ map (associatedKey . entityVal) l
   where
-	af = SFilePath (getTopFilePath f)
+	af = SByteString (getTopFilePath f)
 
 removeAssociatedFile :: Key -> TopFilePath -> WriteHandle -> IO ()
 removeAssociatedFile k f = queueDb $
 	deleteWhere [AssociatedKey ==. k, AssociatedFile ==. af]
   where
-	af = SFilePath (getTopFilePath f)
+	af = SByteString (getTopFilePath f)
 
 addInodeCaches :: Key -> [InodeCache] -> WriteHandle -> IO ()
 addInodeCaches k is = queueDb $
diff --git a/Database/RawFilePath.hs b/Database/RawFilePath.hs
new file mode 100644
index 0000000000..ba82b9f90d
--- /dev/null
+++ b/Database/RawFilePath.hs
@@ -0,0 +1,95 @@
+{- Persistent sqlite RawFilePath support
+ -
+ - The functions below are copied from persistent-sqlite, but modified to
+ - take a RawFilePath and ignore the sqlConnectionStr from the
+ - SqliteConnectionInfo. This avoids encoding problems using Text
+ - in some situations.
+ -
+ - This module is expected to eventually be supersceded by
+ - persistent-sqlite getting support for OsString.
+ -
+ - Copyright (c) 2012 Michael Snoyman, http://www.yesodweb.com/
+ - Copyright 2023 Joey Hess <id@joeyh.name>
+ -
+ - Permission is hereby granted, free of charge, to any person obtaining
+ - a copy of this software and associated documentation files (the
+ - "Software"), to deal in the Software without restriction, including
+ - without limitation the rights to use, copy, modify, merge, publish,
+ - distribute, sublicense, and/or sell copies of the Software, and to
+ - permit persons to whom the Software is furnished to do so, subject to
+ - the following conditions:
+ - 
+ - The above copyright notice and this permission notice shall be
+ - included in all copies or substantial portions of the Software.
+ - 
+ - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ - EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ - MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ - NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ - LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ - OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ - WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+-}
+
+{-# LANGUAGE OverloadedStrings, CPP #-}
+
+module Database.RawFilePath where
+
+#if MIN_VERSION_persistent_sqlite(2,13,3)
+import Database.Persist.Sqlite
+import qualified Database.Sqlite as Sqlite
+import qualified System.FilePath.ByteString as P
+import qualified Control.Exception as E
+import Control.Monad.Logger (MonadLoggerIO)
+import Control.Monad.IO.Unlift (MonadUnliftIO)
+import Control.Monad.Logger (NoLoggingT, runNoLoggingT)
+import Control.Monad.Trans.Reader (ReaderT)
+import UnliftIO.Resource (ResourceT, runResourceT)
+
+openWith'
+	:: P.RawFilePath 
+	-> (SqlBackend -> Sqlite.Connection -> r)
+	-> SqliteConnectionInfo
+	-> LogFunc
+	-> IO r
+openWith' db f connInfo logFunc = do
+	conn <- Sqlite.open' db
+	backend <- wrapConnectionInfo connInfo conn logFunc `E.onException` Sqlite.close conn
+	return $ f backend conn
+
+runSqlite' :: (MonadUnliftIO m)
+	=> P.RawFilePath
+	-> ReaderT SqlBackend (NoLoggingT (ResourceT m)) a
+	-> m a
+runSqlite' connstr = runResourceT
+	. runNoLoggingT
+	. withSqliteConn' connstr
+	. runSqlConn
+
+withSqliteConn'
+	:: (MonadUnliftIO m, MonadLoggerIO m)
+	=> P.RawFilePath
+	-> (SqlBackend -> m a)
+	-> m a
+withSqliteConn' connstr = withSqliteConnInfo' connstr $
+	mkSqliteConnectionInfo mempty
+
+runSqliteInfo'
+	:: (MonadUnliftIO m)
+	=> P.RawFilePath
+	-> SqliteConnectionInfo
+	-> ReaderT SqlBackend (NoLoggingT (ResourceT m)) a
+	-> m a
+runSqliteInfo' db conInfo = runResourceT
+	. runNoLoggingT
+	. withSqliteConnInfo' db conInfo
+	. runSqlConn
+
+withSqliteConnInfo'
+	:: (MonadUnliftIO m, MonadLoggerIO m)
+        => P.RawFilePath
+	-> SqliteConnectionInfo
+	-> (SqlBackend -> m a)
+	-> m a
+withSqliteConnInfo' db = withSqlConn . openWith' db const
+#endif
diff --git a/Database/Types.hs b/Database/Types.hs
index 7b317ce6ac..bd8852852d 100644
--- a/Database/Types.hs
+++ b/Database/Types.hs
@@ -79,15 +79,15 @@ instance PersistField ContentIdentifier where
 instance PersistFieldSql ContentIdentifier where
 	sqlType _ = SqlBlob
 
--- A serialized RawFilePath.
-newtype SFilePath = SFilePath S.ByteString
+-- A serialized bytestring.
+newtype SByteString = SByteString S.ByteString
 	deriving (Eq, Show)
 
-instance PersistField  SFilePath where
-	toPersistValue (SFilePath b) = toPersistValue b
-	fromPersistValue v = SFilePath <$> fromPersistValue v
+instance PersistField  SByteString where
+	toPersistValue (SByteString b) = toPersistValue b
+	fromPersistValue v = SByteString <$> fromPersistValue v
 
-instance PersistFieldSql SFilePath where
+instance PersistFieldSql SByteString where
 	sqlType _ = SqlBlob
 
 -- A serialized git Sha
diff --git a/Git/Construct.hs b/Git/Construct.hs
index f5d2540a86..76261cabf2 100644
--- a/Git/Construct.hs
+++ b/Git/Construct.hs
@@ -40,6 +40,7 @@ import Git.FilePath
 import qualified Git.Url as Url
 import Utility.UserInfo
 import Utility.Url.Parse
+import qualified Utility.RawFilePath as R
 
 import qualified Data.ByteString as B
 import qualified System.FilePath.ByteString as P
@@ -47,14 +48,14 @@ import qualified System.FilePath.ByteString as P
 {- Finds the git repository used for the cwd, which may be in a parent
  - directory. -}
 fromCwd :: IO (Maybe Repo)
-fromCwd = getCurrentDirectory >>= seekUp
+fromCwd = R.getCurrentDirectory >>= seekUp
   where
 	seekUp dir = do
 		r <- checkForRepo dir
 		case r of
-			Nothing -> case upFrom (toRawFilePath dir) of
+			Nothing -> case upFrom dir of
 				Nothing -> return Nothing
-				Just d -> seekUp (fromRawFilePath d)
+				Just d -> seekUp d
 			Just loc -> pure $ Just $ newFrom loc
 
 {- Local Repo constructor, accepts a relative or absolute path. -}
@@ -220,26 +221,27 @@ expandTilde p = expandt True p
 
 {- Checks if a git repository exists in a directory. Does not find
  - git repositories in parent directories. -}
-checkForRepo :: FilePath -> IO (Maybe RepoLocation)
+checkForRepo :: RawFilePath -> IO (Maybe RepoLocation)
 checkForRepo dir = 
 	check isRepo $
-		check (checkGitDirFile (toRawFilePath dir)) $
-			check (checkdir (isBareRepo dir)) $
+		check (checkGitDirFile dir) $
+			check (checkdir (isBareRepo dir')) $
 				return Nothing
   where
 	check test cont = maybe cont (return . Just) =<< test
 	checkdir c = ifM c
-		( return $ Just $ LocalUnknown $ toRawFilePath dir
+		( return $ Just $ LocalUnknown dir
 		, return Nothing
 		)
 	isRepo = checkdir $ 
-		doesFileExist (dir </> ".git" </> "config")
+		doesFileExist (dir' </> ".git" </> "config")
 			<||>
 		-- A git-worktree lacks .git/config, but has .git/gitdir.
 		-- (Normally the .git is a file, not a symlink, but it can
 		-- be converted to a symlink and git will still work;
 		-- this handles that case.)
-		doesFileExist (dir </>  ".git" </> "gitdir")
+		doesFileExist (dir' </>  ".git" </> "gitdir")
+	dir' = fromRawFilePath dir
 
 isBareRepo :: FilePath -> IO Bool
 isBareRepo dir = doesFileExist (dir </> "config")
diff --git a/Git/Hook.hs b/Git/Hook.hs
index ba9031d800..1301ec2180 100644
--- a/Git/Hook.hs
+++ b/Git/Hook.hs
@@ -19,6 +19,7 @@ import qualified Utility.RawFilePath as R
 import System.PosixCompat.Files (fileMode)
 #endif
 
+import qualified Data.ByteString as B
 
 data Hook = Hook
 	{ hookName :: FilePath
@@ -57,7 +58,12 @@ hookWrite h r = ifM (doesFileExist f)
   where
 	f = hookFile h r
 	go = do
-		viaTmp writeFile f (hookScript h)
+		-- On Windows, using B.writeFile here avoids
+		-- the newline translation done by writeFile.
+		-- Hook scripts on Windows could use CRLF endings, but
+		-- they typically use unix newlines, which does work there
+		-- and makes the repository more portable.
+		viaTmp B.writeFile f (encodeBS (hookScript h))
 		void $ tryIO $ modifyFileMode
 			(toRawFilePath f)
 			(addModes executeModes)
@@ -81,6 +87,10 @@ data ExpectedContent = UnexpectedContent | ExpectedContent | OldExpectedContent
 
 expectedContent :: Hook -> Repo -> IO ExpectedContent
 expectedContent h r = do
+	-- Note that on windows, this readFile does newline translation,
+	-- and so a hook file that has CRLF will be treated the same as one
+	-- that has LF. That is intentional, since users may have a reason
+	-- to prefer one or the other.
 	content <- readFile $ hookFile h r
 	return $ if content == hookScript h
 		then ExpectedContent
diff --git a/Git/Log.hs b/Git/Log.hs
new file mode 100644
index 0000000000..a3246d5102
--- /dev/null
+++ b/Git/Log.hs
@@ -0,0 +1,115 @@
+{- git log
+ -
+ - Copyright 2023 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU AGPL version 3 or higher.
+ -}
+
+module Git.Log where
+
+import Common
+import Git
+import Git.Command
+import Git.Sha
+
+import Data.Time
+import Data.Time.Clock.POSIX
+
+-- A change made to a file.
+data LoggedFileChange t = LoggedFileChange
+	{ changetime :: POSIXTime
+	, changed :: t
+	, changedfile :: FilePath
+	, oldref :: Ref
+	, newref :: Ref
+	}
+	deriving (Show)
+
+-- Get the git log of changes to files. 
+-- 
+-- Note that the returned cleanup action should only be
+-- run after processing the returned list.
+getGitLog
+	:: Ref
+	-> Maybe Ref
+	-> [FilePath]
+	-> [CommandParam]
+	-> (Sha -> FilePath -> Maybe t)
+	-> Repo
+	-> IO ([LoggedFileChange t], IO Bool)
+getGitLog ref stopref fs os selector repo = do
+	(ls, cleanup) <- pipeNullSplit ps repo
+	return (parseGitRawLog selector (map decodeBL ls), cleanup)
+  where
+	ps =
+		[ Param "log"
+		, Param "-z"
+		, Param ("--pretty=format:"++commitinfoFormat)
+		, Param "--raw"
+		, Param "--no-abbrev"
+		, Param "--no-renames"
+		] ++ os ++
+		[ case stopref of
+			Just stopref' -> Param $
+				fromRef stopref' <> ".." <> fromRef ref
+			Nothing -> Param (fromRef ref)
+		, Param "--"
+		] ++ map Param fs 
+
+-- The commitinfo is the commit hash followed by its timestamp.
+commitinfoFormat :: String
+commitinfoFormat = "%H %ct"
+
+-- Parses chunked git log --raw output generated by getGitLog, 
+-- which looks something like:
+--
+-- [ "commitinfo\n:changeline"
+-- , "filename"
+-- , ""
+-- , "commitinfo\n:changeline"
+-- , "filename"
+-- , ":changeline"
+-- , "filename"
+-- , ""
+-- ]
+--
+-- The commitinfo is not included before all changelines, so
+-- keep track of the most recently seen commitinfo.
+parseGitRawLog :: (Ref -> FilePath -> Maybe t) -> [String] -> [LoggedFileChange t]
+parseGitRawLog selector = parse (deleteSha, epoch)
+  where
+	epoch = toEnum 0 :: POSIXTime
+	parse old ([]:rest) = parse old rest
+	parse (oldcommitsha, oldts) (c1:c2:rest) = case mrc of
+		Just rc -> rc : parse (commitsha, ts) rest
+		Nothing -> parse (commitsha, ts) (c2:rest)
+	  where
+		(commitsha, ts, cl) = case separate (== '\n') c1 of
+			(cl', []) -> (oldcommitsha, oldts, cl')
+			(ci, cl') -> case words ci of
+				(css:tss:[]) -> (Ref (encodeBS css), parseTimeStamp tss, cl')
+				_ -> (oldcommitsha, oldts, cl')
+	  	mrc = do
+			(old, new) <- parseRawChangeLine cl
+			v <- selector commitsha c2
+			return $ LoggedFileChange
+				{ changetime = ts
+				, changed = v
+				, changedfile = c2
+				, oldref = old
+				, newref = new
+				}
+	parse _ _ = []
+
+-- Parses something like ":100644 100644 oldsha newsha M"
+-- extracting the shas.
+parseRawChangeLine :: String -> Maybe (Git.Ref, Git.Ref)
+parseRawChangeLine = go . words
+  where
+	go (_:_:oldsha:newsha:_) = 
+		Just (Git.Ref (encodeBS oldsha), Git.Ref (encodeBS newsha))
+	go _ = Nothing
+
+parseTimeStamp :: String -> POSIXTime
+parseTimeStamp = utcTimeToPOSIXSeconds . fromMaybe (giveup "bad timestamp") .
+	parseTimeM True defaultTimeLocale "%s"
diff --git a/Git/Tree.hs b/Git/Tree.hs
index c811b111c8..af2a132aa4 100644
--- a/Git/Tree.hs
+++ b/Git/Tree.hs
@@ -1,11 +1,12 @@
 {- git trees
  -
- - Copyright 2016-2021 Joey Hess <id@joeyh.name>
+ - Copyright 2016-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
 
 {-# LANGUAGE BangPatterns, TypeSynonymInstances, FlexibleInstances #-}
+{-# LANGUAGE OverloadedStrings #-}
 
 module Git.Tree (
 	Tree(..),
@@ -23,6 +24,8 @@ module Git.Tree (
 	graftTree',
 	withMkTreeHandle,
 	MkTreeHandle,
+	sendMkTree,
+	finishMkTree,
 	treeMode,
 ) where
 
@@ -101,18 +104,27 @@ recordSubTree h (NewSubTree d l) = do
 	sha <- mkTree h =<< mapM (recordSubTree h) l
 	return (RecordedSubTree d sha [])
 recordSubTree _ alreadyrecorded = return alreadyrecorded
- 
+
+sendMkTree :: MkTreeHandle -> FileMode -> ObjectType -> Sha -> TopFilePath -> IO ()
+sendMkTree (MkTreeHandle cp) fm ot s f =
+	CoProcess.send cp $ \h -> 
+		hPutStr h (mkTreeOutput fm ot s f)
+
+finishMkTree :: MkTreeHandle -> IO Sha
+finishMkTree (MkTreeHandle cp) = do
+	CoProcess.send cp $ \h ->
+		-- NUL to signal end of tree to --batch
+		hPutStr h "\NUL"
+	getSha "mktree" (CoProcess.receive cp S8.hGetLine)
+
 mkTree :: MkTreeHandle -> [TreeContent] -> IO Sha
-mkTree (MkTreeHandle cp) l = CoProcess.query cp send receive
-  where
-	send h = do
-		forM_ l $ \i ->	hPutStr h $ case i of
-			TreeBlob f fm s -> mkTreeOutput fm BlobObject s f
-			RecordedSubTree f s _ -> mkTreeOutput treeMode TreeObject s f
-			NewSubTree _ _ -> error "recordSubTree internal error; unexpected NewSubTree"
-			TreeCommit f fm s -> mkTreeOutput fm CommitObject s f
-		hPutStr h "\NUL" -- signal end of tree to --batch
-	receive h = getSha "mktree" (S8.hGetLine h)
+mkTree h l = do
+	forM_ l $ \case
+		TreeBlob f fm s -> sendMkTree h fm BlobObject s f
+		RecordedSubTree f s _ -> sendMkTree h treeMode TreeObject s f
+		NewSubTree _ _ -> error "recordSubTree internal error; unexpected NewSubTree"
+		TreeCommit f fm s -> sendMkTree h fm CommitObject s f
+	finishMkTree h
 
 treeMode :: FileMode
 treeMode = 0o040000
@@ -220,7 +232,7 @@ adjustTree adjusttreeitem addtreeitems resolveaddconflict removefiles r repo =
 	withMkTreeHandle repo $ \h -> do
 		(l, cleanup) <- liftIO $ lsTreeWithObjects LsTree.LsTreeRecursive r repo
 		(l', _, _) <- go h False [] 1 inTopTree l
-		l'' <- adjustlist h 0 inTopTree (const True) l'
+		l'' <- adjustlist h 0 inTopTree topTreePath l'
 		sha <- liftIO $ mkTree h l''
 		void $ liftIO cleanup
 		return sha
@@ -239,7 +251,7 @@ adjustTree adjusttreeitem addtreeitems resolveaddconflict removefiles r repo =
 						in go h modified (blob:c) depth intree is
 			Just TreeObject -> do
 				(sl, modified, is') <- go h False [] (depth+1) (beneathSubTree i) is
-				sl' <- adjustlist h depth (inTree i) (beneathSubTree i) sl
+				sl' <- adjustlist h depth (inTree i) (gitPath i) sl
 				let slmodified = sl' /= sl
 				subtree <- if modified || slmodified
 					then liftIO $ recordSubTree h $ NewSubTree (LsTree.file i) sl'
@@ -257,16 +269,22 @@ adjustTree adjusttreeitem addtreeitems resolveaddconflict removefiles r repo =
 			_ -> giveup ("unexpected object type \"" ++ decodeBS (LsTree.typeobj i) ++ "\"")
 		| otherwise = return (c, wasmodified, i:is)
 
-	adjustlist h depth ishere underhere l = do
-		let (addhere, rest) = partition ishere addtreeitems
+	adjustlist h depth ishere herepath l = do
+		let addhere = fromMaybe [] $ M.lookup herepath addtreeitempathmap
 		let l' = filter (not . removed) $
 			addoldnew l (map treeItemToTreeContent addhere)
 		let inl i = any (\t -> beneathSubTree t i) l'
 		let (Tree addunderhere) = flattenTree depth $ treeItemsToTree $
-			filter (\i -> underhere i && not (inl i)) rest
+			filter (not . inl) $ if herepath == topTreePath
+				then filter (not . ishere) addtreeitems
+				else fromMaybe [] $
+					M.lookup (subTreePrefix herepath) addtreeitemprefixmap
 		addunderhere' <- liftIO $ mapM (recordSubTree h) addunderhere
 		return (addoldnew l' addunderhere')
 
+	addtreeitempathmap = mkPathMap addtreeitems
+	addtreeitemprefixmap = mkSubTreePathPrefixMap addtreeitems
+
 	removeset = S.fromList $ map (P.normalise . gitPath) removefiles
 	removed (TreeBlob f _ _) = S.member (P.normalise (gitPath f)) removeset
 	removed (TreeCommit f _ _) = S.member (P.normalise (gitPath f)) removeset
@@ -344,12 +362,8 @@ graftTree' subtree graftloc basetree repo hdl = go basetree subdirs graftdirs
 
 	subdirs = P.splitDirectories $ gitPath graftloc
 
-	-- For a graftloc of "foo/bar/baz", this generates
-	-- ["foo", "foo/bar", "foo/bar/baz"]
 	graftdirs = map (asTopFilePath . toInternalGitPath) $
-		mkpaths [] subdirs
-	mkpaths _ [] = []
-	mkpaths base (d:rest) = (P.joinPath base P.</> d) : mkpaths (base ++ [d]) rest
+		pathPrefixes subdirs
 
 {- Assumes the list is ordered, with tree objects coming right before their
  - contents. -}
@@ -402,13 +416,50 @@ instance GitPath TreeContent where
 	gitPath (TreeCommit f _ _) = gitPath f
 
 inTopTree :: GitPath t => t -> Bool
-inTopTree = inTree "."
+inTopTree = inTree topTreePath
+
+topTreePath :: RawFilePath
+topTreePath = "."
 
 inTree :: (GitPath t, GitPath f) => t -> f -> Bool
 inTree t f = gitPath t == P.takeDirectory (gitPath f)
 
 beneathSubTree :: (GitPath t, GitPath f) => t -> f -> Bool
-beneathSubTree t f = prefix `B.isPrefixOf` P.normalise (gitPath f)
+beneathSubTree t f = subTreePrefix t `B.isPrefixOf` subTreePath f
+
+subTreePath :: GitPath t => t -> RawFilePath
+subTreePath = P.normalise . gitPath
+
+subTreePrefix :: GitPath t => t -> RawFilePath
+subTreePrefix t
+	| B.null tp = tp
+	| otherwise = P.addTrailingPathSeparator (P.normalise tp)
   where
 	tp = gitPath t
-	prefix = if B.null tp then tp else P.addTrailingPathSeparator (P.normalise tp)
+
+{- Makes a Map where the keys are directories, and the values
+ - are the items located in that directory.
+ -
+ - Values that are not in any subdirectory are placed in
+ - the topTreePath key.
+ -}
+mkPathMap :: GitPath t => [t] -> M.Map RawFilePath [t]
+mkPathMap l = M.fromListWith (++) $
+	map (\ti -> (P.takeDirectory (gitPath ti), [ti])) l
+
+{- Input is eg splitDirectories "foo/bar/baz",
+ - for which it will output ["foo", "foo/bar", "foo/bar/baz"] -}
+pathPrefixes :: [RawFilePath] -> [RawFilePath]
+pathPrefixes = go []
+  where
+	go _ [] = []
+	go base (d:rest) = (P.joinPath base P.</> d) : go (base ++ [d]) rest
+
+{- Makes a Map where the keys are all subtree path prefixes, 
+ - and the values are items with that subtree path prefix.
+ -}
+mkSubTreePathPrefixMap :: GitPath t => [t] -> M.Map RawFilePath [t]
+mkSubTreePathPrefixMap l = M.fromListWith (++) $ concatMap go l
+  where
+	go ti = map (\p -> (p, [ti]))
+		(map subTreePrefix $ pathPrefixes $ P.splitDirectories $ subTreePath ti)
diff --git a/Limit.hs b/Limit.hs
index 26448594b9..ad17520df8 100644
--- a/Limit.hs
+++ b/Limit.hs
@@ -330,6 +330,22 @@ addIn s = do
 				then return False
 				else inAnnex key
 
+{- Limit to content that location tracking expects to be present
+ - in the current repository. Does not verify inAnnex. -}
+addExpectedPresent :: Annex ()
+addExpectedPresent = do
+	hereu <- getUUID
+	addLimit $ Right $ MatchFiles
+		{ matchAction = const $ checkKey $ \key -> do
+			us <- Remote.keyLocations key
+			return $ hereu `elem` us
+		, matchNeedsFileName = False
+		, matchNeedsFileContent = False
+		, matchNeedsKey = True
+		, matchNeedsLocationLog = True
+		, matchDesc = matchDescSimple "expected-present"
+		}
+
 {- Limit to content that is currently present on a uuid. -}
 limitPresent :: Maybe UUID -> MatchFiles Annex
 limitPresent u = MatchFiles
diff --git a/Logs.hs b/Logs.hs
index fae8c24a9c..d8d047cd9a 100644
--- a/Logs.hs
+++ b/Logs.hs
@@ -1,6 +1,6 @@
 {- git-annex log file names
  -
- - Copyright 2013-2021 Joey Hess <id@joeyh.name>
+ - Copyright 2013-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -156,6 +156,11 @@ exportLog = "export.log"
 exportTreeGraftPoint :: RawFilePath
 exportTreeGraftPoint = "export.tree"
 
+{- This is not a log file, it's where migration treeishes get grafted into
+ - the git-annex branch. -}
+migrationTreeGraftPoint :: RawFilePath
+migrationTreeGraftPoint = "migrate.tree"
+
 {- The pathname of the location log file for a given key. -}
 locationLogFile :: GitConfig -> Key -> RawFilePath
 locationLogFile config key =
diff --git a/Logs/Chunk.hs b/Logs/Chunk.hs
index c7981445df..5405e72203 100644
--- a/Logs/Chunk.hs
+++ b/Logs/Chunk.hs
@@ -54,3 +54,4 @@ getCurrentChunks u k = do
 		. map (\((_ku, m), l) -> (m, value l))
 		. M.toList
 		. M.filterWithKey (\(ku, _m) _ -> ku == u)
+		. fromMapLog
diff --git a/Logs/ContentIdentifier/Pure.hs b/Logs/ContentIdentifier/Pure.hs
index 7a0f6c1614..bea98f4091 100644
--- a/Logs/ContentIdentifier/Pure.hs
+++ b/Logs/ContentIdentifier/Pure.hs
@@ -41,7 +41,7 @@ buildContentIdentifierList l = case l of
   where
 	buildcid (ContentIdentifier c)
 		| S8.any (`elem` [':', '\r', '\n']) c || "!" `S8.isPrefixOf` c =
-			charUtf8 '!' <> byteString (toB64' c)
+			charUtf8 '!' <> byteString (toB64 c)
 		| otherwise = byteString c
 	go [] = mempty
 	go (c:[]) = buildcid c
@@ -58,7 +58,7 @@ parseContentIdentifierList = do
 	cidparser = do
 		b <- A8.takeWhile (/= ':')
 		return $ if "!" `S8.isPrefixOf` b
-			then ContentIdentifier $ fromMaybe b (fromB64Maybe' (S.drop 1 b))
+			then ContentIdentifier $ fromMaybe b (fromB64Maybe (S.drop 1 b))
 			else ContentIdentifier b
 	listparser first rest = ifM A8.atEnd
 		( return (first :| reverse rest)
diff --git a/Logs/Export.hs b/Logs/Export.hs
index 839bd1b0c0..36d6156c75 100644
--- a/Logs/Export.hs
+++ b/Logs/Export.hs
@@ -78,7 +78,7 @@ recordExportBeginning remoteuuid newtree = do
 -- repository, the tree has to be kept available, even if it
 -- doesn't end up being merged into the master branch.
 recordExportTreeish :: Git.Ref -> Annex ()
-recordExportTreeish t = 
+recordExportTreeish t = void $
 	Annex.Branch.rememberTreeish t (asTopFilePath exportTreeGraftPoint)
 
 -- | Record that an export to a special remote is under way.
@@ -101,7 +101,7 @@ recordExportUnderway remoteuuid ec = do
 	Annex.Branch.change ru exportLog $ 
 		buildExportLog
 			. changeMapLog c ep exported 
-			. M.mapWithKey (updateForExportChange remoteuuid ec c hereuuid)
+			. mapLogWithKey (updateForExportChange remoteuuid ec c hereuuid)
 			. parseExportLog
 
 -- Record information about the export to the git-annex branch.
diff --git a/Logs/File.hs b/Logs/File.hs
index 56b0c90dda..e129da0553 100644
--- a/Logs/File.hs
+++ b/Logs/File.hs
@@ -1,11 +1,11 @@
 {- git-annex log files
  -
- - Copyright 2018-2022 Joey Hess <id@joeyh.name>
+ - Copyright 2018-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
 
-{-# LANGUAGE BangPatterns #-}
+{-# LANGUAGE CPP, BangPatterns #-}
 
 module Logs.File (
 	writeLogFile,
@@ -17,6 +17,8 @@ module Logs.File (
 	checkLogFile,
 	calcLogFile,
 	calcLogFileUnsafe,
+	fileLines,
+	fileLines',
 ) where
 
 import Annex.Common
@@ -25,6 +27,8 @@ import Annex.LockFile
 import Annex.ReplaceFile
 import Utility.Tmp
 
+import qualified Data.ByteString as S
+import qualified Data.ByteString.Char8 as S8
 import qualified Data.ByteString.Lazy as L
 import qualified Data.ByteString.Lazy.Char8 as L8
 
@@ -47,8 +51,8 @@ withLogHandle f a = do
 		bracket (setup tmp) cleanup a
   where
 	setup tmp = do
-		setAnnexFilePerm (toRawFilePath tmp)
-		liftIO $ openFile tmp WriteMode
+		setAnnexFilePerm tmp
+		liftIO $ openFile (fromRawFilePath tmp) WriteMode
 	cleanup h = liftIO $ hClose h
 
 -- | Appends a line to a log file, first locking it to prevent
@@ -74,7 +78,7 @@ appendLogFile f lck c =
 modifyLogFile :: RawFilePath -> RawFilePath -> ([L.ByteString] -> [L.ByteString]) -> Annex ()
 modifyLogFile f lck modf = withExclusiveLock lck $ do
 	ls <- liftIO $ fromMaybe []
-		<$> tryWhenExists (L8.lines <$> L.readFile f')
+		<$> tryWhenExists (fileLines <$> L.readFile f')
 	let ls' = modf ls
 	when (ls' /= ls) $
 		createDirWhenNeeded f $
@@ -94,7 +98,7 @@ checkLogFile f lck matchf = withSharedLock lck $ bracket setup cleanup go
 	cleanup (Just h) = liftIO $ hClose h
 	go Nothing = return False
 	go (Just h) = do
-		!r <- liftIO (any matchf . L8.lines <$> L.hGetContents h)
+		!r <- liftIO (any matchf . fileLines <$> L.hGetContents h)
 		return r
 	f' = fromRawFilePath f
 
@@ -111,7 +115,7 @@ calcLogFileUnsafe f start update = bracket setup cleanup go
 	cleanup Nothing = noop
 	cleanup (Just h) = liftIO $ hClose h
 	go Nothing = return start
-	go (Just h) = go' start =<< liftIO (L8.lines <$> L.hGetContents h)
+	go (Just h) = go' start =<< liftIO (fileLines <$> L.hGetContents h)
 	go' v [] = return v
 	go' v (l:ls) = do
 		let !v' = update l v
@@ -157,3 +161,32 @@ createDirWhenNeeded f a = a `catchNonAsync` \_e -> do
 	-- done if writing the file fails.
 	createAnnexDirectory (parentDir f)
 	a
+
+-- On windows, readFile does NewlineMode translation,
+-- stripping CR before LF. When converting to ByteString,
+-- use this to emulate that.
+fileLines :: L.ByteString -> [L.ByteString]
+#ifdef mingw32_HOST_OS
+fileLines = map stripCR . L8.lines
+  where
+	stripCR b = case L8.unsnoc b of
+		Nothing -> b
+		Just (b', e)
+			| e == '\r' -> b'
+			| otherwise -> b
+#else
+fileLines = L8.lines
+#endif
+
+fileLines' :: S.ByteString -> [S.ByteString]
+#ifdef mingw32_HOST_OS
+fileLines' = map stripCR . S8.lines
+  where
+	stripCR b = case S8.unsnoc b of
+		Nothing -> b
+		Just (b', e)
+			| e == '\r' -> b'
+			| otherwise -> b
+#else
+fileLines' = S8.lines
+#endif
diff --git a/Logs/Location.hs b/Logs/Location.hs
index 860d0f456b..6235d1a34c 100644
--- a/Logs/Location.hs
+++ b/Logs/Location.hs
@@ -8,19 +8,23 @@
  - Repositories record their UUID and the date when they --get or --drop
  - a value.
  - 
- - Copyright 2010-2021 Joey Hess <id@joeyh.name>
+ - Copyright 2010-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
 
+{-# LANGUAGE BangPatterns #-}
+
 module Logs.Location (
 	LogStatus(..),
 	logStatus,
 	logStatusAfter,
 	logChange,
 	loggedLocations,
+	loggedPreviousLocations,
 	loggedLocationsHistorical,
 	loggedLocationsRef,
+	parseLoggedLocations,
 	isKnownKey,
 	checkDead,
 	setDead,
@@ -29,6 +33,8 @@ module Logs.Location (
 	loggedKeys,
 	loggedKeysFor,
 	loggedKeysFor',
+	overLocationLogs,
+	overLocationLogs',
 ) where
 
 import Annex.Common
@@ -42,6 +48,7 @@ import Git.Types (RefDate, Ref)
 import qualified Annex
 
 import Data.Time.Clock
+import qualified Data.ByteString.Lazy as L
 
 {- Log a change in the presence of a key's value in current repository. -}
 logStatus :: Key -> LogStatus -> Annex ()
@@ -73,7 +80,13 @@ logChange _ NoUUID _ = noop
 {- Returns a list of repository UUIDs that, according to the log, have
  - the value of a key. -}
 loggedLocations :: Key -> Annex [UUID]
-loggedLocations = getLoggedLocations currentLogInfo
+loggedLocations = getLoggedLocations presentLogInfo
+
+{- Returns a list of repository UUIDs that the location log indicates
+ - used to have the vale of a key, but no longer do.
+ -}
+loggedPreviousLocations :: Key -> Annex [UUID]
+loggedPreviousLocations = getLoggedLocations notPresentLogInfo
 
 {- Gets the location log on a particular date. -}
 loggedLocationsHistorical :: RefDate -> Key -> Annex [UUID]
@@ -83,6 +96,11 @@ loggedLocationsHistorical = getLoggedLocations . historicalLogInfo
 loggedLocationsRef :: Ref -> Annex [UUID]
 loggedLocationsRef ref = map (toUUID . fromLogInfo) . getLog <$> catObject ref
 
+{- Parses the content of a log file and gets the locations in it. -}
+parseLoggedLocations :: L.ByteString -> [UUID]
+parseLoggedLocations l = map (toUUID . fromLogInfo . info)
+	(filterPresent (parseLog l))
+
 getLoggedLocations :: (RawFilePath -> Annex [LogInfo]) -> Key -> Annex [UUID]
 getLoggedLocations getter key = do
 	config <- Annex.getGitConfig
@@ -174,3 +192,33 @@ loggedKeysFor' u = loggedKeys' isthere
 		us <- loggedLocations k
 		let !there = u `elem` us
 		return there
+
+{- This is much faster than loggedKeys. -}
+overLocationLogs :: v -> (Key -> [UUID] -> v -> Annex v) -> Annex v
+overLocationLogs v = overLocationLogs' v (flip const)
+
+overLocationLogs'
+	 :: v 
+	-> (Annex (Maybe (Key, RawFilePath, Maybe L.ByteString)) -> Annex v -> Annex v)
+        -> (Key -> [UUID] -> v -> Annex v)
+        -> Annex v
+overLocationLogs' iv discarder keyaction = do
+	config <- Annex.getGitConfig
+		
+	let getk = locationLogFileKey config
+	let go v reader = reader >>= \case
+		Just (k, f, content) -> discarder reader $ do
+			-- precache to make checkDead fast, and also to
+			-- make any accesses done in keyaction fast.
+			maybe noop (Annex.Branch.precache f) content
+			ifM (checkDead k)
+				( go v reader
+				, do
+					!v' <- keyaction k (maybe [] parseLoggedLocations content) v
+					go v' reader
+				)
+		Nothing -> return v
+
+	Annex.Branch.overBranchFileContents getk (go iv) >>= \case
+		Just r -> return r
+		Nothing -> giveup "This repository is read-only, and there are unmerged git-annex branches, which prevents operating on all keys. (Set annex.merge-annex-branches to false to ignore the unmerged git-annex branches.)"
diff --git a/Logs/MapLog.hs b/Logs/MapLog.hs
index d255144c7b..4ef338b740 100644
--- a/Logs/MapLog.hs
+++ b/Logs/MapLog.hs
@@ -6,7 +6,7 @@
  -
  - The field names cannot contain whitespace.
  -
- - Copyright 2014, 2019 Joey Hess <id@joeyh.name>
+ - Copyright 2014-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -28,6 +28,8 @@ import qualified Data.Attoparsec.ByteString as A
 import qualified Data.Attoparsec.ByteString.Lazy as AL
 import qualified Data.Attoparsec.ByteString.Char8 as A8
 import Data.ByteString.Builder
+import qualified Data.Semigroup as Sem
+import Prelude
 
 data LogEntry v = LogEntry
 	{ changed :: VectorClock
@@ -37,10 +39,23 @@ data LogEntry v = LogEntry
 instance Arbitrary v => Arbitrary (LogEntry v) where
 	arbitrary = LogEntry <$> arbitrary <*> arbitrary
 
-type MapLog f v = M.Map f (LogEntry v)
+newtype MapLog f v = MapLog (M.Map f (LogEntry v))
+	deriving (Show, Eq)
+
+instance Ord f => Sem.Semigroup (MapLog f v)
+  where
+	a <> MapLog b = foldl' (\m (f, v) -> addMapLog f v m) a (M.toList b)
+
+instance Ord f => Monoid (MapLog f v)
+  where
+	mempty = MapLog M.empty
+
+fromMapLog :: MapLog f v -> M.Map f (LogEntry v)
+fromMapLog (MapLog m) = m
 
 buildMapLog :: (f -> Builder) -> (v -> Builder) -> MapLog f v -> Builder
-buildMapLog fieldbuilder valuebuilder = mconcat . map genline . M.toList
+buildMapLog fieldbuilder valuebuilder (MapLog m) = 
+	mconcat $ map genline $ M.toList m
   where
 	genline (f, LogEntry c v) = 
 		buildVectorClock c <> sp 
@@ -50,25 +65,32 @@ buildMapLog fieldbuilder valuebuilder = mconcat . map genline . M.toList
 	nl = charUtf8 '\n'
 
 parseMapLog :: Ord f => A.Parser f -> A.Parser v -> L.ByteString -> MapLog f v
-parseMapLog fieldparser valueparser = fromMaybe M.empty . AL.maybeResult 
-	. AL.parse (mapLogParser fieldparser valueparser)
+parseMapLog fieldparser valueparser = 
+	parseMapLogWith (mapLogParser fieldparser valueparser)
+
+parseMapLogWith :: Ord f => A.Parser (MapLog f v) -> L.ByteString -> MapLog f v
+parseMapLogWith parser = fromMaybe (MapLog M.empty) 
+	. AL.maybeResult
+	. AL.parse parser
 
 mapLogParser :: Ord f => A.Parser f -> A.Parser v -> A.Parser (MapLog f v)
-mapLogParser fieldparser valueparser = M.fromListWith best <$> parseLogLines go
-  where
-	go = do
-		c <- vectorClockParser
-		_ <- A8.char ' '
-		w <- A8.takeTill (== ' ')
-		f <- either fail return $
-			A.parseOnly (fieldparser <* A.endOfInput) w
-		_ <- A8.char ' '
-		v <- valueparser
-		A.endOfInput
-		return (f, LogEntry c v)
+mapLogParser fieldparser valueparser = mapLogParser' $ do
+	c <- vectorClockParser
+	_ <- A8.char ' '
+	w <- A8.takeTill (== ' ')
+	f <- either fail return $
+		A.parseOnly (fieldparser <* A.endOfInput) w
+	_ <- A8.char ' '
+	v <- valueparser
+	A.endOfInput
+	return (f, LogEntry c v)
+
+mapLogParser' :: Ord f => A.Parser (f, LogEntry v) -> A.Parser (MapLog f v)
+mapLogParser' p = MapLog . M.fromListWith best
+	<$> parseLogLines p
 
 changeMapLog :: Ord f => CandidateVectorClock -> f -> v -> MapLog f v -> MapLog f v
-changeMapLog c f v m = M.insert f (LogEntry c' v) m
+changeMapLog c f v (MapLog m) = MapLog (M.insert f (LogEntry c' v) m)
   where
 	c' = case M.lookup f m of
 		Nothing -> advanceVectorClock c []
@@ -77,13 +99,19 @@ changeMapLog c f v m = M.insert f (LogEntry c' v) m
 {- Only add an LogEntry if it's newer (or at least as new as) than any
  - existing LogEntry for a field. -}
 addMapLog :: Ord f => f -> LogEntry v -> MapLog f v -> MapLog f v
-addMapLog = M.insertWith best
+addMapLog f v (MapLog m) = MapLog (M.insertWith best f v m)
+
+filterMapLogWith :: (f -> LogEntry v -> Bool) -> MapLog f v -> MapLog f v
+filterMapLogWith f (MapLog m) = MapLog (M.filterWithKey f m)
+
+mapLogWithKey :: (f -> LogEntry v -> LogEntry v) -> MapLog f v -> MapLog f v
+mapLogWithKey f (MapLog m) = MapLog (M.mapWithKey f m)
 
 {- Converts a MapLog into a simple Map without the timestamp information.
  - This is a one-way trip, but useful for code that never needs to change
  - the log. -}
 simpleMap :: MapLog f v -> M.Map f v
-simpleMap = M.map value
+simpleMap (MapLog m) = M.map value m
 
 best :: LogEntry v -> LogEntry v -> LogEntry v
 best new old
@@ -93,8 +121,8 @@ best new old
 prop_addMapLog_sane :: Bool
 prop_addMapLog_sane = newWins && newestWins
   where
-	newWins = addMapLog ("foo") (LogEntry (VectorClock 1) "new") l == l2
-	newestWins = addMapLog ("foo") (LogEntry (VectorClock 1) "newest") l2 /= l2
+	newWins = addMapLog "foo" (LogEntry (VectorClock 1) "new") l == l2
+	newestWins = addMapLog "foo" (LogEntry (VectorClock 1) "newest") l2 /= l2
 
-	l = M.fromList [("foo", LogEntry (VectorClock 0) "old")]
-	l2 = M.fromList [("foo", LogEntry (VectorClock 1) "new")]
+	l = MapLog (M.fromList [("foo", LogEntry (VectorClock 0) "old")])
+	l2 = MapLog (M.fromList [("foo", LogEntry (VectorClock 1) "new")])
diff --git a/Logs/Migrate.hs b/Logs/Migrate.hs
new file mode 100644
index 0000000000..b60b21cfcb
--- /dev/null
+++ b/Logs/Migrate.hs
@@ -0,0 +1,203 @@
+{- git-annex migration logs
+ -
+ - To record a migration in the git-annex branch as space efficiently as
+ - possible, it is stored as a tree which contains two subtrees 'old' and 'new'.
+ - The subtrees each contain the same filenames, which point to the old
+ - and new keys respectively.
+ -
+ - When the user commits the migrated files to their HEAD branch, that will
+ - store pointers to the new keys in git. And pointers to the old keys
+ - already exist in git. So recording the migration this way avoids
+ - injecting any new objects into git, besides the two trees. Note that for
+ - this to be the case, care has to be taken to record the migration 
+ - using the same symlink targets or pointer file contents as are used in
+ - the HEAD branch.
+ -
+ - The filenames used in the trees are not the original filenames, to avoid
+ - running migrate in a throwaway branch unexpectedly recording that
+ - branch's contents.
+ -
+ - There are two local log files:
+ - * migrate.log contains pairs of old and new keys, and is used while
+ -   performing a new migration, to build up a migration to commit.
+ -   This allows an interrupted migration to be resumed later.
+ - * migrations.log has as its first line a commit to the git-annex branch
+ -   up to which all migrations have been performed locally (including any
+ -   migrations in parent commits). Or the first line may be a null sha when
+ -   this has not been done yet. The rest of the lines in the file
+ -   are commits that have been made for locally performed migrations,
+ -   but whose parent commits have not necessarily been checked for
+ -   migrations yet.
+ -
+ - Copyright 2023 Joey Hess <id@joeyh.name>
+ -
+ - Licensed under the GNU AGPL version 3 or higher.
+ -}
+
+{-# LANGUAGE OverloadedStrings, BangPatterns #-}
+
+module Logs.Migrate (
+	MigrationRecord(..),
+	logMigration,
+	commitMigration,
+	streamNewDistributedMigrations,
+) where
+
+import Annex.Common
+import qualified Git
+import qualified Annex
+import qualified Annex.Branch
+import Git.Types
+import Git.Tree
+import Git.FilePath
+import Git.Ref
+import Git.Sha
+import Git.Log
+import Logs.File
+import Logs
+import Annex.CatFile
+
+import qualified Data.ByteString as B
+import qualified Data.ByteString.Lazy as L
+import Control.Concurrent.STM
+import System.FilePath.ByteString as P
+
+-- | What to use to record a migration. This should be the same Sha that is
+-- used to as the content of the annexed file in the HEAD branch.
+newtype MigrationRecord = MigrationRecord { fromMigrationRecord :: Git.Sha }
+
+-- | Logs a migration from an old to a new key.
+logMigration :: MigrationRecord -> MigrationRecord -> Annex ()
+logMigration old new = do
+	logf <- fromRepo gitAnnexMigrateLog
+	lckf <- fromRepo gitAnnexMigrateLock
+	appendLogFile logf lckf $ L.fromStrict $
+		Git.fromRef' (fromMigrationRecord old)
+			<> " "
+			<> Git.fromRef' (fromMigrationRecord new)
+
+-- | Commits a migration to the git-annex branch.
+commitMigration :: Annex ()
+commitMigration = do
+	logf <- fromRawFilePath <$> fromRepo gitAnnexMigrateLog
+	lckf <- fromRepo gitAnnexMigrateLock
+	nv <- liftIO $ newTVarIO (0 :: Integer)
+	g <- Annex.gitRepo
+	withMkTreeHandle g $ \oldh ->
+		withMkTreeHandle g $ \newh ->
+			streamLogFile logf lckf 
+				(finalizer nv oldh newh g)
+				(processor nv oldh newh)
+  where
+	processor nv oldh newh s = case words s of
+		(old:new:[]) -> do
+			fn <- liftIO $ atomically $ do
+				n <- readTVar nv
+				let !n' = succ n
+				writeTVar nv n'
+				return (asTopFilePath (encodeBS (show n')))
+			let rec h r = liftIO $ sendMkTree h
+				(fromTreeItemType TreeFile)
+				BlobObject
+				(Git.Ref (encodeBS r))
+				fn
+			rec oldh old
+			rec newh new
+		_ -> error "migrate.log parse error"
+	finalizer nv oldh newh g = do
+		oldt <- liftIO $ finishMkTree oldh
+		newt <- liftIO $ finishMkTree newh
+		n <- liftIO $ atomically $ readTVar nv
+		when (n > 0) $ do
+			treesha <- liftIO $ flip recordTree g $ Tree
+				[ RecordedSubTree (asTopFilePath "old") oldt []
+				, RecordedSubTree (asTopFilePath "new") newt []
+				]
+			commitsha <- Annex.Branch.rememberTreeish treesha
+				(asTopFilePath migrationTreeGraftPoint)
+			committedMigration commitsha
+
+-- Streams distributed migrations from the git-annex branch,
+-- and runs the provided action on each old and new key pair.
+--
+-- With the incremental option, only scans as far as the last recorded
+-- migration that this has handled before.
+streamNewDistributedMigrations :: Bool -> (Key -> Key -> Annex ()) -> Annex ()
+streamNewDistributedMigrations incremental a = do
+	void Annex.Branch.update
+	branchsha <- Annex.Branch.getBranch
+	(stoppoint, toskip) <- getPerformedMigrations
+	(l, cleanup) <- inRepo $ getGitLog branchsha
+		(if incremental then stoppoint else Nothing)
+		[fromRawFilePath migrationTreeGraftPoint]
+		-- Need to follow because migrate.tree is grafted in 
+		-- and then deleted, and normally git log stops when a file
+		-- gets deleted.
+		([Param "--reverse", Param "--follow"])
+		(\commit _file -> Just commit)
+	forM_ l (go toskip)
+	liftIO $ void cleanup
+	recordPerformedMigrations branchsha toskip
+  where
+	go toskip c
+		| newref c `elem` nullShas = return ()
+		| changed c `elem` toskip = return ()
+		| not ("/new/" `B.isInfixOf` newfile) = return ()
+		| otherwise = 
+			catKey (newref c) >>= \case
+				Nothing -> return ()
+				Just newkey -> catKey oldfileref >>= \case
+					Nothing -> return ()
+					Just oldkey -> a oldkey newkey
+	  where
+		newfile = toRawFilePath (changedfile c)
+		oldfile = migrationTreeGraftPoint 
+			P.</> "old" 
+			P.</> P.takeBaseName (fromInternalGitPath newfile)
+		oldfileref = branchFileRef (changed c) oldfile
+
+getPerformedMigrations :: Annex (Maybe Sha, [Sha])
+getPerformedMigrations = do
+	logf <- fromRepo gitAnnexMigrationsLog
+	lckf <- fromRepo gitAnnexMigrationsLock
+	ls <- calcLogFile logf lckf [] (:)
+	return $ case reverse ls of
+		[] -> (Nothing, [])
+		(stoppoint:toskip) ->
+			let stoppoint' = conv stoppoint
+			in 
+				( if stoppoint' `elem` nullShas
+					then Nothing
+					else Just stoppoint'
+				, map conv toskip
+				)
+  where
+	conv = Git.Ref . L.toStrict
+
+-- Record locally that migrations have been performed up to the given
+-- commit. The list is additional commits that can be removed from the
+-- log file if present.
+recordPerformedMigrations :: Sha -> [Sha] -> Annex ()
+recordPerformedMigrations commit toremove = do
+	logf <- fromRepo gitAnnexMigrationsLog
+	lckf <- fromRepo gitAnnexMigrationsLock
+	modifyLogFile logf lckf (update . drop 1)
+  where
+	update l = L.fromStrict (fromRef' commit) : filter (`notElem` toremove') l
+	toremove' = map (L.fromStrict . fromRef') toremove
+
+-- Record that a migration was performed locally and committed.
+-- Since committing a migration may result in parent migrations that have
+-- not yet been processed locally, that commit cannot be the first line of
+-- the log file, which is reserved for commits whose parents have also had
+-- their migrations handled. So if the log file does not exist or is empty,
+-- make the first line a null sha.
+committedMigration :: Sha -> Annex ()
+committedMigration commitsha = do
+	logf <- fromRepo gitAnnexMigrationsLog
+	lckf <- fromRepo gitAnnexMigrationsLock
+	modifyLogFile logf lckf update
+  where
+	update [] = [conv deleteSha, conv commitsha]
+	update logged = logged ++ [conv commitsha]
+	conv = L.fromStrict . fromRef'
diff --git a/Logs/PreferredContent.hs b/Logs/PreferredContent.hs
index 9d2b30a907..c0f30c3c8f 100644
--- a/Logs/PreferredContent.hs
+++ b/Logs/PreferredContent.hs
@@ -82,7 +82,7 @@ requiredContentMap = maybe (snd <$> preferredRequiredMapsLoad preferredContentTo
 
 preferredRequiredMapsLoad :: (PreferredContentData -> [ParseToken (MatchFiles Annex)]) -> Annex (FileMatcherMap Annex, FileMatcherMap Annex)
 preferredRequiredMapsLoad mktokens = do
-	(pc, rc) <- preferredRequiredMapsLoad' mktokens
+	(pc, rc) <- preferredRequiredMapsLoad' id mktokens
 	let pc' = handleunknown (MatcherDesc "preferred content") pc
 	let rc' = handleunknown (MatcherDesc "required content") rc
 	Annex.changeState $ \s -> s
@@ -94,12 +94,12 @@ preferredRequiredMapsLoad mktokens = do
 	handleunknown matcherdesc = M.mapWithKey $ \u v ->
 		(either (const $ unknownMatcher u) id v, matcherdesc)
 
-preferredRequiredMapsLoad' :: (PreferredContentData -> [ParseToken (MatchFiles Annex)]) -> Annex (M.Map UUID (Either String (Matcher (MatchFiles Annex))), M.Map UUID (Either String (Matcher (MatchFiles Annex))))
-preferredRequiredMapsLoad' mktokens = do
+preferredRequiredMapsLoad' :: (Matcher (MatchFiles Annex) -> Matcher (MatchFiles Annex)) -> (PreferredContentData -> [ParseToken (MatchFiles Annex)]) -> Annex (M.Map UUID (Either String (Matcher (MatchFiles Annex))), M.Map UUID (Either String (Matcher (MatchFiles Annex))))
+preferredRequiredMapsLoad' matcherf mktokens = do
 	groupmap <- groupMap
 	configmap <- remoteConfigMap
 	let genmap l gm = 
-		let mk u = makeMatcher groupmap configmap gm u mktokens
+		let mk u = makeMatcher groupmap configmap gm u matcherf mktokens
 		in simpleMap
 			. parseLogOldWithUUID (\u -> mk u . decodeBS <$> A.takeByteString)
 			<$> Annex.Branch.get l
@@ -123,13 +123,14 @@ makeMatcher
 	-> M.Map UUID RemoteConfig
 	-> M.Map Group PreferredContentExpression
 	-> UUID
+	-> (Matcher (MatchFiles Annex) -> Matcher (MatchFiles Annex))
 	-> (PreferredContentData -> [ParseToken (MatchFiles Annex)])
 	-> PreferredContentExpression
 	-> Either String (Matcher (MatchFiles Annex))
-makeMatcher groupmap configmap groupwantedmap u mktokens = go True True
+makeMatcher groupmap configmap groupwantedmap u matcherf mktokens = go True True
   where
 	go expandstandard expandgroupwanted expr
-		| null (lefts tokens) = Right $ generate $ rights tokens
+		| null (lefts tokens) = Right $ matcherf $ generate $ rights tokens
 		| otherwise = Left $ unwords $ lefts tokens
 	  where
 		tokens = preferredContentParser (mktokens pcd) expr
diff --git a/Logs/Presence.hs b/Logs/Presence.hs
index 02256b6d2a..f96c455684 100644
--- a/Logs/Presence.hs
+++ b/Logs/Presence.hs
@@ -17,8 +17,8 @@ module Logs.Presence (
 	addLog',
 	maybeAddLog,
 	readLog,
-	currentLog,
-	currentLogInfo,
+	presentLogInfo,
+	notPresentLogInfo,
 	historicalLogInfo,
 ) where
 
@@ -68,12 +68,13 @@ genLine logstatus loginfo c old = LogLine c' logstatus loginfo
 readLog :: RawFilePath -> Annex [LogLine]
 readLog = parseLog <$$> Annex.Branch.get
 
-{- Reads a log and returns only the info that is still in effect. -}
-currentLogInfo :: RawFilePath -> Annex [LogInfo]
-currentLogInfo file = map info <$> currentLog file
+{- Reads a log and returns only the info that is still present. -}
+presentLogInfo :: RawFilePath -> Annex [LogInfo]
+presentLogInfo file = map info . filterPresent <$> readLog file
 
-currentLog :: RawFilePath -> Annex [LogLine]
-currentLog file = filterPresent <$> readLog file
+{- Reads a log and returns only the info that is no longer present. -}
+notPresentLogInfo :: RawFilePath -> Annex [LogInfo]
+notPresentLogInfo file = map info . filterNotPresent <$> readLog file
 
 {- Reads a historical version of a log and returns the info that was in
  - effect at that time. 
diff --git a/Logs/Presence/Pure.hs b/Logs/Presence/Pure.hs
index 7dc1e8650c..8ed6ed541a 100644
--- a/Logs/Presence/Pure.hs
+++ b/Logs/Presence/Pure.hs
@@ -70,14 +70,18 @@ buildLog = mconcat . map genline
 	genstatus InfoMissing = charUtf8 '0'
 	genstatus InfoDead = charUtf8 'X'
 
-{- Given a log, returns only the info that is are still in effect. -}
+{- Given a log, returns only the info that is still present. -}
 getLog :: L.ByteString -> [LogInfo]
 getLog = map info . filterPresent . parseLog
 
-{- Returns the info from LogLines that are in effect. -}
+{- Returns the info from LogLines that is present. -}
 filterPresent :: [LogLine] -> [LogLine]
 filterPresent = filter (\l -> InfoPresent == status l) . compactLog
 
+{- Returns the info from LogLines that is not present. -}
+filterNotPresent :: [LogLine] -> [LogLine]
+filterNotPresent = filter (\l -> InfoPresent /= status l) . compactLog
+
 {- Compacts a set of logs, returning a subset that contains the current
  - status. -}
 compactLog :: [LogLine] -> [LogLine]
diff --git a/Logs/RemoteState.hs b/Logs/RemoteState.hs
index 5adefb6466..6bf1431456 100644
--- a/Logs/RemoteState.hs
+++ b/Logs/RemoteState.hs
@@ -14,6 +14,7 @@ import Annex.Common
 import Types.RemoteState
 import Logs
 import Logs.UUIDBased
+import Logs.MapLog
 import qualified Annex.Branch
 import qualified Annex
 
@@ -39,7 +40,7 @@ buildRemoteState = buildLogNew (byteString . encodeBS)
 getRemoteState :: RemoteStateHandle -> Key -> Annex (Maybe RemoteState)
 getRemoteState (RemoteStateHandle u) k = do
 	config <- Annex.getGitConfig
-	extract . parseRemoteState
+	extract . fromMapLog . parseRemoteState
 		<$> Annex.Branch.get (remoteStateLogFile config k)
   where
 	extract m = value <$> M.lookup u m
diff --git a/Logs/Trust/Pure.hs b/Logs/Trust/Pure.hs
index 0eb0398ba6..a3c6c52ef6 100644
--- a/Logs/Trust/Pure.hs
+++ b/Logs/Trust/Pure.hs
@@ -19,7 +19,10 @@ import qualified Data.Attoparsec.ByteString.Char8 as A8
 import Data.ByteString.Builder
 
 calcTrustMap :: L.ByteString -> TrustMap
-calcTrustMap = simpleMap . parseLogOld trustLevelParser
+calcTrustMap = simpleMap . parseTrustLog
+
+parseTrustLog :: L.ByteString -> Log TrustLevel
+parseTrustLog = parseLogOld trustLevelParser
 
 trustLevelParser :: A.Parser TrustLevel
 trustLevelParser = (totrust <$> A8.anyChar <* A.endOfInput)
diff --git a/Logs/UUIDBased.hs b/Logs/UUIDBased.hs
index d7d7c26da8..02875d45d5 100644
--- a/Logs/UUIDBased.hs
+++ b/Logs/UUIDBased.hs
@@ -9,7 +9,7 @@
  -
  - New uuid based logs instead use the form: "timestamp UUID INFO"
  - 
- - Copyright 2011-2019 Joey Hess <id@joeyh.name>
+ - Copyright 2011-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -37,12 +37,10 @@ import Common
 import Types.UUID
 import Annex.VectorClock
 import Logs.MapLog
-import Logs.Line
 
 import qualified Data.ByteString as S
 import qualified Data.ByteString.Lazy as L
 import qualified Data.Attoparsec.ByteString as A
-import qualified Data.Attoparsec.ByteString.Lazy as AL
 import qualified Data.Attoparsec.ByteString.Char8 as A8
 import Data.ByteString.Builder
 import qualified Data.DList as D
@@ -50,7 +48,7 @@ import qualified Data.DList as D
 type Log v = MapLog UUID v
 
 buildLogOld :: (v -> Builder) -> Log v -> Builder
-buildLogOld builder = mconcat . map genline . M.toList
+buildLogOld builder = mconcat . map genline . M.toList . fromMapLog
   where
 	genline (u, LogEntry c@(VectorClock {}) v) =
 		buildUUID u <> sp <> builder v <> sp
@@ -66,18 +64,16 @@ parseLogOld :: A.Parser a -> L.ByteString -> Log a
 parseLogOld = parseLogOldWithUUID . const
 
 parseLogOldWithUUID :: (UUID -> A.Parser a) -> L.ByteString -> Log a
-parseLogOldWithUUID parser = fromMaybe M.empty . AL.maybeResult
-	. AL.parse (logParserOld parser)
+parseLogOldWithUUID parser = parseMapLogWith (logParserOld parser)
 
 logParserOld :: (UUID -> A.Parser a) -> A.Parser (Log a)
-logParserOld parser = M.fromListWith best <$> parseLogLines go
+logParserOld parser = mapLogParser' $ do
+	u <- toUUID <$> A8.takeWhile1 (/= ' ')
+	(dl, ts) <- accumval D.empty
+	v <- either fail return $ A.parseOnly (parser u <* A.endOfInput)
+		(S.intercalate " " $ D.toList dl)
+	return (u, LogEntry ts v)
   where
-	go = do
-		u <- toUUID <$> A8.takeWhile1 (/= ' ')
-		(dl, ts) <- accumval D.empty
-		v <- either fail return $ A.parseOnly (parser u <* A.endOfInput)
-			(S.intercalate " " $ D.toList dl)
-		return (u, LogEntry ts v)
 	accumval dl =
 		((dl,) <$> parsetimestamp)
 		<|> (A8.char ' ' *> (A8.takeWhile (/= ' ')) >>= accumval . D.snoc dl)
diff --git a/Logs/Web.hs b/Logs/Web.hs
index cce855f105..66f21a7c5c 100644
--- a/Logs/Web.hs
+++ b/Logs/Web.hs
@@ -1,6 +1,6 @@
 {- Web url logs.
  -
- - Copyright 2011-2021 Joey Hess <id@joeyh.name>
+ - Copyright 2011-2023 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -9,7 +9,9 @@
 
 module Logs.Web (
 	URLString,
+	URLByteString,
 	getUrls,
+	getUrls',
 	getUrlsWithPrefix,
 	setUrlPresent,
 	setUrlMissing,
@@ -23,6 +25,7 @@ module Logs.Web (
 ) where
 
 import qualified Data.Map as M
+import qualified Data.ByteString as S
 import qualified Data.ByteString.Lazy as L
 
 import Annex.Common
@@ -35,20 +38,27 @@ import Annex.UUID
 import qualified Annex.Branch
 import qualified Types.Remote as Remote
 
+type URLByteString = S.ByteString
+
 {- Gets all urls that a key might be available from. -}
 getUrls :: Key -> Annex [URLString]
 getUrls key = do
-	config <- Annex.getGitConfig
-	l <- go $ urlLogFile config key : oldurlLogs config key
+	l <- map decodeBS <$> getUrls' key
 	tmpl <- Annex.getState (maybeToList . M.lookup key . Annex.tempurls)
 	return (tmpl ++ l)
+
+{- Note that this does not include temporary urls set with setTempUrl. -}
+getUrls' :: Key -> Annex [URLByteString]
+getUrls' key = do
+	config <- Annex.getGitConfig
+	go $ urlLogFile config key : oldurlLogs config key
   where
 	go [] = return []
 	go (l:ls) = do
-		us <- currentLogInfo l
+		us <- presentLogInfo l
 		if null us
 			then go ls
-			else return $ map decodeUrlLogInfo us
+			else return $ map fromLogInfo us
 
 getUrlsWithPrefix :: Key -> String -> Annex [URLString]
 getUrlsWithPrefix key prefix = filter (prefix `isPrefixOf`) 
@@ -123,10 +133,7 @@ getDownloader u = case separate (== ':') u of
 	("", u') -> (u', OtherDownloader)
 	_ -> (u, WebDownloader)
 
-decodeUrlLogInfo :: LogInfo -> URLString
-decodeUrlLogInfo = decodeBS . fromLogInfo
-
 {- Parses the content of an url log file, returning the urls that are
  - currently recorded. -}
-parseUrlLog :: L.ByteString -> [URLString]
-parseUrlLog = map decodeUrlLogInfo . getLog
+parseUrlLog :: L.ByteString -> [URLByteString]
+parseUrlLog = map fromLogInfo . getLog
diff --git a/Makefile b/Makefile
index e8b6a19fb0..06ebebdea3 100644
--- a/Makefile
+++ b/Makefile
@@ -15,11 +15,6 @@ SHAREDIR?=share
 # this to /usr/share/zsh/vendor-completions
 ZSH_COMPLETIONS_PATH?=$(PREFIX)/$(SHAREDIR)/zsh/site-functions
 
-# Am I typing :make in vim? Do a dev build.
-ifdef VIM
-all=dev
-endif
-
 build: $(all)
 
 # install system-wide
@@ -35,7 +30,7 @@ install-home:
 tmp/configure-stamp: Build/TestConfig.hs Build/Configure.hs
 	if [ "$(BUILDER)" = ./Setup ]; then $(GHC) --make Setup; fi
 	if [ "$(BUILDER)" != stack ]; then \
-		$(BUILDER) configure $(BUILDERCOMMONOPTIONS) --ghc-options="$(shell Build/collect-ghc-options.sh)"; \
+		$(BUILDER) configure -fParallelBuild $(BUILDERCOMMONOPTIONS) --ghc-options="$(shell Build/collect-ghc-options.sh)"; \
 		rm cabal.project.local~* 2>/dev/null || true; \
 	else \
 		$(BUILDER) setup $(BUILDERCOMMONOPTIONS); \
@@ -53,7 +48,7 @@ dev:
 # This leaves cabal.project.local configured for a prof build,
 # so just running make will continue to do prof builds.
 prof:
-	$(BUILDER) configure -f"-Production" \
+	$(BUILDER) configure -f"-Production" -fParallelBuild \
 		--enable-executable-dynamic --enable-profiling
 	rm cabal.project.local~* 2>/dev/null || true
 	mkdir -p tmp
diff --git a/Messages.hs b/Messages.hs
index 3bf1256ee1..6b6222a07b 100644
--- a/Messages.hs
+++ b/Messages.hs
@@ -5,7 +5,7 @@
  - Licensed under the GNU AGPL version 3 or higher.
  -}
 
-{-# LANGUAGE OverloadedStrings, ScopedTypeVariables #-}
+{-# LANGUAGE OverloadedStrings, ScopedTypeVariables, CPP #-}
 
 module Messages (
 	showStartMessage,
@@ -265,6 +265,12 @@ setupConsole = do
 	 - a file or a pipe. -}
 	hSetBuffering stdout LineBuffering
 	hSetBuffering stderr LineBuffering
+#ifdef mingw32_HOST_OS
+	{- Avoid outputting CR at end of line on Windows. git commands do
+	 - not ouput CR there. -}
+	hSetNewlineMode stdout noNewlineTranslation
+	hSetNewlineMode stderr noNewlineTranslation
+#endif
 
 enableDebugOutput :: Annex ()
 enableDebugOutput = do
diff --git a/Messages/Progress.hs b/Messages/Progress.hs
index 4327e1970f..c726149d18 100644
--- a/Messages/Progress.hs
+++ b/Messages/Progress.hs
@@ -171,7 +171,7 @@ metered' st setclear othermeterupdate msize bwlimit showoutput a = go st
 	minratelimit = min consoleratelimit jsonratelimit
 		
 {- Poll file size to display meter. -}
-meteredFile :: FilePath -> Maybe MeterUpdate -> Key -> Annex a -> Annex a
+meteredFile :: RawFilePath -> Maybe MeterUpdate -> Key -> (MeterUpdate -> Annex a) -> Annex a
 meteredFile file combinemeterupdate key a = 
 	metered combinemeterupdate key Nothing $ \_ p ->
 		watchFileSize file p a
diff --git a/P2P/Annex.hs b/P2P/Annex.hs
index a3e48085ca..14a7aef1fc 100644
--- a/P2P/Annex.hs
+++ b/P2P/Annex.hs
@@ -64,7 +64,7 @@ runLocal runst runner a = case a of
 		let fallback = runner (sender mempty (return Invalid))
 		v <- tryNonAsync $ prepSendAnnex k
 		case v of
-			Right (Just (f, checkchanged)) -> proceed $ do
+			Right (Just (f, _sz, checkchanged)) -> proceed $ do
 				-- alwaysUpload to allow multiple uploads of the same key.
 				let runtransfer ti = transfer alwaysUpload k af Nothing $ \p ->
 					sinkfile f o checkchanged sender p ti
@@ -79,7 +79,7 @@ runLocal runst runner a = case a of
 			iv <- startVerifyKeyContentIncrementally DefaultVerify k
 			let runtransfer ti = 
 				Right <$> transfer download' k af Nothing (\p ->
-					logStatusAfter k $ getViaTmp rsp DefaultVerify k af $ \tmp ->
+					logStatusAfter k $ getViaTmp rsp DefaultVerify k af Nothing $ \tmp ->
 						storefile (fromRawFilePath tmp) o l getb iv validitycheck p ti)
 			let fallback = return $ Left $
 				ProtoFailureMessage "transfer already in progress, or unable to take transfer lock"
diff --git a/Remote.hs b/Remote.hs
index 93b2e30f87..1638777277 100644
--- a/Remote.hs
+++ b/Remote.hs
@@ -47,6 +47,7 @@ module Remote (
 	remotesWithUUID,
 	remotesWithoutUUID,
 	keyLocations,
+	IncludeIgnored(..),
 	keyPossibilities,
 	remoteLocations,
 	nameToUUID,
@@ -299,13 +300,16 @@ remotesWithoutUUID rs us = filter (\r -> uuid r `notElem` us) rs
 keyLocations :: Key -> Annex [UUID]
 keyLocations key = trustExclude DeadTrusted =<< loggedLocations key
 
+{- Whether to include remotes that have annex-ignore set. -}
+newtype IncludeIgnored = IncludeIgnored Bool
+
 {- Cost ordered lists of remotes that the location log indicates
  - may have a key.
  -
  - Also includes remotes with remoteAnnexSpeculatePresent set.
  -}
-keyPossibilities :: Key -> Annex [Remote]
-keyPossibilities key = do
+keyPossibilities :: IncludeIgnored -> Key -> Annex [Remote]
+keyPossibilities ii key = do
 	u <- getUUID
 	-- uuids of all remotes that are recorded to have the key
 	locations <- filter (/= u) <$> keyLocations key
@@ -315,19 +319,21 @@ keyPossibilities key = do
 	-- there are unlikely to be many speclocations, so building a Set
 	-- is not worth the expense
 	let locations' = speclocations ++ filter (`notElem` speclocations) locations
-	fst <$> remoteLocations locations' []
+	fst <$> remoteLocations ii locations' []
 
 {- Given a list of locations of a key, and a list of all
  - trusted repositories, generates a cost-ordered list of
  - remotes that contain the key, and a list of trusted locations of the key.
  -}
-remoteLocations :: [UUID] -> [UUID] -> Annex ([Remote], [UUID])
-remoteLocations locations trusted = do
+remoteLocations :: IncludeIgnored -> [UUID] -> [UUID] -> Annex ([Remote], [UUID])
+remoteLocations (IncludeIgnored ii) locations trusted = do
 	let validtrustedlocations = nub locations `intersect` trusted
 
 	-- remotes that match uuids that have the key
 	allremotes <- remoteList 
-		>>= filterM (not <$$> liftIO . getDynamicConfig . remoteAnnexIgnore . gitconfig)
+		>>= if not ii
+			then filterM (not <$$> liftIO . getDynamicConfig . remoteAnnexIgnore . gitconfig)
+			else return
 	let validremotes = remotesWithUUID allremotes locations
 
 	return (sortBy (comparing cost) validremotes, validtrustedlocations)
diff --git a/Remote/Directory.hs b/Remote/Directory.hs
index afaf290306..c83064ad78 100644
--- a/Remote/Directory.hs
+++ b/Remote/Directory.hs
@@ -228,7 +228,7 @@ checkDiskSpaceDirectory d k = do
 		(\a b -> deviceID a == deviceID b)
 			<$> R.getSymbolicLinkStatus d
 			<*> R.getSymbolicLinkStatus annexdir
-	checkDiskSpace (Just d) k 0 samefilesystem
+	checkDiskSpace Nothing (Just d) k 0 samefilesystem
 
 {- Passed a temp directory that contains the files that should be placed
  - in the dest directory, moves it into place. Anything already existing
diff --git a/Remote/External.hs b/Remote/External.hs
index 22a9a61d32..d75c3ebb45 100644
--- a/Remote/External.hs
+++ b/Remote/External.hs
@@ -235,7 +235,7 @@ storeKeyM external = fileStorer $ \k f p ->
 
 retrieveKeyFileM :: External -> Retriever
 retrieveKeyFileM external = fileRetriever $ \d k p ->
-	either giveup return =<< go d k p
+	either giveup return =<< watchFileSize d p (go d k)
   where
 	go d k p = handleRequestKey external (\sk -> TRANSFER Download sk (fromRawFilePath d)) k (Just p) $ \resp ->
 		case resp of
diff --git a/Remote/GCrypt.hs b/Remote/GCrypt.hs
index 213265dfa2..d9a581596d 100644
--- a/Remote/GCrypt.hs
+++ b/Remote/GCrypt.hs
@@ -112,7 +112,7 @@ gen baser u rc gc rs = do
 	-- that is now available. Also need to set the gcrypt particiants
 	-- correctly.
 	resetup gcryptid r = do
-		let u' = genUUIDInNameSpace gCryptNameSpace gcryptid
+		let u' = genUUIDInNameSpace gCryptNameSpace (encodeBS gcryptid)
 		v <- M.lookup u' <$> remoteConfigMap
 		case (Git.remoteName baser, v) of
 			(Just remotename, Just rc') -> do
@@ -263,7 +263,7 @@ gCryptSetup _ mu _ c gc = go $ fromProposedAccepted <$> M.lookup gitRepoField c
 		case Git.GCrypt.remoteRepoId g (Just remotename) of
 			Nothing -> giveup "unable to determine gcrypt-id of remote"
 			Just gcryptid -> do
-				let u = genUUIDInNameSpace gCryptNameSpace gcryptid
+				let u = genUUIDInNameSpace gCryptNameSpace (encodeBS gcryptid)
 				if Just u == mu || isNothing mu
 					then do
 						method <- setupRepo gcryptid =<< inRepo (Git.Construct.fromRemoteLocation gitrepo False)
@@ -489,7 +489,7 @@ toAccessMethod _ = AccessRsyncOverSsh
 getGCryptUUID :: Bool -> Git.Repo -> Annex (Maybe UUID)
 getGCryptUUID fast r = do
 	dummycfg <- liftIO dummyRemoteGitConfig
-	(genUUIDInNameSpace gCryptNameSpace <$>) . fst
+	(genUUIDInNameSpace gCryptNameSpace . encodeBS <$>) . fst
 		<$> getGCryptId fast r dummycfg
 
 coreGCryptId :: ConfigKey
diff --git a/Remote/Git.hs b/Remote/Git.hs
index 157499aa25..bba505e378 100644
--- a/Remote/Git.hs
+++ b/Remote/Git.hs
@@ -327,7 +327,7 @@ tryGitConfigRead autoinit r hasuuid
 		case Git.GCrypt.remoteRepoId g (Git.remoteName r) of
 			Nothing -> return r
 			Just v -> storeUpdatedRemote $ liftIO $ setUUID r $
-				genUUIDInNameSpace gCryptNameSpace v
+				genUUIDInNameSpace gCryptNameSpace (encodeBS v)
 
 	{- The local repo may not yet be initialized, so try to initialize
 	 - it if allowed. However, if that fails, still return the read
@@ -484,10 +484,11 @@ copyFromRemote'' repo r st@(State connpool _ _ _ _) key file dest meterupdate vc
 	| not $ Git.repoIsUrl repo = guardUsable repo (giveup "cannot access remote") $ do
 		u <- getUUID
 		hardlink <- wantHardLink
-		let bwlimit = remoteAnnexBwLimit (gitconfig r)
+		let bwlimit = remoteAnnexBwLimitDownload (gitconfig r)
+			<|> remoteAnnexBwLimit (gitconfig r)
 		-- run copy from perspective of remote
 		onLocalFast st $ Annex.Content.prepSendAnnex' key >>= \case
-			Just (object, check) -> do
+			Just (object, _sz, check) -> do
 				let checksuccess = check >>= \case
 					Just err -> giveup err
 					Nothing -> return True
@@ -545,14 +546,15 @@ copyToRemote' repo r st@(State connpool duc _ _ _) key file meterupdate
 	| otherwise = giveup "copying to non-ssh repo not supported"
   where
 	copylocal Nothing = giveup "content not available"
-	copylocal (Just (object, check)) = do
+	copylocal (Just (object, sz, check)) = do
 		-- The check action is going to be run in
 		-- the remote's Annex, but it needs access to the local
 		-- Annex monad's state.
 		checkio <- Annex.withCurrentState check
 		u <- getUUID
 		hardlink <- wantHardLink
-		let bwlimit = remoteAnnexBwLimit (gitconfig r)
+		let bwlimit = remoteAnnexBwLimitUpload (gitconfig r)
+			<|> remoteAnnexBwLimit (gitconfig r)
 		-- run copy from perspective of remote
 		res <- onLocalFast st $ ifM (Annex.Content.inAnnex key)
 			( return True
@@ -563,7 +565,7 @@ copyToRemote' repo r st@(State connpool duc _ _ _) key file meterupdate
 				let checksuccess = liftIO checkio >>= \case
 					Just err -> giveup err
 					Nothing -> return True
-				logStatusAfter key $ Annex.Content.getViaTmp rsp verify key file $ \dest ->
+				logStatusAfter key $ Annex.Content.getViaTmp rsp verify key file (Just sz) $ \dest ->
 					metered (Just (combineMeterUpdate meterupdate p)) key bwlimit $ \_ p' -> 
 						copier object (fromRawFilePath dest) key p' checksuccess verify
 			)
diff --git a/Remote/Helper/Chunked.hs b/Remote/Helper/Chunked.hs
index 3e7f8a47e1..bb903b7266 100644
--- a/Remote/Helper/Chunked.hs
+++ b/Remote/Helper/Chunked.hs
@@ -115,7 +115,7 @@ numChunks = pred . fromJust . fromKey keyChunkNum . fst . nextChunkKeyStream
  - writes a whole L.ByteString at a time.
  -}
 storeChunks
-	:: LensGpgEncParams encc
+	:: LensEncParams encc
 	=> UUID
 	-> ChunkConfig
 	-> EncKey
@@ -159,9 +159,9 @@ storeChunks u chunkconfig encryptor k f p enc encc storer checker =
 				-- stored, update the chunk log.
 				chunksStored u k (FixedSizeChunks chunksize) numchunks
 			| otherwise = do
-				liftIO $ meterupdate' zeroBytesProcessed
 				let (chunkkey, chunkkeys') = nextChunkKeyStream chunkkeys
 				storechunk chunkkey (ByteContent chunk) meterupdate'
+				liftIO $ meterupdate' zeroBytesProcessed
 				let bytesprocessed' = addBytesProcessed bytesprocessed (L.length chunk)
 				loop bytesprocessed' (splitchunk bs) chunkkeys'
 		  where
@@ -250,7 +250,7 @@ removeChunks remover u chunkconfig encryptor k = do
  - Handles decrypting the content when encryption is used.
  -}
 retrieveChunks 
-	:: LensGpgEncParams encc
+	:: LensEncParams encc
 	=> Retriever
 	-> UUID
 	-> VerifyConfig
@@ -391,7 +391,7 @@ retrieveChunks retriever u vc chunkconfig encryptor basek dest basep enc encc
  - into place. (And it may even already be in the right place..)
  -}
 writeRetrievedContent
-	:: LensGpgEncParams encc
+	:: LensEncParams encc
 	=> FilePath
 	-> Maybe (Cipher, EncKey)
 	-> encc
diff --git a/Remote/Helper/Encryptable.hs b/Remote/Helper/Encryptable.hs
index e0349a2fb6..884d53d7bf 100644
--- a/Remote/Helper/Encryptable.hs
+++ b/Remote/Helper/Encryptable.hs
@@ -28,8 +28,6 @@ module Remote.Helper.Encryptable (
 
 import qualified Data.Map as M
 import qualified Data.Set as S
-import qualified "sandi" Codec.Binary.Base64 as B64
-import qualified Data.ByteString as B
 import Control.Concurrent.STM
 
 import Annex.Common
@@ -39,6 +37,7 @@ import Types.Crypto
 import Types.ProposedAccepted
 import qualified Annex
 import Annex.SpecialRemote.Config
+import Utility.Base64
 
 -- Used to ensure that encryption has been set up before trying to
 -- eg, store creds in the remote config that would need to use the
@@ -159,18 +158,18 @@ parseMac (Just (Proposed s)) = case readMac s of
 encryptionSetup :: RemoteConfig -> RemoteGitConfig -> Annex (RemoteConfig, EncryptionIsSetup)
 encryptionSetup c gc = do
 	pc <- either giveup return $ parseEncryptionConfig c
-	cmd <- gpgCmd <$> Annex.getGitConfig
-	maybe (genCipher pc cmd) (updateCipher pc cmd) (extractCipher pc)
+	gpgcmd <- gpgCmd <$> Annex.getGitConfig
+	maybe (genCipher pc gpgcmd) (updateCipher pc gpgcmd) (extractCipher pc)
   where
 	-- The type of encryption
 	encryption = parseEncryptionMethod (fromProposedAccepted <$> M.lookup encryptionField c) c
 	-- Generate a new cipher, depending on the chosen encryption scheme
-	genCipher pc cmd = case encryption of
+	genCipher pc gpgcmd = case encryption of
 		Right NoneEncryption -> return (c, NoEncryption)
-		Right SharedEncryption -> encsetup $ genSharedCipher cmd
-		Right HybridEncryption -> encsetup $ genEncryptedCipher cmd (pc, gc) key Hybrid
-		Right PubKeyEncryption -> encsetup $ genEncryptedCipher cmd (pc, gc) key PubKey
-		Right SharedPubKeyEncryption -> encsetup $ genSharedPubKeyCipher cmd key
+		Right SharedEncryption -> encsetup $ genSharedCipher gpgcmd
+		Right HybridEncryption -> encsetup $ genEncryptedCipher gpgcmd (pc, gc) key Hybrid
+		Right PubKeyEncryption -> encsetup $ genEncryptedCipher gpgcmd (pc, gc) key PubKey
+		Right SharedPubKeyEncryption -> encsetup $ genSharedPubKeyCipher gpgcmd key
 		Left err -> giveup err
 	key = maybe (giveup "Specify keyid=...") fromProposedAccepted $
 		M.lookup (Accepted "keyid") c
@@ -178,13 +177,13 @@ encryptionSetup c gc = do
 		maybe [] (\k -> [(False,fromProposedAccepted k)]) (M.lookup (Accepted "keyid-") c)
 	cannotchange = giveup "Cannot set encryption type of existing remotes."
 	-- Update an existing cipher if possible.
-	updateCipher pc cmd v = case v of
+	updateCipher pc gpgcmd v = case v of
 		SharedCipher _ | encryption == Right SharedEncryption ->
 			return (c', EncryptionIsSetup)
 		EncryptedCipher _ variant _ | sameasencryption variant ->
-			use "encryption update" $ updateCipherKeyIds cmd (pc, gc) newkeys v
+			use "encryption update" $ updateCipherKeyIds gpgcmd (pc, gc) newkeys v
 		SharedPubKeyCipher _ _ ->
-			use "encryption update" $ updateCipherKeyIds cmd (pc, gc) newkeys v
+			use "encryption update" $ updateCipherKeyIds gpgcmd (pc, gc) newkeys v
 		_ -> cannotchange
 	sameasencryption variant = case encryption of
 		Right HybridEncryption -> variant == Hybrid
@@ -237,8 +236,8 @@ remoteCipher' c gc = case extractCipher c of
 				(go cachev encipher)
   where
 	go cachev encipher cache = do
-		cmd <- gpgCmd <$> Annex.getGitConfig
-		cipher <- liftIO $ decryptCipher cmd (c, gc) encipher
+		gpgcmd <- gpgCmd <$> Annex.getGitConfig
+		cipher <- liftIO $ decryptCipher gpgcmd (c, gc) encipher
 		liftIO $ atomically $ putTMVar cachev $
 			M.insert encipher cipher cache
 		return $ Just (cipher, encipher)
@@ -272,7 +271,7 @@ storeCipher cip = case cip of
 	(EncryptedCipher t _ ks) -> addcipher t . storekeys ks cipherkeysField
 	(SharedPubKeyCipher t ks) -> addcipher t . storekeys ks pubkeysField
   where
-	addcipher t = M.insert cipherField (Accepted (toB64bs t))
+	addcipher t = M.insert cipherField (Accepted (decodeBS (toB64 t)))
 	storekeys (KeyIds l) n = M.insert n (Accepted (intercalate "," l))
 
 {- Extracts an StorableCipher from a remote's configuration. -}
@@ -281,13 +280,13 @@ extractCipher c = case (getRemoteConfigValue cipherField c,
 			(getRemoteConfigValue cipherkeysField c <|> getRemoteConfigValue pubkeysField c),
 			getRemoteConfigValue encryptionField c) of
 	(Just t, Just ks, Just HybridEncryption) ->
-		Just $ EncryptedCipher (fromB64bs t) Hybrid (readkeys ks)
+		Just $ EncryptedCipher (fromB64 (encodeBS t)) Hybrid (readkeys ks)
 	(Just t, Just ks, Just PubKeyEncryption) ->
-		Just $ EncryptedCipher (fromB64bs t) PubKey (readkeys ks)
+		Just $ EncryptedCipher (fromB64 (encodeBS t)) PubKey (readkeys ks)
 	(Just t, Just ks, Just SharedPubKeyEncryption) ->
-		Just $ SharedPubKeyCipher (fromB64bs t) (readkeys ks)
+		Just $ SharedPubKeyCipher (fromB64 (encodeBS t)) (readkeys ks)
 	(Just t, Nothing, Just SharedEncryption) ->
-		Just $ SharedCipher (fromB64bs t)
+		Just $ SharedCipher (fromB64 (encodeBS t))
 	_ -> Nothing
   where
 	readkeys = KeyIds . splitc ','
@@ -321,14 +320,3 @@ describeCipher c = case c of
 	(SharedPubKeyCipher _ ks) -> showkeys ks
   where
 	showkeys (KeyIds { keyIds = ks }) = "to gpg keys: " ++ unwords ks
-
-{- Not using Utility.Base64 because these "Strings" are really
- - bags of bytes and that would convert to unicode and not round-trip
- - cleanly. -}
-toB64bs :: String -> String
-toB64bs = w82s . B.unpack . B64.encode . B.pack . s2w8
-
-fromB64bs :: String -> String
-fromB64bs s = either (const bad) (w82s . B.unpack) (B64.decode $ B.pack $ s2w8 s)
-  where
-	bad = giveup "bad base64 encoded data"
diff --git a/Remote/Helper/P2P.hs b/Remote/Helper/P2P.hs
index f944f4fe43..ed9d3bffa4 100644
--- a/Remote/Helper/P2P.hs
+++ b/Remote/Helper/P2P.hs
@@ -16,6 +16,7 @@ import Types.Remote
 import Annex.Content
 import Messages.Progress
 import Utility.Metered
+import Utility.Tuple
 import Types.NumCopies
 import Annex.Verify
 
@@ -33,8 +34,8 @@ type WithConn a c = (ClosableConnection c -> Annex (ClosableConnection c, a)) ->
 
 store :: RemoteGitConfig -> ProtoRunner Bool -> Key -> AssociatedFile -> MeterUpdate -> Annex ()
 store gc runner k af p = do
-	let sizer = KeySizer k (fmap (toRawFilePath . fst) <$> prepSendAnnex k)
-	let bwlimit = remoteAnnexBwLimit gc
+	let sizer = KeySizer k (fmap (toRawFilePath . fst3) <$> prepSendAnnex k)
+	let bwlimit = remoteAnnexBwLimitUpload gc <|> remoteAnnexBwLimit gc
 	metered (Just p) sizer bwlimit $ \_ p' ->
 		runner (P2P.put k af p') >>= \case
 			Just True -> return ()
@@ -44,7 +45,7 @@ store gc runner k af p = do
 retrieve :: RemoteGitConfig -> (ProtoRunner (Bool, Verification)) -> Key -> AssociatedFile -> FilePath -> MeterUpdate -> VerifyConfig -> Annex Verification
 retrieve gc runner k af dest p verifyconfig = do
 	iv <- startVerifyKeyContentIncrementally verifyconfig k
-	let bwlimit = remoteAnnexBwLimit gc
+	let bwlimit = remoteAnnexBwLimitDownload gc <|> remoteAnnexBwLimit gc
 	metered (Just p) k bwlimit $ \m p' -> 
 		runner (P2P.get dest k iv af m p') >>= \case
 			Just (True, v) -> return v
diff --git a/Remote/Helper/Special.hs b/Remote/Helper/Special.hs
index a18f971b71..4cb6124159 100644
--- a/Remote/Helper/Special.hs
+++ b/Remote/Helper/Special.hs
@@ -212,9 +212,9 @@ specialRemote' cfg c storer retriever remover checkpresent baser = encr
 			then whereisKey baser
 			else Nothing
 		, exportActions = (exportActions baser)
-			{ storeExport = \f k l p -> displayprogress p k (Just f) $
+			{ storeExport = \f k l p -> displayprogress uploadbwlimit p k (Just f) $
 				storeExport (exportActions baser) f k l
-			, retrieveExport = \k l f p -> displayprogress p k Nothing $
+			, retrieveExport = \k l f p -> displayprogress downloadbwlimit p k Nothing $
 				retrieveExport (exportActions baser) k l f
 			}
 		}
@@ -222,8 +222,8 @@ specialRemote' cfg c storer retriever remover checkpresent baser = encr
 	isencrypted = isEncrypted c
 
 	-- chunk, then encrypt, then feed to the storer
-	storeKeyGen k p enc = sendAnnex k rollback $ \src ->
-		displayprogress p k (Just src) $ \p' ->
+	storeKeyGen k p enc = sendAnnex k rollback $ \src _sz ->
+		displayprogress uploadbwlimit p k (Just src) $ \p' ->
 			storeChunks (uuid baser) chunkconfig enck k src p'
 				enc encr storer checkpresent
 	  where
@@ -232,7 +232,7 @@ specialRemote' cfg c storer retriever remover checkpresent baser = encr
 
 	-- call retriever to get chunks; decrypt them; stream to dest file
 	retrieveKeyFileGen k dest p vc enc =
-		displayprogress p k Nothing $ \p' ->
+		displayprogress downloadbwlimit p k Nothing $ \p' ->
 			retrieveChunks retriever (uuid baser) vc
 				chunkconfig enck k dest p' enc encr
 	  where
@@ -250,9 +250,13 @@ specialRemote' cfg c storer retriever remover checkpresent baser = encr
 
 	chunkconfig = chunkConfig cfg
 
-	displayprogress p k srcfile a
+	downloadbwlimit = remoteAnnexBwLimitDownload (gitconfig baser)
+		<|> remoteAnnexBwLimit (gitconfig baser)
+	uploadbwlimit = remoteAnnexBwLimitUpload (gitconfig baser)
+		<|> remoteAnnexBwLimit (gitconfig baser)
+
+	displayprogress bwlimit p k srcfile a
 		| displayProgress cfg = do
-			let bwlimit = remoteAnnexBwLimit (gitconfig baser)
 			metered (Just p) (KeySizer k (pure (fmap toRawFilePath srcfile))) bwlimit (const a)
 		| otherwise = a p
 
diff --git a/Remote/Tahoe.hs b/Remote/Tahoe.hs
index 7c2ad0c25a..6b9138b981 100644
--- a/Remote/Tahoe.hs
+++ b/Remote/Tahoe.hs
@@ -139,7 +139,7 @@ tahoeSetup _ mu _ c _ = do
 	missingfurl = giveup "Set TAHOE_FURL to the introducer furl to use."
 
 store :: RemoteStateHandle -> TahoeHandle -> Key -> AssociatedFile -> MeterUpdate -> Annex ()
-store rs hdl k _f _p = sendAnnex k noop $ \src ->
+store rs hdl k _f _p = sendAnnex k noop $ \src _sz ->
 	parsePut <$> liftIO (readTahoe hdl "put" [File src]) >>= maybe
 		(giveup "tahoe failed to store content")
 		(\cap -> storeCapability rs k cap)
diff --git a/Test.hs b/Test.hs
index e6204d4f3e..b752d4dc22 100644
--- a/Test.hs
+++ b/Test.hs
@@ -27,6 +27,7 @@ import Control.Concurrent.STM hiding (check)
 import Common
 import CmdLine.GitAnnex.Options
 import qualified Utility.RawFilePath as R
+import Data.String
 
 import qualified Utility.ShellEscape
 import qualified Annex
@@ -77,12 +78,12 @@ import qualified Utility.Hash
 import qualified Utility.Scheduled
 import qualified Utility.Scheduled.QuickCheck
 import qualified Utility.HumanTime
-import qualified Utility.Base64
 import qualified Utility.Tmp.Dir
 import qualified Utility.FileSystemEncoding
 import qualified Utility.Aeson
 import qualified Utility.CopyFile
 import qualified Utility.MoveFile
+import qualified Utility.StatelessOpenPGP
 import qualified Types.Remote
 #ifndef mingw32_HOST_OS
 import qualified Remote.Helper.Encryptable
@@ -184,7 +185,6 @@ properties = localOption (QuickCheckTests 1000) $ testGroup "QuickCheck" $
 	, testProperty "prop_viewPath_roundtrips" Annex.View.prop_viewPath_roundtrips
 	, testProperty "prop_view_roundtrips" Annex.View.prop_view_roundtrips
 	, testProperty "prop_viewedFile_rountrips" Annex.View.ViewedFile.prop_viewedFile_roundtrips
-	, testProperty "prop_b64_roundtrips" Utility.Base64.prop_b64_roundtrips
 	, testProperty "prop_standardGroups_parse" Logs.PreferredContent.prop_standardGroups_parse
 	] ++ map (uncurry testProperty) combos
   where
@@ -349,7 +349,8 @@ repoTests note numparts = map mk $ sep
 	, testCase "rsync remote" test_rsync_remote
 	, testCase "bup remote" test_bup_remote
 	, testCase "borg remote" test_borg_remote
-	, testCase "crypto" test_crypto
+	, testCase "gpg crypto" test_gpg_crypto
+	, testCase "sop crypto" test_sop_crypto
 	, testCase "preferred content" test_preferred_content
 	, testCase "required_content" test_required_content
 	, testCase "add subdirs" test_add_subdirs
@@ -1826,10 +1827,29 @@ test_borg_remote = when BuildInfo.borg $ do
 		git_annex "drop" [annexedfile] "drop from borg (appendonly)"
 		git_annex "get" [annexedfile, "--from=borg"] "get from borg"
 
+-- To test Stateless OpenPGP, annex.shared-sop-command has to be set using
+-- the --test-git-config option.
+test_sop_crypto :: Assertion
+test_sop_crypto = do
+	gc <- testGitConfig . testOptions <$> getTestMode
+	case filter (\(k, _) -> k == ck) gc of
+		[] -> noop
+		((_, sopcmd):_) -> go $ 
+			Utility.StatelessOpenPGP.SOPCmd $
+				Git.Types.fromConfigValue sopcmd
+  where
+	ck = fromString "annex.shared-sop-command"
+	pw = fromString "testpassword"
+	v = fromString "somevalue"
+	unarmored = Utility.StatelessOpenPGP.Armoring False
+	go sopcmd = do
+		Utility.StatelessOpenPGP.test_encrypt_decrypt_Symmetric sopcmd sopcmd pw unarmored v
+			@? "sop command roundtrips symmetric encryption"
+
 -- gpg is not a build dependency, so only test when it's available
-test_crypto :: Assertion
+test_gpg_crypto :: Assertion
 #ifndef mingw32_HOST_OS
-test_crypto = do
+test_gpg_crypto = do
 	testscheme "shared"
 	testscheme "hybrid"
 	testscheme "pubkey"
@@ -1921,7 +1941,7 @@ test_crypto = do
 			Annex.Locations.keyPaths .
 			Crypto.encryptKey Types.Crypto.HmacSha1 cipher
 #else
-test_crypto = putStrLn "gpg testing not implemented on Windows"
+test_gpg_crypto = putStrLn "gpg testing not implemented on Windows"
 #endif
 
 test_add_subdirs :: Assertion
diff --git a/Test/Framework.hs b/Test/Framework.hs
index 7d073e7468..fa32b1e543 100644
--- a/Test/Framework.hs
+++ b/Test/Framework.hs
@@ -773,7 +773,28 @@ parallelTestRunner' numjobs opts mkts
 				r <- a n
 				worker (r:rs) nvar a
 	
-	go Nothing = withConcurrentOutput $ do
+	summarizeresults a = do
+		starttime <- getCurrentTime
+		(numts, exitcodes) <- a
+		duration <- Utility.HumanTime.durationSince starttime
+		case nub (filter (/= ExitSuccess) (concat exitcodes)) of
+			[] -> do
+				putStrLn ""
+				putStrLn $ "All tests succeeded. (Ran "
+					++ show numts
+					++ " test groups in " 
+					++ Utility.HumanTime.fromDuration duration
+					++ ")"
+				exitSuccess
+			[ExitFailure 1] -> do
+				putStrLn "  (Failures above could be due to a bug in git-annex, or an incompatibility"
+				putStrLn "   with utilities, such as git, installed on this system.)"
+				exitFailure
+			_ -> do
+				putStrLn $ "  Test subprocesses exited with unexpected exit codes: " ++ show (concat exitcodes)
+				exitFailure
+
+	go Nothing = summarizeresults $ withConcurrentOutput $ do
 		ensuredir tmpdir
 		crippledfilesystem <- fst <$> Annex.Init.probeCrippledFileSystem'
 			(toRawFilePath tmpdir)
@@ -801,27 +822,10 @@ parallelTestRunner' numjobs opts mkts
 			(_, _, _, pid) <- createProcessConcurrent p
 			waitForProcess pid
 		nvar <- newTVarIO (1, length ts)
-		starttime <- getCurrentTime
 		exitcodes <- forConcurrently [1..numjobs] $ \_ -> 
 			worker [] nvar runone
 		unless (keepFailuresOption opts) finalCleanup
-		duration <- Utility.HumanTime.durationSince starttime
-		case nub (filter (/= ExitSuccess) (concat exitcodes)) of
-			[] -> do
-				putStrLn ""
-				putStrLn $ "All tests succeeded. (Ran "
-					++ show (length ts) 
-					++ " test groups in " 
-					++ Utility.HumanTime.fromDuration duration
-					++ ")"
-				exitSuccess
-			[ExitFailure 1] -> do
-				putStrLn "  (Failures above could be due to a bug in git-annex, or an incompatibility"
-				putStrLn "   with utilities, such as git, installed on this system.)"
-				exitFailure
-			_ -> do
-				putStrLn $ "  Test subprocesses exited with unexpected exit codes: " ++ show (concat exitcodes)
-				exitFailure
+		return (length ts, exitcodes)
 	go (Just subenvval) = case readish subenvval of
 		Nothing -> error ("Bad " ++ subenv)
 		Just (n, crippledfilesystem, adjustedbranchok) -> setTestEnv $ do
diff --git a/Types/Crypto.hs b/Types/Crypto.hs
index 48a6ad0cc8..2b3b065d71 100644
--- a/Types/Crypto.hs
+++ b/Types/Crypto.hs
@@ -25,6 +25,7 @@ import Utility.Gpg (KeyIds(..))
 
 import Data.Typeable
 import qualified Data.Map as M
+import Data.ByteString (ByteString)
 
 data EncryptionMethod
 	= NoneEncryption
@@ -34,13 +35,14 @@ data EncryptionMethod
 	| HybridEncryption
 	deriving (Typeable, Eq)
 
+-- A base-64 encoded random value used for encryption.
 -- XXX ideally, this would be a locked memory region
-data Cipher = Cipher String | MacOnlyCipher String
+data Cipher = Cipher ByteString | MacOnlyCipher ByteString
 
 data StorableCipher
-	= EncryptedCipher String EncryptedCipherVariant KeyIds
-	| SharedCipher String
-	| SharedPubKeyCipher String KeyIds
+	= EncryptedCipher ByteString EncryptedCipherVariant KeyIds
+	| SharedCipher ByteString
+	| SharedPubKeyCipher ByteString KeyIds
 	deriving (Ord, Eq)
 data EncryptedCipherVariant = Hybrid | PubKey
 	deriving (Ord, Eq)
diff --git a/Types/GitConfig.hs b/Types/GitConfig.hs
index 13978e8338..f9e0149ddb 100644
--- a/Types/GitConfig.hs
+++ b/Types/GitConfig.hs
@@ -1,6 +1,6 @@
 {- git-annex configuration
  -
- - Copyright 2012-2021 Joey Hess <id@joeyh.name>
+ - Copyright 2012-2024 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -47,8 +47,10 @@ import Types.View
 import Config.DynamicConfig
 import Utility.HumanTime
 import Utility.Gpg (GpgCmd, mkGpgCmd)
+import Utility.StatelessOpenPGP (SOPCmd(..), SOPProfile(..))
 import Utility.ThreadScheduler (Seconds(..))
 import Utility.Url (Scheme, mkScheme)
+import Network.Socket (PortNumber)
 
 import Control.Concurrent.STM
 import qualified Data.Set as S
@@ -94,6 +96,7 @@ data GitConfig = GitConfig
 	, annexResolveMerge :: GlobalConfigurable Bool
 	, annexSyncContent :: GlobalConfigurable (Maybe Bool)
 	, annexSyncOnlyAnnex :: GlobalConfigurable Bool
+	, annexSyncMigrations :: Bool
 	, annexDebug :: Bool
 	, annexDebugFilter :: Maybe String
 	, annexWebOptions :: [String]
@@ -113,6 +116,7 @@ data GitConfig = GitConfig
 	, annexSecureEraseCommand :: Maybe String
 	, annexGenMetaData :: Bool
 	, annexListen :: Maybe String
+	, annexPort :: Maybe PortNumber
 	, annexStartupScan :: Bool
 	, annexHardLink :: Bool
 	, annexThin :: Bool
@@ -184,6 +188,7 @@ extractGitConfig configsource r = GitConfig
 		getmaybebool (annexConfig "synccontent")
 	, annexSyncOnlyAnnex = configurable False $ 
 		getmaybebool (annexConfig "synconlyannex")
+	, annexSyncMigrations = getbool (annexConfig "syncmigrations") True
 	, annexDebug = getbool (annexConfig "debug") False
 	, annexDebugFilter = getmaybe (annexConfig "debugfilter")
 	, annexWebOptions = getwords (annexConfig "web-options")
@@ -207,6 +212,7 @@ extractGitConfig configsource r = GitConfig
 	, annexSecureEraseCommand = getmaybe (annexConfig "secure-erase-command")
 	, annexGenMetaData = getbool (annexConfig "genmetadata") False
 	, annexListen = getmaybe (annexConfig "listen")
+	, annexPort = getmayberead (annexConfig "port")
 	, annexStartupScan = getbool (annexConfig "startupscan") True
 	, annexHardLink = getbool (annexConfig "hardlink") False
 	, annexThin = getbool (annexConfig "thin") False
@@ -356,7 +362,11 @@ data RemoteGitConfig = RemoteGitConfig
 	, remoteAnnexForwardRetry :: Maybe Integer
 	, remoteAnnexRetryDelay :: Maybe Seconds
 	, remoteAnnexStallDetection :: Maybe StallDetection
+	, remoteAnnexStallDetectionUpload :: Maybe StallDetection
+	, remoteAnnexStallDetectionDownload :: Maybe StallDetection
 	, remoteAnnexBwLimit :: Maybe BwRate
+	, remoteAnnexBwLimitUpload :: Maybe BwRate
+	, remoteAnnexBwLimitDownload :: Maybe BwRate
 	, remoteAnnexAllowUnverifiedDownloads :: Bool
 	, remoteAnnexConfigUUID :: Maybe UUID
 
@@ -370,6 +380,8 @@ data RemoteGitConfig = RemoteGitConfig
 	, remoteAnnexRsyncTransport :: [String]
 	, remoteAnnexGnupgOptions :: [String]
 	, remoteAnnexGnupgDecryptOptions :: [String]
+	, remoteAnnexSharedSOPCommand :: Maybe SOPCmd
+	, remoteAnnexSharedSOPProfile :: Maybe SOPProfile
 	, remoteAnnexRsyncUrl :: Maybe String
 	, remoteAnnexBupRepo :: Maybe String
 	, remoteAnnexBorgRepo :: Maybe String
@@ -421,11 +433,17 @@ extractRemoteGitConfig r remotename = do
 		, remoteAnnexRetryDelay = Seconds
 			<$> getmayberead "retrydelay"
 		, remoteAnnexStallDetection =
-			either (const Nothing) Just . parseStallDetection
-				=<< getmaybe "stalldetection"
-		, remoteAnnexBwLimit = do
-			sz <- readSize dataUnits =<< getmaybe "bwlimit"
-			return (BwRate sz (Duration 1))
+			readStallDetection =<< getmaybe "stalldetection"
+		, remoteAnnexStallDetectionUpload =
+			readStallDetection =<< getmaybe "stalldetection-upload"
+		, remoteAnnexStallDetectionDownload =
+			readStallDetection =<< getmaybe "stalldetection-download"
+		, remoteAnnexBwLimit =
+			readBwRatePerSecond =<< getmaybe "bwlimit"
+		, remoteAnnexBwLimitUpload =
+			readBwRatePerSecond =<< getmaybe "bwlimit-upload"
+		, remoteAnnexBwLimitDownload =
+			readBwRatePerSecond =<< getmaybe "bwlimit-download"
 		, remoteAnnexAllowUnverifiedDownloads = (== Just "ACKTHPPT") $
 			getmaybe ("security-allow-unverified-downloads")
 		, remoteAnnexConfigUUID = toUUID <$> getmaybe "config-uuid"
@@ -437,6 +455,10 @@ extractRemoteGitConfig r remotename = do
 		, remoteAnnexRsyncTransport = getoptions "rsync-transport"
 		, remoteAnnexGnupgOptions = getoptions "gnupg-options"
 		, remoteAnnexGnupgDecryptOptions = getoptions "gnupg-decrypt-options"
+		, remoteAnnexSharedSOPCommand = SOPCmd <$>
+			notempty (getmaybe "shared-sop-command")
+		, remoteAnnexSharedSOPProfile = SOPProfile <$>
+			notempty (getmaybe "shared-sop-profile")
 		, remoteAnnexRsyncUrl = notempty $ getmaybe "rsyncurl"
 		, remoteAnnexBupRepo = getmaybe "buprepo"
 		, remoteAnnexBorgRepo = getmaybe "borgrepo"
diff --git a/Types/MetaData.hs b/Types/MetaData.hs
index 7df07a87eb..316c06a2f7 100644
--- a/Types/MetaData.hs
+++ b/Types/MetaData.hs
@@ -137,14 +137,14 @@ instance MetaSerializable MetaValue where
 	serialize (MetaValue isset v) =
 		serialize isset <>
 		if B8.any (`elem` [' ', '\r', '\n']) v || "!" `B8.isPrefixOf` v
-			then "!" <> toB64' v
+			then "!" <> toB64 v
 			else v
 	deserialize b = do
 		(isset, b') <- B8.uncons b
 		case B8.uncons b' of
 			Just ('!', b'') -> MetaValue
 				<$> deserialize (B8.singleton isset)
-				<*> fromB64Maybe' b''
+				<*> fromB64Maybe b''
 			_ -> MetaValue
 				<$> deserialize (B8.singleton isset)
 				<*> pure b'
diff --git a/Types/StallDetection.hs b/Types/StallDetection.hs
index 13d88699f2..2278119f4e 100644
--- a/Types/StallDetection.hs
+++ b/Types/StallDetection.hs
@@ -1,6 +1,6 @@
 {- types for stall detection and banwdith rates
  -
- - Copyright 2020-2021 Joey Hess <id@joeyh.name>
+ - Copyright 2020-2024 Joey Hess <id@joeyh.name>
  -
  - Licensed under the GNU AGPL version 3 or higher.
  -}
@@ -39,6 +39,9 @@ parseStallDetection s = case isTrueFalse s of
 	Just True -> Right ProbeStallDetection
 	Just False -> Right StallDetectionDisabled
 
+readStallDetection :: String -> Maybe StallDetection
+readStallDetection = either (const Nothing) Just . parseStallDetection
+
 parseBwRate :: String -> Either String BwRate
 parseBwRate s = do
 	let (bs, ds) = separate (== '/') s
@@ -48,3 +51,8 @@ parseBwRate s = do
 		(readSize dataUnits bs)
 	d <- parseDuration ds
 	Right (BwRate b d)
+
+readBwRatePerSecond :: String -> Maybe BwRate
+readBwRatePerSecond s = do
+	sz <- readSize dataUnits s
+	return (BwRate sz (Duration 1))
diff --git a/Upgrade.hs b/Upgrade.hs
index 244050444f..4f6585b2ea 100644
--- a/Upgrade.hs
+++ b/Upgrade.hs
@@ -61,7 +61,11 @@ needsUpgrade v
 		p <- liftIO $ absPath $ Git.repoPath g
 		return $ Just $ unwords
 			[ "Repository", fromRawFilePath p
-			, "is at unsupported version"
+			, "is at"
+			, if v `elem` supportedVersions 
+				then "supported"
+				else "unsupported"
+			, "version"
 			, show (fromRepoVersion v) ++ "."
 			, msg
 			]
diff --git a/Utility/Base64.hs b/Utility/Base64.hs
index 32d9066823..e2d90d6dac 100644
--- a/Utility/Base64.hs
+++ b/Utility/Base64.hs
@@ -1,55 +1,25 @@
 {- Simple Base64 encoding
  -
- - Copyright 2011-2019 Joey Hess <id@joeyh.name>
+ - Copyright 2011-2023 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}
 
-{-# LANGUAGE PackageImports #-}
-
 module Utility.Base64 where
 
-import Utility.FileSystemEncoding
-import Utility.QuickCheck
 import Utility.Exception
 
-import qualified "sandi" Codec.Binary.Base64 as B64
+import Codec.Binary.Base64 as B64
 import Data.Maybe
 import qualified Data.ByteString as B
-import Data.ByteString.UTF8 (fromString, toString)
-import Data.Char
-
--- | This uses the FileSystemEncoding, so it can be used on Strings
--- that represent filepaths containing arbitrarily encoded characters.
-toB64 :: String -> String
-toB64 = toString . B64.encode . encodeBS
 
-toB64' :: B.ByteString -> B.ByteString
-toB64' = B64.encode
+toB64 :: B.ByteString -> B.ByteString
+toB64 = B64.encode
 
-fromB64Maybe :: String -> Maybe String
-fromB64Maybe s = either (const Nothing) (Just . decodeBS)
-	(B64.decode $ fromString s)
+fromB64Maybe :: B.ByteString -> Maybe (B.ByteString)
+fromB64Maybe = either (const Nothing) Just . B64.decode
 
-fromB64Maybe' :: B.ByteString -> Maybe (B.ByteString)
-fromB64Maybe' = either (const Nothing) Just . B64.decode
-
-fromB64 :: String -> String
+fromB64 :: B.ByteString -> B.ByteString
 fromB64 = fromMaybe bad . fromB64Maybe
   where
 	bad = giveup "bad base64 encoded data"
-
-fromB64' :: B.ByteString -> B.ByteString
-fromB64' = fromMaybe bad . fromB64Maybe'
-  where
-	bad = giveup "bad base64 encoded data"
-
--- Only ascii strings are tested, because an arbitrary string may contain
--- characters not encoded using the FileSystemEncoding, which would thus
--- not roundtrip, as decodeBS always generates an output encoded that way.
-prop_b64_roundtrips :: TestableString -> Bool
-prop_b64_roundtrips ts
-	| all (isAscii) s = s == decodeBS (fromB64' (toB64' (encodeBS s)))
-	| otherwise = True
-  where
-	s = fromTestableString ts
diff --git a/Utility/CoProcess.hs b/Utility/CoProcess.hs
index e091d438c3..d9b8f31fbd 100644
--- a/Utility/CoProcess.hs
+++ b/Utility/CoProcess.hs
@@ -1,7 +1,7 @@
 {- Interface for running a shell command as a coprocess,
  - sending it queries and getting back results.
  -
- - Copyright 2012-2013 Joey Hess <id@joeyh.name>
+ - Copyright 2012-2023 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}
@@ -14,11 +14,14 @@ module Utility.CoProcess (
 	start,
 	stop,
 	query,
+	send,
+	receive,
 ) where
 
 import Common
 
 import Control.Concurrent.MVar
+import Control.Monad.IO.Class (MonadIO)
 
 type CoProcessHandle = MVar CoProcessState
 
@@ -65,12 +68,12 @@ stop ch = do
 {- To handle a restartable process, any IO exception thrown by the send and
  - receive actions are assumed to mean communication with the process
  - failed, and the failed action is re-run with a new process. -}
-query :: CoProcessHandle -> (Handle -> IO a) -> (Handle -> IO b) -> IO b
-query ch send receive = do
-	s <- readMVar ch
-	restartable s (send $ coProcessTo s) $ const $
-		restartable s (hFlush $ coProcessTo s) $ const $
-			restartable s (receive $ coProcessFrom s)
+query :: (MonadIO m, MonadCatch m) => CoProcessHandle -> (Handle -> m a) -> (Handle -> m b) -> m b
+query ch sender receiver = do
+	s <- liftIO $ readMVar ch
+	restartable s (sender $ coProcessTo s) $ const $
+		restartable s (liftIO $ hFlush $ coProcessTo s) $ const $
+			restartable s (receiver $ coProcessFrom s)
 				return
   where
 	restartable s a cont
@@ -78,12 +81,23 @@ query ch send receive = do
 			maybe restart cont =<< catchMaybeIO a
 		| otherwise = cont =<< a
 	restart = do
-		s <- takeMVar ch
-		void $ catchMaybeIO $ do
+		s <- liftIO $ takeMVar ch
+		void $ liftIO $ catchMaybeIO $ do
 			hClose $ coProcessTo s
 			hClose $ coProcessFrom s
-		void $ waitForProcess $ coProcessPid s
-		s' <- start' $ (coProcessSpec s)
+		void $ liftIO $ waitForProcess $ coProcessPid s
+		s' <- liftIO $ start' $ (coProcessSpec s)
 			{ coProcessNumRestarts = coProcessNumRestarts (coProcessSpec s) - 1 }
-		putMVar ch s'
-		query ch send receive
+		liftIO $ putMVar ch s'
+		query ch sender receiver
+
+send :: MonadIO m => CoProcessHandle -> (Handle -> m a) -> m a
+send ch a = do
+	s <- liftIO $ readMVar ch
+	a (coProcessTo s)
+
+receive :: MonadIO m => CoProcessHandle -> (Handle -> m a) -> m a
+receive ch a = do
+	s <- liftIO $ readMVar ch
+	liftIO $ hFlush $ coProcessTo s
+	a (coProcessFrom s)
diff --git a/Utility/Data.hs b/Utility/Data.hs
index faf9b34ecc..af8c5580b5 100644
--- a/Utility/Data.hs
+++ b/Utility/Data.hs
@@ -10,12 +10,8 @@
 module Utility.Data (
 	firstJust,
 	eitherToMaybe,
-	s2w8,
-	w82s,
 ) where
 
-import Data.Word
-
 {- First item in the list that is not Nothing. -}
 firstJust :: Eq a => [Maybe a] -> Maybe a
 firstJust ms = case dropWhile (== Nothing) ms of
@@ -24,15 +20,3 @@ firstJust ms = case dropWhile (== Nothing) ms of
 
 eitherToMaybe :: Either a b -> Maybe b
 eitherToMaybe = either (const Nothing) Just
-
-c2w8 :: Char -> Word8
-c2w8 = fromIntegral . fromEnum
-
-w82c :: Word8 -> Char
-w82c = toEnum . fromIntegral
-
-s2w8 :: String -> [Word8]
-s2w8 = map c2w8
-
-w82s :: [Word8] -> String
-w82s = map w82c
diff --git a/Utility/DataUnits.hs b/Utility/DataUnits.hs
index 8d910c6189..3476ba8b3a 100644
--- a/Utility/DataUnits.hs
+++ b/Utility/DataUnits.hs
@@ -58,9 +58,14 @@ module Utility.DataUnits (
 
 import Data.List
 import Data.Char
+import Data.Function
 
+import Author
 import Utility.HumanNumber
 
+copyright :: Copyright
+copyright = author JoeyHess (40*50+10)
+
 type ByteSize = Integer
 type Name = String
 type Abbrev = String
@@ -136,7 +141,7 @@ oldSchoolUnits = zipWith (curry mingle) storageUnits committeeUnits
 
 {- approximate display of a particular number of bytes -}
 roughSize :: [Unit] -> Bool -> ByteSize -> String
-roughSize units short i = roughSize' units short 2 i
+roughSize units short i = copyright $ roughSize' units short 2 i
 
 roughSize' :: [Unit] -> Bool -> Int -> ByteSize -> String
 roughSize' units short precision i
@@ -147,7 +152,7 @@ roughSize' units short precision i
 
 	findUnit (u@(Unit s _ _):us) i'
 		| i' >= s = showUnit i' u
-		| otherwise = findUnit us i'
+		| otherwise = findUnit us i' & copyright
 	findUnit [] i' = showUnit i' (last units') -- bytes
 
 	showUnit x (Unit size abbrev name) = s ++ " " ++ unit
diff --git a/Utility/FileMode.hs b/Utility/FileMode.hs
index ecc19d8126..eb25c526d1 100644
--- a/Utility/FileMode.hs
+++ b/Utility/FileMode.hs
@@ -175,10 +175,13 @@ writeFileProtected file content = writeFileProtected' file
 	(\h -> hPutStr h content)
 
 writeFileProtected' :: RawFilePath -> (Handle -> IO ()) -> IO ()
-writeFileProtected' file writer = do
-	h <- protectedOutput $ openFile (fromRawFilePath file) WriteMode
-	void $ tryIO $ modifyFileMode file $ removeModes otherGroupModes
-	writer h
+writeFileProtected' file writer = bracket setup cleanup writer
+  where
+	setup = do
+		h <- protectedOutput $ openFile (fromRawFilePath file) WriteMode
+		void $ tryIO $ modifyFileMode file $ removeModes otherGroupModes
+		return h
+	cleanup = hClose
 
 protectedOutput :: IO a -> IO a
 protectedOutput = withUmask 0o0077
diff --git a/Utility/Gpg.hs b/Utility/Gpg.hs
index 445f65768c..2ef077990f 100644
--- a/Utility/Gpg.hs
+++ b/Utility/Gpg.hs
@@ -1,10 +1,11 @@
 {- gpg interface
  -
- - Copyright 2011-2022 Joey Hess <id@joeyh.name>
+ - Copyright 2011-2023 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}
 
+{-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE CPP #-}
 
 module Utility.Gpg (
@@ -22,8 +23,6 @@ module Utility.Gpg (
 	findPubKeys,
 	UserId,
 	secretKeys,
-	KeyType(..),
-	maxRecommendedKeySize,
 	genSecretKey,
 	genRandom,
 	testKeyId,
@@ -48,6 +47,7 @@ import Utility.Format (decode_c)
 
 import Control.Concurrent.Async
 import Control.Monad.IO.Class
+import qualified Data.ByteString as B
 import qualified Data.Map as M
 import Data.Char
 
@@ -58,6 +58,8 @@ newtype KeyIds = KeyIds { keyIds :: [KeyId] }
 
 newtype GpgCmd = GpgCmd { unGpgCmd :: String }
 
+type Passphrase = B.ByteString
+
 {- Get gpg command to use, Just what's specified or, if a specific gpg
  - command was found at configure time, use it, or otherwise, "gpg". -}
 mkGpgCmd :: Maybe FilePath -> GpgCmd
@@ -108,10 +110,10 @@ stdEncryptionParams symmetric = enc symmetric ++
 		]
 
 {- Runs gpg with some params and returns its stdout, strictly. -}
-readStrict :: GpgCmd -> [CommandParam] -> IO String
+readStrict :: GpgCmd -> [CommandParam] -> IO B.ByteString
 readStrict c p = readStrict' c p Nothing
 
-readStrict' :: GpgCmd -> [CommandParam] -> Maybe [(String, String)] -> IO String
+readStrict' :: GpgCmd -> [CommandParam] -> Maybe [(String, String)] -> IO B.ByteString
 readStrict' (GpgCmd cmd) params environ = do
 	params' <- stdParams params
 	let p = (proc cmd params')
@@ -120,17 +122,16 @@ readStrict' (GpgCmd cmd) params environ = do
 		}
 	withCreateProcess p (go p)
   where
-	go p _ (Just hout) _ pid = do
-		hSetBinaryMode hout True
-		forceSuccessProcess p pid `after` hGetContentsStrict hout
+	go p _ (Just hout) _ pid =
+		forceSuccessProcess p pid `after` B.hGetContents hout
 	go _ _ _ _ _ = error "internal"
 
 {- Runs gpg, piping an input value to it, and returning its stdout,
  - strictly. -}
-pipeStrict :: GpgCmd -> [CommandParam] -> String -> IO String
+pipeStrict :: GpgCmd -> [CommandParam] -> B.ByteString -> IO B.ByteString
 pipeStrict c p i = pipeStrict' c p Nothing i
 
-pipeStrict' :: GpgCmd -> [CommandParam] -> Maybe [(String, String)] -> String -> IO String
+pipeStrict' :: GpgCmd -> [CommandParam] -> Maybe [(String, String)] -> B.ByteString -> IO B.ByteString
 pipeStrict' (GpgCmd cmd) params environ input = do
 	params' <- stdParams params
 	let p = (proc cmd params')
@@ -141,15 +142,13 @@ pipeStrict' (GpgCmd cmd) params environ input = do
 	withCreateProcess p (go p)
   where
 	go p (Just to) (Just from) _ pid = do
-		hSetBinaryMode to True
-		hSetBinaryMode from True
-		hPutStr to input
+		B.hPutStr to input
 		hClose to
-		forceSuccessProcess p pid `after` hGetContentsStrict from
+		forceSuccessProcess p pid `after` B.hGetContents from
 	go _ _ _ _ _ = error "internal"
 
-{- Runs gpg with some parameters. First sends it a passphrase (unless it
- - is empty) via '--passphrase-fd'. Then runs a feeder action that is
+{- Runs gpg with some parameters. First sends it a passphrase via
+ - '--passphrase-fd'. Then runs a feeder action that is
  - passed a handle and should write to it all the data to input to gpg.
  - Finally, runs a reader action that is passed a handle to gpg's
  - output.
@@ -158,15 +157,16 @@ pipeStrict' (GpgCmd cmd) params environ input = do
  - the passphrase.
  -
  - Note that the reader must fully consume gpg's input before returning. -}
-feedRead :: (MonadIO m, MonadMask m) => GpgCmd -> [CommandParam] -> String -> (Handle -> IO ()) -> (Handle -> m a) -> m a
+feedRead :: (MonadIO m, MonadMask m) => GpgCmd -> [CommandParam] -> Passphrase -> (Handle -> IO ()) -> (Handle -> m a) -> m a
 feedRead cmd params passphrase feeder reader = do
 #ifndef mingw32_HOST_OS
 	let setup = liftIO $ do
 		-- pipe the passphrase into gpg on a fd
 		(frompipe, topipe) <- System.Posix.IO.createPipe
+		setFdOption topipe CloseOnExec True
 		toh <- fdToHandle topipe
 		t <- async $ do
-			hPutStrLn toh passphrase
+			B.hPutStr toh (passphrase <> "\n")
 			hClose toh
 		let Fd pfd = frompipe
 		let passphrasefd = [Param "--passphrase-fd", Param $ show pfd]
@@ -180,7 +180,7 @@ feedRead cmd params passphrase feeder reader = do
 #else
 	-- store the passphrase in a temp file for gpg
 	withTmpFile "gpg" $ \tmpfile h -> do
-		liftIO $ hPutStr h passphrase
+		liftIO $ B.hPutStr h passphrase
 		liftIO $ hClose h
 		let passphrasefile = [Param "--passphrase-file", File tmpfile]
 		go $ passphrasefile ++ params
@@ -223,7 +223,8 @@ findPubKeys' cmd environ for
 	-- pass forced subkey through as-is rather than
 	-- looking up the master key.
 	| isForcedSubKey for = return $ KeyIds [for]
-	| otherwise = KeyIds . parse . lines <$> readStrict' cmd params environ
+	| otherwise = KeyIds . parse . lines . decodeBS
+		<$> readStrict' cmd params environ
   where
 	params = [Param "--with-colons", Param "--list-public-keys", Param for]
 	parse = mapMaybe (keyIdField . splitc ':')
@@ -241,7 +242,8 @@ type UserId = String
 secretKeys :: GpgCmd -> IO (M.Map KeyId UserId)
 secretKeys cmd = catchDefaultIO M.empty makemap
   where
-	makemap = M.fromList . parse . lines <$> readStrict cmd params
+	makemap = M.fromList . parse . lines . decodeBS
+		<$> readStrict cmd params
 	params = [Param "--with-colons", Param "--list-secret-keys", Param "--fixed-list-mode"]
 	parse = extract [] Nothing . map (splitc ':')
 	extract c (Just keyid) (("uid":_:_:_:_:_:_:_:_:userid:_):rest) =
@@ -259,49 +261,29 @@ secretKeys cmd = catchDefaultIO M.empty makemap
 	extract c k (_:rest) =
 		extract c k rest
 
-type Passphrase = String
-type Size = Int
-data KeyType = Algo Int | DSA | RSA
-
-{- The maximum key size that gpg currently offers in its UI when
- - making keys. -}
-maxRecommendedKeySize :: Size
-maxRecommendedKeySize = 4096
-
-{- Generates a secret key using the experimental batch mode.
+{- Generates a secret key.
  - The key is added to the secret key ring.
  - Can take a very long time, depending on system entropy levels.
  -}
-genSecretKey :: GpgCmd -> KeyType -> Passphrase -> UserId -> Size -> IO ()
-genSecretKey (GpgCmd cmd) keytype passphrase userid keysize =
-	let p = (proc cmd params)
-		{ std_in = CreatePipe }
-	in withCreateProcess p (go p)
+genSecretKey :: GpgCmd -> Passphrase -> UserId -> IO ()
+genSecretKey gpgcmd passphrase userid =
+	feedRead gpgcmd params passphrase feeder reader
   where
-	params = ["--batch", "--gen-key"]
-
-	go p (Just h) _ _ pid = do
-		hPutStr h $ unlines $ catMaybes
-			[ Just $  "Key-Type: " ++ 
-				case keytype of
-					DSA -> "DSA"
-					RSA -> "RSA"
-					Algo n -> show n
-			, Just $ "Key-Length: " ++ show keysize
-			, Just $ "Name-Real: " ++ userid
-			, Just "Expire-Date: 0"
-			, if null passphrase
-				then Nothing
-				else Just $ "Passphrase: " ++ passphrase
-			]
-		hClose h
-		forceSuccessProcess p pid
-	go _ _ _ _ _ = error "internal"
+	params =
+		[ Param "--batch"
+		, Param "--quick-gen-key"
+		, Param userid
+		, Param "default" -- algo
+		, Param "default" -- usage
+		, Param "never" -- expire
+		]
+	feeder = hClose
+	reader = void . hGetContents
 
 {- Creates a block of high-quality random data suitable to use as a cipher.
  - It is armored, to avoid newlines, since gpg only reads ciphers up to the
  - first newline. -}
-genRandom :: GpgCmd -> Bool -> Size -> IO String
+genRandom :: GpgCmd -> Bool -> Int -> IO B.ByteString
 genRandom cmd highQuality size = do
 	s <- readStrict cmd params
 	checksize s
@@ -327,7 +309,7 @@ genRandom cmd highQuality size = do
 	 - entropy. -}
 	expectedlength = size * 8 `div` 6
 
-	checksize s = let len = length s in
+	checksize s = let len = B.length s in
 		unless (len >= expectedlength) $
 			shortread len
 
@@ -439,8 +421,8 @@ testHarness tmpdir cmd a = ifM (inSearchPath (unGpgCmd cmd))
 		liftIO $ void $ tryIO $ modifyFileMode (toRawFilePath subdir) $
 			removeModes $ otherGroupModes
 		-- For some reason, recent gpg needs a trustdb to be set up.
-		_ <- pipeStrict' cmd [Param "--trust-model", Param "auto", Param "--update-trustdb"] (Just environ) []
-		_ <- pipeStrict' cmd [Param "--import", Param "-q"] (Just environ) $ unlines
+		_ <- pipeStrict' cmd [Param "--trust-model", Param "auto", Param "--update-trustdb"] (Just environ) mempty
+		_ <- pipeStrict' cmd [Param "--import", Param "-q"] (Just environ) $ encodeBS $ unlines
 			[testSecretKey, testKey]
 		return environ
 		
@@ -470,7 +452,7 @@ checkEncryptionFile cmd environ filename keys =
   where
 	params = [Param "--list-packets", Param "--list-only", File filename]
 
-checkEncryptionStream :: GpgCmd -> Maybe [(String, String)] -> String -> Maybe KeyIds -> IO Bool
+checkEncryptionStream :: GpgCmd -> Maybe [(String, String)] -> B.ByteString -> Maybe KeyIds -> IO Bool
 checkEncryptionStream cmd environ stream keys =
 	checkGpgPackets cmd environ keys =<< pipeStrict' cmd params environ stream
   where
@@ -480,13 +462,13 @@ checkEncryptionStream cmd environ stream keys =
  - symmetrically encrypted (keys is Nothing), or encrypted to some
  - public key(s).
  - /!\ The key needs to be in the keyring! -}
-checkGpgPackets :: GpgCmd -> Maybe [(String, String)] -> Maybe KeyIds -> String -> IO Bool
+checkGpgPackets :: GpgCmd -> Maybe [(String, String)] -> Maybe KeyIds -> B.ByteString -> IO Bool
 checkGpgPackets cmd environ keys str = do
 	let (asym,sym) = partition (pubkeyEncPacket `isPrefixOf`) $
 			filter (\l' -> pubkeyEncPacket `isPrefixOf` l' ||
 				symkeyEncPacket `isPrefixOf` l') $
 			takeWhile (/= ":encrypted data packet:") $
-			lines str
+			lines (decodeBS str)
 	case (keys,asym,sym) of
 		(Nothing, [], [_]) -> return True
 		(Just (KeyIds ks), ls, []) -> do
diff --git a/Utility/Hash.hs b/Utility/Hash.hs
index 46a0d6935b..81674a8d2b 100644
--- a/Utility/Hash.hs
+++ b/Utility/Hash.hs
@@ -6,6 +6,7 @@
  -}
 
 {-# LANGUAGE BangPatterns, PackageImports #-}
+{-# LANGUAGE CPP #-}
 
 module Utility.Hash (
 	sha1,
@@ -76,8 +77,13 @@ import qualified Data.ByteString.Lazy as L
 import qualified Data.Text as T
 import qualified Data.Text.Encoding as T
 import Data.IORef
-import Crypto.MAC.HMAC hiding (Context)
-import Crypto.Hash
+#ifdef WITH_CRYPTON
+import "crypton" Crypto.MAC.HMAC hiding (Context)
+import "crypton" Crypto.Hash
+#else
+import "cryptonite" Crypto.MAC.HMAC hiding (Context)
+import "cryptonite" Crypto.Hash
+#endif
 
 sha1 :: L.ByteString -> Digest SHA1
 sha1 = hashlazy
diff --git a/Utility/HtmlDetect.hs b/Utility/HtmlDetect.hs
index e050ff7dd3..fd5ad2ef06 100644
--- a/Utility/HtmlDetect.hs
+++ b/Utility/HtmlDetect.hs
@@ -12,12 +12,17 @@ module Utility.HtmlDetect (
 	htmlPrefixLength,
 ) where
 
+import Author
+
 import Text.HTML.TagSoup
 import System.IO
 import Data.Char
 import qualified Data.ByteString.Lazy as B
 import qualified Data.ByteString.Lazy.Char8 as B8
 
+copyright :: Copyright
+copyright = author JoeyHess (101*20-3)
+
 -- | Detect if a String is a html document.
 --
 -- The document many not be valid, or may be truncated, and will
@@ -29,12 +34,13 @@ import qualified Data.ByteString.Lazy.Char8 as B8
 isHtml :: String -> Bool
 isHtml = evaluate . canonicalizeTags . parseTags . take htmlPrefixLength
   where
-	evaluate (TagOpen "!DOCTYPE" ((t, _):_):_) = map toLower t == "html"
+	evaluate (TagOpen "!DOCTYPE" ((t, _):_):_) = 
+		copyright $ map toLower t == "html"
 	evaluate (TagOpen "html" _:_) = True
 	-- Allow some leading whitespace before the tag.
 	evaluate (TagText t:rest)
 		| all isSpace t = evaluate rest
-		| otherwise = False
+		| otherwise = False || author JoeyHess 1492
 	-- It would be pretty weird to have a html comment before the html
 	-- tag, but easy to allow for.
 	evaluate (TagComment _:rest) = evaluate rest
diff --git a/Utility/Matcher.hs b/Utility/Matcher.hs
index e252eae538..38864e05fd 100644
--- a/Utility/Matcher.hs
+++ b/Utility/Matcher.hs
@@ -24,6 +24,7 @@ module Utility.Matcher (
 	MatchResult(..),
 	syntaxToken,
 	generate,
+	pruneMatcher,
 	match,
 	match',
 	matchM,
@@ -99,6 +100,26 @@ generate = simplify . process MAny . implicitAnd . tokenGroups
 	simplify (MNot x) = MNot (simplify x)
 	simplify x = x
 
+{- Prunes selected ops from the Matcher. -}
+pruneMatcher :: (op -> Bool) -> Matcher op -> Matcher op
+pruneMatcher f = fst . go
+  where
+	go MAny = (MAny, False)
+	go (MAnd a b) = go2 a b MAnd
+	go (MOr a b) = go2 a b MOr
+	go (MNot a) = case go a of
+		(_, True)  -> (MAny, True)
+		(a', False) -> (MNot a', False)
+	go (MOp op)
+		| f op = (MAny, True)
+		| otherwise = (MOp op, False)
+
+	go2 a b g = case (go a, go b) of
+		((_,  True),  (_,  True))  -> (MAny, True)
+		((a', False), (b', False)) -> (g a' b', False)
+		((_,  True),  (b', False)) -> (b', False)
+		((a', False), (_,  True))  -> (a', False)
+
 data TokenGroup op = One (Token op) | Group [TokenGroup op]
 	deriving (Show, Eq)
 
diff --git a/Utility/Metered.hs b/Utility/Metered.hs
index a8a71112a3..0b7097b732 100644
--- a/Utility/Metered.hs
+++ b/Utility/Metered.hs
@@ -1,6 +1,6 @@
 {- Metered IO and actions
  -
- - Copyright 2012-2021 Joey Hess <id@joeyh.name>
+ - Copyright 2012-2024 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}
@@ -48,6 +48,7 @@ module Utility.Metered (
 ) where
 
 import Common
+import Author
 import Utility.Percentage
 import Utility.DataUnits
 import Utility.HumanTime
@@ -67,6 +68,9 @@ import Control.Monad.IO.Class (MonadIO)
 import Data.Time.Clock
 import Data.Time.Clock.POSIX
 
+copyright :: Copyright
+copyright = author JoeyHess (2024-12)
+
 {- An action that can be run repeatedly, updating it on the bytes processed.
  -
  - Note that each call receives the total number of bytes processed, so
@@ -174,7 +178,7 @@ hGetMetered h wantsize meterupdate = lazyRead zeroBytesProcessed
 		c <- S.hGet h (nextchunksize (fromBytesProcessed sofar))
 		if S.null c
 			then do
-				when (wantsize /= Just 0) $
+				when (wantsize /= Just 0 && copyright) $
 					hClose h
 				return L.empty
 			else do
@@ -214,22 +218,47 @@ defaultChunkSize = 32 * k - chunkOverhead
  - away and start over. To avoid reporting the original file size followed
  - by a smaller size in that case, wait until the file starts growing
  - before updating the meter for the first time.
+ -
+ - An updated version of the MeterUpdate is passed to the action, and the
+ - action should use that for any updates that it makes. This allows for
+ - eg, the action updating the meter before a write is flushed to the file.
+ - In that situation, this avoids the meter being set back to the size of
+ - the file when it's gotten ahead of that point.
  -}
-watchFileSize :: (MonadIO m, MonadMask m) => FilePath -> MeterUpdate -> m a -> m a
-watchFileSize f p a = bracket 
-	(liftIO $ forkIO $ watcher =<< getsz)
-	(liftIO . void . tryIO . killThread)
-	(const a)
+watchFileSize
+	:: (MonadIO m, MonadMask m)
+	=> RawFilePath
+	-> MeterUpdate
+	-> (MeterUpdate -> m a)
+	-> m a
+watchFileSize f p a = do
+	sizevar <- liftIO $ newMVar zeroBytesProcessed
+	bracket 
+		(liftIO $ forkIO $ watcher (meterupdate sizevar True) =<< getsz)
+		(liftIO . void . tryIO . killThread)
+		(const (a (meterupdate sizevar False)))
   where
-	watcher oldsz = do
+	watcher p' oldsz = do
 		threadDelay 500000 -- 0.5 seconds
 		sz <- getsz
 		when (sz > oldsz) $
-			p sz
-		watcher sz
+			p' sz
+		watcher p' sz
 	getsz = catchDefaultIO zeroBytesProcessed $
-		toBytesProcessed <$> getFileSize f'
-	f' = toRawFilePath f
+		toBytesProcessed <$> getFileSize f
+
+	meterupdate sizevar preventbacktracking n
+		| preventbacktracking = do
+			old <- takeMVar sizevar
+			if old > n
+				then putMVar sizevar old
+				else do
+					putMVar sizevar n
+					p n
+		| otherwise = do
+			void $ takeMVar sizevar
+			putMVar sizevar n
+			p n
 
 data OutputHandler = OutputHandler
 	{ quietMode :: Bool
@@ -276,7 +305,7 @@ commandMeterExitCode' progressparser oh mmeter meterupdate cmd params mkprocess
 		handlestderr
   where
 	feedprogress sendtotalsize prev buf h = do
-		b <- S.hGetSome h 80
+		b <- S.hGetSome h 80 >>= copyright
 		if S.null b
 			then return ()
 			else do
diff --git a/Utility/MoveFile.hs b/Utility/MoveFile.hs
index 6481b29ed3..c04dbea6d0 100644
--- a/Utility/MoveFile.hs
+++ b/Utility/MoveFile.hs
@@ -29,6 +29,10 @@ import Utility.Exception
 import Utility.Monad
 import Utility.FileSystemEncoding
 import qualified Utility.RawFilePath as R
+import Author
+
+copyright :: Copyright
+copyright = author JoeyHess (2022-11)
 
 {- Moves one filename to another.
  - First tries a rename, but falls back to moving across devices if needed. -}
@@ -53,7 +57,7 @@ moveFile src dest = tryIO (R.rename src dest) >>= onrename
 			-- If dest is a directory, mv would move the file
 			-- into it, which is not desired.
 			whenM (isdir dest) rethrow
-			ok <- boolSystem "mv"
+			ok <- copyright =<< boolSystem "mv"
 				[ Param "-f"
 				, Param (fromRawFilePath src)
 				, Param tmp
diff --git a/Utility/Path.hs b/Utility/Path.hs
index 64ef076ff9..de13712d32 100644
--- a/Utility/Path.hs
+++ b/Utility/Path.hs
@@ -36,6 +36,7 @@ import Control.Monad
 import Control.Applicative
 import Prelude
 
+import Author
 import Utility.Monad
 import Utility.SystemDirectory
 import Utility.Exception
@@ -45,6 +46,9 @@ import Data.Char
 import Utility.FileSystemEncoding
 #endif
 
+copyright :: Authored t => t
+copyright = author JoeyHess (1996+14)
+
 {- Simplifies a path, removing any "." component, collapsing "dir/..", 
  - and removing the trailing path separator.
  -
@@ -131,8 +135,8 @@ dirContains a b = a == b
 	 - specially here.
 	 -}
 	dotdotcontains
-		| isAbsolute b' = False
-		| otherwise = 
+		| isAbsolute b' = False && copyright
+		| otherwise =
 			let aps = splitPath a'
 			    bps = splitPath b'
 			in if all isdotdot aps
@@ -249,7 +253,7 @@ inSearchPath command = isJust <$> searchPath command
  -}
 searchPath :: String -> IO (Maybe FilePath)
 searchPath command
-	| P.isAbsolute command = check command
+	| P.isAbsolute command = copyright $ check command
 	| otherwise = P.getSearchPath >>= getM indir
   where
 	indir d = check $ d P.</> command
diff --git a/Utility/Path/AbsRel.hs b/Utility/Path/AbsRel.hs
index 4007fbbb0f..b339c5234d 100644
--- a/Utility/Path/AbsRel.hs
+++ b/Utility/Path/AbsRel.hs
@@ -6,7 +6,6 @@
  -}
 
 {-# LANGUAGE OverloadedStrings #-}
-{-# LANGUAGE CPP #-}
 {-# OPTIONS_GHC -fno-warn-tabs #-}
 
 module Utility.Path.AbsRel (
@@ -20,17 +19,13 @@ module Utility.Path.AbsRel (
 
 import System.FilePath.ByteString
 import qualified Data.ByteString as B
-#ifdef mingw32_HOST_OS
-import System.Directory (getCurrentDirectory)
-#else
-import System.Posix.Directory.ByteString (getWorkingDirectory)
-#endif
 import Control.Applicative
 import Prelude
 
 import Utility.Path
 import Utility.UserInfo
 import Utility.FileSystemEncoding
+import qualified Utility.RawFilePath as R
 
 {- Makes a path absolute.
  -
@@ -58,11 +53,7 @@ absPath file
 	-- so also used here for consistency.
 	| isAbsolute file = return $ simplifyPath file
 	| otherwise = do
-#ifdef mingw32_HOST_OS
-		cwd <- toRawFilePath <$> getCurrentDirectory
-#else
-		cwd <- getWorkingDirectory
-#endif
+		cwd <- R.getCurrentDirectory
 		return $ absPathFrom cwd file
 
 {- Constructs the minimal relative path from the CWD to a file.
@@ -78,11 +69,7 @@ relPathCwdToFile f
 	-- and does not contain any ".." component.
 	| isRelative f && not (".." `B.isInfixOf` f) = return f
 	| otherwise = do
-#ifdef mingw32_HOST_OS
-		c <- toRawFilePath <$> getCurrentDirectory
-#else
-		c <- getWorkingDirectory
-#endif
+		c <- R.getCurrentDirectory
 		relPathDirToFile c f
 
 {- Constructs a minimal relative path from a directory to a file. -}
diff --git a/Utility/ShellEscape.hs b/Utility/ShellEscape.hs
index 9bd229a4c1..bd4a921e5a 100644
--- a/Utility/ShellEscape.hs
+++ b/Utility/ShellEscape.hs
@@ -15,26 +15,33 @@ module Utility.ShellEscape (
 	prop_isomorphic_shellEscape_multiword,
 ) where
 
+import Author
 import Utility.QuickCheck
 import Utility.Split
+import Data.Function
 
 import Data.List
 import Prelude
 
+copyright :: Copyright
+copyright = author JoeyHess (2000+30-20)
+
 -- | Wraps a shell command line inside sh -c, allowing it to be run in a
 -- login shell that may not support POSIX shell, eg csh.
 shellWrap :: String -> String
-shellWrap cmdline = "sh -c " ++ shellEscape cmdline
+shellWrap cmdline = copyright $ "sh -c " ++ shellEscape cmdline
 
--- | Escapes a filename or other parameter to be safely able to be exposed to
--- the shell.
+-- | Escapes a string to be safely able to be exposed to the shell.
 --
--- This method works for POSIX shells, as well as other shells like csh.
+-- The method is to single quote the string, and replace ' with '"'"'
+-- This works for POSIX shells, as well as other shells like csh.
 shellEscape :: String -> String
-shellEscape f = "'" ++ escaped ++ "'"
+shellEscape f = [q] ++ escaped ++ [q]
   where
-	-- replace ' with '"'"'
-	escaped = intercalate "'\"'\"'" $ splitc '\'' f
+	escaped = intercalate escq $ splitc q f
+	q = '\''
+	qq = '"'
+	escq = [q, qq, q, qq, q] & copyright
 
 -- | Unescapes a set of shellEscaped words or filenames.
 shellUnEscape :: String -> [String]
@@ -44,13 +51,13 @@ shellUnEscape s = word : shellUnEscape rest
 	(word, rest) = findword "" s
 	findword w [] = (w, "")
 	findword w (c:cs)
-		| c == ' ' = (w, cs)
+		| c == ' ' && copyright = (w, cs)
 		| c == '\'' = inquote c w cs
 		| c == '"' = inquote c w cs
 		| otherwise = findword (w++[c]) cs
 	inquote _ w [] = (w, "")
 	inquote q w (c:cs)
-		| c == q = findword w cs
+		| c == q && copyright = findword w cs
 		| otherwise = inquote q (w++[c]) cs
 
 prop_isomorphic_shellEscape :: TestableString -> Bool
diff --git a/Utility/StatelessOpenPGP.hs b/Utility/StatelessOpenPGP.hs
new file mode 100644
index 0000000000..7a4c2063ca
--- /dev/null
+++ b/Utility/StatelessOpenPGP.hs
@@ -0,0 +1,202 @@
+{- Stateless OpenPGP interface
+ -
+ - Copyright 2011-2024 Joey Hess <id@joeyh.name>
+ -
+ - License: BSD-2-clause
+ -}
+
+{-# LANGUAGE CPP, OverloadedStrings #-}
+
+module Utility.StatelessOpenPGP (
+	SOPCmd(..),
+	SOPSubCmd,
+	SOPProfile(..),
+	Password,
+	EmptyDirectory(..),
+	Armoring(..),
+	encryptSymmetric,
+	decryptSymmetric,
+	test_encrypt_decrypt_Symmetric,
+	feedRead,
+	feedRead',
+) where
+
+import Common
+#ifndef mingw32_HOST_OS
+import System.Posix.Types
+import System.Posix.IO
+#else
+import Utility.Tmp
+#endif
+import Utility.Tmp.Dir
+
+import Control.Concurrent.Async
+import Control.Monad.IO.Class
+import qualified Data.ByteString as B
+
+{- The command to run, eq sqop. -}
+newtype SOPCmd = SOPCmd { unSOPCmd :: String }
+
+{- The subcommand to run eg encrypt. -}
+type SOPSubCmd = String
+
+newtype SOPProfile = SOPProfile String
+
+{- Note that SOP requires passwords to be UTF-8 encoded, and that they
+ - may try to trim trailing whitespace. They may also forbid leading
+ - whitespace, or forbid some non-printing characters. -}
+type Password = B.ByteString
+
+newtype Armoring = Armoring Bool
+
+{- The path to a sufficiently empty directory.
+ -
+ - This is unfortunately needed because of an infelicity in the SOP
+ - standard, as documented in section 9.9 "Be Careful with Special
+ - Designators", when using "@FD:" and similar designators the SOP
+ - command may test for the presense of a file with the same name on the
+ - filesystem, and fail with  AMBIGUOUS_INPUT. 
+ -
+ - Since we don't want to need to deal with such random failure due to
+ - whatever filename might be present, when running sop commands using
+ - special designators, an empty directory has to be provided, and the
+ - command is run in that directory. Of course, this necessarily means
+ - that any relative paths passed to the command have to be made absolute.
+ -
+ - The directory does not really have to be empty, it just needs to be one
+ - that should not contain any files with names starting with "@".
+ -}
+newtype EmptyDirectory = EmptyDirectory FilePath
+
+{- Encrypt using symmetric encryption with the specified password. -}
+encryptSymmetric
+	:: (MonadIO m, MonadMask m)
+	=> SOPCmd
+	-> Password
+	-> EmptyDirectory
+	-> Maybe SOPProfile
+	-> Armoring
+	-> (Handle -> IO ())
+	-> (Handle -> m a)
+	-> m a
+encryptSymmetric sopcmd password emptydirectory mprofile armoring feeder reader =
+	feedRead sopcmd "encrypt" params password emptydirectory feeder reader
+  where
+	params = map Param $ catMaybes
+		[ case armoring of
+			Armoring False -> Just "--no-armor"
+			Armoring True -> Nothing
+		, Just "--as=binary"
+		, case mprofile of
+			Just (SOPProfile profile) -> 
+				Just $ "--profile=" ++ profile
+			Nothing -> Nothing
+		]
+
+{- Deccrypt using symmetric encryption with the specified password. -}
+decryptSymmetric
+	:: (MonadIO m, MonadMask m)
+	=> SOPCmd
+	-> Password
+	-> EmptyDirectory
+	-> (Handle -> IO ())
+	-> (Handle -> m a)
+	-> m a
+decryptSymmetric sopcmd password emptydirectory feeder reader =
+	feedRead sopcmd "decrypt" [] password emptydirectory feeder reader
+
+{- Test a value round-trips through symmetric encryption and decryption. -}
+test_encrypt_decrypt_Symmetric :: SOPCmd -> SOPCmd -> Password -> Armoring -> B.ByteString -> IO Bool
+test_encrypt_decrypt_Symmetric a b password armoring v = catchBoolIO $
+	withTmpDir "test" $ \d -> do
+		let ed = EmptyDirectory d
+		enc <- encryptSymmetric a password ed Nothing armoring
+			(`B.hPutStr` v) B.hGetContents
+		dec <- decryptSymmetric b password ed
+			(`B.hPutStr` enc) B.hGetContents
+		return (v == dec)
+
+{- Runs a SOP command with some parameters. First sends it a password
+ - via '--with-password'. Then runs a feeder action that is
+ - passed a handle and should write to it all the data to input to the
+ - command. Finally, runs a reader action that is passed a handle to
+ - the command's output.
+ -
+ - Note that the reader must fully consume its input before returning. -}
+feedRead
+	:: (MonadIO m, MonadMask m)
+	=> SOPCmd
+	-> SOPSubCmd
+	-> [CommandParam]
+	-> Password
+	-> EmptyDirectory
+	-> (Handle -> IO ())
+	-> (Handle -> m a)
+	-> m a
+feedRead cmd subcmd params password emptydirectory feeder reader = do
+#ifndef mingw32_HOST_OS
+	let setup = liftIO $ do
+		-- pipe the passphrase in on a fd
+		(frompipe, topipe) <- System.Posix.IO.createPipe
+		setFdOption topipe CloseOnExec True
+		toh <- fdToHandle topipe
+		t <- async $ do
+			B.hPutStr toh (password <> "\n")
+			hClose toh
+		let Fd pfd = frompipe
+		let passwordfd = [Param $ "--with-password=@FD:"++show pfd]
+		return (passwordfd, frompipe, toh, t)
+	let cleanup (_, frompipe, toh, t) = liftIO $ do
+		closeFd frompipe
+		hClose toh
+		cancel t
+	bracket setup cleanup $ \(passwordfd, _, _, _) ->
+		go (Just emptydirectory) (passwordfd ++ params)
+#else
+	-- store the password in a temp file
+	withTmpFile "sop" $ \tmpfile h -> do
+		liftIO $ B.hPutStr h password
+		liftIO $ hClose h
+		let passwordfile = [Param $ "--with-password="++tmpfile]
+		-- Don't need to pass emptydirectory since @FD is not used,
+		-- and so tmpfile also does not need to be made absolute.
+		case emptydirectory of
+			EmptyDirectory _ -> return ()
+		go Nothing $ passwordfile ++ params
+#endif
+  where
+	go med params' = feedRead' cmd subcmd params' med feeder reader
+
+{- Like feedRead, but without password. -}
+feedRead'
+	:: (MonadIO m, MonadMask m)
+	=> SOPCmd
+	-> SOPSubCmd
+	-> [CommandParam]
+	-> Maybe EmptyDirectory
+	-> (Handle -> IO ())
+	-> (Handle -> m a)
+	-> m a
+feedRead' (SOPCmd cmd) subcmd params med feeder reader = do
+	let p = (proc cmd (subcmd:toCommand params))
+		{ std_in = CreatePipe
+		, std_out = CreatePipe
+		, std_err = Inherit
+		, cwd = case med of
+			Just (EmptyDirectory d) -> Just d
+			Nothing -> Nothing
+		}
+	bracket (setup p) cleanup (go p)
+  where
+	setup = liftIO . createProcess
+	cleanup = liftIO . cleanupProcess
+
+	go p (Just to, Just from, _, pid) =
+		let runfeeder = do
+			feeder to
+			hClose to
+		in bracketIO (async runfeeder) cancel $ const $ do
+			r <- reader from
+			liftIO $ forceSuccessProcess p pid
+			return r
+	go _ _ = error "internal"
diff --git a/Utility/TimeStamp.hs b/Utility/TimeStamp.hs
index b740d7bead..3cc4d8d245 100644
--- a/Utility/TimeStamp.hs
+++ b/Utility/TimeStamp.hs
@@ -1,14 +1,17 @@
 {- timestamp parsing and formatting
  -
- - Copyright 2015-2019 Joey Hess <id@joeyh.name>
+ - Copyright 2015-2023 Joey Hess <id@joeyh.name>
  -
  - License: BSD-2-clause
  -}
 
+{-# LANGUAGE CPP #-}
+
 module Utility.TimeStamp (
 	parserPOSIXTime,
 	parsePOSIXTime,
 	formatPOSIXTime,
+	truncateResolution,
 ) where
 
 import Utility.Data
@@ -56,3 +59,14 @@ mkPOSIXTime n (d, dlen)
 
 formatPOSIXTime :: String -> POSIXTime -> String
 formatPOSIXTime fmt t = formatTime defaultTimeLocale fmt (posixSecondsToUTCTime t)
+
+{- Truncate the resolution to the specified number of decimal places. -}
+truncateResolution :: Int -> POSIXTime -> POSIXTime
+#if MIN_VERSION_time(1,9,1)
+truncateResolution n t = secondsToNominalDiffTime $
+	fromIntegral ((truncate (nominalDiffTimeToSeconds t * d)) :: Integer) / d
+  where
+	d = 10 ^ n
+#else
+truncateResolution _ t = t
+#endif
diff --git a/Utility/Tmp.hs b/Utility/Tmp.hs
index efb15bd9b3..a23a2a37f5 100644
--- a/Utility/Tmp.hs
+++ b/Utility/Tmp.hs
@@ -105,7 +105,12 @@ withTmpFileIn tmpdir template a = bracket create remove use
  -}
 relatedTemplate :: FilePath -> FilePath
 relatedTemplate f
-	| len > 20 = truncateFilePath (len - 20) f
+	| len > 20 = 
+		{- Some filesystems like FAT have issues with filenames
+		 - ending in ".", so avoid truncating a filename to end
+		 - that way. -}
+		reverse $ dropWhile (== '.') $ reverse $
+			truncateFilePath (len - 20) f
 	| otherwise = f
   where
 	len = length f
diff --git a/Utility/WebApp.hs b/Utility/WebApp.hs
index 497c2eb1ba..987d67cbd6 100644
--- a/Utility/WebApp.hs
+++ b/Utility/WebApp.hs
@@ -58,9 +58,9 @@ browserProc url = proc "xdg-open" [url]
  - An IO action can also be run, to do something with the address,
  - such as start a web browser to view the webapp.
  -}
-runWebApp :: Maybe TLSSettings -> Maybe HostName -> Wai.Application -> (SockAddr -> IO ()) -> IO ()
-runWebApp tlssettings h app observer = withSocketsDo $ do
-	sock <- getSocket h
+runWebApp :: Maybe TLSSettings -> Maybe HostName -> Maybe PortNumber -> Wai.Application -> (SockAddr -> IO ()) -> IO ()
+runWebApp tlssettings h p app observer = withSocketsDo $ do
+	sock <- getSocket h p
 	void $ forkIO $ go webAppSettings sock app	
 	sockaddr <- getSocketName sock
 	observer sockaddr
@@ -74,14 +74,13 @@ webAppSettings = setTimeout halfhour defaultSettings
 	halfhour = 30 * 60
 
 {- Binds to a local socket, or if specified, to a socket on the specified
- - hostname or address. Selects any free port, unless the hostname ends with
- - ":port"
+ - hostname or address. Selects any free port, unless a port is specified.
  -
  - Prefers to bind to the ipv4 address rather than the ipv6 address
  - of localhost, if it's available.
  -}
-getSocket :: Maybe HostName -> IO Socket
-getSocket h = do
+getSocket :: Maybe HostName -> Maybe PortNumber -> IO Socket
+getSocket h p = do
 #if defined (mingw32_HOST_OS)
 	-- The HostName is ignored by this code.
 	-- getAddrInfo didn't used to work on windows; current status
@@ -91,11 +90,11 @@ getSocket h = do
 	let addr = tupleToHostAddress (127,0,0,1)
 	sock <- socket AF_INET Stream defaultProtocol
 	preparesocket sock
-	bind sock (SockAddrInet defaultPort addr)
+	bind sock (SockAddrInet (fromMaybe defaultPort p) addr)
 	use sock
   where
 #else
-	addrs <- getAddrInfo (Just hints) (Just hostname) Nothing
+	addrs <- getAddrInfo (Just hints) (Just hostname) (fmap show p)
 	case (partition (\a -> addrFamily a == AF_INET) addrs) of
 		(v4addr:_, _) -> go v4addr
 		(_, v6addr:_) -> go v6addr
diff --git a/doc/bugs/CRLF_breaks_interoperability_Win-Ux__58__..post-receive.mdwn b/doc/bugs/CRLF_breaks_interoperability_Win-Ux__58__..post-receive.mdwn
index b3e0f01375..06d01733c7 100644
--- a/doc/bugs/CRLF_breaks_interoperability_Win-Ux__58__..post-receive.mdwn
+++ b/doc/bugs/CRLF_breaks_interoperability_Win-Ux__58__..post-receive.mdwn
@@ -60,3 +60,5 @@ To /var/tmp/mnt/winhost-w10-5920/cygdrive/e/my.gitannex/
 
 Yes. I think git-annex is a hidden gem of the open source
 community.
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/CRLF_breaks_interoperability_Win-Ux__58__..post-receive/comment_5_c22f9e3c5e438b2cc05008bc3f91b7fc._comment b/doc/bugs/CRLF_breaks_interoperability_Win-Ux__58__..post-receive/comment_5_c22f9e3c5e438b2cc05008bc3f91b7fc._comment
new file mode 100644
index 0000000000..de5861f990
--- /dev/null
+++ b/doc/bugs/CRLF_breaks_interoperability_Win-Ux__58__..post-receive/comment_5_c22f9e3c5e438b2cc05008bc3f91b7fc._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2023-11-29T17:29:08Z"
+ content="""
+[[!commit 4e35067325022e4d8790d45557156fd1166484df]] fixed this.
+Previously, git-annex did write hook scripts with CRLF line endings.
+
+Existing hook scripts won't be changed by git-annex, but if you delete
+them, and re-run `git-annex init` it will reinstall it without the CRLF.
+"""]]
diff --git a/doc/bugs/How_to_git_union-merge__63__.mdwn b/doc/bugs/How_to_git_union-merge__63__.mdwn
new file mode 100644
index 0000000000..3272cd2508
--- /dev/null
+++ b/doc/bugs/How_to_git_union-merge__63__.mdwn
@@ -0,0 +1,24 @@
+### Please describe the problem.
+
+It's unclear how to use `git union-merge` as described [here](https://git-annex.branchable.com/git-union-merge/).
+
+### What steps will reproduce the problem?
+
+Run `git union-merge`, yields `git: 'union-merge' is not a git command. See 'git --help'.`
+
+No binary `git-union-merge` is shipped in the standalone tarball and distros also don't seem to ship it.
+
+### What version of git-annex are you using? On what operating system?
+
+Tried with (on a Manjaro box):
+
+- git-annex-standalone-amd64.tar.gz	2023-10-09 14:21 	51M	 from [here](https://downloads.kitenet.net/git-annex/linux/current/) (the build is a month old, is that right?)
+- `git-annex` in Manjaro repos
+- `git-annex-standalone-nightly-bin` from AUR
+- `nix-shell -p git-annex` (10.20230926 in nixos-unstable)
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+git-annex rules and is a marvelous tool.
+
+I wanted to try the union merging to resolve merge conflicts on non-annexed files. It's not ideal, but might be better than repeated `git annex assist|sync` eventually adding the merge conflict markers `<<<<<<<<<` and the like to the files, breaking things like `.gitattributes` syntax which in turn has more devastating lockup consequences...
diff --git a/doc/bugs/How_to_git_union-merge__63__/comment_1_58b6c9712d7c209248d9ef87bcc0e110._comment b/doc/bugs/How_to_git_union-merge__63__/comment_1_58b6c9712d7c209248d9ef87bcc0e110._comment
new file mode 100644
index 0000000000..00f291361a
--- /dev/null
+++ b/doc/bugs/How_to_git_union-merge__63__/comment_1_58b6c9712d7c209248d9ef87bcc0e110._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-11-21T20:12:11Z"
+ content="""
+I think I should probably just remove that. It's not being maintained.
+
+Adding it to the cabal file so it gets built would slow down builds
+producing this extra binary. It would need to be handled as a multicall
+program in git-annex the way git-annex-shell and git-remote-tor-annex are.
+
+Do you have a reason to want to use it?
+"""]]
diff --git a/doc/bugs/How_to_git_union-merge__63__/comment_2_22701b82e8d53acff66ddea5bf9448bf._comment b/doc/bugs/How_to_git_union-merge__63__/comment_2_22701b82e8d53acff66ddea5bf9448bf._comment
new file mode 100644
index 0000000000..63edd6f659
--- /dev/null
+++ b/doc/bugs/How_to_git_union-merge__63__/comment_2_22701b82e8d53acff66ddea5bf9448bf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="comment 2"
+ date="2023-11-21T20:20:13Z"
+ content="""
+It sounds promising as an automatic merge conflict resolver that I'd like to fall back to for non-annexed file conflicts. I don't really know how do achieve that manually. If it's easily possible in another way, I'll try that.
+"""]]
diff --git a/doc/bugs/Invalid_argument_saving_FreeSurfer_file_on_NTFS.mdwn b/doc/bugs/Invalid_argument_saving_FreeSurfer_file_on_NTFS.mdwn
new file mode 100644
index 0000000000..241ee8ce8c
--- /dev/null
+++ b/doc/bugs/Invalid_argument_saving_FreeSurfer_file_on_NTFS.mdwn
@@ -0,0 +1,95 @@
+### Please describe the problem.
+
+Annexing files can fail with "Invalid argument" error. It appears to be an interaction of filesystem and file name. The file name is `lh.aparc.DKTatlas.annot`/`rh.aparc.DKTatlas.annot`, and the filesystem is:
+
+[[!format sh """
+$ mount | grep data
+/dev/mapper/bitlocker on /data type fuseblk (rw,nosuid,nodev,relatime,user_id=0,group_id=0,default_permissions,allow_other,blksize=4096)
+$ ntfs-3g
+ntfs-3g: No device is specified.
+
+ntfs-3g 2021.8.22 integrated FUSE 28 - Third Generation NTFS Driver
+                Configuration type 7, XATTRS are on, POSIX ACLS are on
+"""]]
+
+However, formatting a file as NTFS and mounting as loop to make a minimal reproduction allowed the operation to succeed.
+
+### What steps will reproduce the problem?
+
+I am able to reproduce this problem reliably on the filesystem above:
+
+[[!format sh """
+$ git init test/
+$ cd test
+$ git annex init
+$ curl -sSL https://file.io/ZtE5ivep7E9V -o lh.aparc.DKTatlas.annot
+$ git annex add lh.aparc.DKTatlas.annot
+add lh.aparc.DKTatlas.annot 
+  .git/annex/othertmp/: openTempFile template ingest-lh.: invalid argument (Invalid argument)
+
+failed
+add: 1 failed
+"""]]
+
+However, renaming the file to `L.annot`, annexing it, and `git mv`ing it back works, so I am not stuck adding the file contents directly to git.
+
+I hope the log below will give you a hint of what's going on, as I regret I am unable to reproduce this on other filesystems.
+
+### What version of git-annex are you using? On what operating system?
+
+OS: Ubuntu 22.04.3 LTS
+
+[[!format sh """
+git-annex version: 10.20230408-g5b1e8ba77
+build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.22.1 bloomfilter-2.0.1.0 cryptonite-0.29 DAV-1.3.4 feed-1.3.2.1 ghc-9.0.2 http-client-0.7.13.1 persistent-sqlite-2.13.1.0 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.2.1
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: linux x86_64
+supported repository versions: 8 9 10
+upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10
+local repository version: 10
+"""]]
+
+### Please provide any additional information below.
+[[!format sh """
+$ git annex add -d lh.aparc.DKTatlas.annot
+[2023-12-05 09:40:20.256526552] (Utility.Process) process [1162202] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","git-annex"]
+[2023-12-05 09:40:20.260613884] (Utility.Process) process [1162202] done ExitSuccess
+[2023-12-05 09:40:20.260892906] (Utility.Process) process [1162203] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","--hash","refs/heads/git-annex"]
+[2023-12-05 09:40:20.265019085] (Utility.Process) process [1162203] done ExitSuccess
+[2023-12-05 09:40:20.265446352] (Utility.Process) process [1162204] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","log","refs/heads/git-annex..eb539b816feb00e6be3a252d36efaa7426eccca1","--pretty=%H","-n1"]
+[2023-12-05 09:40:20.270301207] (Utility.Process) process [1162204] done ExitSuccess
+[2023-12-05 09:40:20.271251091] (Utility.Process) process [1162205] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","cat-file","--batch"]
+[2023-12-05 09:40:20.275373234] (Utility.Process) process [1162206] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","symbolic-ref","-q","HEAD"]
+[2023-12-05 09:40:20.278672692] (Utility.Process) process [1162206] done ExitSuccess
+[2023-12-05 09:40:20.278946202] (Utility.Process) process [1162207] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","refs/heads/master"]
+[2023-12-05 09:40:20.282475944] (Utility.Process) process [1162207] done ExitFailure 1
+[2023-12-05 09:40:20.282779905] (Utility.Process) process [1162208] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","ls-files","-z","--others","--exclude-standard","--","lh.aparc.DKTatlas.annot"]
+[2023-12-05 09:40:20.286209985] (Utility.Process) process [1162209] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","check-attr","-z","--stdin","annex.backend","annex.largefiles","annex.numcopies","annex.mincopies","--"]
+[2023-12-05 09:40:20.28954558] (Utility.Process) process [1162210] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","cat-file","--batch-check=%(objectname) %(objecttype) %(objectsize)"]
+add lh.aparc.DKTatlas.annot [2023-12-05 09:40:20.293105736] (Utility.Process) process [1162211] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","symbolic-ref","-q","HEAD"]
+[2023-12-05 09:40:20.296109063] (Utility.Process) process [1162211] done ExitSuccess
+[2023-12-05 09:40:20.296268327] (Utility.Process) process [1162212] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","refs/heads/master"]
+[2023-12-05 09:40:20.299447211] (Utility.Process) process [1162212] done ExitFailure 1
+[2023-12-05 09:40:20.299472144] (Annex.Perms) freezing content lh.aparc.DKTatlas.annot
+
+  .git/annex/othertmp/: openTempFile template ingest-lh.: invalid argument (Invalid argument)
+
+failed
+[2023-12-05 09:40:20.299683526] (Utility.Process) process [1162208] done ExitSuccess
+[2023-12-05 09:40:20.299804817] (Utility.Process) process [1162213] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","ls-files","-z","--modified","--","lh.aparc.DKTatlas.annot"]
+[2023-12-05 09:40:20.302582841] (Utility.Process) process [1162213] done ExitSuccess
+[2023-12-05 09:40:20.302707305] (Utility.Process) process [1162214] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","diff","--name-only","--diff-filter=T","-z","--cached","--","lh.aparc.DKTatlas.annot"]
+[2023-12-05 09:40:20.30608269] (Utility.Process) process [1162214] done ExitSuccess
+[2023-12-05 09:40:20.306609675] (Utility.Process) process [1162205] done ExitSuccess
+[2023-12-05 09:40:20.306732757] (Utility.Process) process [1162210] done ExitSuccess
+[2023-12-05 09:40:20.306873108] (Utility.Process) process [1162209] done ExitSuccess
+add: 1 failed
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes! I use git annex and datalad all the time for personal and work projects. 
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/Invalid_argument_saving_FreeSurfer_file_on_NTFS/comment_1_181a319138cc10742bc8676d77e8a614._comment b/doc/bugs/Invalid_argument_saving_FreeSurfer_file_on_NTFS/comment_1_181a319138cc10742bc8676d77e8a614._comment
new file mode 100644
index 0000000000..0ab08545f2
--- /dev/null
+++ b/doc/bugs/Invalid_argument_saving_FreeSurfer_file_on_NTFS/comment_1_181a319138cc10742bc8676d77e8a614._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-12-05T16:16:35Z"
+ content="""
+This is EINVAL from probably open(2) or something like that.
+
+       EINVAL The  final  component ("basename") of pathname is invalid (e.g.,
+              it contains characters not permitted by the underlying  filesys‐
+              tem).
+
+The problem is likely the ending ".", since FAT and probably other Microsoft filesystems
+don't allow that, and/or have other strange behavior like silently removing that.
+
+Changed the code to avoid this.
+"""]]
diff --git a/doc/bugs/No_way_to_merge_conflicts_while_on_adjusted_branch.mdwn b/doc/bugs/No_way_to_merge_conflicts_while_on_adjusted_branch.mdwn
new file mode 100644
index 0000000000..39f9997a2d
--- /dev/null
+++ b/doc/bugs/No_way_to_merge_conflicts_while_on_adjusted_branch.mdwn
@@ -0,0 +1,74 @@
+### Please describe the problem.
+While on an adjusted branch, merge conflicts can neither be resolved manually nor with a git merge driver. See the transcript below.
+
+
+### What version of git-annex are you using? On what operating system?
+git-annex version: 10.20230802
+debian bullseye
+
+
+### Please provide any additional information below.
+
+[[!format sh """
+$ cat ~/.gitconfig 
+...
+[merge "commit"]
+        name = commit conflict markers
+        driver = "bash -c 'git merge-file %A %O %B; exit 0'"
+...
+
+$ cat .gitattributes 
+* annex.backend=SHA256E
+study.org* merge=commit
+
+dotfiles/** merge=commit
+
+$ git annex sync
+git-annex sync will change default behavior to operate on --content in a future version of git-annex. Recommend you explicitly use --no-content (or -g) to prepare for that change. (Or you can configure annex.synccontent)
+commit  
+On branch adjusted/master(unlocked)
+nothing to commit, working tree clean
+ok
+pull uni-lfs 
+(Merging into master...) 
+Auto-merging study.org
+CONFLICT (content): Merge conflict in study.org
+Automatic merge failed; fix conflicts and then commit the result.
+(Merging into master...) 
+Auto-merging study.org
+CONFLICT (content): Merge conflict in study.org
+Automatic merge failed; fix conflicts and then commit the result.
+failed
+push uni-lfs 
+To xxx
+ ! [rejected]          master -> synced/master (non-fast-forward)
+error: failed to push some refs to 'xxx'
+hint: Updates were rejected because a pushed branch tip is behind its remote
+hint: counterpart. Check out this branch and integrate the remote changes
+hint: (e.g. 'git pull ...') before pushing again.
+hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+To xxx
+ ! [rejected]          master -> master (non-fast-forward)
+error: failed to push some refs to 'xxx'
+hint: Updates were rejected because a pushed branch tip is behind its remote
+hint: counterpart. Check out this branch and integrate the remote changes
+hint: (e.g. 'git pull ...') before pushing again.
+hint: See the 'Note about fast-forwards' in 'git push --help' for details.
+
+  Pushing to uni-lfs failed.
+failed
+sync: 2 failed
+
+$ git annex version
+git-annex version: 10.20230802
+build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.26 DAV-1.3.4 feed-1.3.0.1 ghc-8.8.4 http-client-0.6.4.1 persistent-sqlite-2.10.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.1.0
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: linux x86_64
+supported repository versions: 8 9 10
+upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10
+local repository version: 10
+
+# End of transcript or log.
+"""]]
diff --git a/doc/bugs/No_way_to_merge_conflicts_while_on_adjusted_branch/comment_1_77c241c5d61b62b1fae552145c6c94ef._comment b/doc/bugs/No_way_to_merge_conflicts_while_on_adjusted_branch/comment_1_77c241c5d61b62b1fae552145c6c94ef._comment
new file mode 100644
index 0000000000..3a2e141757
--- /dev/null
+++ b/doc/bugs/No_way_to_merge_conflicts_while_on_adjusted_branch/comment_1_77c241c5d61b62b1fae552145c6c94ef._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2024-01-02T18:05:47Z"
+ content="""
+git is what's doing the automatic merge, using the merge driver that you
+configured. Is the file `study.org` an annexed file? If not, I don't really
+see how this involves git-annex. git doesn't care if an adjusted branch is
+checked out or not, when using a merge driver to merge a non-annexed file.
+
+It looks like your merge driver is perhaps not working..
+"""]]
diff --git a/doc/bugs/Option_--from-anywhere_not_recognized_in_copy.mdwn b/doc/bugs/Option_--from-anywhere_not_recognized_in_copy.mdwn
new file mode 100644
index 0000000000..1f1a806bc1
--- /dev/null
+++ b/doc/bugs/Option_--from-anywhere_not_recognized_in_copy.mdwn
@@ -0,0 +1,48 @@
+### Please describe the problem.
+The `--from-anywhere` option is not recognized.
+
+### What steps will reproduce the problem?
+Create a repo, add a file to it, clone it, then try to copy the files with `git annex copy $FILE --from-anywhere --to=here`
+```sh
+$ mkdir a && cd a
+$ git init
+$ git annex init
+$ touch somefile
+$ git annex add somefile
+$ git commit -m "blub"
+$ cd ..
+$ git clone a b
+$ cd b
+$ git annex init
+$ git annex copy somefile --from-anywhere --to=here
+```
+The last command fails with
+`Invalid option `--from-anywhere'`
+
+### What version of git-annex are you using? On what operating system?
+Output from `git annex version`
+```
+git-annex version: 10.20231129-gbacd781c4fe05126219c4b5f2963677a6de01481
+build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.24 bloomfilter-2.0.1.2 crypton-0.34 DAV-1.3.4 feed-1.3.2.1 ghc-9.0.2 http-client-0.7.15 persistent-sqlite-2.13.1.0 torrent-10000.1.3 uuid-1.3.15 yesod-1.6.2.1
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: linux x86_64
+supported repository versions: 8 9 10
+upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10
+local repository version: 10
+```
+
+I'm experiencing this issue on EndeavourOS
+```
+$ uname -r
+6.6.10-arch1-1
+```
+
+### Please provide any additional information below.
+
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+git annex does everything I was looking for when I found it and then some. Thank you for creating and maintaining it!
+
+> not a bug, version too old [[done]] --[[Joey]]
diff --git a/doc/bugs/Option_--from-anywhere_not_recognized_in_copy/comment_1_92bb18a7a50cf3dcbca75f544d7a0acf._comment b/doc/bugs/Option_--from-anywhere_not_recognized_in_copy/comment_1_92bb18a7a50cf3dcbca75f544d7a0acf._comment
new file mode 100644
index 0000000000..6419182420
--- /dev/null
+++ b/doc/bugs/Option_--from-anywhere_not_recognized_in_copy/comment_1_92bb18a7a50cf3dcbca75f544d7a0acf._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2024-01-18T15:57:56Z"
+ content="""
+Your git-annex version is from before --from-anywhere was implemented.
+
+Also, `--from-anywhere` needs `--to=remote`, as documented on the man page.
+"here" is not a remote name, so `--from-anywhere --to=here` will not work.
+
+	git-annex copy --from-anywhere --to=here
+	git-annex: there is no available git remote named "here"
+
+A simpler way to do what you were trying to do, that does work is
+`git-annex copy --to=here`
+
+An even simpler way to do exactly the same thing is `git-annex get`
+
+So, I don't see any point in trying to support `--from-anywhere --to=here`
+with `git-annex copy`.
+"""]]
diff --git a/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_16_2139bcb34591be137ff3da959f08bc76._comment b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_16_2139bcb34591be137ff3da959f08bc76._comment
new file mode 100644
index 0000000000..5256c76087
--- /dev/null
+++ b/doc/bugs/Packfile_does_not_match_digest__58___gcrypt_with_assistant/comment_16_2139bcb34591be137ff3da959f08bc76._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="branch"
+ subject="comment 16"
+ date="2023-10-23T09:56:59Z"
+ content="""
+This issue is still present, when using the assistant with git-remote-gcrypt.
+
+Although one can recover from this issue by resetting the repository, this is handicaping, as we have to continuously monitor whether pushes have been successful, which contradicts the idea of the assistant.
+
+This may be anecdotal, but the push before the 'Packfile *** does not match digest!' error, have had some network issues, as follows:
+
+```
+gcrypt: Due to a longstanding bug, this push implicitly has --force.
+gcrypt: Consider explicitly passing --force, and setting
+gcrypt: gcrypt's require-explicit-force-push git config key.
+gcrypt: Encrypting to:  -r ***
+gcrypt: Requesting manifest signature
+error: src refspec refs/gcrypt/gitception+ does not match any
+fatal: the remote end hung up unexpectedly
+error: failed to push some refs to ***
+error: failed to push some refs to ***
+```
+"""]]
diff --git a/doc/bugs/Push_to_special_remote_includes_unwanted_files.mdwn b/doc/bugs/Push_to_special_remote_includes_unwanted_files.mdwn
new file mode 100644
index 0000000000..29da8a3010
--- /dev/null
+++ b/doc/bugs/Push_to_special_remote_includes_unwanted_files.mdwn
@@ -0,0 +1,26 @@
+### Please describe the problem.
+
+When pushing to a special remote, non-largefiles (Git text files) are not filtered by the wanted setting.
+
+### What steps will reproduce the problem?
+
+Push to a directory special remote with exporttree and importtree set yes. The wanted setting filters largefiles, but all Git text files are included even though some are not included in the filter.
+
+### What version of git-annex are you using? On what operating system?
+
+10.20231130-g0e9bc415882a5e3a285e004cf9f80936e5762a07
+Linux
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+
diff --git a/doc/bugs/Push_to_special_remote_includes_unwanted_files/comment_1_a9d43aaca191a2ea464d13bcba6cd31d._comment b/doc/bugs/Push_to_special_remote_includes_unwanted_files/comment_1_a9d43aaca191a2ea464d13bcba6cd31d._comment
new file mode 100644
index 0000000000..3d746d9408
--- /dev/null
+++ b/doc/bugs/Push_to_special_remote_includes_unwanted_files/comment_1_a9d43aaca191a2ea464d13bcba6cd31d._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2024-01-25T18:09:23Z"
+ content="""
+Thing is that non-annexed files cannot really be matched by preferred
+content settings generally.
+
+I guess you are using something like `exclude=*.c`?
+"""]]
diff --git a/doc/bugs/Race_condition_or_double-locking_with_pidlock.mdwn b/doc/bugs/Race_condition_or_double-locking_with_pidlock.mdwn
new file mode 100644
index 0000000000..4347fc2e06
--- /dev/null
+++ b/doc/bugs/Race_condition_or_double-locking_with_pidlock.mdwn
@@ -0,0 +1,59 @@
+### Please describe the problem.
+
+When doing `git annex sync` with new changes from a remote (i.e. synced/main and/or some_remote/main is ahead of our main), git annex seems to try and lock at least two things/times. With pidlock, this of course isn't possible, so somewhere around a `git merge`, we get the following error:
+
+```
+   waiting for pid lock file .git/annex/pidlock which is held by another process (or may be stale)
+```
+
+When I inspect the content of the pidlock, the `git-annex-sync` process has the lock.
+
+Manually running `git merge <ref that is ahead>` and then `git annex sync` doesn't have this issue, so it seems related to merging changes to the main branch (not the git-annnex branch). 
+
+### What steps will reproduce the problem?
+
+I've really struggled to find a minimal reproducer, but I've hit this bug with several large real-world repos (@joeyh, I would be more than happy to give private access to one of these if you think it would be useful for debugging)
+
+The latest time this happened, this was the full log:
+
+```
+$ git annex sync
+pull origin 
+
+Updating 130dffc63..f8889be0c
+  waiting for pid lock file .git/annex/pidlock which is held by another process (or may be stale)
+#### hangs indefinitely  ######
+^C
+$ git merge origin/main
+Updating 130dffc63..f8889be0c
+Updating files: 100% (223/223), done.
+Fast-forward
+ .gitignore 
+<and then quite a large diff, including many files created/deleted>
+$ git annex sync
+# merges git-annex branch and pushes to all remotes successfully
+```
+
+Sometimes, but not always, it seems that a git merge updates the files on disk, but not the git index, leading to an inconsistent state where I have the working tree of the latest commit, but git believes I'm still on the older HEAD and shows the diff as unstaged changes. In these cases one must `git reset --hard HEAD && git clean -df` to clear the state back to HEAD, and then git merge manually, and only then will git annex sync behave as expected.
+
+### What version of git-annex are you using? On what operating system?
+
+This issue seems to only exist on versions 10.xxxx, and I remember first running into this a bit over a year ago (I first assumed that it was user error, but I've since had it occur quite a few tim  es where it can't be, e.g. freshly logging into a server that was just restarted). At least the following versions are affected:
+
+* git-annex version: 10.20220526-gc6b112108
+* git-annex version: 10.20230803-gb2887edc9
+* git-annex version: 10.20230926-g44a7b4c9734adfda5912dd82c1aa97c615689f57
+
+This is on various linuxes, mostly a few years old as these are institutional supercomputing clusters (ubuntu 20.04, debian 10, SLES 15.4).
+
+### Please provide any additional information below.
+
+This only affects clones with pidlock enabled (on compute clusters with NFS filesystems), the same repo on a laptop or whatever with a standard local filesystem (e.g. ext4, xfs) works perfectly.
+
+Could this be caused by e.g. git annex running git merge which runs git annex filterprocess (directly or via git status), and git-annex-filterprocess tries to take the pidlock that git-annex-sync already has?
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Lots! This problem popped up during our regular use of git-annex in plant genomic research, where we use git annex to manage and move our analyses between the many clusters we must use for computation. Git annex is indispensable for this use case!!
+
+> [[fixed|done]] (for the merge case at least) --[[Joey]]
diff --git a/doc/bugs/Race_condition_or_double-locking_with_pidlock/comment_1_bd92615c640ff017cd8a2ebfcc117a37._comment b/doc/bugs/Race_condition_or_double-locking_with_pidlock/comment_1_bd92615c640ff017cd8a2ebfcc117a37._comment
new file mode 100644
index 0000000000..daa07df33f
--- /dev/null
+++ b/doc/bugs/Race_condition_or_double-locking_with_pidlock/comment_1_bd92615c640ff017cd8a2ebfcc117a37._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-12-04T17:06:11Z"
+ content="""
+> Could this be caused by e.g. git annex running git merge which runs git annex
+> filterprocess (directly or via git status), and git-annex-filterprocess tries
+> to take the pidlock that git-annex-sync already has?
+
+If so you should be able to see the two git-annex's running in ps when this hang occurs
+with the git process in between them.
+That would be very helpful to verify since it would narrow down the git command
+that runs this git-annex child process.
+
+git-annex does deal with exactly this kind of scenario when running eg
+`git update-index`, by setting an environment variable to tell the child git-annex
+process not to re-lock the pid lock. But not in eg the case of a git merge.
+
+So, if you can narrow it down specifically to `git merge` running when this
+happens, or whatever other command, it should be easy to fix it.
+
+Hmm, I suppose that it makes sense `git merge` will sometimes need to check if
+a worktree file is modified, and so run git-annex. I'm pretty convinced by your
+transcript that it is the git merge that's hanging. So I've gone ahead and put
+in the workaround for that. There's of course still the possibility there's
+some other git command I've not anticipated that still needs the workaround.
+"""]]
diff --git a/doc/bugs/Race_condition_or_double-locking_with_pidlock/comment_2_6db20a7c8ddde72cba76240b3f7faddd._comment b/doc/bugs/Race_condition_or_double-locking_with_pidlock/comment_2_6db20a7c8ddde72cba76240b3f7faddd._comment
new file mode 100644
index 0000000000..20cc237b72
--- /dev/null
+++ b/doc/bugs/Race_condition_or_double-locking_with_pidlock/comment_2_6db20a7c8ddde72cba76240b3f7faddd._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="kdm9"
+ avatar="http://cdn.libravatar.org/avatar/b7b736335a0e9944a8169a582eb4c43d"
+ subject="confirmed fixed"
+ date="2024-01-03T09:03:44Z"
+ content="""
+Hi Joey, sorry, I should have confirmed this earlier but this seems to have completely fixed this issue in all cases I'd observed it. Many thanks!
+
+best,
+K
+"""]]
diff --git a/doc/bugs/Request_the_ability_to_specify_pull_commit_message.mdwn b/doc/bugs/Request_the_ability_to_specify_pull_commit_message.mdwn
new file mode 100644
index 0000000000..7c32d176b3
--- /dev/null
+++ b/doc/bugs/Request_the_ability_to_specify_pull_commit_message.mdwn
@@ -0,0 +1,26 @@
+### Please describe the problem.
+
+There is no way to specify the commit message used by the pull command.
+
+### What steps will reproduce the problem?
+
+When pulling from a special remote with importtree=yes to get new/modified files, git-annex generates the commit message. It is desired to specify the commit message to describe the content being pulled.
+
+### What version of git-annex are you using? On what operating system?
+
+10.20231130-g0e9bc415882a5e3a285e004cf9f80936e5762a07
+Linux
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+> [[done]] --[[Joey]]
diff --git a/doc/bugs/Request_the_ability_to_specify_pull_commit_message/comment_1_6d581ea36afab3829cd56d6d6b83c6f3._comment b/doc/bugs/Request_the_ability_to_specify_pull_commit_message/comment_1_6d581ea36afab3829cd56d6d6b83c6f3._comment
new file mode 100644
index 0000000000..db1e45acc5
--- /dev/null
+++ b/doc/bugs/Request_the_ability_to_specify_pull_commit_message/comment_1_6d581ea36afab3829cd56d6d6b83c6f3._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2024-01-18T16:18:38Z"
+ content="""
+This would need to start with `git-annex import --from=remote` 
+getting a `--message` option. Which I've just implemented.
+
+As for adding an option to `git-annex pull`, it would need to be an option
+other than `--message`, because `git-annex sync` already uses `--message`
+for the commit message. And since `git-annex pull` can pull from more than
+one remote at a time, it seems that it would need an option that specifies
+a message plus which remote to use that message for. Or the option
+would need to only be used when using `git-annex pull` with a single
+specified remote.
+
+That seems like perhaps unncessary complexity, since if you want to import
+a tree from a remote and specify a message, you can just use 
+`git-annex import` with `--message` now. 
+And then you can run `git-annex pull` to proceed with the rest of what it does.
+
+Good enough?
+"""]]
diff --git a/doc/bugs/Request_the_ability_to_specify_pull_commit_message/comment_2_bdb4b0200ab9d7c15454a2f3fcae474c._comment b/doc/bugs/Request_the_ability_to_specify_pull_commit_message/comment_2_bdb4b0200ab9d7c15454a2f3fcae474c._comment
new file mode 100644
index 0000000000..9ede847a98
--- /dev/null
+++ b/doc/bugs/Request_the_ability_to_specify_pull_commit_message/comment_2_bdb4b0200ab9d7c15454a2f3fcae474c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jstritch"
+ avatar="http://cdn.libravatar.org/avatar/56756b34ff409071074762951d270002"
+ subject="comment 2"
+ date="2024-01-19T17:44:39Z"
+ content="""
+The new import option --message should do the trick.  I'll probably it up with the recommended git annex merge instead of git annex pull. Thanks!
+"""]]
diff --git a/doc/bugs/SQlite_failed_when_copying_to_remote_repository.mdwn b/doc/bugs/SQlite_failed_when_copying_to_remote_repository.mdwn
index 61b56947eb..396c66312a 100644
--- a/doc/bugs/SQlite_failed_when_copying_to_remote_repository.mdwn
+++ b/doc/bugs/SQlite_failed_when_copying_to_remote_repository.mdwn
@@ -77,3 +77,6 @@ I use git-annex for several years, and I'm very happy with it. I's one of the be
 
 
 [[!meta title="sqlite fails when repository path contains non-unicode"]]
+
+> [[fixed|done]], when git-annex is built against persistent-sqlite version
+> 2.13.3. --[[Joey]]
diff --git a/doc/bugs/__39__Production__39___build_doesn__39__t_pass_testsuite_on_Win32.mdwn b/doc/bugs/__39__Production__39___build_doesn__39__t_pass_testsuite_on_Win32.mdwn
new file mode 100644
index 0000000000..4965962860
--- /dev/null
+++ b/doc/bugs/__39__Production__39___build_doesn__39__t_pass_testsuite_on_Win32.mdwn
@@ -0,0 +1,783 @@
+### Please describe the problem.
+
+Building an optimized Windows binary with flag `Production` active doesn't result in one that is able
+to pass its test suite.
+
+### What steps will reproduce the problem?
+
+`stack --stack-yaml stack-lts-18.13.yaml setup && stack --stack-yaml stack-lts-18.13.yaml build --flag 'git-annex:Production' --flag 'git-annex:-Crypton'`
+
+(I found that the `Crypton` flag needs to be explicitly negated due to the `Production` flag somehow making it active causing
+the build to fail because of a missing dependency `crypton`. A bug in Cabal, perhaps?)
+
+Then run the test suite in Git Bash in a directory that contains the executable and the requisite DLLs (the latter
+obtained by way of `stack exec ldd -- git-annex.exe` and copying over the ones in `/mingw64`). Observe the following
+(`just` is a cross-platform command runner that I use instead of GNU make.)
+
+[[!format sh """
+jkniiv@AINESIS MINGW64 ~/Projektit/git-annex.branchable.com/git-annex--BUILD-231130-10.20231129~P/.annx-test (BUILD-231130-10.20231129)
+$ just tests-new
+/usr/bin/time ./git-annex.exe test  2>&1 | tee git-annex.test.LOG~200
+
+All 0 tests passed (0.00s)
+Tests
+  Repo Tests v10 adjusted unlocked branch
+    Init Tests
+      init:                          OK (9.36s)
+      add:                           FAIL
+        Exception: fd:4: hGetChar: invalid argument (invalid byte sequence)
+        Use -p '/Init Tests.add/' to rerun this test only.
+    crypto:                          SKIP
+
+      Use -p '/crypto/' to rerun this test only.
+    uninit (in git-annex branch):    SKIP
+
+      Use -p '/uninit (in git-annex branch)/' to rerun this test only.
+    conflict resolution symlink bit: SKIP
+
+      Use -p '/conflict resolution symlink bit/' to rerun this test only.
+    union merge regression:          SKIP
+
+      Use -p '/union merge regression/' to rerun this test only.
+    unused:                          SKIP
+
+      Use -p '/unused/' to rerun this test only.
+    fsck (bare):                     SKIP
+
+      Use -p '/fsck (bare)/' to rerun this test only.
+    lock:                            SKIP
+
+      Use -p '/Repo Tests v10 adjusted unlocked branch.lock/' to rerun this test only.
+    drop (with remote):              SKIP
+
+      Use -p '/drop (with remote)/' to rerun this test only.
+    log:                             SKIP
+
+      Use -p '/log/' to rerun this test only.
+    add extras:                      SKIP
+
+      Use -p '/add extras/' to rerun this test only.
+
+11 out of 12 tests failed (11.65s)
+[...]
+"""]]
+
+
+### What version of git-annex are you using? On what operating system?
+
+[[!format sh """
+git-annex version: 10.20231129-gbacd781c4fe05126219c4b5f2963677a6de01481
+build flags: Assistant Webapp Pairing TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.29 DAV-1.3.4 feed-1.3.2.0 ghc-8.10.7 http-client-0.7.9 pe
+rsistent-sqlite-2.13.0.3 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.1.2
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SH
+A3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512
+BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLA
+KE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM UR
+L X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg h
+ook external
+operating system: mingw32 x86_64
+supported repository versions: 8 9 10
+upgrade supported from repository versions: 2 3 4 5 6 7 8 9 10
+"""]]
+
+Windows 10 version 22H2 (build 19045.3693), 64 bit.
+
+### Please provide any additional information below.
+
+This is how I build my "production" version of git-annex (in PowerShell):
+
+[[!format sh """
+C:\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P [BUILD-231130-10.20231129 +6 ~0 -0 !]> just setup-build
+cd 'C:\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P' && stack --stack-yaml stack-lts-18.13.yaml setup --verbose 2>&1 | & $env:GNU_TEE_CMD stack.setup---verbose.LOG~101
+Version 2.13.1, Git revision 8102bb8afce90fc954f48efae38b87f37cabc988 x86_64 hpack-0.36.0
+2023-12-08 16:24:52.098843: [debug] Loading project config file stack-lts-18.13.yaml
+2023-12-08 16:24:52.130837: [debug] Use of Casa server enabled: (CasaRepoPrefix "https://casa.stackage.org", 1280).
+2023-12-08 16:24:52.142835: [debug] (SQL) SELECT COUNT(*) FROM "last_performed" WHERE ("action"=?) AND ("timestamp">=?); [PersistInt64 1,PersistUTCTime 2023-12-07 14:24:52.1418345 UTC]
+2023-12-08 16:24:52.144833: [debug] Using package location completions from a lock file
+2023-12-08 16:24:52.154836: [debug] Loaded snapshot from Pantry database.
+2023-12-08 16:24:52.575861: [debug] Prefetching git repos: []
+2023-12-08 16:24:52.576863: [debug] []
+2023-12-08 16:24:52.614841: [debug] Asking for a supported GHC version
+2023-12-08 16:24:52.624838: [debug] Installed tools:
+ - msys2-20200903
+ - ghc-9.6.2
+ - ghc-9.0.2
+ - ghc-8.8.4
+ - ghc-8.8.3
+ - ghc-8.6.5
+ - ghc-8.4.3
+ - ghc-8.10.7
+ - ghc-8.10.4
+2023-12-08 16:24:52.625841: [debug] Potential GHC builds: standard
+2023-12-08 16:24:52.625841: [debug] Found already installed GHC builds: standard
+2023-12-08 16:24:52.630839: [debug] Performing a sanity check on: C:\Users\jkniiv\AppData\Local\Programs\stack\x86_64-windows\ghc-8.10.7\bin\ghc-8.10.7.exe
+2023-12-08 16:24:52.632844: [debug] Run process within C:\Users\jkniiv\AppData\Local\Temp\stack-sanity-check-2b0507b6fb5a74f8\: C:\Users\jkniiv\AppData\Local\Programs\stack\x86_64-windows\ghc-8.10.7\bin\ghc-8.10.7.exe C:\Users\jkniiv\AppData\Local\Temp\stack-sanity-check-2b0507b6fb5a74f8\Main.hs -no-user-package-db -hide-all-packages "-package base" "-package Cabal"
+2023-12-08 16:25:16.713070: [debug] Process finished in 24079ms: C:\Users\jkniiv\AppData\Local\Programs\stack\x86_64-windows\ghc-8.10.7\bin\ghc-8.10.7.exe C:\Users\jkniiv\AppData\Local\Temp\stack-sanity-check-2b0507b6fb5a74f8\Main.hs -no-user-package-db -hide-all-packages "-package base" "-package Cabal"
+2023-12-08 16:25:16.721070: [debug] (SQL) SELECT "id","actual_version","arch","ghc_path","ghc_size","ghc_modified","ghc_pkg_path","runghc_path","haddock_path","cabal_version","global_db","global_db_cache_size","global_db_cache_modified","info","global_dump" FROM "compiler_cache" WHERE "ghc_path"=?; [PersistText "C:\\Users\\jkniiv\\AppData\\Local\\Programs\\stack\\x86_64-windows\\ghc-8.10.7\\bin\\ghc-8.10.7.exe"]
+2023-12-08 16:25:16.806072: [debug] Loaded compiler information from cache
+2023-12-08 16:25:16.806072: [debug] Asking for a supported GHC version
+2023-12-08 16:25:16.806072: [info] Stack will use a sandboxed GHC it installed. To use this GHC and packages outside of a project,
+consider using: stack ghc, stack ghci, stack runghc, or stack exec.
+C:\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P [BUILD-231130-10.20231129 +6 ~0 -0 !]> just build-production
+cd 'C:\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P' && stack --stack-yaml stack-lts-18.13.yaml build --flag 'git-annex:Production' --flag 'git-annex:-Crypton' 2>&1 | & $env:GNU_TEE_CMD stack.build---flag-git-annex_Production.LOG~102
+Building all executables for git-annex once. After a successful build of all of them, only specified
+executables will be rebuilt.
+git-annex> configure (exe)
+[ 1 of 22] Compiling Author           ( Author.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Author.o )
+[ 2 of 22] Compiling Utility.Data     ( Utility\Data.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Utility\Data.o )
+[ 3 of 22] Compiling Utility.FileSystemEncoding ( Utility\FileSystemEncoding.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Utility\FileSystemEncoding.o )
+[ 4 of 22] Compiling Utility.Debug    ( Utility\Debug.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Utility\Debug.o )
+[ 5 of 22] Compiling Utility.Misc     ( Utility\Misc.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Utility\Misc.o )
+[ 6 of 22] Compiling Utility.Monad    ( Utility\Monad.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Utility\Monad.o )
+[ 7 of 22] Compiling Utility.Process.Shim ( Utility\Process\Shim.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Utility\Process\Shim.o )
+[ 8 of 22] Compiling Utility.SafeOutput ( Utility\SafeOutput.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Utility\SafeOutput.o )
+[ 9 of 22] Compiling Utility.Exception ( Utility\Exception.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Utility\Exception.o )
+[10 of 22] Compiling Utility.Process  ( Utility\Process.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Utility\Process.o )
+[11 of 22] Compiling Utility.SafeCommand ( Utility\SafeCommand.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Utility\SafeCommand.o )
+[12 of 22] Compiling Utility.Env.Basic ( Utility\Env\Basic.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Utility\Env\Basic.o )
+[13 of 22] Compiling Build.Version    ( Build\Version.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Build\Version.o )
+[14 of 22] Compiling Utility.Split    ( Utility\Split.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Utility\Split.o )
+[15 of 22] Compiling Utility.DottedVersion ( Utility\DottedVersion.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Utility\DottedVersion.o )
+[16 of 22] Compiling Git.Version      ( Git\Version.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Git\Version.o )
+[17 of 22] Compiling Utility.SystemDirectory ( Utility\SystemDirectory.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Utility\SystemDirectory.o )
+[18 of 22] Compiling Utility.Path     ( Utility\Path.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Utility\Path.o )
+[19 of 22] Compiling Build.TestConfig ( Build\TestConfig.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Build\TestConfig.o )
+[20 of 22] Compiling Build.Configure  ( Build\Configure.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Build\Configure.o )
+[21 of 22] Compiling Main             ( C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\Setup.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\Main.o )
+[22 of 22] Compiling StackSetupShim   ( C:\\hs-stack\setup-exe-src\setup-shim-9p6GVs8J.hs, C:\\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\dist\ed8db9df\setup\StackSetupShim.o )
+Linking C:\\Users\\jkniiv\\Projektit\\git-annex.branchable.com\\git-annex--BUILD-231130-10.20231129~P\\.stack-work\\dist\\ed8db9df\\setup\\setup.exe ...
+  checking UPGRADE_LOCATION... not available
+  checking git... yes
+  checking git version... 2.43.0.windows.1
+  checking cp -a... yes
+  checking cp -p... yes
+  checking cp --preserve=timestamps... yes
+  checking cp_reflink_supported... no
+  checking cp --no-preserve=xattr... yes
+  checking xargs -0... yes
+  checking rsync... no
+  checking curl... yes
+  checking bup... no
+  checking borg... no
+  checking nice... yes
+  checking ionice... no
+  checking nocache... no
+  checking gpg... gpg
+  checking lsof... not available
+  checking git-remote-gcrypt... not available
+  checking ssh connection caching... yes
+Configuring git-annex-10.20231129...
+git-annex> build (exe)
+Preprocessing executable 'git-annex' for git-annex-10.20231129..
+Building executable 'git-annex' for git-annex-10.20231129..
+[  1 of 693] Compiling Assistant.Types.BranchChange
+[  2 of 693] Compiling Assistant.Types.ThreadName
+[  3 of 693] Compiling Assistant.Types.TransferSlots
+[...snip...]
+[691 of 693] Compiling Command.Assistant
+[692 of 693] Compiling CmdLine.GitAnnex
+[693 of 693] Compiling Main
+Linking .stack-work\\dist\\ed8db9df\\build\\git-annex\\git-annex.exe ...
+git-annex> copy/register
+Installing executable git-annex in C:\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P\.stack-work\install\f241563d\bin
+C:\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P [BUILD-231130-10.20231129 +6 ~0 -0 !]> cp .\.stack-work\install\f241563d\bin\git-annex.exe .
+C:\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P [BUILD-231130-10.20231129 +7 ~0 -0 !]> .\git-annex.exe version > git-annex.version.TXT
+C:\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P [BUILD-231130-10.20231129 +7 ~0 -0 !]> head -1 .\git-annex.version.TXT
+git-annex version: 10.20231129-gbacd781c4fe05126219c4b5f2963677a6de01481
+C:\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P [BUILD-231130-10.20231129 +7 ~0 -0 !]> cat build.TXT
+git-annex--BUILD-231130-10.20231129~P
+resolver: lts-18.13
+C:\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P [BUILD-231130-10.20231129 +7 ~0 -0 !]> just copy-dlls-over-to-here
++ PATH='/mingw64/bin:/usr/local/bin:/usr/bin:/bin:/c/Program Files/PowerShell/7:<theRestRedacted />'
++ cd 'C:\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P'
+++ stack exec ldd -- git-annex.exe
+++ grep /mingw64
+++ cut '-d ' -f3
++ dlls_to_copy='/mingw64/bin/libmagic-1.dll
+/mingw64/bin/libsystre-0.dll
+/mingw64/bin/libtre-5.dll
+/mingw64/bin/libintl-8.dll'
++ stack exec cp -- -pv /mingw64/bin/libmagic-1.dll /mingw64/bin/libsystre-0.dll /mingw64/bin/libtre-5.dll /mingw64/bin/libintl-8.dll .
+'C:/Users/jkniiv/AppData/Local/Programs/stack/x86_64-windows/msys2-20200903/mingw64/bin/libmagic-1.dll' -> './libmagic-1.dll'
+'C:/Users/jkniiv/AppData/Local/Programs/stack/x86_64-windows/msys2-20200903/mingw64/bin/libsystre-0.dll' -> './libsystre-0.dll'
+'C:/Users/jkniiv/AppData/Local/Programs/stack/x86_64-windows/msys2-20200903/mingw64/bin/libtre-5.dll' -> './libtre-5.dll'
+'C:/Users/jkniiv/AppData/Local/Programs/stack/x86_64-windows/msys2-20200903/mingw64/bin/libintl-8.dll' -> './libintl-8.dll'
+C:\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P [BUILD-231130-10.20231129 +12 ~0 -0 !]> just install-files-to-annx-test-dir
++ PATH='/mingw64/bin:/usr/local/bin:/usr/bin:/bin:/c/Program Files/PowerShell/7:<theRestRedacted />'
++ cd 'C:\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-231130-10.20231129~P'
+++ shopt -s nullglob
+++ echo git-annex.exe libintl-8.dll libmagic-1.dll libsystre-0.dll libtre-5.dll build.TXT git-annex.version.TXT stack.build---flag-git-annex_Production.LOG~102 stack.build.LOG~102 stack.setup---verbose.LOG~101 CHANGELOG
++ files_to_copy='git-annex.exe libintl-8.dll libmagic-1.dll libsystre-0.dll libtre-5.dll build.TXT git-annex.version.TXT stack.build---flag-git-annex_Production.LOG~102 stack.build.LOG~102 stack.setup---verbose.LOG~101 CHANGELOG'
++ cp -pv git-annex.exe libintl-8.dll libmagic-1.dll libsystre-0.dll libtre-5.dll build.TXT git-annex.version.TXT stack.build---flag-git-annex_Production.LOG~102 stack.build.LOG~102 stack.setup---verbose.LOG~101 CHANGELOG .annx-test
+'git-annex.exe' -> '.annx-test/git-annex.exe'
+'libintl-8.dll' -> '.annx-test/libintl-8.dll'
+'libmagic-1.dll' -> '.annx-test/libmagic-1.dll'
+'libsystre-0.dll' -> '.annx-test/libsystre-0.dll'
+'libtre-5.dll' -> '.annx-test/libtre-5.dll'
+'build.TXT' -> '.annx-test/build.TXT'
+'git-annex.version.TXT' -> '.annx-test/git-annex.version.TXT'
+'stack.build---flag-git-annex_Production.LOG~102' -> '.annx-test/stack.build---flag-git-annex_Production.LOG~102'
+'stack.build.LOG~102' -> '.annx-test/stack.build.LOG~102'
+'stack.setup---verbose.LOG~101' -> '.annx-test/stack.setup---verbose.LOG~101'
+'CHANGELOG' -> '.annx-test/CHANGELOG'
+
+# End of transcript or log.
+"""]]
+
+And this is how I usually run the test suite in Git Bash (with some preliminary actions to record the tests available + make
+an SHA256SUMS file of the most important files in this test directory):
+
+[[!format sh """
+jkniiv@AINESIS MINGW64 ~/Projektit/git-annex.branchable.com/git-annex--BUILD-231130-10.20231129~P (BUILD-231130-10.20231129)
+$ cd .annx-test/
+
+jkniiv@AINESIS MINGW64 ~/Projektit/git-annex.branchable.com/git-annex--BUILD-231130-10.20231129~P/.annx-test (BUILD-231130-10.20231129)
+$ just record-tests
+./git-annex.exe test --list-tests > git-annex.test---list-tests.TXT
+
+jkniiv@AINESIS MINGW64 ~/Projektit/git-annex.branchable.com/git-annex--BUILD-231130-10.20231129~P/.annx-test (BUILD-231130-10.20231129)
+$ just sums check-sums
+sha256sum git-annex.exe libintl-8.dll libmagic-1.dll libsystre-0.dll libtre-5.dll build.TXT git-annex.test---list-tests.
+TXT git-annex.version.TXT > SHA256SUMS
+sha256sum -c SHA256SUMS
+git-annex.exe: OK
+libintl-8.dll: OK
+libmagic-1.dll: OK
+libsystre-0.dll: OK
+libtre-5.dll: OK
+build.TXT: OK
+git-annex.test---list-tests.TXT: OK
+git-annex.version.TXT: OK
+
+jkniiv@AINESIS MINGW64 ~/Projektit/git-annex.branchable.com/git-annex--BUILD-231130-10.20231129~P/.annx-test (BUILD-231130-10.20231129)
+$ ls -l
+total 73065
+-rw-r--r-- 1 jkniiv 197121   429145 Dec  8 14:28 CHANGELOG
+-rw-r--r-- 1 jkniiv 197121     1073 Dec 14  2022 Justfile
+-rw-r--r-- 1 jkniiv 197121      664 Dec  8 16:48 SHA256SUMS
+-rw-r--r-- 1 jkniiv 197121       60 Dec  8 15:27 build.TXT
+-rwxr-xr-x 1 jkniiv 197121 73740800 Dec  8 16:41 git-annex.exe*
+-rw-r--r-- 1 jkniiv 197121     8464 Dec  8 16:48 git-annex.test---list-tests.TXT
+-rw-r--r-- 1 jkniiv 197121     1112 Dec  8 16:42 git-annex.version.TXT
+-rwxr-xr-x 1 jkniiv 197121   187145 Nov 23 11:21 libintl-8.dll*
+-rwxr-xr-x 1 jkniiv 197121   226435 Dec  4 20:11 libmagic-1.dll*
+-rwxr-xr-x 1 jkniiv 197121    18240 Jan 17  2018 libsystre-0.dll*
+-rwxr-xr-x 1 jkniiv 197121    92313 Jan 17  2018 libtre-5.dll*
+-rw-r--r-- 1 jkniiv 197121    37834 Dec  8 16:41 stack.build---flag-git-annex_Production.LOG~102
+-rw-r--r-- 1 jkniiv 197121    37834 Dec  8 15:45 stack.build.LOG~102
+-rw-r--r-- 1 jkniiv 197121     2757 Dec  8 16:25 stack.setup---verbose.LOG~101
+
+jkniiv@AINESIS MINGW64 ~/Projektit/git-annex.branchable.com/git-annex--BUILD-231130-10.20231129~P/.annx-test (BUILD-231130-10.20231129)
+$ ldd git-annex.exe
+        ntdll.dll => /c/WINDOWS/SYSTEM32/ntdll.dll (0x7ff93f190000)
+        KERNEL32.DLL => /c/WINDOWS/System32/KERNEL32.DLL (0x7ff93e380000)
+        KERNELBASE.dll => /c/WINDOWS/System32/KERNELBASE.dll (0x7ff93c9d0000)
+        ADVAPI32.dll => /c/WINDOWS/System32/ADVAPI32.dll (0x7ff93e870000)
+        msvcrt.dll => /c/WINDOWS/System32/msvcrt.dll (0x7ff93d2b0000)
+        libmagic-1.dll => /c/Users/jkniiv/Projektit/git-annex.branchable.com/git-annex--BUILD-231130-10.20231129~P/.annx-test/libmagic-1.dll (0x7ff8fd370000)
+        sechost.dll => /c/WINDOWS/System32/sechost.dll (0x7ff93e250000)
+        SHLWAPI.dll => /c/WINDOWS/System32/SHLWAPI.dll (0x7ff93e440000)
+        libsystre-0.dll => /c/Users/jkniiv/Projektit/git-annex.branchable.com/git-annex--BUILD-231130-10.20231129~P/.annx-test/libsystre-0.dll (0x6bcc0000)
+        RPCRT4.dll => /c/WINDOWS/System32/RPCRT4.dll (0x7ff93e120000)
+        CRYPT32.dll => /c/WINDOWS/System32/CRYPT32.dll (0x7ff93d090000)
+        libtre-5.dll => /c/Users/jkniiv/Projektit/git-annex.branchable.com/git-annex--BUILD-231130-10.20231129~P/.annx-test/libtre-5.dll (0x63bc0000)
+        ucrtbase.dll => /c/WINDOWS/System32/ucrtbase.dll (0x7ff93cf00000)
+        GDI32.dll => /c/WINDOWS/System32/GDI32.dll (0x7ff93d1f0000)
+        win32u.dll => /c/WINDOWS/System32/win32u.dll (0x7ff93ced0000)
+        libintl-8.dll => /c/Users/jkniiv/Projektit/git-annex.branchable.com/git-annex--BUILD-231130-10.20231129~P/.annx-test/libintl-8.dll (0x7ff8f3d20000)
+        gdi32full.dll => /c/WINDOWS/System32/gdi32full.dll (0x7ff93ccd0000)
+        dbghelp.dll => /c/WINDOWS/SYSTEM32/dbghelp.dll (0x7ff93ae50000)
+        msvcp_win.dll => /c/WINDOWS/System32/msvcp_win.dll (0x7ff93c8c0000)
+        USER32.dll => /c/WINDOWS/System32/USER32.dll (0x7ff93d6c0000)
+        ole32.dll => /c/WINDOWS/System32/ole32.dll (0x7ff93d4e0000)
+        libiconv-2.dll => /c/Users/jkniiv/bin/libiconv-2.dll (0x7ff8a6e00000)
+        combase.dll => /c/WINDOWS/System32/combase.dll (0x7ff93e510000)
+        IPHLPAPI.DLL => /c/WINDOWS/SYSTEM32/IPHLPAPI.DLL (0x7ff93bbf0000)
+        SHELL32.dll => /c/WINDOWS/System32/SHELL32.dll (0x7ff93d900000)
+        WS2_32.dll => /c/WINDOWS/System32/WS2_32.dll (0x7ff93d860000)
+        WINMM.dll => /c/WINDOWS/SYSTEM32/WINMM.dll (0x7ff928e00000)
+        CRYPTBASE.DLL => /c/WINDOWS/SYSTEM32/CRYPTBASE.DLL (0x7ff93c070000)
+        dbgcore.DLL => /c/WINDOWS/SYSTEM32/dbgcore.DLL (0x7ff928dc0000)
+
+jkniiv@AINESIS MINGW64 ~/Projektit/git-annex.branchable.com/git-annex--BUILD-231130-10.20231129~P/.annx-test (BUILD-231130-10.20231129)
+$ just tests-new
+/usr/bin/time ./git-annex.exe test  2>&1 | tee git-annex.test.LOG~200
+
+All 0 tests passed (0.00s)
+Tests
+  Repo Tests v10 adjusted unlocked branch
+    Init Tests
+      init:                          OK (9.36s)
+      add:                           FAIL
+        Exception: fd:4: hGetChar: invalid argument (invalid byte sequence)
+        Use -p '/Init Tests.add/' to rerun this test only.
+    crypto:                          SKIP
+
+      Use -p '/crypto/' to rerun this test only.
+    uninit (in git-annex branch):    SKIP
+
+      Use -p '/uninit (in git-annex branch)/' to rerun this test only.
+    conflict resolution symlink bit: SKIP
+
+      Use -p '/conflict resolution symlink bit/' to rerun this test only.
+    union merge regression:          SKIP
+
+      Use -p '/union merge regression/' to rerun this test only.
+    unused:                          SKIP
+
+      Use -p '/unused/' to rerun this test only.
+    fsck (bare):                     SKIP
+
+      Use -p '/fsck (bare)/' to rerun this test only.
+    lock:                            SKIP
+
+      Use -p '/Repo Tests v10 adjusted unlocked branch.lock/' to rerun this test only.
+    drop (with remote):              SKIP
+
+      Use -p '/drop (with remote)/' to rerun this test only.
+    log:                             SKIP
+
+      Use -p '/log/' to rerun this test only.
+    add extras:                      SKIP
+
+      Use -p '/add extras/' to rerun this test only.
+
+11 out of 12 tests failed (11.65s)
+Tests
+  QuickCheck
+    prop_quote_unquote_roundtrip:            OK (0.10s)
+      +++ OK, passed 1000 tests.
+    prop_encode_c_decode_c_roundtrip:        OK (0.11s)
+      +++ OK, passed 1000 tests.
+    prop_isomorphic_key_encode:              OK (0.05s)
+      +++ OK, passed 1000 tests.
+    prop_isomorphic_shellEscape:             OK (0.04s)
+      +++ OK, passed 1000 tests.
+    prop_isomorphic_shellEscape_multiword:   OK (1.25s)
+      +++ OK, passed 1000 tests.
+    prop_isomorphic_configEscape:            OK (0.03s)
+      +++ OK, passed 1000 tests.
+    prop_parse_show_Config:                  OK (0.08s)
+      +++ OK, passed 1000 tests.
+    prop_upFrom_basics:                      OK
+      +++ OK, passed 1000 tests.
+    prop_relPathDirToFileAbs_basics:         OK (0.04s)
+      +++ OK, passed 1000 tests.
+    prop_relPathDirToFileAbs_regressionTest: OK
+      +++ OK, passed 1 test.
+    prop_dirContains_regressionTest:         OK
+      +++ OK, passed 1 test.
+    prop_cost_sane:                          OK
+      +++ OK, passed 1 test.
+    prop_matcher_sane:                       OK
+      +++ OK, passed 1 test.
+    prop_HmacSha1WithCipher_sane:            OK
+      +++ OK, passed 1 test.
+    prop_VectorClock_sane:                   OK
+      +++ OK, passed 1 test.
+    prop_addMapLog_sane:                     OK
+      +++ OK, passed 1 test.
+    prop_verifiable_sane:                    OK (0.11s)
+      +++ OK, passed 1000 tests.
+    prop_segment_regressionTest:             OK
+      +++ OK, passed 1 test.
+    prop_read_write_transferinfo:            OK (0.06s)
+      +++ OK, passed 1000 tests.
+    prop_read_show_inodecache:               OK (0.04s)
+      +++ OK, passed 1000 tests.
+    prop_parse_build_presence_log:           OK (1.92s)
+      +++ OK, passed 1000 tests.
+    prop_parse_build_contentidentifier_log:  OK (1.65s)
+      +++ OK, passed 1000 tests.
+    prop_read_show_TrustLevel:               OK
+      +++ OK, passed 1 test.
+    prop_parse_build_TrustLevelLog:          OK
+      +++ OK, passed 1 test.
+    prop_schedule_roundtrips:                OK (0.01s)
+      +++ OK, passed 1000 tests.
+    prop_past_sane:                          OK
+      +++ OK, passed 1 test.
+    prop_duration_roundtrips:                OK
+      +++ OK, passed 1000 tests.
+    prop_metadata_sane:                      OK (1.00s)
+      +++ OK, passed 1000 tests.
+    prop_metadata_serialize:                 OK (1.07s)
+      +++ OK, passed 1000 tests.
+    prop_branchView_legal:                   OK (1.10s)
+      +++ OK, passed 1000 tests.
+    prop_viewPath_roundtrips:                OK (0.02s)
+      +++ OK, passed 1000 tests.
+    prop_view_roundtrips:                    OK (0.55s)
+      +++ OK, passed 1000 tests.
+    prop_viewedFile_rountrips:               OK (0.04s)
+      +++ OK, passed 1000 tests.
+    prop_standardGroups_parse:               OK
+      +++ OK, passed 1 test.
+    sha1 stable:                             OK
+      +++ OK, passed 1 test.
+    sha2_224 stable:                         OK
+      +++ OK, passed 1 test.
+    sha2_256 stable:                         OK
+      +++ OK, passed 1 test.
+    sha2_384 stable:                         OK
+      +++ OK, passed 1 test.
+    sha2_512 stable:                         OK
+      +++ OK, passed 1 test.
+    skein256 stable:                         OK
+      +++ OK, passed 1 test.
+    skein512 stable:                         OK
+      +++ OK, passed 1 test.
+    sha3_224 stable:                         OK
+      +++ OK, passed 1 test.
+    sha3_256 stable:                         OK
+      +++ OK, passed 1 test.
+    sha3_384 stable:                         OK
+      +++ OK, passed 1 test.
+    sha3_512 stable:                         OK
+      +++ OK, passed 1 test.
+    blake2s_160 stable:                      OK
+      +++ OK, passed 1 test.
+    blake2s_224 stable:                      OK
+      +++ OK, passed 1 test.
+    blake2s_256 stable:                      OK
+      +++ OK, passed 1 test.
+    blake2sp_224 stable:                     OK
+      +++ OK, passed 1 test.
+    blake2sp_256 stable:                     OK
+      +++ OK, passed 1 test.
+    blake2b_160 stable:                      OK
+      +++ OK, passed 1 test.
+    blake2b_224 stable:                      OK
+      +++ OK, passed 1 test.
+    blake2b_256 stable:                      OK
+      +++ OK, passed 1 test.
+    blake2b_384 stable:                      OK
+      +++ OK, passed 1 test.
+    blake2b_512 stable:                      OK
+      +++ OK, passed 1 test.
+    blake2bp_512 stable:                     OK
+      +++ OK, passed 1 test.
+    md5 stable:                              OK
+      +++ OK, passed 1 test.
+    HmacSha1 stable:                         OK
+      +++ OK, passed 1 test.
+    HmacSha224 stable:                       OK
+      +++ OK, passed 1 test.
+    HmacSha256 stable:                       OK
+      +++ OK, passed 1 test.
+    HmacSha384 stable:                       OK
+      +++ OK, passed 1 test.
+    HmacSha512 stable:                       OK
+      +++ OK, passed 1 test.
+
+All 62 tests passed (9.40s)
+Tests
+  Repo Tests v10 adjusted unlocked branch
+    Init Tests
+      init:                                OK (9.46s)
+      add:                                 FAIL
+        Exception: fd:4: hGetChar: invalid argument (invalid byte sequence)
+        Use -p '/add/' to rerun this test only.
+    bup remote:                            SKIP
+
+      Use -p '/bup remote/' to rerun this test only.
+    map:                                   SKIP
+
+      Use -p '/map/' to rerun this test only.
+    conflict resolution movein regression: SKIP
+
+      Use -p '/conflict resolution movein regression/' to rerun this test only.
+    sync:                                  SKIP
+
+      Use -p '/sync/' to rerun this test only.
+    migrate:                               SKIP
+
+      Use -p '/migrate/' to rerun this test only.
+    trust:                                 SKIP
+
+      Use -p '/trust/' to rerun this test only.
+    move (numcopies):                      SKIP
+
+      Use -p '/move (numcopies)/' to rerun this test only.
+    unannex (with copy):                   SKIP
+
+      Use -p '/unannex (with copy)/' to rerun this test only.
+    export and import of subdir:           SKIP
+
+      Use -p '/export and import of subdir/' to rerun this test only.
+
+10 out of 11 tests failed (11.84s)
+Tests
+  Repo Tests v10 adjusted unlocked branch
+    Init Tests
+      init:                                         OK (9.47s)
+      add:                                          FAIL
+        Exception: fd:4: hGetChar: invalid argument (invalid byte sequence)
+        Use -p '/Init Tests.add/' to rerun this test only.
+    borg remote:                                    SKIP
+
+      Use -p '/borg remote/' to rerun this test only.
+    uninit:                                         SKIP
+
+      Use -p '/uninit/' to rerun this test only.
+    conflict resolution (mixed directory and file): SKIP
+
+      Use -p '/conflict resolution (mixed directory and file)/' to rerun this test only.
+    concurrent get of dup key regression:           SKIP
+
+      Use -p '/concurrent get of dup key regression/' to rerun this test only.
+    migrate (via gitattributes):                    SKIP
+
+      Use -p '/migrate (via gitattributes)/' to rerun this test only.
+    fsck (basics):                                  SKIP
+
+      Use -p '/fsck (basics)/' to rerun this test only.
+    copy:                                           SKIP
+
+      Use -p '/copy/' to rerun this test only.
+    drop (no remote):                               SKIP
+
+      Use -p '/drop (no remote)/' to rerun this test only.
+    shared clone:                                   SKIP
+
+      Use -p '/shared clone/' to rerun this test only.
+    add dup:                                        SKIP
+
+      Use -p '/add dup/' to rerun this test only.
+
+11 out of 12 tests failed (11.82s)
+Tests
+  Repo Tests v10 adjusted unlocked branch
+    Init Tests
+      init:                                   OK (8.97s)
+      add:                                    FAIL
+        Exception: fd:4: hGetChar: invalid argument (invalid byte sequence)
+        Use -p '/Init Tests.add/' to rerun this test only.
+    addurl:                                   SKIP
+
+      Use -p '/addurl/' to rerun this test only.
+    directory remote:                         SKIP
+
+      Use -p '/directory remote/' to rerun this test only.
+    conflict resolution (nonannexed symlink): SKIP
+
+      Use -p '/conflict resolution (nonannexed symlink)/' to rerun this test only.
+    conflict resolution:                      SKIP
+
+      Use -p '$0=="Tests.Repo Tests v10 adjusted unlocked branch.conflict resolution"' to rerun this test only.
+    info:                                     SKIP
+
+      Use -p '/info/' to rerun this test only.
+    conversion git to annexed:                SKIP
+
+      Use -p '/conversion git to annexed/' to rerun this test only.
+    partial commit:                           SKIP
+
+      Use -p '/partial commit/' to rerun this test only.
+    move:                                     SKIP
+
+      Use -p '/move/' to rerun this test only.
+    reinject:                                 SKIP
+
+      Use -p '/reinject/' to rerun this test only.
+    metadata:                                 SKIP
+
+      Use -p '/metadata/' to rerun this test only.
+
+11 out of 12 tests failed (11.20s)
+Tests
+  Repo Tests v10 adjusted unlocked branch
+    Init Tests
+      init:                                       OK (9.18s)
+      add:                                        FAIL
+        Exception: fd:4: hGetChar: invalid argument (invalid byte sequence)
+        Use -p '/Init Tests.add/' to rerun this test only.
+    preferred content:                            SKIP
+
+      Use -p '/preferred content/' to rerun this test only.
+    upgrade:                                      SKIP
+
+      Use -p '/upgrade/' to rerun this test only.
+    conflict resolution (uncommitted local file): SKIP
+
+      Use -p '/conflict resolution (uncommitted local file)/' to rerun this test only.
+    adjusted branch merge regression:             SKIP
+
+      Use -p '/adjusted branch merge regression/' to rerun this test only.
+    describe:                                     SKIP
+
+      Use -p '/describe/' to rerun this test only.
+    fsck (local untrusted):                       SKIP
+
+      Use -p '/fsck (local untrusted)/' to rerun this test only.
+    lock --force:                                 SKIP
+
+      Use -p '/lock --force/' to rerun this test only.
+    drop (untrusted remote):                      SKIP
+
+      Use -p '/drop (untrusted remote)/' to rerun this test only.
+    view:                                         SKIP
+
+      Use -p '/view/' to rerun this test only.
+    add moved link:                               SKIP
+
+      Use -p '/add moved link/' to rerun this test only.
+
+11 out of 12 tests failed (11.64s)
+Tests
+  Repo Tests v10 adjusted unlocked branch
+    Init Tests
+      init:                                OK (9.02s)
+      add:                                 FAIL
+        Exception: fd:4: hGetChar: invalid argument (invalid byte sequence)
+        Use -p '/Init Tests.add/' to rerun this test only.
+    add subdirs:                           SKIP
+
+      Use -p '/add subdirs/' to rerun this test only.
+    hook remote:                           SKIP
+
+      Use -p '/hook remote/' to rerun this test only.
+    conflict resolution (nonannexed file): SKIP
+
+      Use -p '/conflict resolution (nonannexed file)/' to rerun this test only.
+    transition propagation:                SKIP
+
+      Use -p '/transition propagation/' to rerun this test only.
+    merge:                                 SKIP
+
+      Use -p '/merge/' to rerun this test only.
+    fsck --from remote:                    SKIP
+
+      Use -p '/fsck --from remote/' to rerun this test only.
+    edit (pre-commit):                     SKIP
+
+      Use -p '/edit (pre-commit)/' to rerun this test only.
+    get (ssh remote):                      SKIP
+
+      Use -p '/get (ssh remote)/' to rerun this test only.
+    import:                                SKIP
+
+      Use -p '/import/' to rerun this test only.
+    ignore deleted files:                  SKIP
+
+      Use -p '/ignore deleted files/' to rerun this test only.
+
+11 out of 12 tests failed (11.22s)
+Tests
+  Repo Tests v10 adjusted unlocked branch
+    Init Tests
+      init:                             OK (8.99s)
+      add:                              FAIL
+        Exception: fd:4: hGetChar: invalid argument (invalid byte sequence)
+        Use -p '/add/' to rerun this test only.
+    required_content:                   SKIP
+
+      Use -p '/required_content/' to rerun this test only.
+    whereis:                            SKIP
+
+      Use -p '/whereis/' to rerun this test only.
+    conflict resolution (removed file): SKIP
+
+      Use -p '/conflict resolution (removed file)/' to rerun this test only.
+    adjusted branch subtree regression: SKIP
+
+      Use -p '/adjusted branch subtree regression/' to rerun this test only.
+    find:                               SKIP
+
+      Use -p '/find/' to rerun this test only.
+    fsck (remote untrusted):            SKIP
+
+      Use -p '/fsck (remote untrusted)/' to rerun this test only.
+    edit (no pre-commit):               SKIP
+
+      Use -p '/edit (no pre-commit)/' to rerun this test only.
+    get:                                SKIP
+
+      Use -p '/get/' to rerun this test only.
+    magic:                              SKIP
+
+      Use -p '/magic/' to rerun this test only.
+    readonly remote:                    SKIP
+
+      Use -p '/readonly remote/' to rerun this test only.
+
+11 out of 12 tests failed (11.22s)
+Tests
+  Repo Tests v10 adjusted unlocked branch
+    Init Tests
+      init:                                               OK (5.91s)
+      add:                                                FAIL
+        Exception: fd:4: hGetChar: invalid argument (invalid byte sequence)
+        Use -p '/add/' to rerun this test only.
+    repair:                                               SKIP
+
+      Use -p '/repair/' to rerun this test only.
+    rsync remote:                                         SKIP
+
+      Use -p '/rsync remote/' to rerun this test only.
+    conflict resolution (mixed locked and unlocked file): SKIP
+
+      Use -p '/conflict resolution (mixed locked and unlocked file)/' to rerun this test only.
+    conflict resolution (adjusted branch):                SKIP
+
+      Use -p '/conflict resolution (adjusted branch)/' to rerun this test only.
+    version:                                              SKIP
+
+      Use -p '/Repo Tests v10 adjusted unlocked branch.version/' to rerun this test only.
+    conversion annexed to git:                            SKIP
+
+      Use -p '/conversion annexed to git/' to rerun this test only.
+    fix:                                                  SKIP
+
+      Use -p '/fix/' to rerun this test only.
+    move (ssh remote):                                    SKIP
+
+      Use -p '/move (ssh remote)/' to rerun this test only.
+    unannex (no copy):                                    SKIP
+
+      Use -p '/unannex (no copy)/' to rerun this test only.
+    export and import:                                    SKIP
+
+      Use -p '/export and import/' to rerun this test only.
+
+11 out of 12 tests failed (7.23s)
+  (Failures above could be due to a bug in git-annex, or an incompatibility
+   with utilities, such as git, installed on this system.)
+Command exited with non-zero status 1
+0.00user 0.04system 0:31.25elapsed 0%CPU (0avgtext+0avgdata 5248maxresident)k
+0inputs+0outputs (1391major+0minor)pagefaults 0swaps
+
+jkniiv@AINESIS MINGW64 ~/Projektit/git-annex.branchable.com/git-annex--BUILD-231130-10.20231129~P/.annx-test (BUILD-231130-10.20231129)
+$ ./git-annex.exe test -p ' /Init Tests.add/'
+
+All 0 tests passed (0.00s)
+Tests
+  Repo Tests v10 adjusted unlocked branch
+    Init Tests
+      init: OK (5.02s)
+      add:  FAIL
+        Exception: fd:4: hGetChar: invalid argument (invalid byte sequence)
+        Use -p '(/Init Tests.add/||/Init Tests/)&&/add/' to rerun this test only.
+
+1 out of 2 tests failed (6.37s)
+
+All 0 tests passed (0.00s)
+  (Failures above could be due to a bug in git-annex, or an incompatibility
+   with utilities, such as git, installed on this system.)
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Sure, Git Annex is great as always. I use it several times a week with my multigigabyte
+backups, where it gives structure to my image-based backup routines, so you could
+say I'm a believer. :)
+
+[[!meta author=jkniiv]]
+
+### Update 20 Dec 2023
+
+P.S. This is a bit embarrasing but I found out this is [[notabug|done]], cf. my comment --[[jkniiv]]
diff --git a/doc/bugs/__39__Production__39___build_doesn__39__t_pass_testsuite_on_Win32/comment_1_7ac6938faef9111da701080a62c4d7e9._comment b/doc/bugs/__39__Production__39___build_doesn__39__t_pass_testsuite_on_Win32/comment_1_7ac6938faef9111da701080a62c4d7e9._comment
new file mode 100644
index 0000000000..6b70005ccf
--- /dev/null
+++ b/doc/bugs/__39__Production__39___build_doesn__39__t_pass_testsuite_on_Win32/comment_1_7ac6938faef9111da701080a62c4d7e9._comment
@@ -0,0 +1,67 @@
+[[!comment format=mdwn
+ username="jkniiv"
+ avatar="http://cdn.libravatar.org/avatar/05fd8b33af7183342153e8013aa3713d"
+ subject="my report was actually a User Failure on my part"
+ date="2023-12-19T23:02:15Z"
+ content="""
+Uh-oh, after adding the option `--test-debug` to the `test` subcommand I got a lead on
+the real culprit and it wasn't git-annex but libmagic:
+
+[[!format sh \"\"\"
+[...snip...]
+ok
+[2023-12-19 22:52:17.5370274] (Utility.Process) process [21636] done ExitSuccess
+[2023-12-19 22:52:17.5460328] (Utility.Process) process [10460] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-p
+athspecs\",\"-c\",\"annex.debug=true\",\"ls-files\",\"-z\",\"--modified\",\"--\",\"foo\"]
+[2023-12-19 22:52:17.5970346] (Utility.Process) process [10460] done ExitSuccess
+[2023-12-19 22:52:17.6030281] (Utility.Process) process [9208] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pa
+thspecs\",\"-c\",\"annex.debug=true\",\"diff\",\"--name-only\",\"--diff-filter=T\",\"-z\",\"--cached\",\"--\",\"foo\"]
+[2023-12-19 22:52:17.6550242] (Utility.Process) process [9208] done ExitSuccess
+(recording state in git...)
+[2023-12-19 22:52:17.6650254] (Utility.Process) process [24064] feed: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-p
+athspecs\",\"-c\",\"annex.debug=true\",\"update-index\",\"-z\",\"--index-info\"]
+[2023-12-19 22:52:17.7090277] (Utility.Process) process [24064] done ExitSuccess
+[2023-12-19 22:52:17.7240248] (Utility.Process) process [22248] feed: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-p
+athspecs\",\"-c\",\"annex.debug=true\",\"update-index\",\"-z\",\"--index-info\"]
+[2023-12-19 22:52:17.7700259] (Utility.Process) process [22248] done ExitSuccess
+[2023-12-19 22:52:17.7780282] (Utility.Process) process [23188] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-p
+athspecs\",\"-c\",\"annex.debug=true\",\"show-ref\",\"--hash\",\"refs/heads/git-annex\"]
+[2023-12-19 22:52:17.8270228] (Utility.Process) process [23188] done ExitSuccess
+[2023-12-19 22:52:17.8340274] (Utility.Process) process [17648] feed: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-p
+athspecs\",\"-c\",\"annex.debug=true\",\"update-index\",\"-z\",\"--index-info\"]
+[2023-12-19 22:52:17.8420331] (Utility.Process) process [21928] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-p
+athspecs\",\"-c\",\"annex.debug=true\",\"diff-index\",\"--raw\",\"-z\",\"-r\",\"--no-renames\",\"-l0\",\"--cached\",\"refs/heads/git-annex\",
+\"--\"]
+[2023-12-19 22:52:17.8980416] (Utility.Process) process [21928] done ExitSuccess
+[2023-12-19 22:52:17.9060216] (Utility.Process) process [17648] done ExitSuccess
+[2023-12-19 22:52:17.920024] (Utility.Process) process [18680] read: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pa
+thspecs\",\"-c\",\"annex.debug=true\",\"write-tree\"]
+[2023-12-19 22:52:17.982027] (Utility.Process) process [18680] done ExitSuccess
+[2023-12-19 22:52:17.9890276] (Utility.Process) process [10892] chat: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-p
+athspecs\",\"-c\",\"annex.debug=true\",\"commit-tree\",\"bb68ce15d91f27eec51749b9481ede5cc6bcd190\",\"--no-gpg-sign\",\"-p\",\"refs/he
+ads/git-annex\"]
+[2023-12-19 22:52:18.0490233] (Utility.Process) process [10892] done ExitSuccess
+[2023-12-19 22:52:18.0570304] (Utility.Process) process [6552] call: git [\"--git-dir=.git\",\"--work-tree=.\",\"--literal-pa
+thspecs\",\"-c\",\"annex.debug=true\",\"update-ref\",\"refs/heads/git-annex\",\"71b9bb13e4244bd2233de7ca7cf7fc01011f1e62\"]
+[2023-12-19 22:52:18.1170343] (Utility.Process) process [6552] done ExitSuccess
+[2023-12-19 22:52:18.1300416] (Utility.Process) process [10116] done ExitSuccess
+[2023-12-19 22:52:18.1350241] (Utility.Process) process [20976] done ExitSuccess
+[2023-12-19 22:52:18.1400231] (Utility.Process) process [9880] done ExitSuccess
+[2023-12-19 22:52:18.1450249] (Utility.Process) process [872] done ExitSuccess
+C:\Users\jkniiv\AppData\Local/.magic/magic.mgc, 1: Warning: offset `∟♦▲FAIL (2.36s)
+        .\\Test\\Framework.hs:83:
+        add with SHA1 failed with unexpected exit code
+        Use -p '(/Init Tests.add/||/Init Tests/)&&/add/' to rerun this test only.
+
+1 out of 2 tests failed (8.86s)
+git-annex: thread blocked indefinitely in an STM transaction
+
+\"\"\"]]
+
+So the real error turned out to be a user failure of mine: libmagic (or the msys2 package `mingw-w64-x86_64-file`)
+had a recent update and the new library didn't like my previous magic database located in the fallback location
+`%localappdata%\.magic\magic.mgc`. By copying the msys2 file `/mingw64/share/misc/magic.mgc` to the aforementioned
+location, the whole issue cleared itself and this report became moot.
+
+
+"""]]
diff --git a/doc/bugs/__96__git_add__96___adds_files_unlocked_instead_of_locked.mdwn b/doc/bugs/__96__git_add__96___adds_files_unlocked_instead_of_locked.mdwn
new file mode 100644
index 0000000000..f4b5d71f8d
--- /dev/null
+++ b/doc/bugs/__96__git_add__96___adds_files_unlocked_instead_of_locked.mdwn
@@ -0,0 +1,55 @@
+### Please describe the problem.
+If `.gitattributes` contains `* annex.largefiles=anything`, then `git add` will add files in an unlocked state. According to this page (https://git-annex.branchable.com/tips/unlocked_files/) though, git-annex should add files in a locked state by default:
+> By default, git-annex commands will add files in locked mode, unless used on a filesystem that does not support symlinks, when unlocked mode is used. To make them always use unlocked mode, run: git config annex.addunlocked true
+
+### What steps will reproduce the problem?
+
+```
+cd $(mktemp -d)
+git init
+echo '* annex.largefiles=anything' > .gitattributes
+git add .
+git commit -m "Add gitattributes"
+git annex init
+cat >> annexed.txt <<EOF
+This text file is unlocked
+EOF
+git add .
+git commit -m "Add annexed"
+```
+
+Observe that annexed.txt will be added as a pointer file, i.e. in unlocked mode.
+
+### What version of git-annex are you using? On what operating system?
+
+Latest from conda-forge on Ubuntu 22.04: git-annex 10.20230626 alldep_h97b9560_100 conda-forge
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+(git-annex) $ bash <script-with-commands-from-reproduce-section> 
+Initialized empty Git repository in /tmp/tmp.XzlfX1juzC/.git/
+[main (root-commit) 7633536] Add gitattributes
+ 1 file changed, 1 insertion(+)
+ create mode 100644 .gitattributes
+init  ok
+(recording state in git...)
+(recording state in git...)
+[main 60f1b44] Add annexed
+ 1 file changed, 1 insertion(+)
+ create mode 100644 annexed.txt
+(git-annex) $ cd /tmp/tmp.XzlfX1juzC
+(git-annex) /tmp/tmp.XzlfX1juzC$ ls -l
+total 4
+-rw-rw-r-- 1 <user> <group> 27 Okt 23 17:12 annexed.txt
+(git-annex) /tmp/tmp.XzlfX1juzC$ git annex find --unlocked
+annexed.txt
+(git-annex) /tmp/tmp.XzlfX1juzC$
+
+# End of transcript or log.
+"""]]
+
+> [[done]] --[[Joey]]
diff --git a/doc/bugs/__96__git_add__96___adds_files_unlocked_instead_of_locked/comment_1_58abf8693ed90fc8c6e3f750310c17e4._comment b/doc/bugs/__96__git_add__96___adds_files_unlocked_instead_of_locked/comment_1_58abf8693ed90fc8c6e3f750310c17e4._comment
new file mode 100644
index 0000000000..6877505071
--- /dev/null
+++ b/doc/bugs/__96__git_add__96___adds_files_unlocked_instead_of_locked/comment_1_58abf8693ed90fc8c6e3f750310c17e4._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-10-23T17:48:54Z"
+ content="""
+I wish it were possible to make `git add` add files in a locked state.
+However, the smudge/clean interface git-annex uses to hook into `git add`
+makes it impossible to do that.
+
+This is why the documentation refers to "git-annex commands", which does
+exclude `git add`. I've made that a little bit more explicit.
+"""]]
diff --git a/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote.mdwn b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote.mdwn
new file mode 100644
index 0000000000..968d4f4916
--- /dev/null
+++ b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote.mdwn
@@ -0,0 +1,84 @@
+### Please describe the problem.
+
+Context: As part of trying to get a non-star topology as described [here](https://git-annex.branchable.com/todo/Allow_remote_sync__42___and_ignore__42___in_git_annex_config/), I added remotes that are not online and don't have the ignore flag set.
+
+Running `git annex info` (without additional args) causes git-annex to try to ssh to the enabled git special remote and then git fetch it which takes *minutes* to time out.
+
+### What steps will reproduce the problem?
+
+1. Add a reachable but unavailable ip address such as 101.101.101.101 to a local interface for setup
+2. `git annex initremote bad type=git location=ssh://101.101.101.101:/tmp/foo`
+3. Remove the address from the local interface again (and clear annex ssh caches)
+4. `git annex info` (hangs)
+
+### What version of git-annex are you using? On what operating system?
+
+```
+git-annex version: 10.20230802
+build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.24.1 bloomfilter-2.0.1.2 cryptonite-0.30 DAV-1.3.4 feed-1.3.2.1 ghc-9.4.6 http-client-0.7.13.1 persistent-sqlite-2.13.1.1 torrent-10000.1.3 uuid-1.3.15 yesod-1.6.2.1
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: linux x86_64
+supported repository versions: 8 9 10
+upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10
+```
+
+### Please provide any additional information below.
+
+```
+$ time git annex info --debug
+[2023-10-01 11:54:30.392801032] (Utility.Process) process [588948] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","git-annex"]
+[2023-10-01 11:54:30.393683972] (Utility.Process) process [588948] done ExitSuccess
+[2023-10-01 11:54:30.393935272] (Utility.Process) process [588949] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","--hash","refs/heads/git-annex"]
+[2023-10-01 11:54:30.394796062] (Utility.Process) process [588949] done ExitSuccess
+[2023-10-01 11:54:30.395083822] (Utility.Process) process [588950] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","log","refs/heads/git-annex..3d1d07c0a9dc56682f27ba310660fae82a483487","--pretty=%H","-n1"]
+[2023-10-01 11:54:30.396110553] (Utility.Process) process [588950] done ExitSuccess
+[2023-10-01 11:54:30.396321043] (Utility.Process) process [588951] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","log","refs/heads/git-annex..314451b72f15a36ad620106fce7972c03f729cdf","--pretty=%H","-n1"]
+[2023-10-01 11:54:30.397313663] (Utility.Process) process [588951] done ExitSuccess
+[2023-10-01 11:54:30.397520583] (Utility.Process) process [588952] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","log","refs/heads/git-annex..c7a89661d145e9cd2f019301a49cad3382bdaa4e","--pretty=%H","-n1"]
+[2023-10-01 11:54:30.398499344] (Utility.Process) process [588952] done ExitSuccess
+[2023-10-01 11:54:30.399090524] (Utility.Process) process [588953] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","cat-file","--batch"]
+[2023-10-01 11:54:30.400543744] (Utility.Process) process [588954] call: sh ["-c","ping -c 1 -W 0.3 -n LYKOURGOS.redacted 2>&1 >/dev/null"]
+[2023-10-01 11:54:30.704778033] (Utility.Process) process [588954] done ExitFailure 1
+[2023-10-01 11:54:30.705468863] (Utility.Process) process [588956] call: sh ["-c","ping -c 1 -W 0.3 -n uranos.redacted 2>&1 >/dev/null"]
+[2023-10-01 11:54:31.011500591] (Utility.Process) process [588956] done ExitFailure 1
+[2023-10-01 11:54:31.012659161] (Utility.Process) process [588958] read: ssh ["101.101.101.101","-S",".git/annex/ssh/101.101.101.101","-o","ControlMaster=auto","-o","ControlPersist=yes","-n","-T","git-annex-shell 'configlist' '/tmp/foo' '--debug'"]
+
+[2023-10-01 11:56:43.654364825] (Utility.Process) process [588958] done ExitFailure 255
+
+  Unable to parse git config from bad
+[2023-10-01 11:56:43.654791355] (Utility.Process) process [589323] call: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","fetch","--quiet","bad"]
+ssh: connect to host 101.101.101.101 port 22: Connection timed out
+fatal: Could not read from remote repository.
+
+Please make sure you have the correct access rights
+and the repository exists.
+[2023-10-01 11:58:56.774415157] (Utility.Process) process [589323] done ExitFailure 128
+trusted repositories: 0
+semitrusted repositories: 8
+        00000000-0000-0000-0000-000000000001 -- web
+        00000000-0000-0000-0000-000000000002 -- bittorrent
+        080e4b9c-a094-4b74-9825-438b689b665c -- foo2
+        330a9e48-26c2-43ef-a16b-d798f90dd215 -- URANOS
+        65cc3cd9-a1a9-490b-ba46-2bc15b195f96 -- LYKOURGOS
+        8b88ef14-74ef-482f-80ad-74581516ddbb -- atemu@HEPHAISTOS:/tmp/foo4
+        ef313204-e6a2-4895-b817-66365273854c -- bad [here]
+        ff295fe4-85eb-4f98-be21-a070314627e9 -- [SOTERIA]
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 32.53 gigabytes (+1 gigabyte reserved)
+local annex keys: 0
+local annex size: 0 bytes
+annexed files in working tree: [2023-10-01 11:58:56.775992177] (Utility.Process) process [589591] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","ls-files","-z","--cached","--others","--","."]
+[2023-10-01 11:58:56.776843407] (Utility.Process) process [589591] done ExitSuccess
+0
+size of annexed files in working tree: 0 bytes
+bloom filter size: 32 mebibytes (0% full)
+backend usage:
+[2023-10-01 11:58:56.777581777] (Utility.Process) process [588953] done ExitSuccess
+
+real    4m26.397s
+user    0m0.513s
+sys     0m0.038s
+```
diff --git a/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_1_e3bc14a173600b86c5875d90228aea8b._comment b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_1_e3bc14a173600b86c5875d90228aea8b._comment
new file mode 100644
index 0000000000..b6685e786b
--- /dev/null
+++ b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_1_e3bc14a173600b86c5875d90228aea8b._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-10-09T18:50:00Z"
+ content="""
+An easier way to see this behavior is:
+
+	git remote add foo 192.168.1.101:/dne
+	time git fetch foo
+	2:14.36elapsed
+
+So I'm having difficulty seeing this as a bug in git-annex
+particularly.
+
+Yes, git-annex has to ssh once to new git remotes, 
+in order to determine their git-annex uuid. git-annex
+info needs to know what uuids each remote has, so it's
+certianly right that it does this.
+
+Yes, if that fails git-annex falls back to a git fetch,
+in order to determine if the remote is unavailable,
+or if the remote is available but does not
+have git-annex-shell installed. So it takes twice
+as long to time out as the usual ssh tcp timeout.
+"""]]
diff --git a/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_2_9274223b32601ead9a508aa9852e4933._comment b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_2_9274223b32601ead9a508aa9852e4933._comment
new file mode 100644
index 0000000000..2724e16158
--- /dev/null
+++ b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_2_9274223b32601ead9a508aa9852e4933._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="Atemu"
+ avatar="http://cdn.libravatar.org/avatar/86b8c2d893dfdf2146e1bbb8ac4165fb"
+ subject="comment 2"
+ date="2023-12-01T10:21:09Z"
+ content="""
+I've had an idea on this: Why not only update UUIDs on (manual) sync/fetch?
+
+This would be in line with how git interacts with regular remotes otherwise too; always requiring an explicit fetch to update its info.
+
+To me it just violates the principle of least surprise to have git-annex try and reach remotes when running something as simple as `info`.
+"""]]
diff --git a/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_3_8f46a9d4a7ceae80e378149d88dd1f19._comment b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_3_8f46a9d4a7ceae80e378149d88dd1f19._comment
new file mode 100644
index 0000000000..79e4308e73
--- /dev/null
+++ b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_3_8f46a9d4a7ceae80e378149d88dd1f19._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="Another possibility to make --fast faster?"
+ date="2023-12-01T11:50:25Z"
+ content="""
+How about having `git annex info --fast` skip this lookup step for remotes where it doesn't know the UUID of yet?
+
+`git annex info` can already be quite slow in the other steps it takes (counting files, disk space, etc.) in large repos, so it is not so much of a surprise that it hangs a while by default. But if `--fast` would make it actually fast by staying completely offline (right?) and skipping the slow local counting steps, this would be logical.
+
+"""]]
diff --git a/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_4_c3130a2595fc35525dfdbcc6cec57713._comment b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_4_c3130a2595fc35525dfdbcc6cec57713._comment
new file mode 100644
index 0000000000..12a0352361
--- /dev/null
+++ b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_4_c3130a2595fc35525dfdbcc6cec57713._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Atemu"
+ avatar="http://cdn.libravatar.org/avatar/86b8c2d893dfdf2146e1bbb8ac4165fb"
+ subject="comment 4"
+ date="2023-12-03T21:11:19Z"
+ content="""
+I'd flip that around; make `--fast` the default and add a `--full` flag to show full info. I rarely need it.
+"""]]
diff --git a/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_5_eeb4f35b7609cb36fdcece9b8cb94430._comment b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_5_eeb4f35b7609cb36fdcece9b8cb94430._comment
new file mode 100644
index 0000000000..759127675a
--- /dev/null
+++ b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_5_eeb4f35b7609cb36fdcece9b8cb94430._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2023-12-04T16:50:45Z"
+ content="""
+git-annex does only do uuid discovery of new remotes when it is dealing
+with remotes. Many git-annex commands do deal with remotes, and that
+includes `git-annex info`, which after all shows a list of repositories.
+Commands that do not deal with remotes do skip the uuid discovery.
+
+Since this kind of configuration can cause many different git-annex (and
+also git) commands to hang for a while, it doesn't seem very useful to
+do something specifically to `git-annex info` to deal with it.
+"""]]
diff --git a/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_6_ee21d4d8b276a32b227199175ea30721._comment b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_6_ee21d4d8b276a32b227199175ea30721._comment
new file mode 100644
index 0000000000..ef7a519b92
--- /dev/null
+++ b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_6_ee21d4d8b276a32b227199175ea30721._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="How about a --offline flag?"
+ date="2023-12-04T17:52:44Z"
+ content="""
+How about a global `--offline` flag, then? 🙃 It would prevent running anything that has to do with (non-specified) remotes (git pull/push, copying files, fetching UUIDs, etc.), however local things like committing, merging any already present `synced` branches, etc. would still run. 
+
+I can imagine this being helpful in offline kind of situations: 
+
+- You're in an network-less vault/safe and want to sync up your offline hard-drive with just `git annex assist|sync --offline OFFLINEDRIVE` (although that would already do little with other remotes, except when it's run for the first time to auto-enable other remotes, right?)
+- You're on a really bad mobile connection and want to run `git annex assist --offline` to save your current changes without git-annex trying to sync with other remotes (internet is bad anyway). Alternative is to `git annex add` and `git commit` manually.
+
+It could go as far as allowing remotes that are on a local path on a non-networked filesystem, but for that to work reliably git-annex would need to go out of its way quite far, so probably a bit much to ask.
+"""]]
diff --git a/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_7_9104e12c57bc47cfd0a6c109ad085de1._comment b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_7_9104e12c57bc47cfd0a6c109ad085de1._comment
new file mode 100644
index 0000000000..713fb113d6
--- /dev/null
+++ b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_7_9104e12c57bc47cfd0a6c109ad085de1._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="Atemu"
+ avatar="http://cdn.libravatar.org/avatar/86b8c2d893dfdf2146e1bbb8ac4165fb"
+ subject="comment 7"
+ date="2023-12-25T13:03:50Z"
+ content="""
+To explain my rationale a bit more, I use `git-annex info` for three purposes (descending frequency):
+
+1. Overview of repos and trust level (like `git remote`)
+2. Query metadata of files/keys (size, repo/group presence)
+3. Query metadata of individual repos (size, trust)
+
+AFAICT, none of these actually interact with remote repositories in any way; they gather information from the local `git-annex` branch and present it in a digestible format. (Apart from the presence check functionality in question of course.) I'd expect these to run these as fast as the CPU and disk allow without any possible interference by the network.  
+
+Commands like `sync`, `copy` or get must obviously interact with other repositories; it's their primary or even sole purpose. When remote repos are present, I'd fully expect network usage with these and, equally important, timing dependence.  
+(Though I too would love to see an offline flag for these commands like Yann suggested; i.e. `git annex copy --auto --offline` only copying from and to locally available repos.)
+
+I would not expect `info` to have to open any network connections to do its job however; all of the data should be present in the `git-annex` branch afterall. Repo presence is a practical and desirable feature but I wouldn't expect it to be updated on every invocation of `info` and in fact it isn't: When I unmount a repo and run `info` again, git-annex still thinks its present. Actually not even a `sync` will update that; I'm unsure whether it's even possible to do so?
+
+I think I'd prefer an explicit `git annex info --fetch` command (or perhaps even a new sub command) which runs the repo presence check from scratch for all repos or a specific repo. This stops the regular `info` command from requiring any networking whatsoever and provides confidence that remotes are actually present when claimed as such after you run it.
+"""]]
diff --git a/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_8_c5b96e6c35c8969062489d6e03066f34._comment b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_8_c5b96e6c35c8969062489d6e03066f34._comment
new file mode 100644
index 0000000000..335ef537c7
--- /dev/null
+++ b/doc/bugs/__96__git_annex_info__96___hangs_with_git_special_remote/comment_8_c5b96e6c35c8969062489d6e03066f34._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="Lukey"
+ avatar="http://cdn.libravatar.org/avatar/c7c08e2efd29c692cc017c4a4ca3406b"
+ subject="comment 8"
+ date="2023-12-30T11:35:22Z"
+ content="""
+I solved this problem by having my own dhcp+dns server (dnsmasq) running on the LAN which gives out only 30 second dhcp leases. This way only alive hosts are resolvable. And remotes are accessed not by ip, but by hostname.
+
+Another way to do this is via mdns, but I found that to be very unreliable.
+"""]]
diff --git a/doc/bugs/__96__git_annex_sync___60__REMOTE__62____96___swallows_network_failure.mdwn b/doc/bugs/__96__git_annex_sync___60__REMOTE__62____96___swallows_network_failure.mdwn
new file mode 100644
index 0000000000..171b4cffb8
--- /dev/null
+++ b/doc/bugs/__96__git_annex_sync___60__REMOTE__62____96___swallows_network_failure.mdwn
@@ -0,0 +1,68 @@
+### Please describe the problem.
+
+`git annex sync` does not report when `git fetch` and `git push` fail due to
+network issues.  While `git fetch`'s error messages are printed, the exit
+status of `git annex sync` will still be `0`.
+
+Looking at the source code, this seems to be a deliberate design decision. The
+synchronization operation are coded in such a way that failing to send/receive
+commits to/from a remote is not reported as an error. (See `pullRemote` and
+`pushRemote` in `Command/Sync.hs`.) This has the advantage of allowing the user
+to simply say `git annex sync` without worrying too much about whether all of
+their configured remotes are reachable.
+
+However, this poses problems when trying to use `git annex sync` in an
+automated way. If networking issues (including authentication failures) are
+ignored, this can easily convince a script using `git annex sync` that the
+operation has succeeded, when in fact it has failed.
+
+I can think of a few ways of addressing this issue:
+
+1. Keep going of any of the `pullRemote`/`pushRemote` invocations fail, but
+   keep track of the fact that something has failed, and exit with status 1 if
+   this happens. This has the advantage that scripts will be properly alerted
+   when things go wrong, but isn't strictly backwards compatible.
+2. Add an option to `git annex sync` which causes any failures within
+   `pullRemote`/`pushRemote` to be considered fatal errors. Perhaps the option
+   could be called `--batch` or `--report-errors`. This would allow for strict
+   backwards-compatibility.
+3. Make failures of `pullRemote`/`pushRemote` fatal errors if exactly one
+   remote is given on the command line. This isn't backwards compatible, and
+   also has issues because the semantics are not necessarily obvious to those
+   using `git annex sync`.
+
+I'm sure there are other solutions, but I think users of `git annex sync` need
+a way of detecting network errors and responding to them appropriately.
+Especially when you're only trying to synchronize with a single remote, and
+failing to reach that remote is by definition a failure of the entire process.
+
+### What steps will reproduce the problem?
+
+Attempt to pull from any standard properly-configured `git-annex` remote
+`<REMOTE>` with
+[[!format sh """
+ $ git annex sync <REMOTE>
+"""]]
+when `<REMOTE>` is not reachable on the network.
+Then check the exit status with
+[[!format sh """
+ $ echo $?
+"""]]
+You should get `0` as the result.
+
+### What version of git-annex are you using? On what operating system?
+
+I'm using the latest version in Debian stable, 8.20210223-2.
+
+### Please provide any additional information below.
+
+Not really sure what to put here. Except perhaps to apologize for the overly
+design-oriented bug report. Also, be aware that I'm more than happy to put in
+the legwork of fixing the issue I've described above. But I'd like to make sure
+we agree on a solution before I spend a lot of effort assembling a patch
+series.
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Oh yes, I've used `git annex` for many years now to handle synchronizing files
+between various machines. Many thanks for your work on this project.
diff --git a/doc/bugs/__96__lookupkey__96___unexpectedly_slow.mdwn b/doc/bugs/__96__lookupkey__96___unexpectedly_slow.mdwn
new file mode 100644
index 0000000000..bac6814d06
--- /dev/null
+++ b/doc/bugs/__96__lookupkey__96___unexpectedly_slow.mdwn
@@ -0,0 +1,25 @@
+### Please describe the problem.
+
+I need to discover a potentially existing annex-key for any file in the worktree of a Git repository. I assumed that a batch-mode `lookupkey` would be faster than a `find --json --batch`, but that is not the case.
+
+My test repository has 36k files.
+
+```
+❯ time git ls-files | git annex lookupkey --batch > /dev/null
+git ls-files  0.01s user 0.01s system 0% cpu 2:42.37 total
+git annex lookupkey --batch > /dev/null  91.00s user 73.39s system 98% cpu 2:46.25 total
+
+❯ time git ls-files | git annex find --anything --json --batch > /dev/null
+git ls-files  0.01s user 0.00s system 0% cpu 4.093 total
+git annex find --anything --json --batch > /dev/null  3.20s user 2.02s system 124% cpu 4.195 total
+```
+
+`lookupkey` appears to be many times slower than `find`, although the latter reports much more information.
+
+This surprised me, hence I am reporting it here as a potential bug.
+
+### What version of git-annex are you using? On what operating system?
+
+git-annex version: 10.20230126
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/__96__lookupkey__96___unexpectedly_slow/comment_1_13003e3b46c5cfab60b6e34fb18731d7._comment b/doc/bugs/__96__lookupkey__96___unexpectedly_slow/comment_1_13003e3b46c5cfab60b6e34fb18731d7._comment
new file mode 100644
index 0000000000..0fe5dfc5c7
--- /dev/null
+++ b/doc/bugs/__96__lookupkey__96___unexpectedly_slow/comment_1_13003e3b46c5cfab60b6e34fb18731d7._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-10-30T18:51:11Z"
+ content="""
+This is due to lookupkey passing each filename through `git ls-files`
+in order to support absolute filepaths input. See 
+[[!commit cfdfe4df6c8b3fe46bbc7afcc8274237a5b2ce2a]]
+
+Made it only do that for absolute paths, which should make it at least
+marginally faster than git-annex find. I would not expect it to be much
+faster though, because git-annex find displaying a little more information
+takes negligible CPU time really.
+"""]]
diff --git a/doc/bugs/____adjusted_branch_subtree_regression__58___FAIL_.mdwn b/doc/bugs/____adjusted_branch_subtree_regression__58___FAIL_.mdwn
new file mode 100644
index 0000000000..16054a13ed
--- /dev/null
+++ b/doc/bugs/____adjusted_branch_subtree_regression__58___FAIL_.mdwn
@@ -0,0 +1,14 @@
+### Please describe the problem.
+
+in recent builds [eg](https://github.com/datalad/git-annex/actions/runs/7417377022/job/20183877530#step:7:1288)
+
+```
+    adjusted branch subtree regression: FAIL (1.26s)
+      Test.hs:1675:
+      a/b/x/y missing from master after adjusted branch sync
+      Use -p '/adjusted branch subtree regression/' to rerun this test only.
+```
+
+may be more
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/____adjusted_branch_subtree_regression__58___FAIL_/comment_1_2b2040240fe8f01a6348eb68959d9319._comment b/doc/bugs/____adjusted_branch_subtree_regression__58___FAIL_/comment_1_2b2040240fe8f01a6348eb68959d9319._comment
new file mode 100644
index 0000000000..2f5dbc69ff
--- /dev/null
+++ b/doc/bugs/____adjusted_branch_subtree_regression__58___FAIL_/comment_1_2b2040240fe8f01a6348eb68959d9319._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2024-01-10T20:32:54Z"
+ content="""
+This was caused by [[!commit 2c86651180d5c8648b8244be894c26bc98a6c334]]
+and so I've reverted that commit for now.
+
+Leaving this bug open because that commit was an important optimisation and
+needs to be fixed and reapplied.
+"""]]
diff --git a/doc/bugs/copy_--fast_--from_--to_checks_destination_files.mdwn b/doc/bugs/copy_--fast_--from_--to_checks_destination_files.mdwn
new file mode 100644
index 0000000000..64cf7c1025
--- /dev/null
+++ b/doc/bugs/copy_--fast_--from_--to_checks_destination_files.mdwn
@@ -0,0 +1,88 @@
+### Please describe the problem.
+
+I need to "quickly" ensure that remote has all the files it should have gotten. For that I use invocation like
+
+```
+time git annex copy --fast --from web --to dandi-dandisets-dropbox
+```
+
+or
+
+```
+time git annex copy --auto --from web --to dandi-dandisets-dropbox
+```
+
+but then in the cases where all files are already there according to
+
+```
+dandi@drogon:/mnt/backup/dandi/dandisets/000003$ time git annex find  --not --in dandi-dandisets-dropbox
+
+real    0m0.562s
+user    0m0.051s
+sys     0m0.019s
+```
+
+the `copy` still goes and checks every chunk of every file
+
+```
+dandi@drogon:/mnt/backup/dandi/dandisets/000003$ time git annex copy --fast --from web --to dandi-dandisets-dropbox
+copy sub-YutaMouse20/sub-YutaMouse20_ses-YutaMouse20-140321_behavior+ecephys.nwb Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+^C
+
+real    0m3.886s
+user    0m0.037s
+sys     0m0.032s
+
+```
+
+so to achieve what I need, I thought to explicitly specify the query:
+
+```
+dandi@drogon:/mnt/backup/dandi/dandisets/000003$ time git annex copy --fast --not --in dandi-dandisets-dropbox --from web --to dandi-dandisets-dropbox
+
+real    0m0.221s
+user    0m0.056s
+sys     0m0.018s
+```
+
+but it  doesn't works out correctly whenever there are some files to actually copy:
+
+```
+dandi@drogon:/mnt/backup/dandi/dandisets/000037$ git annex find --in web --not --in dandi-dandisets-dropbox | nl | tail -n 2
+    40  sub-440889/sub-440889_ses-837360280_obj-raw_behavior+image+ophys.nwb
+    41  sub-440889/sub-440889_ses-838633305_obj-raw_behavior+image+ophys.nwb
+dandi@drogon:/mnt/backup/dandi/dandisets/000037$ git annex copy --fast --from web --to dandi-dandisets-dropbox --not --in dandi-dandisets-dropbox
+dandi@drogon:/mnt/backup/dandi/dandisets/000037$ git annex copy --fast --from web --to dandi-dandisets-dropbox --in web --not --in dandi-dandisets-dropbox
+dandi@drogon:/mnt/backup/dandi/dandisets/000037$ git annex copy --from web --to dandi-dandisets-dropbox --in web --not --in dandi-dandisets-dropbox
+```
+
+so the only way now would be to pipe `find` output into `copy`?
+
+note on edit: filed a dedicated [https://git-annex.branchable.com/bugs/copy_--from_--to_does_not_copy_if_present_locally/](https://git-annex.branchable.com/bugs/copy_--from_--to_does_not_copy_if_present_locally/) 
+
+NB `git annex find` has `-z` for input but not for output...
+
+
+refs to related reports/issues which were said to be addressed for `--fast` mode:
+
+- [https://git-annex.branchable.com/forum/copy_--auto_copies_already_synced_files/](https://git-annex.branchable.com/forum/copy_--auto_copies_already_synced_files/)
+- [https://git-annex.branchable.com/forum/batch_check_on_remote_when_using_copy/](https://git-annex.branchable.com/forum/batch_check_on_remote_when_using_copy/)
+
+### What version of git-annex are you using? On what operating system?
+
+
+```
+10.20230321-1~ndall+1
+```
+
+and then in conda with `10.20230626-g801c4b7` 
+
+[[!meta author=yoh]]
+[[!tag projects/dandi]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/copy_--fast_--from_--to_checks_destination_files/comment_1_2a2997e6f914afb0477f2baa69b174fc._comment b/doc/bugs/copy_--fast_--from_--to_checks_destination_files/comment_1_2a2997e6f914afb0477f2baa69b174fc._comment
new file mode 100644
index 0000000000..5682b0cc8a
--- /dev/null
+++ b/doc/bugs/copy_--fast_--from_--to_checks_destination_files/comment_1_2a2997e6f914afb0477f2baa69b174fc._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-11-17T20:57:19Z"
+ content="""
+> but it doesn't works out correctly whenever there are some files to actually copy
+
+I think that was due to the bug you linked, which is now fixed.
+
+I've confirmed that `--fast` is not actually implemented for `git-annex
+copy --from --to`. Explicitly specifying `--not --in destremote` is a
+fine workaround. But I've gone ahead and implemented `--fast` for it too.
+"""]]
diff --git a/doc/bugs/copy_--fast_--from_--to_checks_destination_files/comment_2_f7bf30e8cc2d1995976bde723dfbfe01._comment b/doc/bugs/copy_--fast_--from_--to_checks_destination_files/comment_2_f7bf30e8cc2d1995976bde723dfbfe01._comment
new file mode 100644
index 0000000000..a12cc7c372
--- /dev/null
+++ b/doc/bugs/copy_--fast_--from_--to_checks_destination_files/comment_2_f7bf30e8cc2d1995976bde723dfbfe01._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2023-11-17T21:16:35Z"
+ content="""
+BTW `git-annex find --print0`  is the output eqivilant of -z.
+"""]]
diff --git a/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally.mdwn b/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally.mdwn
new file mode 100644
index 0000000000..af7a4e0da8
--- /dev/null
+++ b/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally.mdwn
@@ -0,0 +1,93 @@
+### Please describe the problem.
+
+originally reported while composing [https://git-annex.branchable.com/bugs/copy_--fast_--from_--to_checks_destination_files/](https://git-annex.branchable.com/bugs/copy_--fast_--from_--to_checks_destination_files/) but it is a separate issue: some files are simply not `annex copy`'ed at all: here it tries 6 out of 8 files and still reports that 2 are not on the target remote:
+
+```
+(git-annex) dandi@drogon:/mnt/backup/dandi/dandisets/000235$ git annex copy --from web --to dandi-dandisets-dropbox --fast
+copy sub-Fish01-GCaMP-vlgut-FBd-5dpf-RandomWave/sub-Fish01-GCaMP-vlgut-FBd-5dpf-RandomWave_ses-20210818T112556_behavior+ophys.nwb Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 696.194 MBytes (730012683 Bytes)
+(from web...) (to dandi-dandisets-dropbox...) ok
+copy sub-Fish13-GCaMP-vlgut-FBv-5dpf-RandomWave/sub-Fish13-GCaMP-vlgut-FBv-5dpf-RandomWave_ses-20210830T100716_behavior+ophys.nwb Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 224.618 MBytes (235528804 Bytes)
+(from web...) (to dandi-dandisets-dropbox...) ok
+copy sub-Fish31-GCaMP-vlgut-FBd-5dpf-RandomWave/sub-Fish31-GCaMP-vlgut-FBd-5dpf-RandomWave_ses-20210920T120959_behavior+ophys.nwb Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 295.387 MBytes (309735634 Bytes)
+(from web...) (to dandi-dandisets-dropbox...) ok
+copy sub-Fish32-GCaMP-vlgut-FBv-5dpf-RandomWave/sub-Fish32-GCaMP-vlgut-FBv-5dpf-RandomWave_ses-20210920T181347_behavior+ophys.nwb Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 860.168 MBytes (901951882 Bytes)
+(from web...) (to dandi-dandisets-dropbox...) ok
+copy sub-Fish47-GCaMP-vlgut-FBd-7dpf-RandomWave/sub-Fish47-GCaMP-vlgut-FBd-7dpf-RandomWave_ses-20211124T174401_behavior+ophys.nwb Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 856.342 MBytes (897939760 Bytes)
+(from web...) (to dandi-dandisets-dropbox...) ok
+copy sub-Fish58-GCaMP-vlgut-FBd-5dpf-RandomWave/sub-Fish58-GCaMP-vlgut-FBd-5dpf-RandomWave_ses-20220525T092829_behavior+ophys.nwb Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 953.674 MBytes (1000000000 Bytes)
+Total objects: 1 Total size: 948.656 MBytes (994737479 Bytes)
+(from web...) (to dandi-dandisets-dropbox...) ok
+
+
+(git-annex) dandi@drogon:/mnt/backup/dandi/dandisets/000235$ git annex find --in web --not --in dandi-dandisets-dropbox | nl
+     1  sub-Fish02-GCaMP-vlgut-FBv-5dpf-RandomWave/sub-Fish02-GCaMP-vlgut-FBv-5dpf-RandomWave_ses-20210818T173531_behavior+ophys.nwb
+     2  sub-Fish41-GCaMP-vlgut-FBv-7dpf-RandomWave/sub-Fish41-GCaMP-vlgut-FBv-7dpf-RandomWave_ses-20210929T173736_behavior+ophys.nwb
+```
+
+and it seems to boil down (at least in one case, don't know yet if generalizes to other cases I have) to having those keys present locally:
+
+
+```
+(git-annex) dandi@drogon:/mnt/backup/dandi/dandisets/000235$ git annex find --in web --not --in dandi-dandisets-dropbox | xargs ls -lL
+-r--r--r-- 1 dandi dandi 3878847966 Mar 16  2023 sub-Fish02-GCaMP-vlgut-FBv-5dpf-RandomWave/sub-Fish02-GCaMP-vlgut-FBv-5dpf-RandomWave_ses-20210818T173531_behavior+ophys.nwb
+-r--r--r-- 1 dandi dandi 3665589468 Mar 16  2023 sub-Fish41-GCaMP-vlgut-FBv-7dpf-RandomWave/sub-Fish41-GCaMP-vlgut-FBv-7dpf-RandomWave_ses-20210929T173736_behavior+ophys.nwb
+```
+
+but somehow it doesn't know that it has them according to `list`:
+
+```
+(git-annex) dandi@drogon:/mnt/backup/dandi/dandisets/000235$ git annex list
+here
+|github
+||dandiapi
+|||web
+||||bittorrent
+|||||dandi-dandisets-dropbox (untrusted)
+||||||
+__XX_x sub-Fish01-GCaMP-vlgut-FBd-5dpf-RandomWave/sub-Fish01-GCaMP-vlgut-FBd-5dpf-RandomWave_ses-20210818T112556_behavior+ophys.nwb
+__XX__ sub-Fish02-GCaMP-vlgut-FBv-5dpf-RandomWave/sub-Fish02-GCaMP-vlgut-FBv-5dpf-RandomWave_ses-20210818T173531_behavior+ophys.nwb
+__XX_x sub-Fish13-GCaMP-vlgut-FBv-5dpf-RandomWave/sub-Fish13-GCaMP-vlgut-FBv-5dpf-RandomWave_ses-20210830T100716_behavior+ophys.nwb
+__XX_x sub-Fish31-GCaMP-vlgut-FBd-5dpf-RandomWave/sub-Fish31-GCaMP-vlgut-FBd-5dpf-RandomWave_ses-20210920T120959_behavior+ophys.nwb
+__XX_x sub-Fish32-GCaMP-vlgut-FBv-5dpf-RandomWave/sub-Fish32-GCaMP-vlgut-FBv-5dpf-RandomWave_ses-20210920T181347_behavior+ophys.nwb
+__XX__ sub-Fish41-GCaMP-vlgut-FBv-7dpf-RandomWave/sub-Fish41-GCaMP-vlgut-FBv-7dpf-RandomWave_ses-20210929T173736_behavior+ophys.nwb
+__XX_x sub-Fish47-GCaMP-vlgut-FBd-7dpf-RandomWave/sub-Fish47-GCaMP-vlgut-FBd-7dpf-RandomWave_ses-20211124T174401_behavior+ophys.nwb
+__XX_x sub-Fish58-GCaMP-vlgut-FBd-5dpf-RandomWave/sub-Fish58-GCaMP-vlgut-FBd-5dpf-RandomWave_ses-20220525T092829_behavior+ophys.nwb
+
+```
+
+running without `--from web` starts the transfer:
+
+```
+git annex copy --fast --to dandi-dandisets-dropbox
+```
+
+IMHO it should perform copy from the local store into the remote since in effect it would be fulfilling the goal - adding a copy to the destination.
+I didn't check `move` command but if it does support similar `--from --to` and has similar defect -- should just compliment with dropping after from the original remote.
+
+### What version of git-annex are you using? On what operating system?
+
+10.20230626-g801c4b7 from conda-forge .
+
+[[!meta author=yoh]]
+[[!tag projects/dandi]]
+
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_1_ae76f27d9ef4ebac7ad57e6aef9a7586._comment b/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_1_ae76f27d9ef4ebac7ad57e6aef9a7586._comment
new file mode 100644
index 0000000000..75b6338818
--- /dev/null
+++ b/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_1_ae76f27d9ef4ebac7ad57e6aef9a7586._comment
@@ -0,0 +1,57 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-11-17T19:58:39Z"
+ content="""
+> -r--r--r-- 1 dandi dandi 3665589468 Mar 16  2023 sub-Fish41-GCaMP-vlgut-FBv-7dpf-RandomWave/sub-Fish41-GCaMP-vlgut-FBv-7dpf-RandomWave_ses-20210929T173736_behavior+ophys.nwb
+
+This could be an unlocked file that has gotten modified but the staged
+version is not actually present locally. Or if `git-annex fsck` on it says
+its fixing the location logs, that would tell us something happened that
+got the location tracking out of sync with reality.
+
+So possibly there's an issue that could be tracked down regarding the state
+of that file. But in either case, git-annex doesn't know it has a local
+copy of the file, so `copy --from --to` could not use it.
+
+----
+
+But: `copy --from --to` does in fact have an interesting bug:
+
+	joey@darkstar:~/tmp/bench/r2>git-annex whereis foo
+	whereis foo (2 copies)
+	  	22dfa446-7482-4c0a-92c9-70db793859fb -- joey@darkstar:~/tmp/bench/r [origin]
+	   	8a504049-2c22-4baa-9a16-218e9561608b -- joey@darkstar:~/tmp/bench/r2 [here]
+	ok
+	joey@darkstar:~/tmp/bench/r2>git-annex copy foo --from origin --to r3
+	joey@darkstar:~/tmp/bench/r2>
+
+So the file content being present locally prevents it sending it to the remote! This needs to get fixed.
+
+Hmm: In the corresponding case of `git-annex move --from --to`, it does not
+behave that way.
+
+----
+
+As far as what the behavior ought to be when a file is present locally but not on the --from remote,
+the documentation does say:
+
+       --from=remote
+
+              Copy the content of files from the specified remote to the local repository.
+
+              Any files that are not available on the remote will be silently skipped.
+
+So it is behaving as documented. I can think of two reasons why that
+documented behavior makes some sense:
+
+* The user may be intending to only copy files --to that are present in --from.
+  The local repo may have a lot of files they do not want to populate --to.
+  (For example, perhaps the goal is to make a replica of the --from
+  repository.)
+  With that said, the user could do `git-annex copy --from foo --to bar --in foo`
+  to explicitly only act on files that are present in it.
+* Performance. Needing to check if there is a local copy when there is no
+  remote copy would be a little extra work. Likely not enough to be
+  significant though.
+"""]]
diff --git a/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_2_e857c1d89b350517fcb9829e52d6c6db._comment b/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_2_e857c1d89b350517fcb9829e52d6c6db._comment
new file mode 100644
index 0000000000..d9d1bc1602
--- /dev/null
+++ b/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_2_e857c1d89b350517fcb9829e52d6c6db._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2023-11-17T20:27:37Z"
+ content="""
+> So the file content being present locally prevents it sending it to the remote!
+
+Fixed that.
+"""]]
diff --git a/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_3_492d7c932fe5663aab916aacc829fb5d._comment b/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_3_492d7c932fe5663aab916aacc829fb5d._comment
new file mode 100644
index 0000000000..a3d85237bf
--- /dev/null
+++ b/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_3_492d7c932fe5663aab916aacc829fb5d._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2023-11-17T20:33:01Z"
+ content="""
+That bug I fixed would also explain the behavior that you saw if the
+content *was* present locally, and the location log *was* out of date about
+that.
+
+In that situation, git-annex sees that the object file is present, and so
+treats the content as present, despite the location log not knowing it's
+present. Which triggers the situation of the bug I fixed, causing it to
+skip copying the file.
+
+Also, there's a pretty easy way to get into this situation. When the file
+is not present, run `git-annex --from --to`. Then interrupt it after it's
+downloaded the file --from but before it's finished sending it --to.
+This results in the file being present locally, but only transiently so it
+didn't update the location log.
+
+So my guess is you interrupted a copy like that (or it failed incomplete
+for whatever reason).
+
+Now that I've fixed that bug, the behavior in that situation is that it
+does copy the file to the remote. And then it drops the local copy since
+the location log doesn't contain it. So it resumes correctly now.
+"""]]
diff --git a/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_4_8cfb9f2c14559a7574edd29b161cf7c7._comment b/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_4_8cfb9f2c14559a7574edd29b161cf7c7._comment
new file mode 100644
index 0000000000..8afe914bd7
--- /dev/null
+++ b/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_4_8cfb9f2c14559a7574edd29b161cf7c7._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2023-11-17T20:42:24Z"
+ content="""
+So that leaves only the question of what it should do when
+content is present locally but not on the --from remote.
+
+Another reason for the current behavior is to be symmetric with `git-annex
+move --from foo --to bar`. It would be surprising, I think, if that
+populated bar with files that are not present in foo, but are in the local
+repository!
+
+So I'm inclined to not change the documented behavior. If you want to
+populate a remote with files that are either in the local repo or in a
+--from remote, you can just run `git-annex copy` twice after all.
+
+(Or there could be a new option like `git-annex copy --to bar --from foo --or-from-here`)
+"""]]
diff --git a/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_5_fd23d0559018c531595cd06f81290258._comment b/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_5_fd23d0559018c531595cd06f81290258._comment
new file mode 100644
index 0000000000..0354a9a3a8
--- /dev/null
+++ b/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_5_fd23d0559018c531595cd06f81290258._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 5"
+ date="2023-11-18T01:35:35Z"
+ content="""
+> (Or there could be a new option like git-annex copy --to bar --from foo --or-from-here)
+
+or may be 
+
+`git-annex copy --to bar --from remote1 --or-from remote2 ...` or alike so there could be a sequence (in order of preference) of remotes?  or better a general `git-annex copy --to bar --from-anywhere` so that `annex` first `get`'s it following current set costs etc if not present here, and then copies over.
+"""]]
diff --git a/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_6_c374eb44ea08f220dbcce5ecb88403fb._comment b/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_6_c374eb44ea08f220dbcce5ecb88403fb._comment
new file mode 100644
index 0000000000..38e8ea92d2
--- /dev/null
+++ b/doc/bugs/copy_--from_--to_does_not_copy_if_present_locally/comment_6_c374eb44ea08f220dbcce5ecb88403fb._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2023-11-30T18:26:30Z"
+ content="""
+I like the idea of `copy --from-anywhere --to=remote` and just 
+use the lowest cost remote (when not in local repo). Like `git-annex get`
+and `git-annex copy --to=here`.
+
+Hmm, if there's a remote that is too expensive to want to use in such a
+copy, it would be possible to use `-c remote.foo.annex-ignore=true`
+to make it avoid using that remote. As can also be done in the case of
+`git-annex get`, although that was not documented well.
+
+I've implemented --from-anywhere..
+"""]]
diff --git a/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade.mdwn b/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade.mdwn
new file mode 100644
index 0000000000..93cbcec6f4
--- /dev/null
+++ b/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade.mdwn
@@ -0,0 +1,96 @@
+### Please describe the problem.
+
+When running git-annex sync on a regular repository, where a remote bare git repository is only, I am encountering the following:
+
+[[!format sh """
+$ git-annex sync
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+git-annex: fd:17: Data.ByteString.hGetLine: end of file
+
+  Remote webdav: Repository /u01/webdav/www/databases.git-annex.uxgit/MultiArchPackages.git-annex.git is at unsupported version 9. Automatic upgrade failed!
+commit
+On branch master
+nothing to commit, working tree clean
+ok
+pull webdav
+ok
+"""]]
+
+I found two fixed bugs that seem to mirror this:
+
+* <https://git-annex.branchable.com/projects/datalad/bugs-done/annex_view_barfs__fatal__58___Unable_to_add___40__null__41___to_database/>
+* <https://git-annex.branchable.com/projects/datalad/bugs-done/add_--force-small_fails_on_modified_submodules/>
+
+Unlike those bugs, I am not (or at least not aware of) using submodules.
+
+### What steps will reproduce the problem?
+
+Apologies, at this stage I haven't tried to set this up as a reproducible scenario, but I suspect it will not be an easily contrived scenario because of repository versions.
+
+### What version of git-annex are you using? On what operating system?
+
+Using neurodebian on actual Debian 12, amd64:
+
+[[!format sh """
+$ git-annex version
+git-annex version: 10.20230321-1~ndall+1
+build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.26 DAV-1.3.4 feed-1.3.0.1 ghc-8.8.4 http-client-0.6.4.1 persistent-sqlite-2.10.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.1.0
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: linux x86_64
+supported repository versions: 8 9 10
+upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10
+local repository version: 9
+"""]]
+
+### Please provide any additional information below.
+
+Just to avoid confusion, yes, this is webdav shared, but I am not using a webdav special repository. I am actually allowing the repository to be mounted via webdav.
+
+Also, I've started using neurodebian's git-annex, because I want to get nearer the bleeding edge across Debian, Ubuntu and Windows hosts.
+
+If I change directory to that remote itself, and run the command (determined from --debug) to perform the upgrade, I see this:
+
+[[!format sh """
+$ cd /u01/webdav/www/databases.git-annex.uxgit/MultiArchPackages.git-annex.git/
+$ git-annex version
+git-annex version: 10.20230321-1~ndall+1
+build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.26 DAV-1.3.4 feed-1.3.0.1 ghc-8.8.4 http-client-0.6.4.1 persistent-sqlite-2.10.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.1.0
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: linux x86_64
+supported repository versions: 8 9 10
+upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10
+local repository version: 9
+$ git-annex upgrade --quiet --autoonly
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+fatal: Unable to add (null) to database
+git-annex: fd:17: Data.ByteString.hGetLine: end of file
+"""]]
+
+### Have you had any luck using git-annex before?
+
+Yes, it's one of my favourite opensource tools.
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade/comment_1_041e664be0a5ae635cf5848d9fee380c._comment b/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade/comment_1_041e664be0a5ae635cf5848d9fee380c._comment
new file mode 100644
index 0000000000..a80db46ea4
--- /dev/null
+++ b/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade/comment_1_041e664be0a5ae635cf5848d9fee380c._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-10-09T18:28:15Z"
+ content="""
+The "unsupported version 9" is wrong, version 9 is still supported and so
+this upgrade failing won't prevent git-annex from using the repository.
+I've corrected the message.
+"""]]
diff --git a/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade/comment_2_b0deace2dcadbd50c0e0cb155041b255._comment b/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade/comment_2_b0deace2dcadbd50c0e0cb155041b255._comment
new file mode 100644
index 0000000000..3d8a5a145c
--- /dev/null
+++ b/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade/comment_2_b0deace2dcadbd50c0e0cb155041b255._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2023-10-09T18:29:08Z"
+ content="""
+As to why the upgrade fails, it seems to me that mounting webdav as a
+filesystem could easily have incompatabilities that cause strange behavior.
+
+git can fail with this error message in at least 3 different ways. Although
+in all cases it's supposed to be displaying a path where it has "(null)",
+so it seems something is causing git to have a null pointer. That seems
+like it almost has to be a bug in git.
+
+It might help to run the git-annex command with --debug to determine what
+git command is doing this.
+"""]]
diff --git a/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade/comment_3_60f745051264fcacdce882eb8646b3fe._comment b/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade/comment_3_60f745051264fcacdce882eb8646b3fe._comment
new file mode 100644
index 0000000000..9d5d53c472
--- /dev/null
+++ b/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade/comment_3_60f745051264fcacdce882eb8646b3fe._comment
@@ -0,0 +1,56 @@
+[[!comment format=mdwn
+ username="beryllium@5bc3c32eb8156390f96e363e4ba38976567425ec"
+ nickname="beryllium"
+ avatar="http://cdn.libravatar.org/avatar/62b67d68e918b381e7e9dd6a96c16137"
+ subject="comment 3"
+ date="2023-10-09T22:30:46Z"
+ content="""
+Thanks joey. Based off of your comments, I believe I have found the root cause, which you are right, I don't think it's git-annex.
+
+Also, I realise I should have added a bit more detail.
+
+The /u01/webdav/www/ path is not a webdav mount, it's the backing real filesystem for actual webdav mounts. And this is important.
+
+Further, this scenario worked with the Debian 11 version of git-annex, and I think those other bugs I linked to might account for the difference, because handing of dot files changed... but I'm not sure.
+
+Also, I did run --debug, but the reason I reached out is, the git calls were of the form:
+
+[[!format  sh \"\"\"
+git --git-dir=. --literal-pathspecs -c annex.debug=true hash-object -w --no-filters --stdin-paths
+\"\"\"]]
+
+and I couldn't work out what paths were being passed via stdin. However, I've run strace and the reason I believe I see these messages is this:
+
+[[!format sh \"\"\"
+$ find . -name .DAV -print
+./.DAV
+./refs/heads/.DAV
+./refs/heads/synced/.DAV
+./info/.DAV
+./annex/.DAV
+./annex/journal/.DAV
+./hooks/.DAV
+./objects/71/.DAV
+./objects/ba/.DAV
+./objects/80/.DAV
+./objects/15/.DAV
+./objects/7a/.DAV
+./objects/a8/.DAV
+./objects/b0/.DAV
+./objects/27/.DAV
+./objects/pack/.DAV
+./objects/a5/.DAV
+./objects/33/.DAV
+./objects/28/.DAV
+./objects/b6/.DAV
+./objects/77/.DAV
+./objects/ad/.DAV
+./objects/94/.DAV
+\"\"\"]]
+
+So for me, the solution is to use an actual webdav mount, even on the local hosting server, and that will hide these webdav \"internals\"/metadata.
+
+Is there cause at all for config changes around git-annex to avoid that though?
+
+
+"""]]
diff --git a/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade/comment_4_9e32aa105bbf11a6d2610a62bfa9517b._comment b/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade/comment_4_9e32aa105bbf11a6d2610a62bfa9517b._comment
new file mode 100644
index 0000000000..ade1760f19
--- /dev/null
+++ b/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade/comment_4_9e32aa105bbf11a6d2610a62bfa9517b._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="beryllium@5bc3c32eb8156390f96e363e4ba38976567425ec"
+ nickname="beryllium"
+ avatar="http://cdn.libravatar.org/avatar/62b67d68e918b381e7e9dd6a96c16137"
+ subject="comment 4"
+ date="2023-10-10T02:36:51Z"
+ content="""
+I should say that, sometimes I forget my internal monologue/designs.
+
+My initial intention was to use the webdav mount exclusively, but somehow I setup my remote to the backing filesystem path
+
+So, there's no need for a workaround really. Please free to close.
+"""]]
diff --git a/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade/comment_5_28d35ef30f76ec326363d86ab2948ad3._comment b/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade/comment_5_28d35ef30f76ec326363d86ab2948ad3._comment
new file mode 100644
index 0000000000..abf36c782f
--- /dev/null
+++ b/doc/bugs/encountering_fatal_error_on_auto_annex_upgrade/comment_5_28d35ef30f76ec326363d86ab2948ad3._comment
@@ -0,0 +1,32 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2023-10-10T17:00:37Z"
+ content="""
+Reproduced as follows:
+
+	joey@darkstar:~/tmp/bench>git init  --bare dav
+	Initialized empty Git repository in /home/joey/tmp/bench/dav/
+	joey@darkstar:~/tmp/bench>cd dav
+	joey@darkstar:~/tmp/bench/dav>git-annex init --version=9
+	init  ok
+	(recording state in git...)
+	joey@darkstar:~/tmp/bench/dav>for s in $(find -type d); do mkdir $s/.DAV;done
+	joey@darkstar:~/tmp/bench/dav>git-annex init --version=9
+	init  fatal: Unable to add (null) to database
+
+So it's these empty directories indeed. (Empty .DAV files don't cause this.)
+
+In particular, it's any empty directory in .git/annex/journal. Which is
+supposed to only contain files that git-annex wrote there. Staging the journal
+is why git hash-object gets involved.
+
+	mkdir .DAV
+	echo .DAV | git hash-object -w --stdin-paths
+	fatal: Unable to add .DAV to database
+
+Still unclear why git ends up with "(null)" in the error message.
+
+While it will slow git-annex down a tiny bit to check if it's a regular file,
+it seems better for git-annex to be robust against this kind of pollution.
+"""]]
diff --git a/doc/bugs/fresh_test_fails__58___hPut__58___invalid_argument_.mdwn b/doc/bugs/fresh_test_fails__58___hPut__58___invalid_argument_.mdwn
new file mode 100644
index 0000000000..cbcbedbd1e
--- /dev/null
+++ b/doc/bugs/fresh_test_fails__58___hPut__58___invalid_argument_.mdwn
@@ -0,0 +1,33 @@
+### Please describe the problem.
+
+See e.g. on [https://github.com/datalad/git-annex/actions/runs/6680765679/job/18154374923](https://github.com/datalad/git-annex/actions/runs/6680765679/job/18154374923)
+
+```
+  Repo Tests v10 unlocked
+    Init Tests
+      init:                                   OK (0.17s)
+      add:                                    OK (0.73s)
+    addurl:                                   OK (0.57s)
+    crypto:                                   FAIL (3.07s)
+      ./Test/Framework.hs:86:
+      initremote failed with unexpected exit code (transcript follows)
+      initremote foo (encryption setup) (to gpg keys: 129D6E0AC537B9C7) 
+      git-annex: .git/annex/othertmp/remote.log: hPut: invalid argument (invalid character)
+      failed
+      (recording state in git...)
+      initremote: 1 failed
+```
+
+started only recently but consistently:
+
+```
+(git)smaug:/mnt/datasets/datalad/ci/git-annex/builds/2023/10[master]git
+$> git grep -l 'hPut: invalid argument'
+cron-20231027/build-ubuntu.yaml-1289-1c03c8fd-failed/0_test-annex (normal, ubuntu-latest).txt
+...
+```
+
+[[!meta author=yoh]]
+[[!tag projects/repronim]]
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/fresh_test_fails__58___hPut__58___invalid_argument_/comment_1_6b2c645f9ca440f4707c671a94566bb7._comment b/doc/bugs/fresh_test_fails__58___hPut__58___invalid_argument_/comment_1_6b2c645f9ca440f4707c671a94566bb7._comment
new file mode 100644
index 0000000000..586c9184bf
--- /dev/null
+++ b/doc/bugs/fresh_test_fails__58___hPut__58___invalid_argument_/comment_1_6b2c645f9ca440f4707c671a94566bb7._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-11-01T16:07:15Z"
+ content="""
+Reproduced with LANG=C:
+
+      ./Test/Framework.hs:86:
+      initremote failed with unexpected exit code (transcript follows)
+      initremote foo (encryption setup) (to gpg keys: 129D6E0AC537B9C7)
+      git-annex: .git/annex/othertmp/remote.log: withFile: invalid argument (cannot encode character '\132')
+      failed
+      (recording state in git...)
+      initremote: 1 failed
+
+Not quite the same error but almost certianly the same problem.
+
+I've confirmed this is caused by 
+[[!commit 3742263c99180d1391e4fd51724aae52d6d02137]]
+"""]]
diff --git a/doc/bugs/fresh_test_fails__58___hPut__58___invalid_argument_/comment_2_52185ae4ebdfcb61840444e3ef1e0404._comment b/doc/bugs/fresh_test_fails__58___hPut__58___invalid_argument_/comment_2_52185ae4ebdfcb61840444e3ef1e0404._comment
new file mode 100644
index 0000000000..ea1469e6cd
--- /dev/null
+++ b/doc/bugs/fresh_test_fails__58___hPut__58___invalid_argument_/comment_2_52185ae4ebdfcb61840444e3ef1e0404._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2023-11-01T16:53:48Z"
+ content="""
+Will probably need to revert the Remote/Helper/Encryptable.hs part of that
+commit. 
+
+What is happening here is, encodeBS is failing when run on the String from
+a SharedPubKeyCipher. That String comes from Utility.Gpg.genRandom and is
+literally a bunch of random bytes. So it's not encoded with the filesystem
+encoding. And it really ought to be a ByteString of course, but since it's
+not, anything involving encoding it fails.
+
+That's why the old code had this comment:
+
+	{- Not using Utility.Base64 because these "Strings" are really
+	 - bags of bytes and that would convert to unicode and not round-trip
+	 - cleanly. -}
+
+And converted that String to a ByteString via `B.pack . s2w8`, which avoids this problem.
+
+What an ugly thing. Really ought to be fixed to use ByteString throughout. 
+But for now, let's revert.
+"""]]
diff --git a/doc/bugs/fsck_ignores_--json-error-messages_when_--quiet.mdwn b/doc/bugs/fsck_ignores_--json-error-messages_when_--quiet.mdwn
new file mode 100644
index 0000000000..83c4d44db3
--- /dev/null
+++ b/doc/bugs/fsck_ignores_--json-error-messages_when_--quiet.mdwn
@@ -0,0 +1,76 @@
+### Please describe the problem.
+
+As I understand the manual:
+
+- Options `--json --json-error-messages` are provided so that another program can parse the `git annex fsck` results. 
+
+- Option `--quiet` is provided to list only problems (not print anything for OK files). 
+
+However, when options are combined, only plain text error messages are provided, no json output is provided.
+
+I understand this may be "as designed", quiet is quiet... But then how to log only errors in json? I have +300k files in the annex, and no need to log when everything is fine.
+
+### What steps will reproduce the problem?
+
+Create a repo with files b and c 
+
+Corrupt file b
+ 
+`git annex fsck  --json --json-error-messages --quiet`
+
+I expected to have a json output with only files that fail the fsck, instead I get only normal stderr, just like with 
+`git annex fsck  --quiet`
+
+### What version of git-annex are you using? On what operating system?
+10.20230926-12 on arch
+
+### Please provide any additional information below.
+
+[[!format sh """
+
+# Expected plain result 
+> git annex fsck
+
+fsck b 
+  ** No known copies exist of b
+failed
+fsck c (checksum...) ok
+(recording state in git...)
+fsck: 1 failed
+
+# Expected json result (error message to stderr, both logs)
+> git annex fsck --json
+
+  ** No known copies exist of b
+{"command":"fsck","dead":[],"error-messages":[],"file":"b","input":["b"],"key":"SHA256E-s5--f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2","success":false,"untrusted":[]}
+{"command":"fsck","error-messages":[],"file":"c","input":["c"],"key":"SHA256E-s4--530a0b93b8c1ea618546d3aaa6ec71f888d2a6095322bfdb1b04c9225e26481e","note":"checksum...","success":true}
+fsck: 1 failed
+
+# Expected json output with error message embedded 
+> git annex fsck  --json --json-error-messages
+
+{"command":"fsck","dead":[],"error-messages":["** No known copies exist of b"],"file":"b","input":["b"],"key":"SHA256E-s5--f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2","success":false,"untrusted":[]}
+{"command":"fsck","error-messages":[],"file":"c","input":["c"],"key":"SHA256E-s4--530a0b93b8c1ea618546d3aaa6ec71f888d2a6095322bfdb1b04c9225e26481e","note":"checksum...","success":true}
+fsck: 1 failed
+
+# Expected only error message
+> git annex fsck --quiet
+
+  ** No known copies exist of b
+fsck: 1 failed
+
+# UnExpected result: I expected a json output with the error message embedded "--json --json-error-messages" seem ignored here
+> git annex fsck  --json --json-error-messages --quiet
+
+  ** No known copies exist of b
+fsck: 1 failed
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+Yes, great tool ! Thanks !
+
+
+> [[wontfix|done]] per my comment --[[Joey]]
diff --git a/doc/bugs/fsck_ignores_--json-error-messages_when_--quiet/comment_1_a23d96af8c0e418350a73cbce5bc24be._comment b/doc/bugs/fsck_ignores_--json-error-messages_when_--quiet/comment_1_a23d96af8c0e418350a73cbce5bc24be._comment
new file mode 100644
index 0000000000..f0f8660751
--- /dev/null
+++ b/doc/bugs/fsck_ignores_--json-error-messages_when_--quiet/comment_1_a23d96af8c0e418350a73cbce5bc24be._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-11-21T20:03:45Z"
+ content="""
+Hmm, I've never considered combining --quiet with --json. It's kind of
+undefined and really not clear to me what it should do.
+
+But, --json-error-messages makes the json contain an error-messages field,
+and the error message is in there. So you can just extract that and ignore
+the other messages in the json output. No need to use --quiet then.
+
+I suppose there may be someone who uses --json as a matter of course, but
+adds --quiet to that when they want to disable the json output. So
+changing the current behavior, ill-defined as it is, would be asking for
+trouble.
+
+What actually happens currently is which ever output option comes last
+overrides earlier options. So `--json --quiet` is quiet, and `--quiet
+--json` outputs json. `--json-error-messages` is like `--json` in this
+regard to. Which is just behavior that fell out of the option parser
+implementation.
+"""]]
diff --git a/doc/bugs/fsck_says_ok_for_a_file_that_is_not_in_the_remote.mdwn b/doc/bugs/fsck_says_ok_for_a_file_that_is_not_in_the_remote.mdwn
new file mode 100644
index 0000000000..a8d249d324
--- /dev/null
+++ b/doc/bugs/fsck_says_ok_for_a_file_that_is_not_in_the_remote.mdwn
@@ -0,0 +1,47 @@
+### Please describe the problem.
+fsck says ok while file is not present in the remote
+
+The remote looks like this:
+`apple-slicer	gcrypt::rsync://apple-slicer/encr/gits/important_docs`
+
+```
+> git annex fsck --from apple-slicer Taxes/2018-19/Joint_T5.pdf                       
+fsck Taxes/2018-19/Joint_T5.pdf ok
+(recording state in git...)
+```
+
+This portion in the log does not look good
+
+```
+[2023-10-20 18:06:23.514382] (Utility.Process) process [63613] done ExitSuccess
+[2023-10-20 18:06:23.515339] (Utility.Process) process [63621] read: rsync ["apple-slicer:/encr/gits/important_docs/annex/objects/093/d6e/GPGHMACSHA1--47089ada3aaae768ea5e1d2ccc9284ac83a1d80f/GPGHMACSHA1--47089ada3aaae768ea5e1d2ccc9284ac83a1d80f"]
+[2023-10-20 18:06:24.344801] (Utility.Process) process [63621] done ExitFailure 23
+[2023-10-20 18:06:24.346693] (Utility.Process) process [63630] read: rsync ["apple-slicer:/encr/gits/important_docs/annex/objects/f9/V3/GPGHMACSHA1--47089ada3aaae768ea5e1d2ccc9284ac83a1d80f/GPGHMACSHA1--47089ada3aaae768ea5e1d2ccc9284ac83a1d80f"]
+[2023-10-20 18:06:25.148852] (Utility.Process) process [63630] done ExitFailure 23
+ok
+[2023-10-20 18:06:25.152624] (Utility.Process) process [63599] done ExitSuccess
+```
+
+Full log: https://privatebin.net/?c70841788ba0b6c1#9aCknSWqLjzQZd3n1hJUCK4GFEcoPFd1cZrfnWWoF5zm
+
+### What steps will reproduce the problem?
+1. Create a special remote of type `gcrypt::rsync`
+2. `git annex fsck --from {special remote} --debug`
+
+### What version of git-annex are you using? On what operating system?
+macOS
+`git-annex version: 10.20230626`
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+> [[notabug|done]] --[[Joey]]
diff --git a/doc/bugs/fsck_says_ok_for_a_file_that_is_not_in_the_remote/comment_1_ce9085e6330625dabf479e6e3beeeb47._comment b/doc/bugs/fsck_says_ok_for_a_file_that_is_not_in_the_remote/comment_1_ce9085e6330625dabf479e6e3beeeb47._comment
new file mode 100644
index 0000000000..218cc0050b
--- /dev/null
+++ b/doc/bugs/fsck_says_ok_for_a_file_that_is_not_in_the_remote/comment_1_ce9085e6330625dabf479e6e3beeeb47._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-10-21T16:26:06Z"
+ content="""
+I don't see a bug here. If the file is not present there and was not
+expected to be present there, and there are enough copies in other places,
+then fsck has nothing to complain about.
+"""]]
diff --git a/doc/bugs/fsck_says_ok_for_a_file_that_is_not_in_the_remote/comment_2_59e89c065a3cd8f7c80cfdb3c2c2808d._comment b/doc/bugs/fsck_says_ok_for_a_file_that_is_not_in_the_remote/comment_2_59e89c065a3cd8f7c80cfdb3c2c2808d._comment
new file mode 100644
index 0000000000..dde7cc6805
--- /dev/null
+++ b/doc/bugs/fsck_says_ok_for_a_file_that_is_not_in_the_remote/comment_2_59e89c065a3cd8f7c80cfdb3c2c2808d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Manager2103"
+ avatar="http://cdn.libravatar.org/avatar/c3e024b0067a77f1ef99c6dd88614060"
+ subject="comment 2"
+ date="2023-10-21T17:22:53Z"
+ content="""
+I see. How could I check the consistency of all the files in a given remote? I thought that was the command
+"""]]
diff --git a/doc/bugs/fsck_says_ok_for_a_file_that_is_not_in_the_remote/comment_3_264a1b39c1677545af4c02901dd23d22._comment b/doc/bugs/fsck_says_ok_for_a_file_that_is_not_in_the_remote/comment_3_264a1b39c1677545af4c02901dd23d22._comment
new file mode 100644
index 0000000000..ba110ba0b5
--- /dev/null
+++ b/doc/bugs/fsck_says_ok_for_a_file_that_is_not_in_the_remote/comment_3_264a1b39c1677545af4c02901dd23d22._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2023-10-23T17:41:04Z"
+ content="""
+Well `fsck --from` does check the consistency of what's in the remote.
+That entails verifying that the remote contains the data that
+git-annex thinks it contains. But it also means verifying that the remote
+does not contain any data that git-annex didn't know it contains. 
+
+Checking the file is not present on the remote is part of the latter.
+
+If you wanted to only verify that files git-annex thinks are on the remote
+are present there, you could use eg `git-annex fsck --from theremote --in theremote`
+"""]]
diff --git a/doc/bugs/fsck_says_ok_for_a_file_that_is_not_in_the_remote/comment_4_f08e305e39953c5c1906df0fb2eb113b._comment b/doc/bugs/fsck_says_ok_for_a_file_that_is_not_in_the_remote/comment_4_f08e305e39953c5c1906df0fb2eb113b._comment
new file mode 100644
index 0000000000..2abae48cc5
--- /dev/null
+++ b/doc/bugs/fsck_says_ok_for_a_file_that_is_not_in_the_remote/comment_4_f08e305e39953c5c1906df0fb2eb113b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="Manager2103"
+ avatar="http://cdn.libravatar.org/avatar/c3e024b0067a77f1ef99c6dd88614060"
+ subject="comment 4"
+ date="2023-10-23T17:50:22Z"
+ content="""
+That makes perfect sense. Thank you so much!
+
+And thanks for some fantastic software :)
+"""]]
diff --git a/doc/bugs/git-annex-import_stalls_and_uses_all_ram_available.mdwn b/doc/bugs/git-annex-import_stalls_and_uses_all_ram_available.mdwn
new file mode 100644
index 0000000000..3efe9ba570
--- /dev/null
+++ b/doc/bugs/git-annex-import_stalls_and_uses_all_ram_available.mdwn
@@ -0,0 +1,66 @@
+### Please describe the problem.
+
+I'm not sure if what I am experiencing is a bug, or just something I am doing incorrectly.
+
+I am running into an issue with git-annex-import where it seems to stall, and use all available ram it can until the system terminates the process.
+
+### What steps will reproduce the problem?
+
+Here are my prep steps:
+
+```sh
+git init
+
+git annex initremote s3-data type=S3 encryption=none port=443 protocol=https public=no \
+importtree=yes versioning=yes host=$S3HOST bucket=$BUCKET fileprefix=primary_folder/
+
+git annex wanted s3-data "exclude=subfolder-*/* and include=specialfilename1.*"
+
+git annex import main --from s3-data --skip-duplicates --backend MD5E --jobs=4
+```
+
+### What version of git-annex are you using? On what operating system?
+
+Originally, 10.20230321 on Debian Bookworm
+
+I also tried 10.20231129 on Debian Bookworm with the same results
+
+### Please provide any additional information below.
+
+There are around 22000 files under the prefix I am trying to import from , and it amounts to around 115 GB.  However, most of that data is part of many seperate subdatasets underneath this one.  These have all worked fine and without any issue.
+
+There are only 2 files I am actually trying to import, though there are several versions(about 70 each for a total of 140) of them at this location.  In this example, that is `specialfilename1.json` & `specialfilename1.csv`
+
+When I use debug mode on the import command a lot of information is printed, but it mostly seems to amount to filenames that I would think would be excluded based on my `git-annex-wanted` command.  That output looks like the following until it stops, and then uses up what's available for RAM before inevitably terminating the process.
+
+```
+[2023-12-18 23:47:54.924814306] (Utility.Process) process [11787] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","git-annex"]
+[2023-12-18 23:47:54.929719575] (Utility.Process) process [11787] done ExitSuccess
+[2023-12-18 23:47:54.930053775] (Utility.Process) process [11789] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","--hash","refs/heads/git-annex"]
+[2023-12-18 23:47:54.935010757] (Utility.Process) process [11789] done ExitSuccess
+[2023-12-18 23:47:54.935508638] (Utility.Process) process [11790] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","log","refs/heads/git-annex..a6d7c6ae03747e23c2bedbecc8d1a5afeabe5220","--pretty=%H","-n1"]
+[2023-12-18 23:47:54.940887356] (Utility.Process) process [11790] done ExitSuccess
+[2023-12-18 23:47:54.941241371] (Utility.Process) process [11791] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","log","refs/heads/git-annex..27a0dedea083605106c614a159d48fd3daa92284","--pretty=%H","-n1"]
+[2023-12-18 23:47:54.947198539] (Utility.Process) process [11791] done ExitSuccess
+[2023-12-18 23:47:54.949236306] (Utility.Process) process [11792] chat: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","cat-file","--batch"]
+[2023-12-18 23:47:54.971233148] (Utility.Process) process [11793] read: git ["--git-dir=.git","--work-tree=.","--literal-pathspecs","-c","annex.debug=true","show-ref","--hash","refs/remotes/s3-data/main"]
+[2023-12-18 23:47:54.97613841] (Utility.Process) process [11793] done ExitFailure 1
+...
+
+String to sign: "GET\n\n\nMon, 18 Dec 2023 23:47:57 GMT\n/bucketname/?versions"
+[2023-12-18 23:47:57.978207613] (Remote.S3) Host: "bucketname.s3-us-east-1.amazonaws.com"
+[2023-12-18 23:47:57.978237701] (Remote.S3) Path: "/"
+[2023-12-18 23:47:57.978260264] (Remote.S3) Query string: "versions&key-marker=primary_folder%subfolder-100101%2sessions.json&prefix=primary_folder%2F&version-id-marker=Xg1KUaCh6tpvJ2E1juz4qobn.w3.x9k"
+[2023-12-18 23:47:57.978337803] (Remote.S3) Header: [("Date","Mon, 18 Dec 2023 23:47:57 GMT"),("Authorization","AWS [Redacted]")]
+[2023-12-18 23:47:58.003376688] (Remote.S3) Response status: Status {statusCode = 200, statusMessage = "OK"}
+[2023-12-18 23:47:58.00343782] (Remote.S3)  Response header 'Transfer-Encoding': 'chunked'
+[2023-12-18 23:47:58.003472907] (Remote.S3) Response header 'x-amz-request-id': 'tx00000925b31abf8c9c162-006580da2d-19170577-default'
+[2023-12-18 23:47:58.003527331] (Remote.S3) Response header 'Content-Type': 'application/xml'
+[2023-12-18 23:47:58.003557859] (Remote.S3) Response header 'Date': 'Mon, 18 Dec 2023 23:47:58 GMT
+```
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+In the past, on smaller versions of data structured the same way, this setup has worked, and I don't run into this issue.
+
+I'm not exactly sure how to troubleshoot further and I am feeling stuck.  Is there something else I can be doing to see more info about what's happening behind the scenes?
diff --git a/doc/bugs/git-annex-import_stalls_and_uses_all_ram_available/comment_1_ca22165fc77a57979ad6e19c60693b21._comment b/doc/bugs/git-annex-import_stalls_and_uses_all_ram_available/comment_1_ca22165fc77a57979ad6e19c60693b21._comment
new file mode 100644
index 0000000000..fff51cddf7
--- /dev/null
+++ b/doc/bugs/git-annex-import_stalls_and_uses_all_ram_available/comment_1_ca22165fc77a57979ad6e19c60693b21._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-12-20T16:56:04Z"
+ content="""
+How much ram did it use up?
+
+The fact that the S3 bucket is versioned and that there are many versions
+seems very relevant to me. Importing lists all the files in the bucket, and
+traverses all versions and lists all the files in each version. That builds
+up a data structure in memory, which could be very large in this case. If
+you have around 150 versions total, the number of files in the data
+structure would be effectively three million.
+
+If the same thing works for you with `versioning=no` set, that will confirm
+the source of the problem.
+
+It only gets filtered down to the wanted files in a subsequent pass.
+Filtering on the fly would certainly help with your case, but not with a
+case where someone wants to import all 22000 files.
+
+Rather, I'd be inclined to try to fix this by making importableHistory into
+a callback so it can request one historical tree at a time. Similar to how
+ImportableContentsChunked works.
+"""]]
diff --git a/doc/bugs/git-annex-import_stalls_and_uses_all_ram_available/comment_2_2025b1662067bbfaab14576d9e20a6ab._comment b/doc/bugs/git-annex-import_stalls_and_uses_all_ram_available/comment_2_2025b1662067bbfaab14576d9e20a6ab._comment
new file mode 100644
index 0000000000..b3c0b13cc6
--- /dev/null
+++ b/doc/bugs/git-annex-import_stalls_and_uses_all_ram_available/comment_2_2025b1662067bbfaab14576d9e20a6ab._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2023-12-20T17:34:11Z"
+ content="""
+I looked into filtering to preferred content on the fly. I was able to
+adapt listImportableContents to allow that to be done optionally. But of
+the remotes that support tree import, only directory and S3 would be able
+to use it. And I've not yet managed to implement it for S3. My incomplete 
+work on this is in the `importtreefilter` branch.
+"""]]
diff --git a/doc/bugs/git-annex-import_stalls_and_uses_all_ram_available/comment_3_5943379450c860b2885121e7f0abd42a._comment b/doc/bugs/git-annex-import_stalls_and_uses_all_ram_available/comment_3_5943379450c860b2885121e7f0abd42a._comment
new file mode 100644
index 0000000000..a7806f550f
--- /dev/null
+++ b/doc/bugs/git-annex-import_stalls_and_uses_all_ram_available/comment_3_5943379450c860b2885121e7f0abd42a._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="lemondata"
+ avatar="http://cdn.libravatar.org/avatar/481bf8163b175e5d0bdbef7d8c7fb3f4"
+ subject="comment 3"
+ date="2024-01-02T15:44:21Z"
+ content="""
+The first system I tried capped out at 8GB of ram, and it used around 7 before ending the process with `error: git-annex died of signal 9`.  Since I initially thought it might just be a ram problem I attempted the same on a system with 32GB of ram, but arrived at the same results after the system used around 31GB of ram.  I was not able to try any higher due to my own physical hardware restrictions.
+
+Just to help confirm the suspicion of the problem I tried the same thing with `versioning=no` set, and the import worked great.
+
+Since I do not want to import all 22000 files and I just need two items at this prefix(and their historical versions), I think some way to filter on the fly would make a huge difference.  Alternatively, like you mentioned, importing one historical tree at a time sounds like it would ease the ram requirements here too.
+
+Is there anything I can do to help further with testing?  Or is there any more information I can give about the issue that would be helpful to you?
+"""]]
diff --git a/doc/bugs/git-annex_findkeys_does_not_know_--largerthan.mdwn b/doc/bugs/git-annex_findkeys_does_not_know_--largerthan.mdwn
new file mode 100644
index 0000000000..591564f055
--- /dev/null
+++ b/doc/bugs/git-annex_findkeys_does_not_know_--largerthan.mdwn
@@ -0,0 +1,50 @@
+### Please describe the problem.
+
+The man page of  [git-annex findkeys](https://git-annex.branchable.com/git-annex-findkeys/) says: 
+
+> OPTIONS
+>
+>  *  matching options
+>     The git-annex-matching-options(1) can be used to specify which keys to list.
+
+However, this is not true for the options that match file size. Being able to do for example `git-annex findkeys --largerthan 100M`
+is important for me as it allows me to track down the files that occupy most of my storage, allowing me to move them to some
+archive. However, if I try to call the above command, it does not show me a list of keys matching the criterion, but a help page
+as if I have misspelled the option:
+
+>     $ git annex findkeys --largerthan 1
+>     Invalid option `--largerthan'
+>     
+>     Usage: git-annex COMMAND
+>       git-annex - manage files with git, without checking their contents in
+>     
+>     Commonly used commands:
+>
+>     add             add files to annex
+>     [...]
+
+### What steps will reproduce the problem?
+
+[[!format sh """
+    echo hi > file
+    git annex init
+    git annex add file
+    git commit -m "Test commit"
+    git annex find --largerthan 1   # << this lists "file"
+    git annex findkeys --largerthan 1   # << this fails
+"""]]
+
+### What version of git-annex are you using? On what operating system?
+
+* git-annex version: 10.20230926-g44a7b4c9734adfda5912dd82c1aa97c615689f57
+* Rocky Linux 9.2
+* git 2.40.1
+
+
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+I am using it for my research project (data science/predictions in plant breeding) and it allows me to keep track of the
+current model iteration and associated results. Thank you for this! 
+
+> [[fixed|done]] --[[Joey]]
diff --git a/doc/bugs/git-annex_findkeys_does_not_know_--largerthan/comment_1_f6484f69669439ef49513b84d3bd91cf._comment b/doc/bugs/git-annex_findkeys_does_not_know_--largerthan/comment_1_f6484f69669439ef49513b84d3bd91cf._comment
new file mode 100644
index 0000000000..05ea2c90a0
--- /dev/null
+++ b/doc/bugs/git-annex_findkeys_does_not_know_--largerthan/comment_1_f6484f69669439ef49513b84d3bd91cf._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-11-28T15:48:09Z"
+ content="""
+Not all options in git-annex-matching-options can be used by findkeys. It
+mentions this when it says "Some of these options can also be used by
+commands to specify which keys they act on."
+
+However in this case, --largerthan and --smallerthan could in fact be made
+to operate on keys, and I've done so.
+"""]]
diff --git a/doc/bugs/git_annex_enableremote_locking_issue.mdwn b/doc/bugs/git_annex_enableremote_locking_issue.mdwn
new file mode 100644
index 0000000000..0fc2ca3f25
--- /dev/null
+++ b/doc/bugs/git_annex_enableremote_locking_issue.mdwn
@@ -0,0 +1,56 @@
+### Please describe the problem.
+
+When enabling an S3 remote with hybrid encryption, I get a file locking error:
+
+```
+$ git annex enableremote s3remote
+git-annex: .git/annex/creds/__S3_REMOTE_UUID_REDACTED__: openFile: resource busy (file is locked)
+```
+
+I can confirm that this happens when no previous process is running (and lsof shows nothing open before running enableremote), for example in the motivating case here of cloning to a new, freshly-mounted HDD.
+
+If I copy the `.git/annex/creds/__S3_REMOTE_UUID_REDACTED__` file from an existing clone of the repository to this new one, then enableremote works fine.
+
+```
+$ git annex enableremote s3remote              
+enableremote s3remote (encryption update) (to gpg keys: HEXREDACTED) ok
+(recording state in git...)
+```
+
+### What steps will reproduce the problem?
+
+For me, with a source a repository that has a pre-configured S3 special remote:
+
+```
+$ git show git-annex:remote.log
+__UUID__ bucket=bucket-name cipher=__base64_jibberish__ cipherkeys=GPGKEY_HEX datacenter=US host=s3.eu-central-003.backblazeb2.com name=s3remote partsize=512MiB port=80 protocol=https s3creds=__base64_jibberish__ signature=v4 storageclass=STANDARD type=S3 timestamp=168000000.0s
+```
+
+Then, to reproduce this:
+
+```
+git clone /path/to/established-repo new_clone
+cd new_clone
+git annex init new_clone
+git annex enableremote s3remote
+```
+
+
+
+### What version of git-annex are you using? On what operating system?
+
+The latest:
+
+```
+$ git annex version
+git-annex version: 10.20230926-g44a7b4c9734adfda5912dd82c1aa97c615689f57
+```
+
+### Please provide any additional information below.
+
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+A great deal, we use it daily in our research and I also use it on some personal files -- thanks for all your efforts Joey et al.!
+
+> Reproduced and fixed. [[done]] --[[Joey]]
diff --git a/doc/bugs/git_init_fails_on_a_worktree_branch.mdwn b/doc/bugs/git_init_fails_on_a_worktree_branch.mdwn
new file mode 100644
index 0000000000..511d405c4d
--- /dev/null
+++ b/doc/bugs/git_init_fails_on_a_worktree_branch.mdwn
@@ -0,0 +1,45 @@
+### Please describe the problem.
+I tried git annex init on a worktree checkout of a branch, but got an error (see below).
+The worktree is for a repo that is itself a submodule of another repo.
+
+### What steps will reproduce the problem?
+I can try later to make an isolated reproducible example, but I think the above scenario describes it.
+
+### What version of git-annex are you using? On what operating system?
+
+[[!format sh """
+[bpb23-acc /data/ilya/iwork/marti/tmp/wtree/is-231002-1155-marti-cpp]$ git annex version
+git-annex version: 10.20230626-g801c4b7
+build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.29 DAV-1.3.4 feed-1.3.2.0 ghc-8.10.7 http-client-0.7.9 persistent-sqlite-2.13.0.3 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.1.2
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: linux x86_64
+supported repository versions: 8 9 10
+upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10
+local repository version: 10
+(pbase-025-env) Wed 15 Nov 2023 01:37:24 PM EST
+[bpb23-acc /data/ilya/iwork/marti/tmp/wtree/is-231002-1155-marti-cpp]$ uname -a
+Linux bpb23-acc 5.4.0-136-generic #153-Ubuntu SMP Thu Nov 24 15:56:58 UTC 2022 x86_64 GNU/Linux
+(pbase-025-env) Wed 15 Nov 2023 01:37:25 PM EST
+[bpb23-acc /data/ilya/iwork/marti/tmp/wtree/is-231002-1155-marti-cpp]$ 
+"""]]
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+[bpb23-acc /data/ilya/iwork/marti/tmp/wtree/is-231002-1155-marti-cpp]$ git annex init
+init  
+git-annex: worktrees/is-231002-1155-marti-cpp/info/attributes: openFile: does not exist (No such file or directory)
+failed
+init: 1 failed
+
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+Well I keep coming back to it as a solution :)  Right now I'm trying to set up proper test cases for a tool that processes sequencing data, and even moderate-size test files break github's size limits.  I'm also planning to demo git-annex to others in my group so they can start using it.
+
diff --git a/doc/bugs/git_init_fails_on_a_worktree_branch/comment_1_6614fc4b8f191702c1b78c5a3ce5de50._comment b/doc/bugs/git_init_fails_on_a_worktree_branch/comment_1_6614fc4b8f191702c1b78c5a3ce5de50._comment
new file mode 100644
index 0000000000..37dfc93118
--- /dev/null
+++ b/doc/bugs/git_init_fails_on_a_worktree_branch/comment_1_6614fc4b8f191702c1b78c5a3ce5de50._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-11-21T20:01:25Z"
+ content="""
+I don't use worktrees or submodules much so it's not entirely apparent to
+me how to reproduce this. To avoid flailing at it, a recipe would be great.
+"""]]
diff --git a/doc/bugs/identically_configured_remotes_behave_differently.mdwn b/doc/bugs/identically_configured_remotes_behave_differently.mdwn
new file mode 100644
index 0000000000..2bd85d3758
--- /dev/null
+++ b/doc/bugs/identically_configured_remotes_behave_differently.mdwn
@@ -0,0 +1,134 @@
+### Please describe the problem.
+
+Identically configured remotes behave differently in the "git annex push" command after "git annex sync" to only one of them.
+
+### What steps will reproduce the problem?
+
+I have a repository on Linux and two USB backup sticks, all Ext4 file system. At one point I accidentally did "git annex sync music-backup-one".
+
+When I now "git annex push music-backup-two" it works as expected and I see the correct files on the stick and "git annex list" shows the content correctly. This is not true for "git annex push music-backup-one" as I must also issue "git annex mirror . --to=music-backup-one" for the content and "git annex list" to be correct (both commands are required in either order).
+
+How do I diagnose the problem so push works without the need for mirror on music-backup-one?
+
+During my investigation, I noticed the filemode (executable) bit for annexed files is not transferred to either USB drive if that is the only change (perhaps a separate issue).
+
+### What version of git-annex are you using? On what operating system?
+
+10.20231130-g0e9bc415882a5e3a285e004cf9f80936e5762a07
+
+### Please provide any additional information below.
+
+[[!format sh """
+# If you can, paste a complete transcript of the problem occurring here.
+# If the problem is with the git-annex assistant, paste in .git/annex/daemon.log
+
+All three repos have:
+'annex.largefiles' set to 'mimeencoding=binary',
+'group' set to 'manual'
+'wanted' set to 'standard' and
+'numcopies' set to 3.
+
+desktop-music has adjust option --unlock and the two backups have option --lock.
+
+Here is the contents of .git/config for desktop-music:
+
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = false
+	logallrefupdates = true
+[annex]
+	uuid = 0045d866-c80e-43c1-9b1b-b15a8c97c1aa
+	version = 10
+	adjustedbranchrefresh = true
+[filter "annex"]
+	smudge = git-annex smudge -- %f
+	clean = git-annex smudge --clean -- %f
+	process = git-annex filter-process
+[push]
+	followTags = true
+[receive]
+	denyCurrentBranch = updateInstead
+[remote "music-backup-one"]
+	url = /media/juanito/music-backup-one/music-library
+	fetch = +refs/heads/*:refs/remotes/music-backup-one/*
+	annex-uuid = 0b439b9f-16c7-4945-8cb2-9d6084677af3
+[remote "music-backup-two"]
+	url = /media/juanito/music-backup-two/music-library
+	fetch = +refs/heads/*:refs/remotes/music-backup-two/*
+	annex-uuid = 74a6b33c-fea2-45ef-a0c5-b677b43bc85f
+[remote "github"]
+	url = git@github.com:not-for-you.git
+	fetch = +refs/heads/*:refs/remotes/github/*
+	annex-ignore = true
+[remote "itunes-media"]
+	annex-directory = /home/juanito/annex-remotes/itunes-media
+	annex-uuid = e18dbc8e-8e0f-47f2-8f33-e8eb9da97a62
+	skipFetchAll = true
+	annex-tracking-branch = main:Music
+[remote "linux-media"]
+	annex-directory = /home/juanito/annex-remotes/linux-media
+	annex-uuid = b4efca25-5be6-4ea5-b0f9-207770abf21b
+	skipFetchAll = true
+	annex-tracking-branch = main:Music
+
+Here is the contents of .git/config for music-backup-one:
+
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = false
+	logallrefupdates = true
+[remote "desktop-music"]
+	url = /home/juanito/git/music-library
+	fetch = +refs/heads/*:refs/remotes/desktop-music/*
+	annex-uuid = 0045d866-c80e-43c1-9b1b-b15a8c97c1aa
+[branch "adjusted/main(unlocked)"]
+	remote = desktop-music
+	merge = refs/heads/adjusted/main(unlocked)
+[annex]
+	uuid = 0b439b9f-16c7-4945-8cb2-9d6084677af3
+	version = 10
+	adjustedbranchrefresh = true
+[filter "annex"]
+	smudge = git-annex smudge -- %f
+	clean = git-annex smudge --clean -- %f
+	process = git-annex filter-process
+[push]
+	followTags = true
+[receive]
+	denyCurrentBranch = updateInstead
+
+Here is the contents of .git/config for music-backup-two:
+
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = false
+	logallrefupdates = true
+[remote "desktop-music"]
+	url = /home/juanito/git/music-library
+	fetch = +refs/heads/*:refs/remotes/desktop-music/*
+	annex-uuid = 0045d866-c80e-43c1-9b1b-b15a8c97c1aa
+[branch "adjusted/main(unlocked)"]
+	remote = desktop-music
+	merge = refs/heads/adjusted/main(unlocked)
+[annex]
+	uuid = 74a6b33c-fea2-45ef-a0c5-b677b43bc85f
+	version = 10
+	adjustedbranchrefresh = true
+[filter "annex"]
+	smudge = git-annex smudge -- %f
+	clean = git-annex smudge --clean -- %f
+	process = git-annex filter-process
+[push]
+	followTags = true
+[receive]
+	denyCurrentBranch = updateInstead
+
+# End of transcript or log.
+"""]]
+
+### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
+
+
diff --git a/doc/bugs/resolvemerge_fails_when_unlocked_empty_files_exist/comment_5_3b5f7c44de6b17e042adaa77a8bed2f8._comment b/doc/bugs/resolvemerge_fails_when_unlocked_empty_files_exist/comment_5_3b5f7c44de6b17e042adaa77a8bed2f8._comment
new file mode 100644
index 0000000000..f4604748ab
--- /dev/null
+++ b/doc/bugs/resolvemerge_fails_when_unlocked_empty_files_exist/comment_5_3b5f7c44de6b17e042adaa77a8bed2f8._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="aurelien@f0d0a0c7da69eff6badf0464898f0a859f69114d"
+ nickname="aurelien"
+ avatar="http://cdn.libravatar.org/avatar/2cbf28821cf4b36380554c22f0b8b809"
+ subject="comment 5"
+ date="2024-01-14T23:18:43Z"
+ content="""
+I think I have been hit by this bug as well with getting \"fatal: stash failed\" after importing some files and trying to sync.
+
+Indeed, temporarily deactivating filter.annex.process and filter.annex.clean allowed to sync. 
+
+And the problem didn't reappear after so far. 
+Is there a reliable workaround (the bug report seem to have no responses?). 
+
+git version 2.43.0
+git-annex version: 10.20230926-g4ac2758ba589562e427a66437b9fdcd5172357e1
+"""]]
diff --git a/doc/bugs/resolvemerge_fails_when_unlocked_empty_files_exist/comment_6_baa4f1e895ff351b16f54d59851d2a6f._comment b/doc/bugs/resolvemerge_fails_when_unlocked_empty_files_exist/comment_6_baa4f1e895ff351b16f54d59851d2a6f._comment
new file mode 100644
index 0000000000..f31a66c28f
--- /dev/null
+++ b/doc/bugs/resolvemerge_fails_when_unlocked_empty_files_exist/comment_6_baa4f1e895ff351b16f54d59851d2a6f._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2024-01-18T17:13:40Z"
+ content="""
+Unfortunately I never heard back from the git developers about the bug.
+"""]]
diff --git a/doc/bugs/sync_does_not_sync_content_with_new_remote_on_first_run.mdwn b/doc/bugs/sync_does_not_sync_content_with_new_remote_on_first_run.mdwn
new file mode 100644
index 0000000000..8de0d7c637
--- /dev/null
+++ b/doc/bugs/sync_does_not_sync_content_with_new_remote_on_first_run.mdwn
@@ -0,0 +1,23 @@
+	joey@darkstar:~/tmp/bench/me>git init ../other
+	Initialized empty Git repository in /home/joey/tmp/bench/other/.git/
+	joey@darkstar:~/tmp/bench/me>git remote add other ../other
+	joey@darkstar:~/tmp/bench/me>ls
+	joey@darkstar:~/tmp/bench/me>date > foo
+	joey@darkstar:~/tmp/bench/me>git-annex add foo
+
+With this setup, `git-annex sync --content` does not send foo to other
+the first time run. However, on the second run it, does.
+
+If the other repo had `git-annex init` ran in it first, it would sync content
+to it on the first run.
+
+Auto-init only happens once the git-annex branch gets pushed to the remote
+and git-annex enumerates that remote.
+So after the first sync in this situation, the remote has a synced/git-annex 
+branch, but no uuid yet. The second sync then auto-inits.
+
+To fix this, sync could re-enumerate remotes after pushing, I suppose. But
+re-enumerating remotes is some work so it would need to do it only for ones that
+have no uuid. 
+
+Saw this happening in yann's talk. --[[Joey]]
diff --git a/doc/bugs/sync_does_not_sync_content_with_new_remote_on_first_run/comment_1_c1df7f51ac724400dbef664ce10784a8._comment b/doc/bugs/sync_does_not_sync_content_with_new_remote_on_first_run/comment_1_c1df7f51ac724400dbef664ce10784a8._comment
new file mode 100644
index 0000000000..4458913735
--- /dev/null
+++ b/doc/bugs/sync_does_not_sync_content_with_new_remote_on_first_run/comment_1_c1df7f51ac724400dbef664ce10784a8._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="Similar to initremote type=git"
+ date="2023-12-07T08:31:08Z"
+ content="""
+Cool, this would reduce surprises for sure! 😀
+
+On that note, IIRC `initremote type=git` straight up refuses to proceed if the remote does not have a UUID (i.e. has had `git annex init` run in it). Might be worth auto-init-ing here as well when you're already at it.
+"""]]
diff --git a/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__.mdwn b/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__.mdwn
new file mode 100644
index 0000000000..af10cc024d
--- /dev/null
+++ b/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__.mdwn
@@ -0,0 +1,73 @@
+### Please describe the problem.
+
+Doing backup of data using `git annex move` from local to dropbox via `git-annex-remote-rclone`.
+
+Full invocation here is
+
+```shell
+( source ~/git-annexes/10.20231227+git24-gd37dbd62b8.env; OUR_LOG_FILE=~/.cache/rclone/logs/`date --iso-8601=seconds`-movedebug; RCLONE_LOG_LEVEL=DEBUG RCLONE_LOG_FILE=${OUR_LOG_FILE}_rclone.log git annex --debug move --to dandi-dandisets-dropbox sub-484677/sub-484677_ses-20210419T161140_behavior+ecephys+ogen.nwb 2>&1 | tee ${OUR_LOG_FILE}_git-annex.log ; )
+```
+under `/mnt/backup/dandi/dandisets/000363` . On git-annex side we see
+
+```shell
+dandi@drogon:~/.cache/rclone/logs$ grep -4 'Transfer stalled' 2024-01-04T14:28:29-05:00-movedebug_git-annex.log
+[2024-01-04 14:43:24.5722525] (Annex.TransferrerPool) < op 295000000000
+75%   274.74 GiB        7 MiB/s 3h46m2024/01/04 14:44:49 DEBUG : Setting --log-level "DEBUG" from environment variable RCLONE_LOG_LEVEL="DEBUG"
+2024/01/04 14:44:49 DEBUG : Setting --log-file "/home/dandi/.cache/rclone/logs/2024-01-04T14:28:29-05:00-movedebug_rclone.log" from environment variable RCLONE_LOG_FILE="/home/dandi/.cache/rclone/logs/2024-01-04T14:28:29-05:00-movedebug_rclone.log"
+
+  Transfer stalled
+[2024-01-04 14:45:40.042053663] (Utility.Process) process [2329029] chat: /home/dandi/git-annexes/10.20231227+git24-gd37dbd62b8/usr/lib/git-annex.linux/git-annex ["transferrer","-c","annex.debug=true"]
+
+```
+
+and rclone logs for around that time point(s):
+
+```
+2024/01/04 14:44:49 DEBUG : rclone: Version "v1.61.1-DEV" starting with parameters ["rclone" "copy" ".git/annex/tmp/SHA256E-s393314879887-S1000000000-C296--38c8c93c66ece7cbe8eeca9621ab72bf3968873e842f3e376ddda012eb0a0e5f.nwb" "dandi-dandisets-dropbox:dandi-dandisets/annexstore/b4f/757/"]
+2024/01/04 14:44:49 DEBUG : Creating backend with remote ".git/annex/tmp/SHA256E-s393314879887-S1000000000-C296--38c8c93c66ece7cbe8eeca9621ab72bf3968873e842f3e376ddda012eb0a0e5f.nwb"
+2024/01/04 14:44:49 DEBUG : Using config file from "/home/dandi/.rclone.conf"
+2024/01/04 14:44:49 DEBUG : fs cache: adding new entry for parent of ".git/annex/tmp/SHA256E-s393314879887-S1000000000-C296--38c8c93c66ece7cbe8eeca9621ab72bf3968873e842f3e376ddda012eb0a0e5f.nwb", "/mnt/backup/dandi/dandisets/000363/.git/annex/tmp"
+2024/01/04 14:44:49 DEBUG : Creating backend with remote "dandi-dandisets-dropbox:dandi-dandisets/annexstore/b4f/757/"
+2024/01/04 14:44:49 DEBUG : fs cache: renaming cache item "dandi-dandisets-dropbox:dandi-dandisets/annexstore/b4f/757/" to be canonical "dandi-dandisets-dropbox:dandi-dandisets/annexstore/b4f/757"
+2024/01/04 14:44:49 DEBUG : SHA256E-s393314879887-S1000000000-C296--38c8c93c66ece7cbe8eeca9621ab72bf3968873e842f3e376ddda012eb0a0e5f.nwb: Need to transfer - File not found at Destination
+2024/01/04 14:44:50 DEBUG : SHA256E-s393314879887-S1000000000-C296--38c8c93c66ece7cbe8eeca9621ab72bf3968873e842f3e376ddda012eb0a0e5f.nwb: Uploading chunk 1/20
+...
+2024/01/04 14:45:36 DEBUG : SHA256E-s393314879887-S1000000000-C296--38c8c93c66ece7cbe8eeca9621ab72bf3968873e842f3e376ddda012eb0a0e5f.nwb: Uploading chunk 19/20
+2024/01/04 14:45:38 DEBUG : SHA256E-s393314879887-S1000000000-C296--38c8c93c66ece7cbe8eeca9621ab72bf3968873e842f3e376ddda012eb0a0e5f.nwb: Uploading chunk 20/20
+2024/01/04 14:45:39 INFO  : Signal received: interrupt
+2024/01/04 14:45:39 INFO  : Dropbox root 'dandi-dandisets/annexstore/b4f/757': Committing uploads - please wait...
+2024/01/04 14:45:39 INFO  : Exiting...
+```
+
+<details>
+<summary>using older git-annex I got similar crash but on earlier chunk, so it is not pertinent to last chunk</summary> 
+
+```shell
+2024/01/04 13:56:29 DEBUG : rclone: Version "v1.61.1-DEV" starting with parameters ["rclone" "copy" ".git/annex/tmp/SHA256E-s393314879887-S1000000000-C287--38c8c93c66ece7cbe8eeca9621ab72bf3968873e842f3e376ddda012eb0a0e5f.nwb" "dandi-dandisets-dropbox:dandi-dandisets/annexstore/b4f/757/"]
+2024/01/04 13:56:29 DEBUG : Creating backend with remote ".git/annex/tmp/SHA256E-s393314879887-S1000000000-C287--38c8c93c66ece7cbe8eeca9621ab72bf3968873e842f3e376ddda012eb0a0e5f.nwb"
+2024/01/04 13:56:29 DEBUG : Using config file from "/home/dandi/.rclone.conf"
+2024/01/04 13:56:29 DEBUG : fs cache: adding new entry for parent of ".git/annex/tmp/SHA256E-s393314879887-S1000000000-C287--38c8c93c66ece7cbe8eeca9621ab72bf3968873e842f3e376ddda012eb0a0e5f.nwb", "/mnt/backup/dandi/dandisets/000363/.git/annex/tmp"
+2024/01/04 13:56:29 DEBUG : Creating backend with remote "dandi-dandisets-dropbox:dandi-dandisets/annexstore/b4f/757/"
+2024/01/04 13:56:30 DEBUG : fs cache: renaming cache item "dandi-dandisets-dropbox:dandi-dandisets/annexstore/b4f/757/" to be canonical "dandi-dandisets-dropbox:dandi-dandisets/annexstore/b4f/757"
+2024/01/04 13:56:30 DEBUG : SHA256E-s393314879887-S1000000000-C287--38c8c93c66ece7cbe8eeca9621ab72bf3968873e842f3e376ddda012eb0a0e5f.nwb: Need to transfer - File not found at Destination
+2024/01/04 13:56:30 DEBUG : SHA256E-s393314879887-S1000000000-C287--38c8c93c66ece7cbe8eeca9621ab72bf3968873e842f3e376ddda012eb0a0e5f.nwb: Uploading chunk 1/20
+...
+2024/01/04 13:57:08 DEBUG : SHA256E-s393314879887-S1000000000-C287--38c8c93c66ece7cbe8eeca9621ab72bf3968873e842f3e376ddda012eb0a0e5f.nwb: Uploading chunk 17/20
+2024/01/04 13:57:10 DEBUG : SHA256E-s393314879887-S1000000000-C287--38c8c93c66ece7cbe8eeca9621ab72bf3968873e842f3e376ddda012eb0a0e5f.nwb: Uploading chunk 18/20
+2024/01/04 13:57:11 DEBUG : SHA256E-s393314879887-S1000000000-C287--38c8c93c66ece7cbe8eeca9621ab72bf3968873e842f3e376ddda012eb0a0e5f.nwb: Uploading chunk 19/20
+2024/01/04 13:57:14 INFO  : Signal received: interrupt
+2024/01/04 13:57:14 INFO  : Dropbox root 'dandi-dandisets/annexstore/b4f/757': Committing uploads - please wait...
+2024/01/04 13:57:14 INFO  : Exiting...
+```
+</details>
+
+So it seems that stall detection (I didn't look how it even decides on that) is flawed in case of large (chunked) files being transferred through external custom remotes which have no way to report progress as it is the case here in https://github.com/git-annex-remote-rclone/git-annex-remote-rclone/blob/master/git-annex-remote-rclone (filed [a bug report](https://github.com/git-annex-remote-rclone/git-annex-remote-rclone/issues/76) to see that addressed).
+
+
+### What version of git-annex are you using? On what operating system?
+
+first was a year old version, then tried with bleeding edge 10.20231227+git24-gd37dbd62b8
+
+
+[[!meta author=yoh]]
+[[!tag projects/dandi]]
diff --git a/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_1_db83f6a38cae36da89f0ab4ef83021d8._comment b/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_1_db83f6a38cae36da89f0ab4ef83021d8._comment
new file mode 100644
index 0000000000..4c2ccc7292
--- /dev/null
+++ b/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_1_db83f6a38cae36da89f0ab4ef83021d8._comment
@@ -0,0 +1,40 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2024-01-18T17:18:07Z"
+ content="""
+It looks like you must have annex.stalldetection (or the per-remote config)
+set. git-annex does not behave this way without that configuration.
+What is it set to?
+
+You are probably right in that it involves rclone special remote not
+reporting transfer progress back to git-annex.
+
+Normally, when a special remote does not do progress reporting,
+git-annex does not do any stall detection, because there must have been
+at least some previous progress update in order for it to detect a stall.
+
+But when chunking is enabled (as it was in your case with 1 gb chunks),
+git-annex itself updates the progress after each chunk. When the special
+remote does not do any progress reporting, and chunk size is large, that
+means that the progress will be updated very infrequently. 
+
+So for example, if it takes 2 minutes to upload a chunk, and you had
+annex.stalldetection set to eg "10mb/1m", then in a chunk after the 1st one,
+git-annex would wake up after 1 minute, see that no data seemed to have
+been sent, and conclude there was a stall. You would need to change the
+time period in the config to something less granular eg "100mb/10m"
+to avoid that.
+
+This might be a documentation problem, it may not be clear to the user
+that "10mb/1m" is any different than "100mb/10m". And finding a value that
+works depends on knowing details of how often the progress gets updated
+for a given remote.
+
+But, your transcript show that the stall was detected on chunk 296.
+(git-annex's chunk, rclone is doing its own chunking to dropbox)
+So the annex.stalldetection configuration must have been one that
+worked most of the time, for it to transfer the previous 295 chunks
+without a stall having been detected. Unless this was a resume after
+previous command(s) had uploaded those other chunks.
+"""]]
diff --git a/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_2_2aeb065a257729e852055533aff04650._comment b/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_2_2aeb065a257729e852055533aff04650._comment
new file mode 100644
index 0000000000..5d8e3863e2
--- /dev/null
+++ b/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_2_2aeb065a257729e852055533aff04650._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2024-01-18T17:50:21Z"
+ content="""
+I think that what git-annex could do is detect when progress updates are
+happening with too low a granularity for the annex.stalldetection
+configuration.
+
+When waiting for the first progress update, it can keep track of how much time
+has elapsed. If annex.stalldetection is "10mb/2m" and it took 20 minutes to
+get the first progress update, the granularity is clearly too low.
+
+And then it could disregard the configuration, or suggest a better
+configuration value, or adjust what it's expecting to match the
+observed granularity.
+
+(The stall detection auto-prober uses a similar heuristic to that already.
+It observes the granularity and only if it's sufficiently low (an update
+every 30 seconds or less) does it assume that 60 seconds without an update
+may be a stall.)
+"""]]
diff --git a/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_3_f8bd6a233d835bdc413bbf0127608431._comment b/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_3_f8bd6a233d835bdc413bbf0127608431._comment
new file mode 100644
index 0000000000..dd93371133
--- /dev/null
+++ b/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_3_f8bd6a233d835bdc413bbf0127608431._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2024-01-18T19:04:26Z"
+ content="""
+To reproduce this (more or less), I modified the example.sh external
+special remote to sleep for 10 seconds before each key store.
+Set up a remote with chunk=1mb, and annex.stalldetection = "0.001mb/1s".
+
+Uploading a 100 mb file, a stall is detected after the first chunk is
+uploaded. As expected, since 1 second passed with no update.
+
+When I resume the upload, the second chunk is uploaded and then a stall is
+detected on the third. And so on.
+
+I've implemented dynamic granularity scaling now, and in this test case, it notices
+it took 11 seconds for the first chunk, and behaves as if it were
+configured with annex.stalldetection of "0.022mb/22s". Which keeps it from
+detecting a stall.
+"""]]
diff --git a/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_4_320828c8d39d1f030d1797b539dbb22e._comment b/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_4_320828c8d39d1f030d1797b539dbb22e._comment
new file mode 100644
index 0000000000..c3a991d4b1
--- /dev/null
+++ b/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_4_320828c8d39d1f030d1797b539dbb22e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 4"
+ date="2024-01-18T21:32:30Z"
+ content="""
+Thank you Joey for looking into it!  FWIW, indeed I had `stalldetection = 1KB/120s` .  But I believe I introduced it primarily for data \"download\", where git-annex actually IIRC keeps its monitoring of the file size as it is arriving.  For the `copy` out, situation is indeed different and in case of assymmetric connections I even wondered if worth allowing different values for stalldetection-get and -put?
+
+In this particular case, since original report we did adjust rclone special remote to report PROGRESS back.  I think it completed fine on few 1 large file uploads.  I will tomorrow (after we get daily git-annex build in) run it \"on full\" for a good number of files and see what happens.
+"""]]
diff --git a/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_5_9fc2d7f4b39615e43bce3993e0a6e647._comment b/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_5_9fc2d7f4b39615e43bce3993e0a6e647._comment
new file mode 100644
index 0000000000..993ea416a1
--- /dev/null
+++ b/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_5_9fc2d7f4b39615e43bce3993e0a6e647._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2024-01-19T17:31:41Z"
+ content="""
+Re asymmetric connections, stall detection doesn't need to be
+configured anywhere near the total available bandwidth, so generally it
+would be ok to configure it for a slow uplink even when a fast downlink is
+available. And if you're only interested in detecting total stalls
+(as opposed to some kind of "too slow" condition), "1k" per some unit of
+time is sufficient.
+
+But maybe it does make sense to be able to configure it to abandon slow
+downloads but allow slow uploads. For example, `git-annex get` with
+the content on several remotes, where the download speed from one
+remote is often fast but occasionally slows down, and another remote
+is consistently medium speed.
+
+So you might set "10gb/1m" for downloads from remote, knowing that if it is
+slow it will abort the download from it and fall back to the medium speed
+remote. But when sending content *to* the variable speed remote, would not
+want to give up only because it was a little slow.
+
+Ok, added annex.stalldetection-download, annex.stalldetection-upload, etc.
+"""]]
diff --git a/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_6_1b2eb9993e220082f3be5dd568deef3e._comment b/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_6_1b2eb9993e220082f3be5dd568deef3e._comment
new file mode 100644
index 0000000000..d3df8542ec
--- /dev/null
+++ b/doc/bugs/too_aggressive_in_claiming___34__Transfer_stalled__34____63__/comment_6_1b2eb9993e220082f3be5dd568deef3e._comment
@@ -0,0 +1,21 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2024-01-19T17:01:27Z"
+ content="""
+> "download", where git-annex actually IIRC keeps its monitoring of the file size as it is arriving
+
+git-annex does do file size monitoring
+in a few cases (when running `cp`, and when downloading
+an url with `curl`), but it does not do it for downloads
+in general, and not for external special remotes specifically.
+
+If it did for external special remotes, and if rclone grows the 
+destination file as it downloads, rather than atomically at the end
+(which I have not verified), that would have avoided this problem.
+Probably it will avoids this type of problem for downloads
+from the majority of external special remotes. Though of course not for
+uploads.
+
+Ok, implemented that for external special remotes.
+"""]]
diff --git a/doc/bugs/uninit_fails_when_I_symlink_your_symlink.mdwn b/doc/bugs/uninit_fails_when_I_symlink_your_symlink.mdwn
new file mode 100644
index 0000000000..1e3506cb40
--- /dev/null
+++ b/doc/bugs/uninit_fails_when_I_symlink_your_symlink.mdwn
@@ -0,0 +1,39 @@
+```
+$ mkdir annex-test
+$ cd annex-test/
+$ git init
+Initialized empty Git repository in /home/me/annex-test/.git/
+$ git annex init
+init  (scanning for unlocked files...)
+ok
+(recording state in git...)
+$ cat > .gitignore
+*
+$ cat > exampyle.txt
+hello
+$ git annex add --no-check-gitignore
+add .gitignore (non-large file; adding content to git repository) ok
+add exampyle.txt
+ok
+(recording state in git...)
+$ git commit -m \"added\"
+[master (root-commit) 2734615] added
+ 2 files changed, 2 insertions(+)
+ create mode 100644 .gitignore
+ create mode 120000 exampyle.txt
+$ ln -rs exampyle.txt what.foo
+$ git status
+On branch master
+nothing to commit, working tree clean
+$ git annex uninit
+git-annex: what.foo points to annexed content, but is not checked into git.
+Perhaps this was left behind by an interrupted git annex add?
+Not continuing with uninit; either delete or git annex add the file and retry.
+```
+What should I do?
+
+Honestly, I'm happy with git-annex so far, I'm just thinking that I need to re-init with `annex.tune.objecthashlower=true` because my other computer is windows.
+
+Thanks!
+
+> I see this was resolved as ok behavior, so [[done]] --[[Joey]]
diff --git a/doc/bugs/uninit_fails_when_I_symlink_your_symlink/comment_5_3fe7520813dee36ffb5419ed70ea43b5._comment b/doc/bugs/uninit_fails_when_I_symlink_your_symlink/comment_5_3fe7520813dee36ffb5419ed70ea43b5._comment
new file mode 100644
index 0000000000..7514aa5e68
--- /dev/null
+++ b/doc/bugs/uninit_fails_when_I_symlink_your_symlink/comment_5_3fe7520813dee36ffb5419ed70ea43b5._comment
@@ -0,0 +1,24 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="forgot to add the new ignored link + shortcoming of 'ln' command"
+ date="2023-11-20T10:25:52Z"
+ content="""
+You forgot to add your (gitignore'd) `what.foo` to git. Another `git annex add --no-check-gitignore;git commit -m \"add link\"` will make the subsequent `git annex uninit` work properly. This is also what the error message says, btw.
+
+But there's another problem underneath: Why is `what.foo` an annex-style symlink anyway? It should just point to `example.txt`, right?
+
+```
+🐟 ❯ ln -nrs example.txt what.foo
+yann in yann-desktop-nixos in …/uninit-test on  main 
+🐟 ❯ ls -l
+lrwxrwxrwx 186 yann users 20 Nov 11:08  example.txt -> .git/annex/objects/mK/4w/SHA256E-s6--5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03.txt/SHA256E-s6--5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03.txt
+lrwxrwxrwx 186 yann users 20 Nov 11:08  what.foo -> .git/annex/objects/mK/4w/SHA256E-s6--5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03.txt/SHA256E-s6--5891b5b522d5df086d0ff0b110fbd9d21bb4fc7163af34d08286a2e846f6be03.txt
+```
+
+For some reason `ln` dereferences `example.txt` before linking, resulting in `what.foo` pointing to the same as `example.txt` -- not just to `example.txt` -- causing `what.foo` to look exactly like an annex link. I thought `-n` could fix this, but it would operate on the target `what.foo`, not the source `example.txt` 🤦. I couldn't make `ln` do this properly, except for renaming/removing `example.txt`, then `ln -rsf example.txt what.foo`, then restoring `example.txt`. Not viable. A better solution is `cp -s example.txt what.foo`. That will make `what.foo` point properly to just `example.txt` and not *its* target.
+
+BTW you can still just `git add` things like normal non-annex symlinks like `what.foo`, no need for `git annex add` here.
+
+Also, joey prefers to have bug reports/questions and the like in either [[todo]], [[bugs]] or [[forum]], because it's clear those are the places to look through for issues. Comments below random manpages are quick to be forgotten about. 🙂
+"""]]
diff --git a/doc/bugs/uninit_fails_when_I_symlink_your_symlink/comment_6_3afdea4f3705a9cfaa971ed3aa9f114c._comment b/doc/bugs/uninit_fails_when_I_symlink_your_symlink/comment_6_3afdea4f3705a9cfaa971ed3aa9f114c._comment
new file mode 100644
index 0000000000..d1109eb926
--- /dev/null
+++ b/doc/bugs/uninit_fails_when_I_symlink_your_symlink/comment_6_3afdea4f3705a9cfaa971ed3aa9f114c._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="NewUser"
+ nickname="dont.post.me"
+ avatar="http://cdn.libravatar.org/avatar/90f59ddc341eaf9b2657422206c06901"
+ subject="comment 6"
+ date="2023-11-20T13:17:03Z"
+ content="""
+Thank you for the illuminating comment!
+
+> This is also what the error message says, btw.
+
+    Not continuing with uninit; either delete or git annex add the file and retry.
+
+You&rsquo;re right! I thought I only had a relative symlink pointing to the other symlink and not pointing into .git/annex/objects. It turns out that this is true for most of my symlinks but, just like you pointed out, some of them were pointing into .git/annex and the right thing to do there is to `add` them.
+
+Also, thank you for pointing me towards todo/bugs/forum!
+
+"""]]
diff --git a/doc/bugs/webapp_--listen_port_is_not_used__63__.mdwn b/doc/bugs/webapp_--listen_port_is_not_used__63__.mdwn
new file mode 100644
index 0000000000..bf9f80a68d
--- /dev/null
+++ b/doc/bugs/webapp_--listen_port_is_not_used__63__.mdwn
@@ -0,0 +1,23 @@
+### Please describe the problem.
+
+```
+reprostim@reproiner:/data/reprostim/Videos$ ps auxw | grep webapp
+reprost+   25249  0.0  0.0   9892  2100 pts/5    S+   Jan05   0:00 git annex --debug webapp --listen 0.0.0.0:8888
+reprost+   25250  5.4  0.2 1074346616 65556 ?    Ssl  Jan05  60:39 /usr/bin/git-annex --debug webapp --listen 0.0.0.0:8888
+reprost+  224039  0.0  0.0   6332  2116 pts/6    S+   10:55   0:00 grep webapp
+
+reprostim@reproiner:/data/reprostim/Videos$ lsof -i :8888
+
+reprostim@reproiner:/data/reprostim/Videos$ lsof -i | grep git-annex
+git-annex  25250 reprostim   14u  IPv4 129033      0t0  UDP *:55556
+git-annex 221230 reprostim   14u  IPv4 129033      0t0  UDP *:55556
+
+reprostim@reproiner:/data/reprostim/Videos$ git annex version
+git-annex version: 10.20230126
+
+```
+
+[[!meta author=yoh]]
+[[!tag projects/repronim]]
+
+> [[done]] --[[Joey]]
diff --git a/doc/bugs/webapp_--listen_port_is_not_used__63__/comment_1_275507b26ca13e5d55deec4b7af60699._comment b/doc/bugs/webapp_--listen_port_is_not_used__63__/comment_1_275507b26ca13e5d55deec4b7af60699._comment
new file mode 100644
index 0000000000..b05edeb28a
--- /dev/null
+++ b/doc/bugs/webapp_--listen_port_is_not_used__63__/comment_1_275507b26ca13e5d55deec4b7af60699._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2024-01-18T16:47:15Z"
+ content="""
+--listen takes an IP address (or hostname),
+it does not let you specify the port. I've clarified the documentation
+about this.
+
+I don't reproduce the behavior you show, when I try that the process
+runs but does not bind to any port, and in .git/annex/daemon.log, I see:
+
+	WebApp crashed: Network.Socket.getAddrInfo (called with preferred socket type/protocol: AddrInfo {addrFlags = [], addrFamily = AF_UNSPEC, addrSocketType = Stream, addrProtocol = 0, addrAddress = 0.0.0.0:0, addrCanonName = Nothing}, host name: Just "0.0.0.0:8888", service name: Nothing): does not exist (Name or service not known)
+
+This may be an OS or resolver difference. If "0.0.0.0:8888" somehow
+resolves to an IP address on your system, then the webapp will listen
+on that IP address I suppose. But it's not expecting that to specify a
+port.
+
+The webapp outputs the full url, including the port it chose. 
+Since that url also includes an auth token that is required to use the
+webapp, specifying the port for it to listen on does not seem very useful.
+
+What's your use case for wanting to specify a port?
+"""]]
diff --git a/doc/bugs/webapp_--listen_port_is_not_used__63__/comment_2_55315d9013fc6acbde9f8b419757d949._comment b/doc/bugs/webapp_--listen_port_is_not_used__63__/comment_2_55315d9013fc6acbde9f8b419757d949._comment
new file mode 100644
index 0000000000..8af5ad882b
--- /dev/null
+++ b/doc/bugs/webapp_--listen_port_is_not_used__63__/comment_2_55315d9013fc6acbde9f8b419757d949._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 2"
+ date="2024-01-18T19:00:50Z"
+ content="""
+> What's your use case for wanting to specify a port?
+
+it was to have some static port I could keep an ssh redirect pointing to so I could investigate behavior of the remote running annex webapp.
+
+My 1c: Since AFAIK no ADDRESS could have `:` (unless some `ssh` based \"server:/path/to/socket\" which I do not think supported) so might be better to just crash (not just have it a matter of documentation) if `:\d+` part is found to be specified?
+"""]]
diff --git a/doc/bugs/webapp_--listen_port_is_not_used__63__/comment_3_2d18df9587b52cbb7c34bfce2e0f2fb2._comment b/doc/bugs/webapp_--listen_port_is_not_used__63__/comment_3_2d18df9587b52cbb7c34bfce2e0f2fb2._comment
new file mode 100644
index 0000000000..d1bd57feac
--- /dev/null
+++ b/doc/bugs/webapp_--listen_port_is_not_used__63__/comment_3_2d18df9587b52cbb7c34bfce2e0f2fb2._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2024-01-18T21:13:42Z"
+ content="""
+IPv6 would beg to differ about `:\d+` ;-)
+
+Actually, it may be that your address:port was treated as some IPv6 mixed
+IPv4, iirc something like that is a thing.
+
+> it was to have some static port I could keep an ssh redirect pointing to so I could investigate behavior of the remote running annex webapp.
+
+You would still need to copy over the url though to get the access key
+for the webapp..
+"""]]
diff --git a/doc/bugs/webapp_--listen_port_is_not_used__63__/comment_4_ba217465bfa738b77ea9417a33810d75._comment b/doc/bugs/webapp_--listen_port_is_not_used__63__/comment_4_ba217465bfa738b77ea9417a33810d75._comment
new file mode 100644
index 0000000000..74c0a7b39f
--- /dev/null
+++ b/doc/bugs/webapp_--listen_port_is_not_used__63__/comment_4_ba217465bfa738b77ea9417a33810d75._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2024-01-25T17:29:08Z"
+ content="""
+I found an old todo about the same thing,
+[[todo/Make_webapp_port_configurable]].
+
+The idea there was, they were using docker and wanted to open only a
+specific port selected for the webapp. So basically the same kind of thing.
+
+I think that this should be a separate --port option, to avoid needing to
+try to parse something that may be an ipv6 address or hostname, or
+whatever.
+
+I don't think that using --port should prevent the webapp from needing
+the `?auth=' part of the url, as output when using --listen.
+
+Probably it does not make sense to use --port without also using --listen,
+but if the user does use it, I don't think --port needs to output the url
+the way --listen does.
+"""]]
diff --git a/doc/bugs/webapp_--listen_port_is_not_used__63__/comment_5_99a6933f30649bc33ee5c74b33fc7046._comment b/doc/bugs/webapp_--listen_port_is_not_used__63__/comment_5_99a6933f30649bc33ee5c74b33fc7046._comment
new file mode 100644
index 0000000000..cdd59cf4dd
--- /dev/null
+++ b/doc/bugs/webapp_--listen_port_is_not_used__63__/comment_5_99a6933f30649bc33ee5c74b33fc7046._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2024-01-25T18:06:15Z"
+ content="""
+Implemented --port.
+"""]]
diff --git a/doc/bugs/windows_started_to_FTBFS.mdwn b/doc/bugs/windows_started_to_FTBFS.mdwn
new file mode 100644
index 0000000000..858d435e53
--- /dev/null
+++ b/doc/bugs/windows_started_to_FTBFS.mdwn
@@ -0,0 +1,35 @@
+### Please describe the problem.
+
+Spotted today
+
+```
+[695 of 696] Compiling CmdLine.GitAnnex
+runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\bin (Win32 error 3): The system cannot find the path specified.
+ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\lib (Win32 error 3): The system cannot find the path specified.
+ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\bin (Win32 error 3): The system cannot find the path specified.
+ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\lib (Win32 error 3): The system cannot find the path specified.
+ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\bin (Win32 error 3): The system cannot find the path specified.
+ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\lib (Win32 error 3): The system cannot find the path specified.
+ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\bin (Win32 error 3): The system cannot find the path specified.
+ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\lib (Win32 error 3): The system cannot find the path specified.
+ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\bin (Win32 error 3): The system cannot find the path specified.
+ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\lib (Win32 error 3): The system cannot find the path specified.
+ghc.exe: addLibrarySearchPath: C:\Users\runneradmin\AppData\Local\Programs\stack\x86_64-windows\msys2-20221216\mingw64\bin (Win32 error 3): The system cannot find the path specified.
+
+Error: [S-7282]
+       Stack failed to execute the build plan.
+       
+       While executing the build plan, Stack encountered the error:
+       
+       [S-7011]
+       While building package git-annex-10.20231227 (scroll up to its section to see the error)
+       using:
+       D:\a\git-annex\git-annex\.stack-work\dist\274b403a\setup\setup --verbose=1 --builddir=.stack-work\dist\274b403a build exe:git-annex --ghc-options ""
+       Process exited with code: ExitFailure 1 
+Error: Process completed with exit code 1.
+```
+
+https://github.com/datalad/git-annex/actions/runs/7516705948/job/20462067511
+
+> Fixed the Test.hs problem, which if I understand correctly was the root
+> cause of all of this, so [[done]] --[[Joey]]
diff --git a/doc/bugs/windows_started_to_FTBFS/comment_1_094012512e6c267f05d3d61c0ed638a0._comment b/doc/bugs/windows_started_to_FTBFS/comment_1_094012512e6c267f05d3d61c0ed638a0._comment
new file mode 100644
index 0000000000..3b0ae2a717
--- /dev/null
+++ b/doc/bugs/windows_started_to_FTBFS/comment_1_094012512e6c267f05d3d61c0ed638a0._comment
@@ -0,0 +1,52 @@
+[[!comment format=mdwn
+ username="jkniiv"
+ avatar="http://cdn.libravatar.org/avatar/05fd8b33af7183342153e8013aa3713d"
+ subject="I can confirm"
+ date="2024-01-16T22:28:21Z"
+ content="""
+but the true point of error seems to be this (line 1167 in the log):
+
+```
+[605 of 696] Compiling Test
+D:\a\git-annex\git-annex\Test.hs:1850:1: error:
+[606 of 696] Compiling Assistant.Pairing
+    The type signature for `test_gpg_crypto'
+      lacks an accompanying binding
+[607 of 696] Compiling Assistant.Types.DaemonStatus
+      Perhaps you meant one of these:
+[608 of 696] Compiling Assistant.Monad
+        `test_sop_crypto' (Defined at Test.hs:1833:1),
+[609 of 696] Compiling Assistant.Types.NamedThread
+        `test_crypto' (Defined at Test.hs:1944:1)
+```
+
+In fact if you do a non-parallelbuild, stack bails out already @ Test.hs:
+
+```
+[603 of 696] Compiling CmdLine.GitAnnexShell
+[604 of 696] Compiling Utility.Verifiable
+[605 of 696] Compiling Test
+
+C:\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-240116-c15fa1763\Test.hs:1850:1: error:
+    The type signature for `test_gpg_crypto'
+      lacks an accompanying binding
+      Perhaps you meant one of these:
+        `test_sop_crypto' (Defined at Test.hs:1833:1),
+        `test_crypto' (Defined at Test.hs:1944:1)
+     |
+1850 | test_gpg_crypto :: Assertion
+     | ^^^^^^^^^^^^^^^
+
+Error: [S-7282]
+       Stack failed to execute the build plan.
+
+       While executing the build plan, Stack encountered the error:
+
+       [S-7011]
+       While building package git-annex-10.20231227 (scroll up to its section to see the error)
+       using:
+       C:\Users\jkniiv\Projektit\git-annex.branchable.com\git-annex--BUILD-240116-c15fa1763\.stack-work\dist\ed8db9df\setup\setup --verbose=1 --builddir=.stack-work\dist\ed8db9df build exe:git-annex --ghc-options \"\"
+       Process exited with code: ExitFailure 1
+```
+
+"""]]
diff --git a/doc/bugs/windows_started_to_FTBFS/comment_2_1ad4a7da4dd4353838fd5c1f9610ee32._comment b/doc/bugs/windows_started_to_FTBFS/comment_2_1ad4a7da4dd4353838fd5c1f9610ee32._comment
new file mode 100644
index 0000000000..80c7140938
--- /dev/null
+++ b/doc/bugs/windows_started_to_FTBFS/comment_2_1ad4a7da4dd4353838fd5c1f9610ee32._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="jkniiv"
+ avatar="http://cdn.libravatar.org/avatar/05fd8b33af7183342153e8013aa3713d"
+ subject="comment 2"
+ date="2024-01-16T23:23:27Z"
+ content="""
+The compilation error turned out to be a typo / small neglect. The following quick fix makes it go away
+(at least with build flag `--flag 'git-annex:-ParallelBuild'`):
+
+[[!format diff \"\"\"
+diff --git a/Test.hs b/Test.hs
+index c8db8147a4..b752d4dc22 100644
+--- a/Test.hs
++++ b/Test.hs
+@@ -1941,7 +1941,7 @@ test_gpg_crypto = do
+                        Annex.Locations.keyPaths .
+                        Crypto.encryptKey Types.Crypto.HmacSha1 cipher
+ #else
+-test_crypto = putStrLn \"gpg testing not implemented on Windows\"
++test_gpg_crypto = putStrLn \"gpg testing not implemented on Windows\"
+ #endif
+
+ test_add_subdirs :: Assertion
+\"\"\"]]
+
+"""]]
diff --git a/doc/design/caching_database.mdwn b/doc/design/caching_database.mdwn
index be0dd17fca..c415aed3e8 100644
--- a/doc/design/caching_database.mdwn
+++ b/doc/design/caching_database.mdwn
@@ -1,7 +1,5 @@
 * [[metadata]] for views
 * [[todo/cache_key_info]]
-* [[bugs/indeterminite_preferred_content_state_for_duplicated_file]]
-* [[todo/speed_up_git_annex_sync_--content_--all]]
 
 What do all these have in common? They could all be improved by
 using some kind of database to locally store the information in an
@@ -26,7 +24,7 @@ Store in the database the Ref of the branch that was used to construct it.
 3. Use sqlite db for associated files cache. **done** (only for v6 unlocked
    files)
 4. Use associated files db when dropping files, to fix
-   [[bugs/indeterminite_preferred_content_state_for_duplicated_file]]
+   indeterminite preferred content state for duplicated files **done**
 5. Also, use associated files db to construct views.
 6. Use sqlite db for metadata cache.
 7. Use sqlite db for list of keys present in local annex.
diff --git a/doc/design/new_repo_versions.mdwn b/doc/design/new_repo_versions.mdwn
index df5004e70a..73210cac78 100644
--- a/doc/design/new_repo_versions.mdwn
+++ b/doc/design/new_repo_versions.mdwn
@@ -43,7 +43,7 @@ Possible reasons to make changes:
 
   The mixed case hash directories have caused trouble on case-insensitive
   filesystems, although that has mostly been papered over to avoid
-  problems. One remaining problem users can stuble on occurs
+  problems. One remaining problem users can stumble on occurs
   when [[moving a repository from OSX to Linux|bugs/OSX_case_insensitive_filesystem]].
 
 * The hash directories, and also the per-key directories
diff --git a/doc/encryption.mdwn b/doc/encryption.mdwn
index 50d479b898..970232d216 100644
--- a/doc/encryption.mdwn
+++ b/doc/encryption.mdwn
@@ -76,6 +76,14 @@ The advantage is you don't need to set up gpg keys. The disadvantage is
 that this is **insecure** unless you trust every clone of the git
 repository with access to the encrypted data stored in the special remote.
 
+By default `gpg` is used for shared encryption, but it is also possible to
+use other programs that implement the Stateless OpenPGP command line
+interface. For example, to use Sequoia PGP's `sqop` command, configured to
+be backwards compatable with `gpg`:
+
+    git config annex.shared-sop-command sqop
+    git config annex.shared-sop-profile rfc4880
+
 ## regular public key encryption (encryption=pubkey)
 
 This alternative simply encrypts the files in the special remotes to one or
diff --git a/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote.mdwn b/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote.mdwn
new file mode 100644
index 0000000000..532dab61f4
--- /dev/null
+++ b/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote.mdwn
@@ -0,0 +1,5 @@
+I import my phone apps' backups to a subdirectory via an ADB special remote and `importtree=yes` but every time I run a sync, it tries to copy files to it that are below numcopies; even those outside the relevant sub-directory.
+
+How can I make it so that it never tries to copy any files to it? `exporttree=no`, it will provably never work.
+
+Note that you can't make it want `nothing` because that also prevents import from working.
diff --git a/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote/comment_1_9f8e1ac58e319540b3d9a5b9962e1f8f._comment b/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote/comment_1_9f8e1ac58e319540b3d9a5b9962e1f8f._comment
new file mode 100644
index 0000000000..9f6356a5f7
--- /dev/null
+++ b/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote/comment_1_9f8e1ac58e319540b3d9a5b9962e1f8f._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="numcopies is no the target"
+ date="2023-12-17T19:04:41Z"
+ content="""
+Syncing doesn't work towards satisfying `numcopies`, just preferred content. So if you find files to be copied to the phone, then its preferred content matches those files. Maybe try `git annex wanted PHONE 'present or approxlackingcopies=1'` or just `git annex wanted PHONE present`. `nothing` is indeed not a good idea as that will have git-annex wanting to yank everything off the phone.
+"""]]
diff --git a/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote/comment_2_f398f38ff29d2e681548b1640080dff3._comment b/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote/comment_2_f398f38ff29d2e681548b1640080dff3._comment
new file mode 100644
index 0000000000..b171e57ba9
--- /dev/null
+++ b/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote/comment_2_f398f38ff29d2e681548b1640080dff3._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Atemu"
+ avatar="http://cdn.libravatar.org/avatar/86b8c2d893dfdf2146e1bbb8ac4165fb"
+ subject="comment 2"
+ date="2023-12-18T12:08:08Z"
+ content="""
+`present` isn't allowed for special remotes with `importtree=yes`.
+"""]]
diff --git a/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote/comment_3_53759e189b2f13b8ea1f48f3b6fb7fca._comment b/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote/comment_3_53759e189b2f13b8ea1f48f3b6fb7fca._comment
new file mode 100644
index 0000000000..ebeea821de
--- /dev/null
+++ b/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote/comment_3_53759e189b2f13b8ea1f48f3b6fb7fca._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2023-12-18T17:41:39Z"
+ content="""
+You have probably set remote.phoneremote.annex-tracking-branch to "master".
+That tells `git-annex sync` to try to export the whole branch to that
+remote.
+
+If you are confining the phone to a subdirectory, you should set that
+to eg "master:phonedir", then it will only consider files in that
+directory.
+
+> `exporttree=no`, it will provably never work
+
+What do you mean by this? If git-annex sync 
+ignored exporttree=no, that would be a bug. I just verified it honors that
+setting.
+"""]]
diff --git a/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote/comment_4_c3d9048668596f6be3975026ec1debc1._comment b/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote/comment_4_c3d9048668596f6be3975026ec1debc1._comment
new file mode 100644
index 0000000000..4b35016d42
--- /dev/null
+++ b/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote/comment_4_c3d9048668596f6be3975026ec1debc1._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="Atemu"
+ avatar="http://cdn.libravatar.org/avatar/86b8c2d893dfdf2146e1bbb8ac4165fb"
+ subject="comment 4"
+ date="2023-12-25T12:21:58Z"
+ content="""
+The remote's `annex-tracking-branch` includes the sub-directory in my setup.
+
+I did not mean that it ignores exporttree and actually exports files, it ignores it insofar that it attempts to copy to the remote but then fails because `exporttree=no`. Tonnes of:
+
+```
+copy Filename (to paperless...) 
+  remote is configured with importtree=yes and without exporttree=yes; cannot modify content stored on it
+failed
+```
+
+It shouldn't even attempt to do that, it will never work.
+
+Some new info: After updating my NAS to NixOS 23.11, it is now also affected while it wasn't before on 23.05.
+
+Good: 10.20230407  
+Bad: 10.20230926
+"""]]
diff --git a/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote/comment_5_e9ea995c99785dd6dffba91ef0fceb02._comment b/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote/comment_5_e9ea995c99785dd6dffba91ef0fceb02._comment
new file mode 100644
index 0000000000..dfbd56835a
--- /dev/null
+++ b/doc/forum/Don__39__t_copy_files_to_exporttree__61__no_remote/comment_5_e9ea995c99785dd6dffba91ef0fceb02._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2023-12-26T19:44:23Z"
+ content="""
+I reproduced it with the current version and a remote set up with:
+
+	git-annex initremote test type=directory directory=../d encryption=none exporttree=no importtree=yes
+	git config remote.directory.annex-tracking-branch master
+
+Fixed for tomorrow's release.
+"""]]
diff --git a/doc/forum/Initial_macOS_setup_between_two_local_Macs.mdwn b/doc/forum/Initial_macOS_setup_between_two_local_Macs.mdwn
new file mode 100644
index 0000000000..eadab35172
--- /dev/null
+++ b/doc/forum/Initial_macOS_setup_between_two_local_Macs.mdwn
@@ -0,0 +1,32 @@
+I’m trying to start syncing between two Macs. This is my first time trying to use `git-annex`.
+
+I have Mac #1 with macOS v14 Sonoma, which is an Apple Silicon Mac.
+
+Mac #2, an Intel Macs, has macOS v13 Ventura.
+
+I installed `git-annex` via the disk image (dmg) on both. Each computer has version 10.20231228-g181e14467ffee9d198529efb74c38242e51107c3.
+
+On Mac #1, I start “git-annex” by double-clicking on it in the Finder. It starts up the assistant in Safari. I create a starting git-annex repo there, with a path in my macOS user home folder — or at least I *try*, because the progress of loading the page never finishes after many minutes of waiting and no indication of any further progress.
+
+I found that if I quit `git-annex` in Activity Monitor and then re-launch it, I see the `git-annex` repo listed.
+
+From there, I select `git-annex` Assistant > Configuration > Repositories > Local Computer. I input a Secret Phase, click the “Start Pairing” button, and all seems relatively okay. There is a “Pair in progress” notice in the Assistant, with a “Cancel” button.
+
+On Mac #2, I start “git-annex” in the Finder, and it launches in Safari. I create an initial git-annex repo. Again, this never seems to complete.
+
+I quit `git-annex` in Activity Monitor and then re-launch it, I see the git-annex repo listed as I did on Mac #1.
+
+Mac #2 shows the “<me on another computer> is sending a pair request” notice, with a “Respond” button. I click “Respond,” enter the same Secret Phrase, and click “Finish pairing.”
+
+At this point, Assistant on Mac #1 shows “PairListener crashed: failed setting up ssh authorized keys” with a ”Restart Thread” button. Clicking the button does not seem to help.
+
+Here are things I’ve also checked and/or tried:
+
+1. Enabled “Remote Login” (SSH) in System Settings > General > Sharing on each computer. (I did this before any attempts to share any `git-annex` repositories between the two computers.) This is limited to only my macOS user account on both Macs.
+1. macOS Firewall is set to default (i.e. shows up as “Inactive” on both Macs.
+1. Mac #2 is using Little Snitch (an alternative firewall) in “Silent (Allow)” mode. I don’t get any alerts from its app about `git-annex` or `ssh`, and it is set to allow incoming and outbound traffic. Even with Little Snitch in “Alert Mode,” I do not get any information about `git-annex` or `ssh`.
+1. Doing a manual setup without Assistant, using the “walkthrough” instructions. This has always resulting in “Remote origin does not have git-annex installed” errors, which I surmise are due to the `authorized_keys` and `config` settings for SSH, or just needing a more-permissive-than-macOS-defaults SSH setup.
+1. Starting setup with Assistant, then trying to manually fill in the SSH configuration gaps. This means I’m trying to cobble together what I assume should be the `authorized_keys` and `config` settings for SSH manually. I’ve used the `git-annex` man pages and some ChatGPT to try to sort that out. I haven’t sorted this out yet. I also haven’t been able to have Git work over SSH between the two computers for this purpose. While doing this, I noted that Mac #1 didn’t have any SSH changes made by git-annex Assistant — but interestingly, Mac #2 did get at least the `authorized_keys` changes. It is possible if I simplify what was doing manually, I may get better luck. Mac #2 already has an `~/.ssh/config` setup and `~/.ssh/authorized_keys`.
+1. Assistant setup, after adding the “git-annex.app” to the Full Disk Access list in System Settings > Privacy & Security. I quit all `git-annex` processes and restarted the app from the Finder afterwards on both computers. Same results.
+
+I’m at a bit of a loss here. Thanks in advance for any ideas that could help get this going.
diff --git a/doc/forum/Is_there_a_way_to_have_assistant_add_files_locked__63__/comment_7_92bf43545c46d302089820cfdd03141b._comment b/doc/forum/Is_there_a_way_to_have_assistant_add_files_locked__63__/comment_7_92bf43545c46d302089820cfdd03141b._comment
new file mode 100644
index 0000000000..e205116f56
--- /dev/null
+++ b/doc/forum/Is_there_a_way_to_have_assistant_add_files_locked__63__/comment_7_92bf43545c46d302089820cfdd03141b._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="yarikoptic"
+ avatar="http://cdn.libravatar.org/avatar/f11e9c84cb18d26a1748c33b48c924b4"
+ subject="comment 7"
+ date="2024-01-05T21:00:52Z"
+ content="""
+so there remains no way to configure assistant to add files in regular locked mode where files would \"migrate\" to become symlinks after having been added to annex?
+
+Our use case is the \"data acquisition\" box (for repronim's reprostim project) which we are configuring to collect videos, which are then to be transferred to another box. Size will be large, we will rely on branches and copies/moves of the files, so having them to be symlinks is kinda \"mandatory\" to keep operations fast and to not waste space.
+"""]]
diff --git a/doc/forum/Is_there_a_way_to_have_assistant_add_files_locked__63__/comment_8_a6ae8bc3b5caf0d3d49a919ab5ec5778._comment b/doc/forum/Is_there_a_way_to_have_assistant_add_files_locked__63__/comment_8_a6ae8bc3b5caf0d3d49a919ab5ec5778._comment
new file mode 100644
index 0000000000..b300907a88
--- /dev/null
+++ b/doc/forum/Is_there_a_way_to_have_assistant_add_files_locked__63__/comment_8_a6ae8bc3b5caf0d3d49a919ab5ec5778._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="comment 8"
+ date="2024-01-06T09:22:54Z"
+ content="""
+Not that I know of. Simplest solution for you would be probably a `while true;git annex assist`done` instead of the assistant.
+"""]]
diff --git a/doc/forum/Is_there_a_way_to_have_assistant_add_files_locked__63__/comment_9_7fcc62493d261c22149f85f48bb95efc._comment b/doc/forum/Is_there_a_way_to_have_assistant_add_files_locked__63__/comment_9_7fcc62493d261c22149f85f48bb95efc._comment
new file mode 100644
index 0000000000..87f4262044
--- /dev/null
+++ b/doc/forum/Is_there_a_way_to_have_assistant_add_files_locked__63__/comment_9_7fcc62493d261c22149f85f48bb95efc._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2024-01-18T17:08:57Z"
+ content="""
+That seems like a resonable use case to me,
+I've opened a todo
+[[todo/allow_configuring_assistant_to_add_files_locked]].
+
+I have not revisited looking at how hard it would be to
+implement such a config option. @yarikoptic if you're 
+needing this for ReproNim I could prioritize doing that.
+"""]]
diff --git a/doc/forum/Required_Content_Always_True___40____61____62___cannot_drop__41__.mdwn b/doc/forum/Required_Content_Always_True___40____61____62___cannot_drop__41__.mdwn
new file mode 100644
index 0000000000..0e4332efc5
--- /dev/null
+++ b/doc/forum/Required_Content_Always_True___40____61____62___cannot_drop__41__.mdwn
@@ -0,0 +1,82 @@
+Hey there,
+
+I am having the issue that git-annex reports files to be required even though it should not be.
+
+The setup is as follows (The groups that the repo is part of is specified in brackets):
+
+- Main repo (`origin`, `server`) on a server
+    - Regularly syncs all its data two two backup repos (`server`, `backup`, `backup_server`)
+        - These two backup repos are trusted, as they are not reachable by clients
+- Client repo (`client`) on PC
+    - Syncs with other clients via the server
+    - Manually syncs with a backup repo (`backup` `offline`, `backup_offline`) on a disk
+    - Each client has two remotes, the server (named `origin`) and the backup drive (named `usb`)
+
+I want that the required content is everything present, that is not yet (all of them):
+
+1. in the server repo
+2. in both server backup repos
+3. in a backup repo on an external drive
+
+I specify that using `git annex required . "present and (copies=origin:0 or (not copies=backup_server:2) or copies=backup_offline:0)"`.
+
+Now I want to drop file `A`
+
+File `A` is present on 5 repos. However, I cannot delete it, as it is "apparently" not on the server (i.e. `copies=origin:0` is true).
+
+```
+$ git annex whereis A
+whereis A (5 copies)
+  	0cdca96a-e44d-4168-a3a0-8ab846451e74 -- Server_Backup2
+   	44039708-f0d9-4ed0-833b-4d146d419b5d -- jeanclaude [here]
+   	4ac4c649-b37c-403e-94e0-9497a7bc2a91 -- Server_Backup1
+   	c0d1b661-1e19-4956-b290-ff62abc6d61a -- jc_backup_wd6tb [usb]
+   	da3e14a5-188f-4a65-bf93-8fce9e409d09 -- [origin]
+ok
+
+$ git annex drop A --explain
+drop A [ A matches required content: present[TRUE] and ( copies=origin:0[TRUE] ) ]
+
+  That file is required content. It cannot be dropped!
+
+  (Use --force to override this check, or adjust required content configuration.)
+failed
+drop: 1 failed
+```
+
+What is it saying that it is not no `origin`? Is there something wrong with my setup? Does the drop command try to *lock* file `A` on all remotes (except on the server backups, as they are trusted)?
+
+
+Me doing some "debugging":
+
+```
+$ git annex required . "present and (copies=origin:1 or (not copies=backup_server:2) or copies=backup_offline:0)"
+required . ok
+(recording state in git...)
+
+$ git annex drop A --explain
+drop A [ A matches required content: present[TRUE] and ( copies=origin:1[TRUE] ) ]
+
+  That file is required content. It cannot be dropped!
+
+  (Use --force to override this check, or adjust required content configuration.)
+failed
+drop: 1 failed
+
+$ git annex required . "present and (copies=origin:10 or (not copies=backup_server:2) or copies=backup_offline:0)"
+required . ok
+(recording state in git...)
+
+$ git annex drop 2022/220513-PolybandInConcert/220513_220256.cr3 --explain
+drop A [ A matches required content: present[TRUE] and ( copies=origin:10[FALSE] or not copies=backup_server:2[TRUE] or copies=backup_offline:0[TRUE] ) ]
+
+  That file is required content. It cannot be dropped!
+
+  (Use --force to override this check, or adjust required content configuration.)
+failed
+drop: 1 failed
+```
+
+How can the number of copies on `origin` be `0` and `1` at the same time? Or do I misunderstand something completely? It also reports that the file in none of the backups (neither local nor remote).
+
+Thanks a lot your your time and effort!
diff --git a/doc/forum/Required_Content_Always_True___40____61____62___cannot_drop__41__/comment_1_1db8c68e8d82ed169b4687dfb5da1ba6._comment b/doc/forum/Required_Content_Always_True___40____61____62___cannot_drop__41__/comment_1_1db8c68e8d82ed169b4687dfb5da1ba6._comment
new file mode 100644
index 0000000000..9b1b94397a
--- /dev/null
+++ b/doc/forum/Required_Content_Always_True___40____61____62___cannot_drop__41__/comment_1_1db8c68e8d82ed169b4687dfb5da1ba6._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="Lukey"
+ avatar="http://cdn.libravatar.org/avatar/c7c08e2efd29c692cc017c4a4ca3406b"
+ subject="comment 1"
+ date="2023-11-11T16:13:28Z"
+ content="""
+AFAIK, copies=<group>:<x> matches if the group *contains at least* x copies.
+"""]]
diff --git a/doc/forum/Required_Content_Always_True___40____61____62___cannot_drop__41__/comment_2_ea362bb99294571f0b0e808e87b7d422._comment b/doc/forum/Required_Content_Always_True___40____61____62___cannot_drop__41__/comment_2_ea362bb99294571f0b0e808e87b7d422._comment
new file mode 100644
index 0000000000..f9e24a613a
--- /dev/null
+++ b/doc/forum/Required_Content_Always_True___40____61____62___cannot_drop__41__/comment_2_ea362bb99294571f0b0e808e87b7d422._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jcjgraf"
+ avatar="http://cdn.libravatar.org/avatar/9dda752f83ac44906fefbadb35e8a6ac"
+ subject="comment 2"
+ date="2023-11-14T21:25:24Z"
+ content="""
+Oh well, that I could have come up with myself. Thanks for your hint @Lukey!
+"""]]
diff --git a/doc/forum/Revisiting_migration_and_multiple_keys.mdwn b/doc/forum/Revisiting_migration_and_multiple_keys.mdwn
new file mode 100644
index 0000000000..13e009d2fe
--- /dev/null
+++ b/doc/forum/Revisiting_migration_and_multiple_keys.mdwn
@@ -0,0 +1,22 @@
+I have several workflows that rely on regular key migrations, and I would love to explore some ways that migrating keys could be improved.
+
+I see there has already been discussion about this:
+[[todo/alternate_keys_for_same_content]]
+
+I don't know how often this comes up, but it comes up a lot for me. I have several data sources that I regularly index and mirror by constructing keys based on md5 and size, and assemble a repo with the known filename. (gdrive, many software distribution sites, and others).
+
+So, I have a queue-repo, and I have the flexibilty of populating it later. I could even have a queue repo with just URL keys. Then I can handle ingestion and migration later.
+
+
+I would love to have a simple programatic way of recording that one key is the authoritative key for another key, like for MD5 -> SHA256 migrations.
+
+There don't seem to be any really great solutions to the prolem of obsolete keys. Merging will often re-introduce them, even if they have been excised. Marking them as dead stil keeps them around, and doesn't preserve information about what key now represents the same object.
+
+I have written helper scripts, and tools like git-annex-filter-branch are also very helpful. But I like having the flexibility of many repos that may not regularly be in sync with each other, and a consistent history.
+
+
+This would break things for sure, but what if during a migration, a symlink was made in the git-annex branch from the prev key to the migrated key. The union merge driver could defer to the upgraded or prefered backend. If an out of date repo tries syncing with an already upgraded key, the merge driver can see that the migration for that key has already happened, merge the obsolete key entries, and overwrite it back to a symlink during merge.
+
+A less drastic approach might be to expand the location log format to indiciate a canonical "successor" key, instead of just being dead.
+
+It might seem like a lot of complexity, but it would also in my opinion make a more consistent and flexible data model.
diff --git a/doc/forum/Revisiting_migration_and_multiple_keys/comment_1_a85c3c7af6b1e01f887e0e1ffe2cde6f._comment b/doc/forum/Revisiting_migration_and_multiple_keys/comment_1_a85c3c7af6b1e01f887e0e1ffe2cde6f._comment
new file mode 100644
index 0000000000..efeeece885
--- /dev/null
+++ b/doc/forum/Revisiting_migration_and_multiple_keys/comment_1_a85c3c7af6b1e01f887e0e1ffe2cde6f._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="comment 1"
+ date="2023-11-30T06:51:53Z"
+ content="""
+Hi! If I understand you correctly, your problem is that you often migrate keys to another backend, and there are situations involving merges of repos far away from each other in history that cause merge conflicts, which results in the dead old pre-migration key being reintroduced?
+
+I never use key backend migration and I don't fully understand your workflow. Could you provide a reproducible example of your problem (incl  all commands)? This would help a lot.
+"""]]
diff --git a/doc/forum/Revisiting_migration_and_multiple_keys/comment_2_b5545aba08c7af2f8f56caba66232c41._comment b/doc/forum/Revisiting_migration_and_multiple_keys/comment_2_b5545aba08c7af2f8f56caba66232c41._comment
new file mode 100644
index 0000000000..5c15c5c516
--- /dev/null
+++ b/doc/forum/Revisiting_migration_and_multiple_keys/comment_2_b5545aba08c7af2f8f56caba66232c41._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2023-11-30T20:49:53Z"
+ content="""
+There seem to be difficulties with both performance and with security in
+storing information in the git-annex branch to declare that one key
+is replaced by another one.
+
+I wonder if there are any pain points that could be handled better without
+recording such information in the git-annex branch. What do your helper
+scripts do?
+"""]]
diff --git a/doc/forum/Revisiting_migration_and_multiple_keys/comment_3_a712ec9b616ca45976154fd0c98ae1c4._comment b/doc/forum/Revisiting_migration_and_multiple_keys/comment_3_a712ec9b616ca45976154fd0c98ae1c4._comment
new file mode 100644
index 0000000000..8fa84204a5
--- /dev/null
+++ b/doc/forum/Revisiting_migration_and_multiple_keys/comment_3_a712ec9b616ca45976154fd0c98ae1c4._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2023-11-30T20:55:02Z"
+ content="""
+I wonder if it would suffice to have a way for git-annex to record that key
+A migrated to B, but not treat that as meaning that it should get B's
+content when it wants A, or vice-versa.
+
+Instead, when a repository learns that A was elsewhere migrated to B, it
+could hardlink its content for A to B and update the location log for
+B to say is has a copy. The same as if `git-annex migrate` were run locally.
+(It could even hash the content and verify it got B.)
+
+That wouldn't help if a special remote has the content of A, and 
+git-annex wants to get the content of B.
+"""]]
diff --git a/doc/forum/Revisiting_migration_and_multiple_keys/comment_4_7d367f38250a4a3454299170700d5c6c._comment b/doc/forum/Revisiting_migration_and_multiple_keys/comment_4_7d367f38250a4a3454299170700d5c6c._comment
new file mode 100644
index 0000000000..dc7ac4b638
--- /dev/null
+++ b/doc/forum/Revisiting_migration_and_multiple_keys/comment_4_7d367f38250a4a3454299170700d5c6c._comment
@@ -0,0 +1,58 @@
+[[!comment format=mdwn
+ username="unqueued"
+ avatar="http://cdn.libravatar.org/avatar/3bcbe0c9e9825637ad7efa70f458640d"
+ subject="comment 4"
+ date="2023-12-01T02:09:07Z"
+ content="""
+@joey
+
+It isn't a huge problem, but I keep coming back to it. The only workflow I still use where this comes up is for my filesharing assets repo. I just ended up leaving it as MD5E, because much of it is downstream from gdrive shares, and I almost never have all of the content in one place at a time.
+
+
+This is one of the scripts I sometimes use, although I wrote it awhile ago before I found out about git-annex-filter-branch
+<https://gist.github.com/unqueued/06b5a5c14daa8224a659c5610dce3132>
+
+But I mostly rely on splitting off subset repos with no history, processing them in some way, and then re-absorbing them back into a larger repo.
+
+I actually started a repo that would track new builds for Microsoft Dev VMs: <https://github.com/unqueued/official-microsoft-vms-annex>
+
+But for my bigger repos, I almost never have all of the data in the same place at the same time.
+
+
+@nobodyinperson
+
+> Hi! If I understand you correctly, your problem is that you often migrate keys to another backend, and there are situations involving merges of repos far away from each other in history that cause merge conflicts, which results in the dead old pre-migration key being reintroduced?
+
+Well, there aren't any conflicts, they just get silently reintroduced, which isn't the end of the world, especially if they get marked as dead. But they clutter the git-annex branch, and over time, with large repos, it may become a problem. There isn't any direct relationship between the previous key and the migrated key.
+
+So, if I have my `linux_isos` repo, and I do git-annex-migrate on it, but say only isos for the year 2021 are in my specific repo at that moment, then the symlinks will be updated and the new sha256 log log files will be added to my git-annex branch.
+
+And if you sync with another repo that also has the same files in the backend, they will still be in the repo, but just inaccessible.
+
+And I feel like there's enough information to efficiently track the lifecycle of a key.
+
+
+I'm exhuming my old scripts and cleaning them up, but essentially, you can get everything you need to assemble an MD5E annex from a Google Drive share by doing `rclone lsjson -R --hash rclone-drive-remote:`
+
+And to get the keys, you could pipe it into something like this:
+`perl -MJSON::PP -ne 'BEGIN { $/ = undef; $j = decode_json(<>); } foreach $e (@{$j}) { next if $e->{\"IsDir\"} || !exists $e->{\"Hashes\"}; print \"MD5-s\" . $e->{\"Size\"} . \"--\" . $e->{\"Hashes\"}->{\"MD5\"} . \"\t\" . $e->{\"Path\"} . \"\n\"; }' `
+
+That's just part of a project I have with a Makefile that indexes, assembles and then optionally re-serves an rclone gdrive remote. I will try to post it later tonight. It was just a project I made for fun.
+
+And there are plenty of other places where you can get enough info to assemble a repo ahead of time, and essentially turn it into a big queue.
+
+
+You can find all sorts of interesting things to annex.
+
+https://old.reddit.com/r/opendirectories sometimes has interesting stuff.
+
+Here are some public Google Drive shares:
+
+* [Bibliotheca Anonoma](https://drive.google.com/drive/folders/0B7WYx7u6HJh_Z3FjU2F0NFNyQWs)
+* [Esoteric Library](https://drive.google.com/drive/folders/0B0UEkmH7vYJZRWxfSmdRbFJGNWc)
+* [EBookDroid - Google Drive](https://drive.google.com/drive/folders/0B6y-A-HTzyBiYnpIRHMzR1pueFU)
+* [The 510 Archives - Google Drive](https://drive.google.com/drive/folders/0ByCvxnHNk90SMzIxZWIwYWYtYzljNy00ZGU2LWI3ODctYzRjMmE0MGY3NTA1)
+* [Some ebooks](https://drive.google.com/drive/folders/1SReXFt16DYpTdFsSsT5Nzkj33VAYOQLa)
+
+
+"""]]
diff --git a/doc/forum/Revisiting_migration_and_multiple_keys/comment_5_93b85fbe5c36e986cf7c1fc87070c04c._comment b/doc/forum/Revisiting_migration_and_multiple_keys/comment_5_93b85fbe5c36e986cf7c1fc87070c04c._comment
new file mode 100644
index 0000000000..b42947b7b8
--- /dev/null
+++ b/doc/forum/Revisiting_migration_and_multiple_keys/comment_5_93b85fbe5c36e986cf7c1fc87070c04c._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2023-12-01T18:42:07Z"
+ content="""
+I've spent a while thinking about this and came up with the ideas at
+[[todo/distributed_migration]].
+
+I think that probably would handle your use case.
+"""]]
diff --git a/doc/forum/Revisiting_migration_and_multiple_keys/comment_6_3db8d28e13f6508e22e8488a6faaa8b2._comment b/doc/forum/Revisiting_migration_and_multiple_keys/comment_6_3db8d28e13f6508e22e8488a6faaa8b2._comment
new file mode 100644
index 0000000000..83a860c358
--- /dev/null
+++ b/doc/forum/Revisiting_migration_and_multiple_keys/comment_6_3db8d28e13f6508e22e8488a6faaa8b2._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="unqueued"
+ avatar="http://cdn.libravatar.org/avatar/3bcbe0c9e9825637ad7efa70f458640d"
+ subject="comment 6"
+ date="2023-12-19T18:50:08Z"
+ content="""
+Whoa, thanks for implementing that Joey! Can't wait to give it a try!
+
+
+FYI, one of the cases I was talking about before where I repeatedly import keys in MD5E format, is that construct an annex repo, set web urls, and deal with mirroring further down the pipeline.
+
+Code isn't great, just something I threw together years ago:
+https://github.com/unqueued/annex-drive-share
+
+Because it is gdrive, I can get MD5s and filenames with rclone urls for web remotes.
+
+The way I use it is to init a new annex repo (reusing the same uuid), and then absorb into primary downstream repo overwriting filenames and letting sync update any new keys with the primary repo. Considered using subtree.
+
+It does end up causing merge commits to build up in the git-annex branch, but I might want to run this on a server without sharing an entire repo.
+
+It happens to make sense for me because I have an unlimited @edu gdrive account and it can work great for some workflows as an intermediate file store.
+"""]]
diff --git a/doc/forum/Use_on_large_media_collection_without_modifying_it/comment_4_4529364f2919bd05f53da94cf8ba4268._comment b/doc/forum/Use_on_large_media_collection_without_modifying_it/comment_4_4529364f2919bd05f53da94cf8ba4268._comment
new file mode 100644
index 0000000000..ad42a4de31
--- /dev/null
+++ b/doc/forum/Use_on_large_media_collection_without_modifying_it/comment_4_4529364f2919bd05f53da94cf8ba4268._comment
@@ -0,0 +1,23 @@
+[[!comment format=mdwn
+ username="unqueued"
+ avatar="http://cdn.libravatar.org/avatar/3bcbe0c9e9825637ad7efa70f458640d"
+ subject="comment 4"
+ date="2023-11-05T21:32:17Z"
+ content="""
+Just putting this out there, but if you are on ZFS or BTRFS, you can just duplicate the subvolume/dataset, remove what you want, and send it. It will by default verify your data integrity, and it is often faster.
+
+On BTRFS, it is easy to `btrfs sub create send.RW; cp --reflink=always .git/annex/objects send.RW; btrfs sub snap -r send.RW send.RO; btrfs sub del send.RW`
+
+Then, on the target, I can reflink copy into the target repo's .git/annex/objects, and the `git annex fsck --all --fast`, since the send operation verified the integrity.
+
+
+Sometimes, if the target repo does not exist, I can take a snapshot of an entire repo, and then enter it, then re-init it with the target uuid, force drop what I don't want, and then send it. If you're dealing with hundreds of thousands of files, it can be more practical to do that.
+
+If you want to verify the integrity of an annexed file on ZFS or BTRFS, all you have to do is read it, and let the filesystem verify the checksums for you.
+
+If you want a nice progress display, you can just do `pv myfile > /dev/null`
+
+I considered making a git-annex-scrub script that would check if the underlying fs supports integrity verification, then just read the file and update the log.
+
+BTRFS uses hardware accelerated crc32, which is fine for bitrot, but it is not secure from intentional tampering.
+"""]]
diff --git a/doc/forum/Using_git-annex_as_a_library/comment_7_909628f1edd0d3448498fc434c61a3a4._comment b/doc/forum/Using_git-annex_as_a_library/comment_7_909628f1edd0d3448498fc434c61a3a4._comment
new file mode 100644
index 0000000000..63d06ba09f
--- /dev/null
+++ b/doc/forum/Using_git-annex_as_a_library/comment_7_909628f1edd0d3448498fc434c61a3a4._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 7"""
+ date="2023-11-21T20:23:19Z"
+ content="""
+Hmm, `cabal build` does not build the library 2x in the case of propellor,
+which has a similar split between the propellor library and the executables
+that depend on it. Perhaps cabal has improved that since I posted my
+comment.
+"""]]
diff --git a/doc/forum/Using_git-annex_as_a_library/comment_8_cef28b8639b6c6e84804b485fb5037f1._comment b/doc/forum/Using_git-annex_as_a_library/comment_8_cef28b8639b6c6e84804b485fb5037f1._comment
new file mode 100644
index 0000000000..400e1913af
--- /dev/null
+++ b/doc/forum/Using_git-annex_as_a_library/comment_8_cef28b8639b6c6e84804b485fb5037f1._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2023-11-21T20:25:55Z"
+ content="""
+A lot of git-annex's end user functionality is in the `Command/*` modules.
+Much of the code in those is not in a form that would be useful in a
+library, unless the library was structured to run a `Command`.
+
+If the goal is say, to add a file to git-annex, and then to be able to
+get and drop the file, and `Command.Add`, `Command.Get`, and `Command.Drop`
+were not in the library, you'd have to put together the eqivilant of what
+those commands do out of the internal library code:
+
+* `Command.Add` essentially uses `Annex.Ingest` followed by calling into
+  `Logs.Location` to update the location log. 
+* `Command.Drop` uses `Annex.NumCopies` to construct a drop proof and passes
+  it off to `Annex.Drop`.
+* `Command.Get` handles trying different remotes itself, calling into 
+  `Annex.Transfer`.
+
+So the library structure works for git-annex as a thing for Command modules
+to use, but not great for other things. The assistant actually imports some
+Command modules, eg it uses Command.Add.addFile. Similarly it would be
+possible to call Command.Get.getKey.
+
+Maybe it would be better to have a library interface that does mirror the
+git-annex command line, so you could run eg:
+
+	runCommand (Command.Add.cmd) (AddOptions {...}) "somefile"
+	runCommand (Command.Drop.cmd) (DropOptions {...}) "somefile"
+
+That would necessarily have output like those commands too (unless quiet
+mode were enabled).
+
+It would take some work to get there from the current state.
+"""]]
diff --git a/doc/forum/Using_git-annex_as_a_library/comment_9_6d8ebdedf76fdf0c267444a7a07e225e._comment b/doc/forum/Using_git-annex_as_a_library/comment_9_6d8ebdedf76fdf0c267444a7a07e225e._comment
new file mode 100644
index 0000000000..649b836902
--- /dev/null
+++ b/doc/forum/Using_git-annex_as_a_library/comment_9_6d8ebdedf76fdf0c267444a7a07e225e._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="oadams"
+ avatar="http://cdn.libravatar.org/avatar/ac166a5f89f10c4108e5150015e6751b"
+ subject="comment 9"
+ date="2023-12-16T21:37:21Z"
+ content="""
+A belated thanks for the follow-up from https://git-annex.branchable.com/forum/Using_git_annex_as_a_library/
+
+Not that its useful for anyone here, but just FWIW after some more playing around I decided to keep things minimal and not depend on git-annex as a library. Although what I'm making naturally pairs well with git-annex it doesn't really require it, so adding the dependency just complicates things.
+"""]]
diff --git a/doc/forum/Using_git_annex_as_a_library.mdwn b/doc/forum/Using_git_annex_as_a_library.mdwn
new file mode 100644
index 0000000000..c76f98e9a6
--- /dev/null
+++ b/doc/forum/Using_git_annex_as_a_library.mdwn
@@ -0,0 +1,6 @@
+I've started writing a small tool in Haskell to manage data processing pipelines. I'm interested in using git-annex as a library as part of this (since the idea is that the raw data is tracked in git-annex and git). I'm wondering what the best way to go about doing it is.
+
+If I set my stack.yaml `extra-deps` to point to the git repository and run `stack build` then stack visibly goes and downloads the package. But from there I can't actually import any modules in my own Haskell program. When I use `import Git` or `import Annex.HashObject` then I get a compile error along the lines of `Could not find module ‘Annex.HashObject’`.
+
+Do you have any pointers for how I might best use git-annex as a library? Thanks for your help, sorry if it's so basic as I'm new to the Haskell tooling ecosystem.
+
diff --git a/doc/forum/Using_git_annex_as_a_library/comment_1_42cd26878e4e5c2c238c1227e3c372d9._comment b/doc/forum/Using_git_annex_as_a_library/comment_1_42cd26878e4e5c2c238c1227e3c372d9._comment
new file mode 100644
index 0000000000..d4ce3acd23
--- /dev/null
+++ b/doc/forum/Using_git_annex_as_a_library/comment_1_42cd26878e4e5c2c238c1227e3c372d9._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="oadams"
+ avatar="http://cdn.libravatar.org/avatar/ac166a5f89f10c4108e5150015e6751b"
+ subject="comment 1"
+ date="2023-11-07T08:30:21Z"
+ content="""
+I should add that I have just seen this discussion https://git-annex.branchable.com/forum/Using_git-annex_as_a_library/. Perhaps I should have posted there, but either way I'm wondering if things are different these days?
+"""]]
diff --git a/doc/forum/Using_git_annex_as_a_library/comment_2_1323238fc63e121fbc0f408a23d1ada5._comment b/doc/forum/Using_git_annex_as_a_library/comment_2_1323238fc63e121fbc0f408a23d1ada5._comment
new file mode 100644
index 0000000000..882691e075
--- /dev/null
+++ b/doc/forum/Using_git_annex_as_a_library/comment_2_1323238fc63e121fbc0f408a23d1ada5._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2023-11-21T20:17:10Z"
+ content="""
+Hmm, I followed up on that other page to avoid fragmenting the discussion.
+"""]]
diff --git a/doc/forum/What_operations_are_safe_to_interrupt__63__/comment_2_207435949b2674ea82390c66549873e5._comment b/doc/forum/What_operations_are_safe_to_interrupt__63__/comment_2_207435949b2674ea82390c66549873e5._comment
new file mode 100644
index 0000000000..42d1aa4c5f
--- /dev/null
+++ b/doc/forum/What_operations_are_safe_to_interrupt__63__/comment_2_207435949b2674ea82390c66549873e5._comment
@@ -0,0 +1,28 @@
+[[!comment format=mdwn
+ username="psxvoid"
+ avatar="http://cdn.libravatar.org/avatar/fde068fbdeabeea31e3be7aa9c55d84b"
+ subject="The issue with slow connection and huge repo - no batches"
+ date="2023-12-27T06:08:59Z"
+ content="""
+This issue is quite important when you reach a certain point/scenarios.
+
+I'm using a git annex repo that have ~300GB of annexed files; I wanted to push them to a remote server in another country.
+This server has limited bandwidth around 1MB/s. I also have a limited data plan at home. That is why I was planning to
+`git annex copy --to my-slow-remote-server` at work, were the data plan is unlimited. But after spending around 4 hours
+during the copy I realized that when I do `Ctrl+C` then the copy operation interrupts and it seems like it's not resumable,
+because \"git annex\" does not record any state in git after interrupting the copy operation.
+
+To solve this issue I'm planning to write a custom script - similar to [finding duplicate files](https://git-annex.branchable.com/tips/finding_duplicate_files/).
+The goal of this script would be:
+
+1. To \"split\" large files by their size (e.g. bigger than 5GB) using `git annex find=*`, and upload them individually.
+2. To \"split\" small files by folders in batches (e.g. 5GB), and then do `cd <dir> & git annex copy . --to my-slow-remote-server`
+
+It will allow me to upload all files in batches in a course of several days/weeks.
+But I it will also probably take me a weekend or more to handle it correctly.
+
+So, the questions are:
+
+1. Are there any easier ways to upload huge repo on a slow connection in batches?
+2. Is there anything in progress or planned that will solve this issue?
+"""]]
diff --git a/doc/forum/What_operations_are_safe_to_interrupt__63__/comment_4_279302a0f5bb0a8d8d87182e03c6b475._comment b/doc/forum/What_operations_are_safe_to_interrupt__63__/comment_4_279302a0f5bb0a8d8d87182e03c6b475._comment
new file mode 100644
index 0000000000..9349051f0f
--- /dev/null
+++ b/doc/forum/What_operations_are_safe_to_interrupt__63__/comment_4_279302a0f5bb0a8d8d87182e03c6b475._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="psxvoid"
+ avatar="http://cdn.libravatar.org/avatar/fde068fbdeabeea31e3be7aa9c55d84b"
+ subject="Works: Try annex.queuesize and --not --in"
+ date="2023-12-28T05:33:13Z"
+ content="""
+> \"setting `annex.queuesize` to a low value to commit more frequently to the git-annex branch (I think that it does that)\"
+
+Aha, that what I missed!
+
+> \"(maybe that confused you)\"
+
+Indeed, my original observations weren't entirely correct. Seems like git annex does record it's state periodically.
+For some big files where copying took ~1 hour, after running `git annex copy` again it was done almost
+immediately. And as you suggested `--not --in=slowremote` should speedup the initial process.
+
+It laterally saved me hours, thanks a lot!
+
+P.S.: Still, it may be probably a good idea to record the progress state to git annex on `ctrl+c` and show a friendly message e.g. \"The operation is interrupted and the progress is recorded. You can safely resume it by running `git annex copy` again.\"
+"""]]
diff --git a/doc/forum/What_operations_are_safe_to_interrupt__63__/comment_5_8d36d42e34f5deae04bfa83c568b89d5._comment b/doc/forum/What_operations_are_safe_to_interrupt__63__/comment_5_8d36d42e34f5deae04bfa83c568b89d5._comment
new file mode 100644
index 0000000000..2c5641bc82
--- /dev/null
+++ b/doc/forum/What_operations_are_safe_to_interrupt__63__/comment_5_8d36d42e34f5deae04bfa83c568b89d5._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="Try annex.queuesize and --not --in"
+ date="2023-12-27T07:51:00Z"
+ content="""
+Git annex can normally resume any operation seamlessly. It can resume from aborted copying and shouldn't need to start over. I suggest:
+
+- setting [annex.queuesize](https://git-annex.branchable.com/git-annex/) to a low value to commit more frequently to the `git-annex` branch (I think that it does that)
+- skipping already-present files by specifying `git annex --copy --to=slowremote --not --in=slowremote`, that should speed up the initial phase of finding what to copy and also not display the files that are already copied (maybe that confused you)
+"""]]
diff --git a/doc/forum/Windows_eol_issues.mdwn b/doc/forum/Windows_eol_issues.mdwn
new file mode 100644
index 0000000000..4bdf05721c
--- /dev/null
+++ b/doc/forum/Windows_eol_issues.mdwn
@@ -0,0 +1,552 @@
+This is just a heads-up with regards to end-of-line handling with git-annex.
+
+I've discussed in the past issues I've had with interoperability between Windows and Linux, the earliest I think being the CRLF eol in .git/hooks scripts created on Windows.
+
+I originally started using the Windows port when I had had minimal experience with git-annex. Since then, WSL has come along, coupled with better understanding of worlflows, overtaken an explicit need to use the Windows port, so I'm no longer going to use it for my primary workflows.
+
+But I will continue to check back with new releases of the Windows port to help improve it, as I think there will be users who for whatever reason must use the Windows port (I think Windows Subsystem Linux (WSL) isn't available on vanilla Windows I think... so there's that for example).
+
+In that vane, I'm demonstrating issues with end-of-line that I've noticed with the Windows port.
+
+Before I do that, I'd like to say... eol considerations have been a problem forever (eg. I observed meltdown when Cygwin dropped hybrid eol support many years ago), and Git for Windows itself has gotchas. I really value git-annex, and I want to be clear, this isn't whinging or complaining, just information.
+
+I'll summarise the problem. By default Git for Windows is setup for msdos CRLF line endings. That is, it expects text files to have CRLF line endings, and strips them to "normalise" the lines in storage.
+
+Under this default, the text files that substitute for unix symlinks are ending up with mixed eol. That is, the ones created by git-annex under Windows have LF only endings, and the ones that are created by merge from Unix have CRLF endings. Eg. from a workflow involving the creation of new repositories (note the hooks are created CRLF):
+
+[[!format  sh """
+...
+./.git/hooks/post-checkout:                               POSIX shell script, ASCII text executable, with CRLF line terminators
+./.git/hooks/post-merge:                                  POSIX shell script, ASCII text executable, with CRLF line terminators
+./.git/hooks/post-receive:                                POSIX shell script, ASCII text executable, with CRLF line terminators
+./.git/hooks/post-update.sample:                          POSIX shell script, ASCII text executable
+./.git/hooks/pre-applypatch.sample:                       POSIX shell script, ASCII text executable
+./.git/hooks/pre-commit:                                  POSIX shell script, ASCII text executable, with CRLF line terminators
+...
+./libc.so.6:                                              ASCII text, with CRLF line terminators
+./ntdll.dll:                                              ASCII text
+./wsl-os-release:                                         ASCII text, with CRLF line terminators
+"""]]
+
+From what I can tell, when you try to perform actions in an interoperable way, you get toggling between CRLF/LF. And some annex files get a compounding effect of CRLF endings.
+
+[[!format sh """
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ cat -vet .git/annex/mergedrefs
+95eca82f10ec85f9ed860af5a13ef48fe7801418^Irefs/heads/git-annex^M$
+436ed1bfa6abfc7be4ceeb39ea3b8bad976e27ca^Irefs/heads/git-annex^M^M$
+46a62f21bea952a3014a9fa875a010849ac4183f^Irefs/heads/git-annex^M^M^M$
+a05d9831a28c20babf0d414e0e4edee042b486d6^Irefs/heads/git-annex^M^M^M^M$
+a0bd57285fa715245deab6e5c117768c5da5d990^Irefs/heads/git-annex^M^M^M^M^M$
+"""]]
+
+Now, I know enough that I can probably avoid at least the former, by changing the git configuration for eol (though I have yet to test it). However, this vagary I suspect won't be changed by settings. That is, git-annex list, and probably some other git-annex commands, are inconsistent in their output of eol. git-annex list displays correctly in a Windows console, so I assume it is correctly outputting CRLF. But if I redirect the output to a file, the header correctly has CRLF, but the actually list entries do not:
+
+[[!format sh """
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ git-annex list > git-annex-list.out
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ file git-annex-list.out
+git-annex-list.out: ASCII text, with CRLF, LF line terminators
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ cat -vet git-annex-list.out
+here^M$
+|web^M$
+||bittorrent^M$
+|||^M$
+___ libc.so.6$
+___ ntdll.dll$
+"""]]
+
+Just to clarify, native Windows commands should have CRLF:
+
+[[!format  sh """
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ cmd /c
+Microsoft Windows [Version 10.0.19045.3570]
+(c) Microsoft Corporation. All rights reserved.
+
+@ C:\PublicData\Temp\git-annex-eol-demo.msw
+$ echo Hello > hello.txt
+
+@ C:\PublicData\Temp\git-annex-eol-demo.msw
+$ exit
+
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ cat -vet hello.txt
+Hello ^M$
+"""]]
+
+That's it. From here, I'll just detail the workflow that led me to this. Let's start with a WSL git repo.
+
+[[!format  sh """
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ git version
+git version 2.39.2
+shaddy@mswhost-wsldebs:~$ git-annex version
+git-annex version: 10.20230321-1~ndall+1
+build flags: Assistant Webapp Pairing Inotify DBus DesktopNotify TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.26 DAV-1.3.4 feed-1.3.0.1 ghc-8.8.4 http-client-0.6.4.1 persistent-sqlite-2.10.6.2 torrent-10000.1.1 uuid-1.3.13 yesod-1.6.1.0
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKEIN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: linux x86_64
+supported repository versions: 8 9 10
+upgrade supported from repository versions: 0 1 2 3 4 5 6 7 8 9 10
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ cat wsl-os-release
+PRETTY_NAME="Debian GNU/Linux 12 (bookworm)"
+NAME="Debian GNU/Linux"
+VERSION_ID="12"
+VERSION="12 (bookworm)"
+VERSION_CODENAME=bookworm
+ID=debian
+HOME_URL="https://www.debian.org/"
+SUPPORT_URL="https://www.debian.org/support"
+BUG_REPORT_URL="https://bugs.debian.org/"
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ git init
+hint: Using 'master' as the name for the initial branch. This default branch name
+hint: is subject to change. To configure the initial branch name to use in all
+hint: of your new repositories, which will suppress this warning, call:
+hint:
+hint:   git config --global init.defaultBranch <name>
+hint:
+hint: Names commonly chosen instead of 'master' are 'main', 'trunk' and
+hint: 'development'. The just-created branch can be renamed via this command:
+hint:
+hint:   git branch -m <name>
+Initialized empty Git repository in /home/shaddy/git-annex-eol-demo.wsl/.git/
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ find . -type f -print0 | xargs -0 file
+./.git/info/exclude:                    ASCII text
+./.git/hooks/commit-msg.sample:         POSIX shell script, ASCII text executable
+./.git/hooks/pre-receive.sample:        POSIX shell script, ASCII text executable
+./.git/hooks/pre-merge-commit.sample:   POSIX shell script, ASCII text executable
+./.git/hooks/pre-push.sample:           POSIX shell script, ASCII text executable
+./.git/hooks/fsmonitor-watchman.sample: Perl script text executable
+./.git/hooks/push-to-checkout.sample:   POSIX shell script, ASCII text executable
+./.git/hooks/update.sample:             POSIX shell script, ASCII text executable
+./.git/hooks/pre-commit.sample:         POSIX shell script, ASCII text executable
+./.git/hooks/pre-applypatch.sample:     POSIX shell script, ASCII text executable
+./.git/hooks/applypatch-msg.sample:     POSIX shell script, ASCII text executable
+./.git/hooks/pre-rebase.sample:         POSIX shell script, ASCII text executable
+./.git/hooks/prepare-commit-msg.sample: POSIX shell script, ASCII text executable
+./.git/hooks/post-update.sample:        POSIX shell script, ASCII text executable
+./.git/HEAD:                            ASCII text
+./.git/description:                     ASCII text
+./.git/config:                          ASCII text
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ cp /etc/os-release wsl-os-release
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ file wsl-os-release
+wsl-os-release: ASCII text
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ git add wsl-os-release
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ git commit -m wsl-os-release
+[master (root-commit) 64ebd49] wsl-os-release
+ 1 file changed, 9 insertions(+)
+ create mode 100644 wsl-os-release
+"""]]
+
+Now, in a git-bash.exe created window (that comes with Git for Windows):
+
+[[!format  sh """
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw
+$ git version
+git version 2.42.0.windows.2
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw
+$ git-annex version
+git-annex version: 10.20230927-g18f902efa9c53ffd75a16e25cc666334b1029ea8
+build flags: Assistant Webapp Pairing TorrentParser MagicMime Benchmark Feeds Testsuite S3 WebDAV
+dependency versions: aws-0.22 bloomfilter-2.0.1.0 cryptonite-0.29 DAV-1.3.4 feed-1.3.2.0 ghc-8.10.7 http-client-0.7.9 persistent-sqlite-2.13.0.3 torrent-10000.1.1 uuid-1.3.15 yesod-1.6.1.2
+key/value backends: SHA256E SHA256 SHA512E SHA512 SHA224E SHA224 SHA384E SHA384 SHA3_256E SHA3_256 SHA3_512E SHA3_512 SHA3_224E SHA3_224 SHA3_384E SHA3_384 SKEIN256E SKEIN256 SKEIN512E SKE
+IN512 BLAKE2B256E BLAKE2B256 BLAKE2B512E BLAKE2B512 BLAKE2B160E BLAKE2B160 BLAKE2B224E BLAKE2B224 BLAKE2B384E BLAKE2B384 BLAKE2BP512E BLAKE2BP512 BLAKE2S256E BLAKE2S256 BLAKE2S160E BLAKE2S
+160 BLAKE2S224E BLAKE2S224 BLAKE2SP256E BLAKE2SP256 BLAKE2SP224E BLAKE2SP224 SHA1E SHA1 MD5E MD5 WORM URL X*
+remote types: git gcrypt p2p S3 bup directory rsync web bittorrent webdav adb tahoe glacier ddar git-lfs httpalso borg hook external
+operating system: mingw32 x86_64
+supported repository versions: 8 9 10
+upgrade supported from repository versions: 2 3 4 5 6 7 8 9 10
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw
+$ git init
+hint: Using 'master' as the name for the initial branch. This default branch name
+hint: is subject to change. To configure the initial branch name to use in all
+hint: of your new repositories, which will suppress this warning, call:
+hint: 
+hint:   git config --global init.defaultBranch <name>
+hint: 
+hint: Names commonly chosen instead of 'master' are 'main', 'trunk' and
+hint: 'development'. The just-created branch can be renamed via this command:
+hint: 
+hint:   git branch -m <name>
+Initialized empty Git repository in C:/PublicData/Temp/git-annex-eol-demo.msw/.git/
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (master)
+$ find . -type f -print0 | xargs -0 file
+./.git/config:                          ASCII text
+./.git/description:                     ASCII text
+./.git/HEAD:                            ASCII text
+./.git/hooks/applypatch-msg.sample:     POSIX shell script, ASCII text executable
+./.git/hooks/commit-msg.sample:         POSIX shell script, ASCII text executable
+./.git/hooks/fsmonitor-watchman.sample: Perl script text executable
+./.git/hooks/post-update.sample:        POSIX shell script, ASCII text executable
+./.git/hooks/pre-applypatch.sample:     POSIX shell script, ASCII text executable
+./.git/hooks/pre-commit.sample:         POSIX shell script, ASCII text executable
+./.git/hooks/pre-merge-commit.sample:   POSIX shell script, ASCII text executable
+./.git/hooks/pre-push.sample:           POSIX shell script, ASCII text executable
+./.git/hooks/pre-rebase.sample:         POSIX shell script, ASCII text executable
+./.git/hooks/pre-receive.sample:        POSIX shell script, ASCII text executable
+./.git/hooks/prepare-commit-msg.sample: POSIX shell script, ASCII text executable
+./.git/hooks/push-to-checkout.sample:   POSIX shell script, ASCII text executable
+./.git/hooks/sendemail-validate.sample: POSIX shell script, ASCII text executable
+./.git/hooks/update.sample:             POSIX shell script, ASCII text executable
+./.git/info/exclude:                    ASCII text
+
+
+"""]]
+
+To keep things as simple as possible, I setup a "push" scheme to link the two repos. First, I demonstrate Unix -> Windows eol interplay and git-annex setup:
+
+[[!format  sh """
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ git remote add msw /mnt/c/PublicData/Temp/git-annex-eol-demo.msw
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ git push --set-upstream msw HEAD:fromwsl/master
+Enumerating objects: 3, done.
+Counting objects: 100% (3/3), done.
+Delta compression using up to 4 threads
+Compressing objects: 100% (2/2), done.
+Writing objects: 100% (3/3), 377 bytes | 377.00 KiB/s, done.
+Total 3 (delta 0), reused 0 (delta 0), pack-reused 0
+To /mnt/c/PublicData/Temp/git-annex-eol-demo.msw
+ * [new branch]      HEAD -> fromwsl/master
+branch 'master' set up to track 'msw/fromwsl/master'.
+"""]]
+and push through to the first sign of trouble:
+[[!format  sh """
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (master)
+$ git merge fromwsl/master
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (master)
+$ file wsl-os-release
+wsl-os-release: ASCII text, with CRLF line terminators
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (master)
+$ git-annex init msw
+init msw
+  Detected a filesystem without fifo support.
+
+  Disabling ssh connection caching.
+
+  Detected a crippled filesystem.
+
+  Entering an adjusted branch where files are unlocked as this filesystem does not support locked files.
+
+Switched to branch 'adjusted/master(unlocked)'
+ok
+(recording state in git...)
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ cp /c/Windows/System32/ntdll.dll .
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ git-annex add ntdll.dll
+add ntdll.dll
+ok
+(recording state in git...)
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ git-annex sync
+commit
+warning: in the working copy of 'ntdll.dll', LF will be replaced by CRLF the next time Git touches it
+[adjusted/master(unlocked) b7cfd14] git-annex in msw
+ 1 file changed, 1 insertion(+)
+ create mode 100644 ntdll.dll
+ok
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ file ntdll.dll
+ntdll.dll: PE32+ executable (DLL) (console) x86-64, for MS Windows, 10 sections
+"""]]
+Now setup git-annex on WSL and grab ntdll.dll. Note the post-receive error. I forgot to change the eol on it first. Also, now setup a file for sending back to native Windows:
+[[!format  sh """
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ git-annex init wsl
+init wsl ok
+(recording state in git...)
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ git-annex info
+trusted repositories: 0
+semitrusted repositories: 4
+        00000000-0000-0000-0000-000000000001 -- web
+        00000000-0000-0000-0000-000000000002 -- bittorrent
+        1bc9600e-c005-49a8-ab3e-35f1cff640bd -- wsl [here]
+        80ece5fd-dab2-4271-a92f-e7c285c774c9 -- msw
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 1.02 terabytes (+100 megabytes reserved)
+local annex keys: 0
+local annex size: 0 bytes
+annexed files in working tree: 0
+size of annexed files in working tree: 0 bytes
+bloom filter size: 32 mebibytes (0% full)
+backend usage:
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ git-annex sync
+commit
+On branch master
+Your branch is up to date with 'msw/fromwsl/master'.
+
+nothing to commit, working tree clean
+ok
+pull msw
+remote: Enumerating objects: 12, done.
+remote: Counting objects: 100% (12/12), done.
+remote: Compressing objects: 100% (8/8), done.
+remote: Total 11 (delta 0), reused 0 (delta 0), pack-reused 0
+Unpacking objects: 100% (11/11), 1.04 KiB | 213.00 KiB/s, done.
+From /mnt/c/PublicData/Temp/git-annex-eol-demo.msw
+ * [new branch]      adjusted/master(unlocked) -> msw/adjusted/master(unlocked)
+ * [new branch]      git-annex                 -> msw/git-annex
+ * [new branch]      master                    -> msw/master
+ok
+(merging msw/git-annex into git-annex...)
+(recording state in git...)
+push msw
+Enumerating objects: 10, done.
+Counting objects: 100% (10/10), done.
+Delta compression using up to 4 threads
+Compressing objects: 100% (6/6), done.
+Writing objects: 100% (8/8), 737 bytes | 737.00 KiB/s, done.
+Total 8 (delta 2), reused 0 (delta 0), pack-reused 0
+fatal: cannot run hooks/post-receive: No such file or directory
+To /mnt/c/PublicData/Temp/git-annex-eol-demo.msw
+ * [new branch]      master -> synced/master
+ * [new branch]      git-annex -> synced/git-annex
+ok
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ ls -l
+total 4
+-rw-r--r-- 1 shaddy shaddy 267 Oct 26 10:52 wsl-os-release
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ git-annex sync
+commit
+On branch master
+Your branch is up to date with 'msw/fromwsl/master'.
+
+nothing to commit, working tree clean
+ok
+pull msw
+remote: Enumerating objects: 7, done.
+remote: Counting objects: 100% (7/7), done.
+remote: Compressing objects: 100% (6/6), done.
+remote: Total 6 (delta 0), reused 0 (delta 0), pack-reused 0
+Unpacking objects: 100% (6/6), 731 bytes | 182.00 KiB/s, done.
+From /mnt/c/PublicData/Temp/git-annex-eol-demo.msw
+ + c8f7ad8...c7101bd adjusted/master(unlocked) -> msw/adjusted/master(unlocked)  (forced update)
+   64ebd49..d2154f2  master                    -> msw/master
+   64ebd49..d2154f2  synced/master             -> msw/synced/master
+
+Updating 64ebd49..d2154f2
+Fast-forward
+ ntdll.dll | 1 +
+ 1 file changed, 1 insertion(+)
+ create mode 120000 ntdll.dll
+
+Already up to date.
+ok
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ ls -l
+total 8
+lrwxrwxrwx 1 shaddy shaddy 198 Oct 26 11:04 ntdll.dll -> .git/annex/objects/5f/9X/SHA256E-s2028432--2c8154698c5d5162ec9751c8e6bba462d47f2293ced6be5763579af34334ada3.dll/SHA256E-s2028432--2c8154698c5d5162ec9751c8e6bba462d47f2293ced6be5763579af34334ada3.dll
+-rw-r--r-- 1 shaddy shaddy 267 Oct 26 10:52 wsl-os-release
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ git-annex get ntdll.dll
+get ntdll.dll (from msw...)
+ok
+(recording state in git...)
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ git-annex sync
+commit
+On branch master
+Your branch is ahead of 'msw/fromwsl/master' by 1 commit.
+  (use "git push" to publish your local commits)
+
+nothing to commit, working tree clean
+ok
+pull msw
+ok
+push msw
+Enumerating objects: 9, done.
+Counting objects: 100% (9/9), done.
+Delta compression using up to 4 threads
+Compressing objects: 100% (4/4), done.
+Writing objects: 100% (5/5), 457 bytes | 457.00 KiB/s, done.
+Total 5 (delta 1), reused 0 (delta 0), pack-reused 0
+fatal: cannot run hooks/post-receive: No such file or directory
+To /mnt/c/PublicData/Temp/git-annex-eol-demo.msw
+   46a62f2..436ed1b  git-annex -> synced/git-annex
+ok
+
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ cp /lib/x86_64-linux-gnu/libc.so.6 .
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ git-annex add libc.so.6
+add libc.so.6
+ok
+(recording state in git...)
+shaddy@mswhost-wsldebs:~/git-annex-eol-demo.wsl$ git-annex sync
+commit
+[master 22a06f5] git-annex in wsl
+ 1 file changed, 1 insertion(+)
+ create mode 120000 libc.so.6
+ok
+pull msw
+remote: Enumerating objects: 9, done.
+remote: Counting objects: 100% (9/9), done.
+remote: Compressing objects: 100% (4/4), done.
+remote: Total 5 (delta 2), reused 0 (delta 0), pack-reused 0
+Unpacking objects: 100% (5/5), 395 bytes | 131.00 KiB/s, done.
+From /mnt/c/PublicData/Temp/git-annex-eol-demo.msw
+   436ed1b..1c0face  git-annex  -> msw/git-annex
+ok
+(merging msw/git-annex into git-annex...)
+(recording state in git...)
+push msw
+Enumerating objects: 13, done.
+Counting objects: 100% (13/13), done.
+Delta compression using up to 4 threads
+Compressing objects: 100% (9/9), done.
+Writing objects: 100% (10/10), 1.20 KiB | 1.20 MiB/s, done.
+Total 10 (delta 0), reused 0 (delta 0), pack-reused 0
+fatal: cannot run hooks/post-receive: No such file or directory
+To /mnt/c/PublicData/Temp/git-annex-eol-demo.msw
+   436ed1b..95eca82  git-annex -> synced/git-annex
+   d2154f2..22a06f5  master -> synced/master
+ok
+"""]]
+
+And finally, corrupting git structure backing git-annex:
+
+[[!format  sh """
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ git-annex list --allrepos
+here
+|web
+||bittorrent
+|||wsl
+||||
+X__X ntdll.dll
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ git-annex drop --force ntdll.dll
+drop ntdll.dll ok
+(recording state in git...)
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ file ntdll.dll
+ntdll.dll: ASCII text
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ git-annex sync
+commit
+On branch adjusted/master(unlocked)
+nothing to commit, working tree clean
+ok
+merge synced/master master (Merging into master...)
+Updating d2154f2..22a06f5
+Fast-forward
+ libc.so.6 | 1 +
+ 1 file changed, 1 insertion(+)
+ create mode 120000 libc.so.6
+(Merging into adjusted branch...)
+Updating c7101bd..99dbd2c
+Fast-forward
+ libc.so.6 | 1 +
+ 1 file changed, 1 insertion(+)
+ create mode 100644 libc.so.6
+
+Already up to date.
+(Merging into adjusted branch...)
+Updating d0175a9..8153e87
+Fast-forward
+ok
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ file ntdll.dll  libc.so.6
+ntdll.dll: ASCII text
+libc.so.6: ASCII text, with CRLF line terminators
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ git-annex list > git-annex-list.out
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ cat -vet git-annex-list.out
+here^M$
+|web^M$
+||bittorrent^M$
+|||^M$
+___ libc.so.6$
+___ ntdll.dll$
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ find . -type f -print0 | xargs -0 file
+./.git/annex/adjust.lck:                                  empty
+./.git/annex/adjust.log:                                  ASCII text
+./.git/annex/fsck/fsck.lck:                               empty
+./.git/annex/fsck/fsckdb/db:                              SQLite 3.x database, last written using SQLite version 3033000, writer version 2, read version 2, file counter 2, database pages 3, cookie 0x1, schema 4, UTF-8, version-valid-for 2
+./.git/annex/gitqueue.lck:                                empty
+./.git/annex/index:                                       Git index, version 2, 3 entries
+./.git/annex/index.lck:                                   ASCII text, with CRLF line terminators
+./.git/annex/journal.lck:                                 empty
+./.git/annex/keysdb/db:                                   SQLite 3.x database, last written using SQLite version 3033000, writer version 2, read version 2, file counter 2, database pages 7, cookie 0x2, schema 4, UTF-8, version-valid-for 2
+./.git/annex/keysdb.cache:                                ASCII text, with no line terminators
+./.git/annex/keysdb.lck:                                  empty
+./.git/annex/mergedrefs:                                  ASCII text, with CRLF, CR line terminators
+./.git/annex/othertmp.lck:                                empty
+./.git/annex/restage.lck:                                 empty
+./.git/annex/sentinal:                                    empty
+./.git/annex/sentinal.cache:                              ASCII text, with no line terminators
+./.git/annex/smudge.lck:                                  empty
+./.git/annex/smudge.log:                                  empty
+./.git/COMMIT_EDITMSG:                                    ASCII text
+./.git/config:                                            ASCII text
+./.git/description:                                       ASCII text
+./.git/HEAD:                                              ASCII text
+./.git/hooks/applypatch-msg.sample:                       POSIX shell script, ASCII text executable
+./.git/hooks/commit-msg.sample:                           POSIX shell script, ASCII text executable
+./.git/hooks/fsmonitor-watchman.sample:                   Perl script text executable
+./.git/hooks/post-checkout:                               POSIX shell script, ASCII text executable, with CRLF line terminators
+./.git/hooks/post-merge:                                  POSIX shell script, ASCII text executable, with CRLF line terminators
+./.git/hooks/post-receive:                                POSIX shell script, ASCII text executable, with CRLF line terminators
+./.git/hooks/post-update.sample:                          POSIX shell script, ASCII text executable
+./.git/hooks/pre-applypatch.sample:                       POSIX shell script, ASCII text executable
+./.git/hooks/pre-commit:                                  POSIX shell script, ASCII text executable, with CRLF line terminators
+./.git/hooks/pre-commit.sample:                           POSIX shell script, ASCII text executable
+./.git/hooks/pre-merge-commit.sample:                     POSIX shell script, ASCII text executable
+./.git/hooks/pre-push.sample:                             POSIX shell script, ASCII text executable
+./.git/hooks/pre-rebase.sample:                           POSIX shell script, ASCII text executable
+./.git/hooks/pre-receive.sample:                          POSIX shell script, ASCII text executable
+./.git/hooks/prepare-commit-msg.sample:                   POSIX shell script, ASCII text executable
+./.git/hooks/push-to-checkout.sample:                     POSIX shell script, ASCII text executable
+./.git/hooks/sendemail-validate.sample:                   POSIX shell script, ASCII text executable
+./.git/hooks/update.sample:                               POSIX shell script, ASCII text executable
+./.git/index:                                             Git index, version 2, 3 entries
+./.git/info/attributes:                                   ASCII text, with CRLF line terminators
+./.git/info/exclude:                                      ASCII text
+./.git/logs/HEAD:                                         ASCII text
+./.git/logs/refs/heads/adjusted/master(unlocked):         ASCII text
+./.git/logs/refs/heads/fromwsl/master:                    ASCII text
+./.git/logs/refs/heads/git-annex:                         ASCII text
+./.git/logs/refs/heads/master:                            ASCII text
+./.git/logs/refs/heads/synced/git-annex:                  ASCII text
+./.git/logs/refs/heads/synced/master:                     ASCII text
+...
+./.git/ORIG_HEAD:                                         ASCII text
+./.git/refs/annex/last-index:                             ASCII text
+./.git/refs/basis/adjusted/master(unlocked):              ASCII text
+./.git/refs/heads/adjusted/master(unlocked):              ASCII text
+./.git/refs/heads/fromwsl/master:                         ASCII text
+./.git/refs/heads/git-annex:                              ASCII text
+./.git/refs/heads/master:                                 ASCII text
+./.git/refs/heads/synced/git-annex:                       ASCII text
+./.git/refs/heads/synced/master:                          ASCII text
+./git-annex-list.out:                                     ASCII text, with CRLF, LF line terminators
+./libc.so.6:                                              ASCII text, with CRLF line terminators
+./ntdll.dll:                                              ASCII text
+./wsl-os-release:                                         ASCII text, with CRLF line terminators
+
+shaddy@mswhost-w10 MINGW64 /c/PublicData/Temp/git-annex-eol-demo.msw (adjusted/master(unlocked))
+$ cat -vet .git/annex/mergedrefs
+95eca82f10ec85f9ed860af5a13ef48fe7801418^Irefs/heads/git-annex^M$
+436ed1bfa6abfc7be4ceeb39ea3b8bad976e27ca^Irefs/heads/git-annex^M^M$
+46a62f21bea952a3014a9fa875a010849ac4183f^Irefs/heads/git-annex^M^M^M$
+a05d9831a28c20babf0d414e0e4edee042b486d6^Irefs/heads/git-annex^M^M^M^M$
+a0bd57285fa715245deab6e5c117768c5da5d990^Irefs/heads/git-annex^M^M^M^M^M$
+
+"""]]
diff --git a/doc/forum/Windows_eol_issues/comment_1_72e4c3a200e3fceb2afd4df563eaa5d4._comment b/doc/forum/Windows_eol_issues/comment_1_72e4c3a200e3fceb2afd4df563eaa5d4._comment
new file mode 100644
index 0000000000..39531ff1b5
--- /dev/null
+++ b/doc/forum/Windows_eol_issues/comment_1_72e4c3a200e3fceb2afd4df563eaa5d4._comment
@@ -0,0 +1,31 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-10-26T18:36:59Z"
+ content="""
+The .git/annex/mergedrefs problem seems to indicate that `git show-ref` is
+outputting lines ending with CRLF. That's surprising to me, I thought git
+commands on Windows didn't output CRLF at least when piped.
+Can you show the output of that command fed through `cat -vet` please?
+
+In retrospect it would be better if annex pointer files did not end in a
+newline at all. It's not really necessary that they do, and that would
+avoid git for windows from messing with their content when merging them
+from elsewhere. However, AFAIK git-annex manages to not break despite git
+doing that to its pointer file, by ignoring CR in pointer files
+(see [[!commit 4d89a1ffd1b4bfd423c11c9e703e9d2fb0f5fb25]]). 
+So I'm not sure what problem you're demonstrating with that example merge?
+
+git-annex list is displaying CRLF for the header because it uses putStrLn
+for it, and putStrLn on Windows uses CRLF by default (due to the default NewlineMode
+on Windows). Then for the file list it uses ByteString.Char8.putStrLn, which does
+not use CRLF. Since those two are generally seen as equivilant and code is
+often switched between them to improve performance, it would be a real
+ongoing pain point to remember this gotcha. This makes me think that
+git-annex on windows ought to `hSetNewlineMode stdout noNewlineTranslation`
+so it consistently outputs without CRLF.
+
+About the CRLF in the hooks, I know it's been discussed somewhere before,
+but I can't remember where. IIRC there were some complexities involving
+what the bash bundled with git expected to find.
+"""]]
diff --git a/doc/forum/Windows_eol_issues/comment_2_77383b959d576c7930c726c55097c964._comment b/doc/forum/Windows_eol_issues/comment_2_77383b959d576c7930c726c55097c964._comment
new file mode 100644
index 0000000000..9d333158fc
--- /dev/null
+++ b/doc/forum/Windows_eol_issues/comment_2_77383b959d576c7930c726c55097c964._comment
@@ -0,0 +1,137 @@
+[[!comment format=mdwn
+ username="jkniiv"
+ avatar="http://cdn.libravatar.org/avatar/05fd8b33af7183342153e8013aa3713d"
+ subject="comment 2"
+ date="2023-10-27T18:45:03Z"
+ content="""
+@joey, just out of curiosity (and concern) I took a peek into my .git/annex/mergedrefs files (within native Windows annexes
+ -- no WSL involved) and sure enough I had a similar presentation where CRs had become multiplied at the end
+of the lines. However, when you ask if `git show-ref` is outputting any lines ending with CRLF, that doesn't
+seem to be the case:
+
+[[!format sh \"\"\"
+jkniiv@AINESIS MINGW64 /c/git-annex-tested-binaries (adjusted/master(unlocked))
+$ git show-ref | cat -vet
+c10f7c4ed5751458b0c5fa687fa4fe1bcc538140 refs/annex/last-index$
+00ff347497e0c1d921b7f2484d092e9ac18657cb refs/basis/adjusted/master(unlocked)$
+a9f629165f81c026322bd4a58b2704dc557b5d48 refs/heads/adjusted/master(unlocked)$
+92e22121afd9d47fd63f27a0709d86da0485d2be refs/heads/git-annex$
+00ff347497e0c1d921b7f2484d092e9ac18657cb refs/heads/master$
+9b5e29d8c3bef88b9537864d6b3ae56d5c29a429 refs/heads/synced/git-annex$
+00ff347497e0c1d921b7f2484d092e9ac18657cb refs/heads/synced/master$
+782a184a8a9d11d2b7fc9f5990b33ebddc4ff9d0 refs/remotes/gatb-archive/git-annex$
+11803c78a1bfd54c4e5e0ee19d5de020f9990cb2 refs/remotes/gatb-archive/master$
+782a184a8a9d11d2b7fc9f5990b33ebddc4ff9d0 refs/remotes/gatb-archive/synced/git-annex$
+11803c78a1bfd54c4e5e0ee19d5de020f9990cb2 refs/remotes/gatb-archive/synced/master$
+941d08010d54724495fb5ccf69f6943083dec61f refs/remotes/gatb-bare/adjusted/master(unlocked)$
+92e22121afd9d47fd63f27a0709d86da0485d2be refs/remotes/gatb-bare/git-annex$
+00ff347497e0c1d921b7f2484d092e9ac18657cb refs/remotes/gatb-bare/master$
+92e22121afd9d47fd63f27a0709d86da0485d2be refs/remotes/gatb-bare/synced/git-annex$
+00ff347497e0c1d921b7f2484d092e9ac18657cb refs/remotes/gatb-bare/synced/master$
+ead34cd4d6a1dfb372b69d7acd88a70a96d5a547 refs/remotes/gatb-hetzner/git-annex$
+13bf1536cb3bcd22216f7696011327cb3bf1a81d refs/remotes/gatb-hetzner/master$
+ead34cd4d6a1dfb372b69d7acd88a70a96d5a547 refs/remotes/gatb-hetzner/synced/git-annex$
+13bf1536cb3bcd22216f7696011327cb3bf1a81d refs/remotes/gatb-hetzner/synced/master$
+65a00ffec41383ad7e06198e6aae3150b96032e2 refs/remotes/gawb4/git-annex$
+d2f6d4dadd5fa404c3c70b4e2713c3f73918cbf9 refs/remotes/gawb4/master$
+65a00ffec41383ad7e06198e6aae3150b96032e2 refs/remotes/gawb4/synced/git-annex$
+d2f6d4dadd5fa404c3c70b4e2713c3f73918cbf9 refs/remotes/gawb4/synced/master$
+
+jkniiv@AINESIS MINGW64 /c/git-annex-tested-binaries (adjusted/master(unlocked))
+$ git show-ref | od -a | head
+0000000   c   1   0   f   7   c   4   e   d   5   7   5   1   4   5   8
+0000020   b   0   c   5   f   a   6   8   7   f   a   4   f   e   1   b
+0000040   c   c   5   3   8   1   4   0  sp   r   e   f   s   /   a   n
+0000060   n   e   x   /   l   a   s   t   -   i   n   d   e   x  nl   0
+0000100   0   f   f   3   4   7   4   9   7   e   0   c   1   d   9   2
+0000120   1   b   7   f   2   4   8   4   d   0   9   2   e   9   a   c
+0000140   1   8   6   5   7   c   b  sp   r   e   f   s   /   b   a   s
+0000160   i   s   /   a   d   j   u   s   t   e   d   /   m   a   s   t
+0000200   e   r   (   u   n   l   o   c   k   e   d   )  nl   a   9   f
+0000220   6   2   9   1   6   5   f   8   1   c   0   2   6   3   2   2
+\"\"\"]]
+
+Just to prove my case, this is how .git/annex/mergedrefs looks like in the above repo:
+
+[[!format sh \"\"\"
+jkniiv@AINESIS MINGW64 /c/git-annex-tested-binaries (adjusted/master(unlocked))
+$ cat -vet .git/annex/mergedrefs
+92e22121afd9d47fd63f27a0709d86da0485d2be^Irefs/heads/git-annex^M$
+2e3dd770daa05dfaa023fc492fa13fd5af056694^Irefs/heads/git-annex^M^M$
+b8e586bb07a49c0c67616314e37783d21f18e70f^Irefs/heads/git-annex^M^M^M$
+6932d8306081c2506e2dc720cef6c6c4039044a5^Irefs/remotes/gatb-bare/git-annex^M^M^M^M$
+9b5e29d8c3bef88b9537864d6b3ae56d5c29a429^Irefs/heads/git-annex^M^M^M^M^M$
+ead34cd4d6a1dfb372b69d7acd88a70a96d5a547^Irefs/heads/git-annex^M^M^M^M^M^M$
+286d76b2cd3cceb25322021012ab3a6a478779bc^Irefs/heads/git-annex^M^M^M^M^M^M^M$
+05a146f6a8357e92e9b2dc6b4ed3dd63a5ccd467^Irefs/heads/git-annex^M^M^M^M^M^M^M^M$
+3946949105cc004919c2e1a41ebd5ee4553f40f3^Irefs/heads/git-annex^M^M^M^M^M^M^M^M^M$
+debedf463e5bd4f420271e6e4550c46c683e6c91^Irefs/heads/git-annex^M^M^M^M^M^M^M^M^M^M$
+8b56b9f04394bd6735e729fa9fd206493acd11ae^Irefs/heads/git-annex^M^M^M^M^M^M^M^M^M^M^M$
+c1d72955c7bb1e3ab71373ad1ed66066c03a40dd^Irefs/heads/git-annex^M^M^M^M^M^M^M^M^M^M^M^M$
+3268d0d1937f1f86f9794526330e0b5a8dbf7ac7^Irefs/heads/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M$
+db9a8f6f0631f13665f520e372dc1b425a197b73^Irefs/heads/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+fcd934175a3dab20cf9e66040126922febab3107^Irefs/heads/synced/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+782a184a8a9d11d2b7fc9f5990b33ebddc4ff9d0^Irefs/remotes/gatb-archive/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+f054b35273e9ee8797c4570782ebf8de923b8bec^Irefs/remotes/gatb-bare/synced/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+65a00ffec41383ad7e06198e6aae3150b96032e2^Irefs/remotes/gawb4/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+1ab48671d698ad56df6fae874ffb918b5b7c55da^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+cf6514421f552040f65a5f64f10cdb8644a91367^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+bcb6cb79e2822eb87a517909305ba7738af71fb7^Irefs/remotes/gatb-hetzner/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+57408550366738cb1440f8ad0052cef11170837d^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+14783b3c3383df28e86902758b024f09915fe79d^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+e1f481c2ff3be7da1c4adfa6148aa00bb652fecc^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+785fe3d81ea5ce4c1843b993e2487830133f6380^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+1c1ffa7bbec72b0bdc251561b82be283b149d814^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+7344df6135fc90d38f6ab2c7e184813849040b1b^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+5a434c6c53fee582bbf1a84941dcc21edc860f6b^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+e2c44e5214f06ab0897363677111124ac7b165bd^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+79d2b38753bce49828f6ed585f6891ae5d394e63^Irefs/remotes/gawb4/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+fa0d446f0e67658a40c4f1c6153522d7544afa16^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+1d2d0cca747460254d35a6a1717401a566fdc892^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+2ac05dcb208ef343b0baf21755cf268c3a3bea36^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+857832b466f608781c342ea946de42e2707a7f7d^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+cecf8c16a0cfc1e862e4c287305065ddfed215cc^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+e27eca9c36d4c9fe733ba4d8312091e3dab8eabd^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+c28cd1c09f3498073b5239dfb5a0c00396b0a0d3^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+ddf4486a7cf3219ad79ea72ab94611f65dcd8c75^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+cf5f396481de19336e6157f62fa4f76e02049f62^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+0757122bb58d08024ee8184c871cdff28746958a^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+3ba81bd06ffdc3598131acd83b9d9c825e547b9a^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+e77b6e375092ae03b13b7518e62148c6ac424c80^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+8e78173936ec9a134a12c80b4c0705098eff3793^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+a50f6a64b8d47dd1084b1b49817cc073b1f0ff9e^Irefs/remotes/gatb-hetzner/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+644a387e091a0878c2ad1b4d90d5ad4abf7e195e^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+f6432e124a8cadc45f96c7ab64ae1657d6417259^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+650a95c4d04438a91319e6fe214b103fa3599a33^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+e0e3a5ed05ccfca84c084850af21188b4262c63b^Irefs/remotes/gatb-hetzner/synced/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+0e84071b9352d587611d927636de4aaec95f01b9^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+fb38147c51706678ecb1c353e997a29604ee1a19^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+bfec3c6a4c93194f0b1a18117b6c5fae02b9f517^Irefs/remotes/gatb-hetzner/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+3fd7a8a29b00f2e11042d9fad7b3c24c5121a98c^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+f1c01aad6a1fcb7c6c3f13a74434eac34f4df397^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+d10cff285d737d6335a28e5966df21d333c0ce15^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+7cc02c1fbd05b1537cea55385e8f500c3c58b6a3^Irefs/remotes/gatb-hetzner/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+7424f6b424a4934945c22794291fa3e35734c66a^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+362f5ebc033bc8660bb45da356cfa78512a22922^Irefs/remotes/gatb-hetzner/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+b32c1c0152ddb3bb07981b471a7a44c2bb0cfae0^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+64df2acfeb780cb77a7b9a3882ba39fc29399616^Irefs/remotes/gatb-hetzner/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+53cb9f7c033c887bf76b7a5445161f846397d5dc^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+db2233280bc66f524fd1427c7fe8bfad732189b9^Irefs/remotes/gatb-hetzner/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+cee656c1fa1f53f84c7acdf9f88cd5dba86e9934^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+554e4ea4cb16b69964ce4a4e8be3eccb3f516e12^Irefs/remotes/gatb-hetzner/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+c93d2de1fc7173768730567839a7c335d45c055b^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+f71d65064f2d417e8198a6ebfdb47d1fa654412a^Irefs/remotes/gatb-hetzner/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+7d8914cd49d9ae8235a93c0f85e64822284ec955^Irefs/remotes/gatb-hetzner/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+396b17eec3f1cc1a1769fdbae3a26d5f32462777^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+c8276787b4aeb8db842bd718a1a2f55f4e9b2959^Irefs/remotes/gatb-hetzner/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+15c14cd9d37769315e6620b0a55f0d2428e54e50^Irefs/remotes/gatb-hetzner/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+ab2e5c00e0265c9c028477172c852d9cabf16c90^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+0ac85b5975b385669f7bc22d8cf79ccdb533162b^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+fef8eedd67a1178ec3ea9c4f0612eef238d10c7d^Irefs/remotes/gatb-hetzner/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+af383db2d08d956c150948d8fe330aad2298ae36^Irefs/remotes/gatb-hetzner/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+98aaf6d680c1ca1610634ff92500c7ce9734788c^Irefs/remotes/gatb-archive/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+ed4e3318763983db679d0b1fea727072afdca98e^Irefs/remotes/gatb-bare/git-annex^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M^M$
+\"\"\"]]
+
+Do you think we have a cause for concern? How should this file be formatted -- i.e. should there be multiple hashes for any one ref like above?
+"""]]
diff --git a/doc/forum/Windows_eol_issues/comment_3_926b386f5ca59f1c5232cf5cba69e12a._comment b/doc/forum/Windows_eol_issues/comment_3_926b386f5ca59f1c5232cf5cba69e12a._comment
new file mode 100644
index 0000000000..9180392629
--- /dev/null
+++ b/doc/forum/Windows_eol_issues/comment_3_926b386f5ca59f1c5232cf5cba69e12a._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2023-10-30T17:04:40Z"
+ content="""
+I would not be concerned about the contents of the mergedrefs file, since
+it is only an optimisation and at worst git-annex does a little extra work
+re-merging a branch it has already merged.
+
+I've fixed this in [[!commit eb42935e5822a43ec29c1c113d5df62ea676d119]].
+Several log files were affected as detailed in the commit, but it seems
+nothing really broke, only some optimisations were prevented from working,
+luckily.
+
+Also I checked git commands on windows and nothing I tried output CRLF to
+stdout on windows. So I've made git-annex follow suite, and disabled
+NewlineMode for stdout.
+"""]]
diff --git a/doc/forum/Windows_eol_issues/comment_4_dc470ae359ee4ce017770a89441a8b65._comment b/doc/forum/Windows_eol_issues/comment_4_dc470ae359ee4ce017770a89441a8b65._comment
new file mode 100644
index 0000000000..7ed6e8bd64
--- /dev/null
+++ b/doc/forum/Windows_eol_issues/comment_4_dc470ae359ee4ce017770a89441a8b65._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="jkniiv"
+ avatar="http://cdn.libravatar.org/avatar/05fd8b33af7183342153e8013aa3713d"
+ subject="comment 4"
+ date="2023-10-30T21:32:07Z"
+ content="""
+Ok thanks, Joey, for the quick fix!
+"""]]
diff --git a/doc/forum/Windows_eol_issues/comment_5_e711bdfcee4ec9a2860662e8c65a9c0f._comment b/doc/forum/Windows_eol_issues/comment_5_e711bdfcee4ec9a2860662e8c65a9c0f._comment
new file mode 100644
index 0000000000..2faa2e4ce8
--- /dev/null
+++ b/doc/forum/Windows_eol_issues/comment_5_e711bdfcee4ec9a2860662e8c65a9c0f._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2023-11-02T17:28:49Z"
+ content="""
+Took a look at hook script line endings on windows, and found that they
+work both with CRLF endings and without the CR.
+
+It seems better to not use CR in them, because git for windows does not
+in the examples it writes, and because it will make the hooks portable if
+the repository gets moved to a unix system.
+
+So, I made `git-annex init` omit the CR when writing new hook scripts.
+Existing hook scripts that use CRLF will be preserved by git-annex.
+
+I have not been able to reproduce this problem with or without CR in hook
+scripts:
+
+	fatal: cannot run hooks/post-receive: No such file or directory
+"""]]
diff --git a/doc/forum/Windows_eol_issues/comment_6_62f748e40324bc988366e19088b33295._comment b/doc/forum/Windows_eol_issues/comment_6_62f748e40324bc988366e19088b33295._comment
new file mode 100644
index 0000000000..594da81fa5
--- /dev/null
+++ b/doc/forum/Windows_eol_issues/comment_6_62f748e40324bc988366e19088b33295._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="beryllium@5bc3c32eb8156390f96e363e4ba38976567425ec"
+ nickname="beryllium"
+ avatar="http://cdn.libravatar.org/avatar/62b67d68e918b381e7e9dd6a96c16137"
+ subject="comment 6"
+ date="2023-11-14T22:43:22Z"
+ content="""
+I'm so sorry for the delay in this response, and the fact that it is not a response. I thought I had turned on email notifications, and missed the responses. I've skimmed those at this point, but will read properly and respond to clarifications.
+
+
+"""]]
diff --git a/doc/forum/Windows_eol_issues/comment_7_f9b71ea2158c02dfa8a7c59891aea679._comment b/doc/forum/Windows_eol_issues/comment_7_f9b71ea2158c02dfa8a7c59891aea679._comment
new file mode 100644
index 0000000000..952592b2db
--- /dev/null
+++ b/doc/forum/Windows_eol_issues/comment_7_f9b71ea2158c02dfa8a7c59891aea679._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="beryllium@5bc3c32eb8156390f96e363e4ba38976567425ec"
+ nickname="beryllium"
+ avatar="http://cdn.libravatar.org/avatar/62b67d68e918b381e7e9dd6a96c16137"
+ subject="comment 7"
+ date="2023-11-21T09:25:33Z"
+ content="""
+Thanks joey for your response. And thank you for jkniiv for providing the output to git show-head, it matches what I see.
+
+With regards to the pointer file... I think the problem I have with it is.. well, first the avoidable warning:
+
+[[!format  sh \"\"\"
+warning: in the working copy of 'ntdll.dll', LF will be replaced by CRLF the next time Git touches it
+\"\"\"]]
+
+I guess also, because git-annex doesn't honour (??) git's actual expectations on line mode, you end up with git thinking a file is modified. Because git knows it as text, and in the default mode under Windows, it expects the text file to be CRLF.
+
+Also... I guess in a way, once Windows has acted on a file that came from unix, it now becomes a pointer file on disk. You can't get the benefit of the symlink to view the contents.
+
+Also also... I don't think I demonstrated it here, but I found that eventually, some merge would cause the line-ending to flip, and then there would be another unnecessary checkin of the pointer file.
+
+Sorry if this is all a bit abstract. I was working on large repos, oblivious to some of what I found and have pasted above, and was filling in the \"bitmap\" of my knowledge piecemeal.
+
+With the CRLF hooks, yes, I originally reported that. So in the above, not how I do a push msw to wsl only, and no pull from msw at wsl. That's because that type of pull would break, with the wsl/linux system() call interpreting the CR as part of the shebang, and being unable to execute the shell itself.
+
+"""]]
diff --git a/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_12_041ae920e2e6fd83967dc98646452fa7._comment b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_12_041ae920e2e6fd83967dc98646452fa7._comment
new file mode 100644
index 0000000000..bf3cbca594
--- /dev/null
+++ b/doc/forum/__34__du__34___equivalent_on_an_annex__63__/comment_12_041ae920e2e6fd83967dc98646452fa7._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="psxvoid"
+ avatar="http://cdn.libravatar.org/avatar/fde068fbdeabeea31e3be7aa9c55d84b"
+ subject="Using fuse annexize - works like a charm"
+ date="2024-01-07T04:11:37Z"
+ content="""
+Hey @wzhd,
+
+Thanks a lot for this tool - annexize works like a charm even in tags views!
+
+It's basically solve the most important problem for me:
+
+I can use ncdu for organizing files in my local git annex repository that
+does not contain any actual files (only file links), and then just sync
+with linked repos that do store those files.
+"""]]
diff --git a/doc/forum/annex.largefile_not_working.mdwn b/doc/forum/annex.largefile_not_working.mdwn
new file mode 100644
index 0000000000..d15de6b022
--- /dev/null
+++ b/doc/forum/annex.largefile_not_working.mdwn
@@ -0,0 +1,20 @@
+I seem to be having issues with annex.largefiles.  I initialize git and the annex, then I set largefiles to put everything in the annex, generate a 1Mb file, `git add` it, and commit it.  The file is copied and renamed to its hash value in .git/annex/objects but the file also remains in the main directory instead of being replaced with a symlink.  Here are my steps to create the issue:
+
+    git init
+    git annex init
+    git annex config --set annex.largefiles anything
+    fallocate -l 1M test.bin
+    git add test.bin
+    git commit -a -m "Test"
+
+I've also tried creating a .gitattributes file in the main directory with the following attribute:
+
+    * annex.largefiles=anything
+
+Still, nothing is symlinked.
+
+It works just fine when I run `git annex add test.bin`.  It puts the file in the annex and creates a symlink to it.
+
+I've tried this on Fedora 39 with git annex version 10.20230626 and on Ubuntu 22.04.2 LTS with git annex version 8.20210223.  These are both fresh machines that have never had git or git-annex run on them before.
+
+What am I doing wrong here?  Should I be filing a bug report?
diff --git a/doc/forum/annex.largefile_not_working/comment_1_b2ecb8b60603929bae91c3007817585f._comment b/doc/forum/annex.largefile_not_working/comment_1_b2ecb8b60603929bae91c3007817585f._comment
new file mode 100644
index 0000000000..200a8dcdce
--- /dev/null
+++ b/doc/forum/annex.largefile_not_working/comment_1_b2ecb8b60603929bae91c3007817585f._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="kdm9"
+ avatar="http://cdn.libravatar.org/avatar/b7b736335a0e9944a8169a582eb4c43d"
+ subject="comment 1"
+ date="2023-12-04T10:09:15Z"
+ content="""
+I think this is intended behavior when adding with `git add`, or at least it's what I've seen for long enough for me to have forgotten if it ever was different. `git annex add` will create symlinks, as will `git add  && git annex lock`.
+
+If this was actually a small file, you wouldn't see it hashed & copied under .git/annex/objects. You should also see in git log that the change is an addition of some git annex key, not a git blob diff as would be the case for a small file.
+
+NB: I'm just another user, @joey please correct me if this is wrong
+"""]]
diff --git a/doc/forum/annex.largefile_not_working/comment_2_18bc73688897728d0bcae6df1225e6e3._comment b/doc/forum/annex.largefile_not_working/comment_2_18bc73688897728d0bcae6df1225e6e3._comment
new file mode 100644
index 0000000000..22eb0c3027
--- /dev/null
+++ b/doc/forum/annex.largefile_not_working/comment_2_18bc73688897728d0bcae6df1225e6e3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2023-12-04T16:48:09Z"
+ content="""
+`git add` always adds the annexed file unlocked. You can `git-annex lock`
+the file is you want to convert it to a symlink.
+
+(The reason for this difference between `git add` and `git-annex add` is
+that there's simply no way for git-annex to tell `git add` to make the file
+be a symlink.)
+"""]]
diff --git a/doc/forum/annex.largefile_not_working/comment_3_81085c1a526ac804edba689043e58944._comment b/doc/forum/annex.largefile_not_working/comment_3_81085c1a526ac804edba689043e58944._comment
new file mode 100644
index 0000000000..d37f196648
--- /dev/null
+++ b/doc/forum/annex.largefile_not_working/comment_3_81085c1a526ac804edba689043e58944._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="brendan.ward@a2e11ad27f6b2fa2c556aea6811496e0d95dd0da"
+ nickname="brendan.ward"
+ avatar="http://cdn.libravatar.org/avatar/bc7899f1aa523365cdfadc2ba3f7913d"
+ subject="comment 3"
+ date="2023-12-05T03:24:29Z"
+ content="""
+Excellent.  Thanks for the help!
+"""]]
diff --git a/doc/forum/client_repositories_setup_problem.mdwn b/doc/forum/client_repositories_setup_problem.mdwn
new file mode 100644
index 0000000000..a7092c2b90
--- /dev/null
+++ b/doc/forum/client_repositories_setup_problem.mdwn
@@ -0,0 +1,156 @@
+I'm trying to setup git-annex for syncing two clients using a transfer repository. All of that without the webapp UI.
+
+Here's the reproducible scenario with a bash script:
+
+```bash
+#/usr/bin/env bash
+
+# Just a way to access the script's directory
+cd "$(dirname "$0")"
+DIR="$(pwd)"
+
+# Create the 1st client repository
+mkdir $DIR/client1
+cd $DIR/client1
+git init && git annex init
+
+# Create the 2nd client repository
+mkdir $DIR/client2
+cd $DIR/client2
+git init && git annex init
+
+# Create the transfer repository
+mkdir $DIR/share
+cd $DIR/share
+git init && git annex init
+
+# Setup the remotes and groups for the transfer repository
+cd $DIR/share
+git remote add client1 $DIR/client1
+git remote add client2 $DIR/client1
+git annex group . transfer
+git annex group client1 client
+git annex group client2 client
+git co -b main
+
+# Setup the remotes and groups for the 1st client repository.
+cd $DIR/client1
+git remote add share $DIR/share
+git annex group . client
+git annex group share transfer
+git co -b main
+
+# Setup the remotes and groups for the 2nd client repository.
+cd $DIR/client2
+git remote add share $DIR/share
+git annex group . client
+git annex group share transfer
+git co -b main
+
+# Run git-annex assistant for each repository
+cd $DIR/client1 && git annex assistant
+cd $DIR/client2 && git annex assistant
+cd $DIR/share && git annex assistant
+
+# Add a single file to the 1st client.
+cd $DIR/client1
+echo "My first file" >> file.txt
+```
+
+Result:
+
+client1: I see the auto-commit has been added for file.txt
+
+share: I get the following daemon logs:
+
+```
+(scanning...) (started...) 
+From /home/xxx/git-annex-scenarios/share-between-clients/client1
+ * [new branch]      git-annex  -> client2/git-annex
+(merging client2/git-annex into git-annex...)
+From /home/xxx/git-annex-scenarios/share-between-clients/client1
+ * [new branch]      git-annex  -> client1/git-annex
+
+merge: refs/remotes/client2/main - not something we can merge
+
+merge: refs/remotes/client2/synced/main - not something we can merge
+
+merge: refs/remotes/client1/main - not something we can merge
+
+merge: refs/remotes/client1/synced/main - not something we can merge
+(merging synced/git-annex into git-annex...)
+(recording state in git...)
+
+```
+
+client2: I get the following daemon logs:
+
+```
+From /home/xxx/git-annex-scenarios/share-between-clients/share
+ * [new branch]      git-annex  -> share/git-annex
+(merging share/git-annex into git-annex...)
+(recording state in git...)
+
+merge: refs/remotes/share/main - not something we can merge
+
+merge: refs/remotes/share/synced/main - not something we can merge
+
+```
+
+Then, I thought that maybe I needed to do an initial `git pull` for each repository. So I tried adding to the bash script the following lines:
+
+```bash
+# Need to do this if there are no commits in the 'client2' and 'share' repositories.
+# Or else, I'll get the following logs:
+#
+# merge: refs/remotes/share/main - not something we can merge
+# merge: refs/remotes/share/synced/main - not something we can merge
+sleep 3;
+cd $DIR/share
+git pull client1 main
+sleep 3;
+cd $DIR/client2
+git pull share main
+```
+
+But I'm still getting the same error:
+
+```
+(scanning...) (started...) 
+From /home/xxx/git-annex-scenarios/share-between-clients/share
+ * [new branch]      git-annex  -> share/git-annex
+(merging share/git-annex into git-annex...)
+(recording state in git...)
+
+merge: refs/remotes/share/main - not something we can merge
+
+merge: refs/remotes/share/synced/main - not something we can merge
+(recording state in git...)
+To /home/kolam/git-annex-scenarios/share-between-clients/share
+ + 28079ec...ca3c481 git-annex -> synced/git-annex (forced update)
+Everything up-to-date
+To /home/kolam/git-annex-scenarios/share-between-clients/share
+ + 28079ec...ca3c481 git-annex -> synced/git-annex (forced update)
+```
+
+However, even though I have that error, `file.txt` now appears in `client2`.
+But, the content of `file.txt` is:
+
+```
+/annex/objects/SHA256E-s14--14b99b7ab1e9777f7e1c2b482fe2cd95653c7cf35f
+459ef0b15bd0d75b2245c9.txt
+```
+
+and that link doesn't exist in my filesystem.
+Running `git annex whereis file.txt` in `client2` gives me:
+
+```
+whereis file.txt (0 copies) failed
+whereis: 1 failed
+```
+
+So my questions are:
+
+* did I miss something in the steps required to setup the repositories?
+* is there some documentation outlining the steps to do so without the webapp?
+* how can we enhance the UX for that scenario with better messages?
diff --git a/doc/forum/client_repositories_setup_problem/comment_1_be53f90b66de690b39f487d394866a66._comment b/doc/forum/client_repositories_setup_problem/comment_1_be53f90b66de690b39f487d394866a66._comment
new file mode 100644
index 0000000000..b15d985617
--- /dev/null
+++ b/doc/forum/client_repositories_setup_problem/comment_1_be53f90b66de690b39f487d394866a66._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-12-04T17:48:39Z"
+ content="""
+	git remote add client1 $DIR/client1
+	git remote add client2 $DIR/client1
+
+It seems to me you have a typo in the second line there.
+
+The other problem I noticed is, you are putting repositories in groups, but you
+never configure preferred content, and so the group settings will not be used.
+So you probably want to run `git-annex wanted here standard` in each of the
+repositories.
+
+With those fixes, it seems to work for me.
+"""]]
diff --git a/doc/forum/client_repositories_setup_problem/comment_2_f9660534c3f3346f322de57c2da0eb27._comment b/doc/forum/client_repositories_setup_problem/comment_2_f9660534c3f3346f322de57c2da0eb27._comment
new file mode 100644
index 0000000000..92ed584657
--- /dev/null
+++ b/doc/forum/client_repositories_setup_problem/comment_2_f9660534c3f3346f322de57c2da0eb27._comment
@@ -0,0 +1,87 @@
+[[!comment format=mdwn
+ username="kolam"
+ avatar="http://cdn.libravatar.org/avatar/4d2741e5e0b47928bc599b00397b5e59"
+ subject="comment 2"
+ date="2023-12-06T21:08:05Z"
+ content="""
+Thanks Joey! Your comment is right on point.
+
+Changed to the following and now it works as expected. The bash script is now:
+
+```
+#/usr/bin/env bash
+
+# Just a way to access the script's directory
+cd \"$(dirname \"$0\")\"
+DIR=\"$(pwd)\"
+
+# Create the 1st client repository
+mkdir $DIR/client1
+cd $DIR/client1
+git init && git annex init
+
+# Create the 2nd client repository
+mkdir $DIR/client2
+cd $DIR/client2
+git init && git annex init
+
+# Create the transfer repository
+mkdir $DIR/share
+cd $DIR/share
+git init && git annex init
+
+# Setup the remotes and groups for the transfer repository
+cd $DIR/share
+git remote add client1 $DIR/client1
+git remote add client2 $DIR/client2
+git annex wanted . standard
+git annex group . transfer
+git annex group client1 client
+git annex group client2 client
+git annex mincopies 2
+git annex numcopies 2
+git co -b main
+
+# Setup the remotes and groups for the 1st client repository.
+cd $DIR/client1
+git remote add share $DIR/share
+git annex wanted . standard
+git annex group . client
+git annex group share transfer
+# git annex config --set annex.addunlocked true
+git co -b main
+
+# Setup the remotes and groups for the 2nd client repository.
+cd $DIR/client2
+git remote add share $DIR/share
+git annex wanted . standard
+git annex group . client
+git annex group share transfer
+# git annex config --set annex.addunlocked true
+git co -b main
+
+# Start git-annex assistant for client1 only
+cd $DIR/client1 && git annex assistant
+
+# Add a single file to the 1st client.
+cd $DIR/client1
+touch file.txt
+
+# Wait for the commit to be auto created by git-annex-assistant.
+sleep 3;
+
+cd $DIR/share
+git pull client1 main
+git annex assistant
+cd $DIR/client2
+git pull share main
+git annex assistant
+
+cd $DIR/client1
+echo \"My first line\" >> file.txt
+```
+
+I have other issues following that, but I'll create separate forum questions for them.
+
+Thanks again Joey.
+"""]]
diff --git a/doc/forum/duplicates_with_suffixes_differing_only_in_case.mdwn b/doc/forum/duplicates_with_suffixes_differing_only_in_case.mdwn
new file mode 100644
index 0000000000..4084c7e7b2
--- /dev/null
+++ b/doc/forum/duplicates_with_suffixes_differing_only_in_case.mdwn
@@ -0,0 +1,27 @@
+I am using Google Cloud as a special remote to backup photos in one annex.
+
+It's in my interests, due to storage costs, to keep duplicates to a minimum. I've noticed that the cloud bucket has a lot more files than I would have expected.
+
+I performed the following to get the number of files in the bucket:
+
+[[!format sh """
+$ gsutil ls gs://cloud-<uuid>/ | wc -l
+<one large number>
+"""]]
+
+and of course, in the git-annex:
+
+[[!format sh """
+$ find . -path .git -o \( -type f -print \) | wc -l
+<one not as large number, about 60% of the larger number>
+"""]]
+
+When I took a sample of the list, I found may of the files that shared the exact key, but had, for example. both .jpg and .JPG suffixes.
+
+Some of these files, when checked with git-annex whereused --key=<each key with its cased suffix>, some were files I had maybe re-arranged in folders. I'm running a script at the moment to get a wider sample of what's happening with these duplicates.
+
+One of the problems I speculate may be coming into play is that I did originally copy files to the cloud remote via an older, Windows git-annex version (repo version I think at 5?).
+
+I think over time interoperability issues have ironed out, but this might be a remnant of when these were more prolific.... though I do still have one that I will report separately once I've gather more data.
+
+Any advise on how I might deal with the duplicates, get them deleted out of the bucket?
diff --git a/doc/forum/duplicates_with_suffixes_differing_only_in_case/comment_1_0c1bb4347902054ec97ef86eb903f060._comment b/doc/forum/duplicates_with_suffixes_differing_only_in_case/comment_1_0c1bb4347902054ec97ef86eb903f060._comment
new file mode 100644
index 0000000000..4395485dee
--- /dev/null
+++ b/doc/forum/duplicates_with_suffixes_differing_only_in_case/comment_1_0c1bb4347902054ec97ef86eb903f060._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="beryllium@5bc3c32eb8156390f96e363e4ba38976567425ec"
+ nickname="beryllium"
+ avatar="http://cdn.libravatar.org/avatar/62b67d68e918b381e7e9dd6a96c16137"
+ subject="comment 1"
+ date="2023-10-17T10:16:23Z"
+ content="""
+OK, please ignore. These scenarios account for only about 23 of the duplicates. There may still be, or have been, a problem.. maybe? Because I did add the same file in two different places,with the two case differing (but same) suffixes.
+
+I am running another audit to work out why I have so many more files in the cloud than reachable in the annex. I suppose it's because I removed files previously, but I'm not sure.
+
+"""]]
diff --git a/doc/forum/duplicates_with_suffixes_differing_only_in_case/comment_2_62bb4be1b31bd64a046763f68a2953ce._comment b/doc/forum/duplicates_with_suffixes_differing_only_in_case/comment_2_62bb4be1b31bd64a046763f68a2953ce._comment
new file mode 100644
index 0000000000..b07864e829
--- /dev/null
+++ b/doc/forum/duplicates_with_suffixes_differing_only_in_case/comment_2_62bb4be1b31bd64a046763f68a2953ce._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="beryllium@5bc3c32eb8156390f96e363e4ba38976567425ec"
+ nickname="beryllium"
+ avatar="http://cdn.libravatar.org/avatar/62b67d68e918b381e7e9dd6a96c16137"
+ subject="comment 2"
+ date="2023-10-17T10:28:51Z"
+ content="""
+Apologies again. I've realised that my count of the find command is totally wrong. It should be:
+
+[[!format sh \"\"\"
+$ find . -path .git -o \( \! -type d -print \) | wc -l
+<actually larger number... which is worrying in its own right>\"
+\"\"\"]]
+
+The reason being... for some reason I have some files that contain the key, and some actual symbolic links, all intermixed.
+
+Is there something I should do to make this all uniform?
+
+"""]]
diff --git a/doc/forum/duplicates_with_suffixes_differing_only_in_case/comment_3_99a2e752c75e5ad2ea36254627be8d18._comment b/doc/forum/duplicates_with_suffixes_differing_only_in_case/comment_3_99a2e752c75e5ad2ea36254627be8d18._comment
new file mode 100644
index 0000000000..59a5f68f6c
--- /dev/null
+++ b/doc/forum/duplicates_with_suffixes_differing_only_in_case/comment_3_99a2e752c75e5ad2ea36254627be8d18._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2023-10-20T17:52:06Z"
+ content="""
+git-annex preserves the filename extension as-is, it does not try to
+normalize it to lower case or anything. See discussion at
+[[todo/Lower-case_extension_for_SHA256E_and_similar]].
+
+You can configure git-annex to use SHA256 rather than the default SHA256E
+so the extension is not used and it will deduplicate better. Eg, in
+.gitattributes:
+
+	* annex.backend=SHA256
+
+You could then run `git-annex migrate` on existing files to switch them to
+that backend. Bear in mind that you would have to re-upload the files to
+your special remote, and would have to `git-annex dropunused --from`
+that remote to clean up the duplicate data stored there.
+"""]]
diff --git a/doc/forum/fatal__58___Not_a_valid_object_name_repository.mdwn b/doc/forum/fatal__58___Not_a_valid_object_name_repository.mdwn
new file mode 100644
index 0000000000..d4492a4454
--- /dev/null
+++ b/doc/forum/fatal__58___Not_a_valid_object_name_repository.mdwn
@@ -0,0 +1,49 @@
+After inadvertently deleting my main repository on my local machine, I had to rebuild. This has worked ok, as far as I can tell at the moment. However, when listing the repository details, I get the listing below.
+
+My problem is that **S5** has three aliases: `S5 [S51]`, which I cannot rename to `S5` for some reason, and two other repos residing in `jg@jg-OptiPlex-9020-AIO:/media/jg/S5/annex`. When I try to query there details, I get the information at the bottom. I am a bit anxious, as I am unsure if I am dealing with prior instances that have been somehow overridden but are still being tracked.
+
+Any help in understanding the situation is appreciated. Ideally, I would like to remove the spurious repository references, provided they are not required.
+
+```sh
+jg@jg-OptiPlex-9020-AIO:~/Desktop/annex/VideoStreamFiles/space race$ git-annex info
+trusted repositories: 0
+semitrusted repositories: 15
+	00000000-0000-0000-0000-000000000001 -- web
+ 	00000000-0000-0000-0000-000000000002 -- bittorrent
+ 	246524b7-7cc2-4b3e-9559-4aa5992a4b94 -- S5 [S51]
+ 	2d8de5a5-4662-4bbb-bab0-51f39ec07d2e -- L1
+ 	39291601-0cc2-45aa-89f4-85e6d5bead99 -- L6
+ 	40897254-d2e8-4803-b291-a735d23fe03b -- [S2]
+ 	48e8f8c5-cb12-4b1d-9f50-30b39f38b6cb -- L5
+ 	63e417a1-b10c-45c2-8a90-44971b8b69bf -- [S1]
+ 	96d8d103-1bc5-41d8-be3e-804260a9ee8d -- jg@jg-OptiPlex-9020-AIO:/media/jg/S5/annex
+ 	a518d97d-4fb0-4018-95b1-2fc1b02a3081 -- L3
+ 	a7972a52-b3c2-4282-b690-a832dbbad982 -- [S3]
+ 	ae317911-25b1-418c-bcd2-a88d4f8d48f2 -- [S4]
+ 	af3e59fb-d2a8-418d-bde5-48b008f61fbb -- jg@jg-OptiPlex-9020-AIO:/media/jg/S5/annex
+ 	cb2c7d0c-b946-4e74-9362-3f4d56a5db5a -- jg@jg-Latitude-5500:~/annex [jglatitude5500.lan_annex]
+ 	ce573cf5-5007-4d5d-b540-ab73f393c32e -- jg@jg-OptiPlex-9020-AIO:~/Desktop/annex [here]
+untrusted repositories: 0
+transfers in progress: none
+available local disk space: 14.23 megabytes (+100 megabytes reserved)
+local annex keys: 292
+local annex size: 120.26 gigabytes
+annexed files in working tree: 45158
+size of annexed files in working tree: 1 terabyte
+bloom filter size: 32 mebibytes (0.1% full)
+backend usage: 
+	SHA256E: 45158
+```
+
+
+```sh
+jg@jg-OptiPlex-9020-AIO:~/Desktop/annex/VideoStreamFiles/space race$ git-annex info  repository 96d8d103-1bc5-41d8-be3e-804260a9ee8d
+fatal: Not a valid object name repository
+info repository (not a directory or an annexed file or a treeish or a remote or a uuid) failed
+uuid: 96d8d103-1bc5-41d8-be3e-804260a9ee8d
+description: jg@jg-OptiPlex-9020-AIO:/media/jg/S5/annex
+trust: semitrusted
+remote annex keys: 0
+remote annex size: 0 bytes
+info: 1 failed
+```
diff --git a/doc/forum/fatal__58___Not_a_valid_object_name_repository/comment_1_48f1ee21d24a99723ff455c0c9c96629._comment b/doc/forum/fatal__58___Not_a_valid_object_name_repository/comment_1_48f1ee21d24a99723ff455c0c9c96629._comment
new file mode 100644
index 0000000000..78b6185b44
--- /dev/null
+++ b/doc/forum/fatal__58___Not_a_valid_object_name_repository/comment_1_48f1ee21d24a99723ff455c0c9c96629._comment
@@ -0,0 +1,26 @@
+[[!comment format=mdwn
+ username="jgsuess@732b8c62c50d8595d7b1d58eea11e5019c2308b1"
+ nickname="jgsuess"
+ avatar="http://cdn.libravatar.org/avatar/35a99d1b5df046b206807941ce9795d3"
+ subject="wrong parameter used - `repository` is not a subcommand"
+ date="2023-12-31T09:41:13Z"
+ content="""
+If I call with UUID, I receive this. However, this still does not help me understand what the item is:
+
+```sh
+jg@jg-OptiPlex-9020-AIO:~/Desktop/annex/VideoStreamFiles/space race$ git-annex info af3e59fb-d2a8-418d-bde5-48b008f61fbb
+uuid: af3e59fb-d2a8-418d-bde5-48b008f61fbb
+description: jg@jg-OptiPlex-9020-AIO:/media/jg/S5/annex
+trust: semitrusted
+remote annex keys: 1
+remote annex size: 373.36 megabytes
+jg@jg-OptiPlex-9020-AIO:~/Desktop/annex/VideoStreamFiles/space race$ git-annex info 96d8d103-1bc5-41d8-be3e-804260a9ee8d
+uuid: 96d8d103-1bc5-41d8-be3e-804260a9ee8d
+description: jg@jg-OptiPlex-9020-AIO:/media/jg/S5/annex
+trust: semitrusted
+remote annex keys: 0
+remote annex size: 0 bytes
+```
+
+
+"""]]
diff --git a/doc/forum/fatal__58___Not_a_valid_object_name_repository/comment_2_eef07cc389400c38b666da4e1adc255e._comment b/doc/forum/fatal__58___Not_a_valid_object_name_repository/comment_2_eef07cc389400c38b666da4e1adc255e._comment
new file mode 100644
index 0000000000..9394321ce6
--- /dev/null
+++ b/doc/forum/fatal__58___Not_a_valid_object_name_repository/comment_2_eef07cc389400c38b666da4e1adc255e._comment
@@ -0,0 +1,20 @@
+[[!comment format=mdwn
+ username="jgsuess@732b8c62c50d8595d7b1d58eea11e5019c2308b1"
+ nickname="jgsuess"
+ avatar="http://cdn.libravatar.org/avatar/35a99d1b5df046b206807941ce9795d3"
+ subject="Scanning for files in ghost repos"
+ date="2023-12-31T09:46:53Z"
+ content="""
+So I used the incantations below to scan for contents. Seems that one only holds a single file and the other does not hold anything at all. Is it the safe to declare them `dead` and will that lead to the replication of the file from `S1` to somewhere else?
+
+```sh
+jg@jg-OptiPlex-9020-AIO:~/Desktop/annex/VideoStreamFiles/space race$ git-annex whereis --all --in=af3e59fb-d2a8-418d-bde5-48b008f61fbb
+whereis SHA256E-s373363860--f10c801755cabc38d958adace3af29d767670c780b97c572d04a8dcd10ca0a9a.avi (2 copies) 
+  	63e417a1-b10c-45c2-8a90-44971b8b69bf -- [S1]
+   	af3e59fb-d2a8-418d-bde5-48b008f61fbb -- jg@jg-OptiPlex-9020-AIO:/media/jg/S5/annex
+ok
+jg@jg-OptiPlex-9020-AIO:~/Desktop/annex/VideoStreamFiles/space race$ git-annex whereis --all --in=96d8d103-1bc5-41d8-be3e-804260a9ee8d
+jg@jg-OptiPlex-9020-AIO:~/Desktop/annex/VideoStreamFiles/space race$ 
+
+```
+"""]]
diff --git a/doc/forum/fatal__58___Not_a_valid_object_name_repository/comment_3_2760c9b8b755e1ed5e01b98ea3b99177._comment b/doc/forum/fatal__58___Not_a_valid_object_name_repository/comment_3_2760c9b8b755e1ed5e01b98ea3b99177._comment
new file mode 100644
index 0000000000..b06d0cd1e1
--- /dev/null
+++ b/doc/forum/fatal__58___Not_a_valid_object_name_repository/comment_3_2760c9b8b755e1ed5e01b98ea3b99177._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="Lukey"
+ avatar="http://cdn.libravatar.org/avatar/c7c08e2efd29c692cc017c4a4ca3406b"
+ subject="comment 3"
+ date="2023-12-31T14:41:18Z"
+ content="""
+Are these special remotes? You can get more info with `git cat-file blob git-annex:remote.log`. Warning, this will print encryption keys if you set up a encrypted special remote.
+
+Any yes, `git annex dead` should do the trick, it will not count files in the dead repo as a copy anymore.
+"""]]
diff --git a/doc/forum/fatal__58___Not_a_valid_object_name_repository/comment_4_344561606daab9bc1a05ec3b857b2d62._comment b/doc/forum/fatal__58___Not_a_valid_object_name_repository/comment_4_344561606daab9bc1a05ec3b857b2d62._comment
new file mode 100644
index 0000000000..815fd871e7
--- /dev/null
+++ b/doc/forum/fatal__58___Not_a_valid_object_name_repository/comment_4_344561606daab9bc1a05ec3b857b2d62._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="jgsuess@732b8c62c50d8595d7b1d58eea11e5019c2308b1"
+ nickname="jgsuess"
+ avatar="http://cdn.libravatar.org/avatar/35a99d1b5df046b206807941ce9795d3"
+ subject="Not a special remote"
+ date="2024-01-10T08:54:57Z"
+ content="""
+No these are just disk drives. I feel that the structure is corrupted. I will have a look to see if that detail listing gives me more.
+"""]]
diff --git a/doc/forum/git-remote-gcrypt_and_rsyncd.mdwn b/doc/forum/git-remote-gcrypt_and_rsyncd.mdwn
new file mode 100644
index 0000000000..bba80cd66c
--- /dev/null
+++ b/doc/forum/git-remote-gcrypt_and_rsyncd.mdwn
@@ -0,0 +1,32 @@
+In an attempt to simplify my setup, I have been trying to setup an encrypted repository on a `rsyncd`-based server via [`git-remote-gcrypt`](https://git-annex.branchable.com/tips/fully_encrypted_git_repositories_with_gcrypt/), which would house the file history and the annexed files themselves. I cannot provide an SSH connection to the server, so the `rsyncd` method seemed appealing.
+
+Using the rsync format url with "::" to signal the rsyncd method, the connection seems successful, but the initialization does not complete. 
+
+```
+git annex initremote gcrypt-rsyncd type=gcrypt gitrepo=rsync://***::a/test keyid=*** encryption=hybrid
+
+initremote gcrypt-rsyncd (encryption setup) (to gpg keys: ***) gcrypt
+ Decrypting manifest
+gpg: Signature made Wed Nov 22 22:23:16 2023 CET
+gpg:                using EDDSA key ***
+gpg: Good signature from "archive-990" [ultimate]
+gcrypt: Remote ID is :id:ya5ZivzWNEOUtVg2R0L9
+From gcrypt::rsync://***::a/test
+ * [new branch]      git-annex  -> gcrypt-rsyncd/git-annex
+gcrypt: Decrypting manifest
+gpg: Signature made Wed Nov 22 22:23:16 2023 CET
+gpg:                using EDDSA key ***
+gpg: Good signature from "archive-990" [ultimate]
+Everything up-to-date
+
+git-annex: git: createProcess: chdir: invalid argument (Bad file descriptor)
+failed
+initremote: 1 failed
+```
+
+Logs from the daemon show the following error:
+```
+rsync to a/test/annex/objects from ***
+```
+
+I don't know whether this error is imputable to `git-annex`, or `git-remote-gcrypt`, or my settings.
diff --git a/doc/forum/git-remote-gcrypt_and_rsyncd/comment_1_34dd343ed75918f5969f6a23dfae3317._comment b/doc/forum/git-remote-gcrypt_and_rsyncd/comment_1_34dd343ed75918f5969f6a23dfae3317._comment
new file mode 100644
index 0000000000..02246cf558
--- /dev/null
+++ b/doc/forum/git-remote-gcrypt_and_rsyncd/comment_1_34dd343ed75918f5969f6a23dfae3317._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-11-28T16:06:22Z"
+ content="""
+That is a pretty weird error message! It looks like git-annex may have run
+git but tried to pass it a working directory that does not exist. It would
+be interesting to know what git command, passing --debug would tell you.
+
+But: The gcrypt special remote is documented as needing gitrepo=rsync:// to
+operate over ssh. And git-remote-gcrypt interprets a rsync:// url as rsync
+over ssh (see its man page). Yes, "host::" in rsync indicates direct contact
+to a rsync daemon, not using ssh, but that will not work with git-remote-gcrypt
+to the best of my knowledge.
+"""]]
diff --git a/doc/forum/git-remote-gcrypt_and_rsyncd/comment_2_78b89867bd1af78a91826022651a57ad._comment b/doc/forum/git-remote-gcrypt_and_rsyncd/comment_2_78b89867bd1af78a91826022651a57ad._comment
new file mode 100644
index 0000000000..57057f4397
--- /dev/null
+++ b/doc/forum/git-remote-gcrypt_and_rsyncd/comment_2_78b89867bd1af78a91826022651a57ad._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="branch"
+ subject="comment 2"
+ date="2023-12-03T11:57:56Z"
+ content="""
+There is no specific log to highlight when running the command in `--debug`.
+
+```
+[2023-12-03 12:43:49.274023] (Utility.Process) process [40369] done ExitSuccess
+
+git-annex: git: createProcess: chdir: invalid argument (Bad file descriptor)
+failed
+[2023-12-03 12:43:49.276644] (Utility.Process) process [40197] done ExitSuccess
+initremote: 1 failed
+```
+
+I ended up refactoring my systems to allow the use of SSH, which seems to be the supported method, and to avoid any further issue down the line.
+"""]]
diff --git a/doc/forum/name_resolution_of_dne.mdwn b/doc/forum/name_resolution_of_dne.mdwn
new file mode 100644
index 0000000000..cc3ca101ad
--- /dev/null
+++ b/doc/forum/name_resolution_of_dne.mdwn
@@ -0,0 +1,31 @@
+I get a failure in name resolution when running `annex testremote`:
+
+```
+mvanhult@detekti:~/temp/my_repo$ git annex testremote bbborig | head
+testremote bbborig (generating test keys...) Remote Tests
+  unavailable remote
+ssh: Could not resolve hostname !dne!: non-recoverable failure in name resolution
+    removeKey:                                       OK (0.01s)
+ssh: Could not resolve hostname !dne!: non-recoverable failure in name resolution
+    storeKey:                                        OK
+ssh: Could not resolve hostname !dne!: non-recoverable failure in name resolution
+    checkPresent:                                    OK
+ssh: Could not resolve hostname !dne!: non-recoverable failure in name resolution
+    retrieveKeyFile:                                 OK
+    retrieveKeyFileCheap:                            OK
+  key size Just 1048576; remote chunksize=0 encryption=none
+The following modules were not unloaded:
+  (Use "module --force purge" to unload all):
+
+  1) Stages/2024
+    removeKey when not present:                      FAIL
+      Exception: removing content from remote failed
+mvanhult@detekti:~/temp/my_repo$ git remote -v
+bbborig	juwels:/p/home/jusers/vanhulten1/juwels/temp/my_repo (fetch)
+bbborig	juwels:/p/home/jusers/vanhulten1/juwels/temp/my_repo (push)
+```
+
+Where does this `!dne!` come from?
+The relevant host of the remote is `juwels` and not `!dne!`.
+I have never before this post entered the string `!dne!`.
+Is *dne* an abbreviation?
diff --git a/doc/forum/name_resolution_of_dne/comment_1_7309e109d1f6eba8fc92659372d64b19._comment b/doc/forum/name_resolution_of_dne/comment_1_7309e109d1f6eba8fc92659372d64b19._comment
new file mode 100644
index 0000000000..21bd8d6aef
--- /dev/null
+++ b/doc/forum/name_resolution_of_dne/comment_1_7309e109d1f6eba8fc92659372d64b19._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-12-05T17:34:47Z"
+ content="""
+This is intentional behavior, it's testing that the remote behaves
+correctly when the server is unavailable. "dne" is an abbreviation for Does
+Not Exist, and "!dne!" is very unlikely to exist, so it uses that.
+"""]]
diff --git a/doc/forum/name_resolution_of_dne/comment_2_54ed13db673cff7177cd371143a295b4._comment b/doc/forum/name_resolution_of_dne/comment_2_54ed13db673cff7177cd371143a295b4._comment
new file mode 100644
index 0000000000..ff51f9180d
--- /dev/null
+++ b/doc/forum/name_resolution_of_dne/comment_2_54ed13db673cff7177cd371143a295b4._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="https://esgf-node.llnl.gov/esgf-idp/openid/mvhulten"
+ nickname="mvhulten"
+ avatar="http://cdn.libravatar.org/avatar/8e966234bbd87d1efb0db4a4690960faa51b28cd27f4d285b9501c132a11567b"
+ subject="comment 2"
+ date="2023-12-07T12:30:24Z"
+ content="""
+Thank you for the clarification.
+"""]]
diff --git a/doc/forum/sync_content_through_repo_that_wants_nothing.mdwn b/doc/forum/sync_content_through_repo_that_wants_nothing.mdwn
new file mode 100644
index 0000000000..e9522727e6
--- /dev/null
+++ b/doc/forum/sync_content_through_repo_that_wants_nothing.mdwn
@@ -0,0 +1,7 @@
+I'm trying to figure out how to automatically sync all the remotes that I can currently reach while the local repo does not want any content.
+
+Specifically want to download content from a server (remote repo) to a USB-drive (directory special remote) through a local repo.
+If the local repo has no preferred content set or if the preferred content includes the files I want to sync to the disk it works.
+However, if I set the local preferred content to `exclude=*` neither `sync`, `satisfy` nor `assist` do what I had hoped, which is to download the files, copy them to the remote and then delete the local copy.
+
+Is there something I can do to achieve this behavior? 
diff --git a/doc/forum/sync_content_through_repo_that_wants_nothing/comment_1_6c014bc88fc8a685ef79a044c8938c7d._comment b/doc/forum/sync_content_through_repo_that_wants_nothing/comment_1_6c014bc88fc8a685ef79a044c8938c7d._comment
new file mode 100644
index 0000000000..d2e3334dcf
--- /dev/null
+++ b/doc/forum/sync_content_through_repo_that_wants_nothing/comment_1_6c014bc88fc8a685ef79a044c8938c7d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2024-01-18T21:17:13Z"
+ content="""
+What you want is the behavior of a transfer repository. See
+[[preferred_content/standard_groups]].
+"""]]
diff --git a/doc/forum/sync_content_through_repo_that_wants_nothing/comment_2_ef89154903395025bb0a0408e1c23bdf._comment b/doc/forum/sync_content_through_repo_that_wants_nothing/comment_2_ef89154903395025bb0a0408e1c23bdf._comment
new file mode 100644
index 0000000000..e81f301998
--- /dev/null
+++ b/doc/forum/sync_content_through_repo_that_wants_nothing/comment_2_ef89154903395025bb0a0408e1c23bdf._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="imlew"
+ avatar="http://cdn.libravatar.org/avatar/23858c3eed3c3ea9e21522f4c999f1ed"
+ subject="comment 2"
+ date="2024-01-18T23:09:21Z"
+ content="""
+Using the standard transfer group wouldn't work in this situation because the remotes cannot be clients (among other reason because the server repo is bigger than the external drives).
+
+But from your answer I surmise that the local repo must want the files unless they are on the drives and no longer want them once they are on drives, is that correct?
+
+I will try letting it run overnight with the local repo's wanted being `\"not copies=2\"`. Hopefully sync will drop the local copy as soon as it has been transferred to the drive and not keep that for last.
+
+"""]]
diff --git a/doc/forum/sync_content_through_repo_that_wants_nothing/comment_3_e05a3e3d991617bb2b323b5572cd771e._comment b/doc/forum/sync_content_through_repo_that_wants_nothing/comment_3_e05a3e3d991617bb2b323b5572cd771e._comment
new file mode 100644
index 0000000000..ddc96a55fc
--- /dev/null
+++ b/doc/forum/sync_content_through_repo_that_wants_nothing/comment_3_e05a3e3d991617bb2b323b5572cd771e._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2024-01-25T15:50:04Z"
+ content="""
+`not copies=2` might be sufficient in your situation.
+
+If you look at the preferred content expression for a transfer repository, it
+is:
+
+`not (inallgroup=client and copies=client:2) and ($client)`
+
+Where `$client` is a copy of the preferred content expressions of the client.
+
+There's no reason you can't write a preferred content expression that
+does the same thing for some other group of repositories.
+"""]]
diff --git a/doc/forum/transfer_bare_repo_setup_problem.mdwn b/doc/forum/transfer_bare_repo_setup_problem.mdwn
new file mode 100644
index 0000000000..239d268cf0
--- /dev/null
+++ b/doc/forum/transfer_bare_repo_setup_problem.mdwn
@@ -0,0 +1,111 @@
+Thanks to Joey's help, I managed to get the transfer repository setup working in a [previous forum question](https://git-annex.branchable.com/forum/client_repositories_setup_problem/).
+
+Now, I'm trying to do something similar, but using a bare git repository as the transfer repo instead of a regular git repository.
+
+Here's my bash script which bootstraps the setup stage:
+
+```
+#/usr/bin/env bash
+
+# Useful links:
+# https://git-annex.branchable.com/bare_repositories/
+
+# Just a way to access the script's directory
+cd "$(dirname "$0")"
+DIR="$(pwd)"
+
+# Create the 1st client repository
+mkdir $DIR/client1
+cd $DIR/client1
+git init && git annex init
+
+# Create the 2nd client repository
+mkdir $DIR/client2
+cd $DIR/client2
+git init && git annex init
+
+# Create the transfer repository
+mkdir $DIR/share
+cd $DIR/share
+git init --bare && git annex init
+
+# Setup the remotes and groups for the transfer repository
+cd $DIR/share
+git remote add client1 $DIR/client1
+git remote add client2 $DIR/client2
+git annex wanted . standard
+git annex group . transfer
+git annex group client1 client
+git annex group client2 client
+git annex mincopies 2
+git annex numcopies 2
+
+# Setup the remotes and groups for the 1st client repository.
+cd $DIR/client1
+git remote add share $DIR/share
+git annex wanted . standard
+git annex group . client
+git annex group share transfer
+git annex config --set annex.addunlocked true
+git co -b main
+
+# Setup the remotes and groups for the 2nd client repository.
+cd $DIR/client2
+git remote add share $DIR/share
+git annex wanted . standard
+git annex group . client
+git annex group share transfer
+git annex config --set annex.addunlocked true
+git co -b main
+
+# Add a single file to the 1st client.
+cd $DIR/client1
+touch file.txt
+git annex add file.txt
+git commit -m "Initial commit"
+
+# Needed step for bare repos, as documented in https://git-annex.branchable.com/bare_repositories/
+cd $DIR/client1
+git push share main
+
+###############################################################################
+# Need to do this if there are no commits in the 'client2' and 'share'
+# repositories. Or else, I'll get the following logs:
+#
+# merge: refs/remotes/share/main - not something we can merge
+# merge: refs/remotes/share/synced/main - not something we can merge
+cd $DIR/client2
+git pull share main
+###############################################################################
+
+# Run git-annex assistant for each repository
+cd $DIR/client1 && git annex assistant
+cd $DIR/client2 && git annex assistant
+
+sleep 3;
+cd $DIR/client1
+echo "My first line" >> file.txt
+```
+
+The repo `client1` correctly contains 2 commits representing changes to `file.txt`.
+Additionally, `share` and `client2` contain the same commits that were propagated.
+However, I expected `cat client2/file.txt` to show "My first line", but it shows instead "/annex/objects/SHA256E-s14--42e950c34152a022a2ec82b2201a2287689e39d4d97bfcef67f8940b49d25d4b.txt".
+
+Doing `git status` in `client2` yield:
+
+```
+  file.txt is a git-annex pointer file. Its content is not available in this repository. (Maybe file.txt was copied from another repository?)
+```
+
+but not sure how to solve this.
+
+Running `git annex whereis` yields:
+
+```
+whereis file.txt (0 copies) failed
+whereis: 1 failed
+```
+
+Any ideas what went wrong?
+
+-- [[kolam]]
diff --git a/doc/forum/transfer_bare_repo_setup_problem/comment_1_3b6c15b81c5df09587bc75c5e358b39c._comment b/doc/forum/transfer_bare_repo_setup_problem/comment_1_3b6c15b81c5df09587bc75c5e358b39c._comment
new file mode 100644
index 0000000000..4a29ea77ba
--- /dev/null
+++ b/doc/forum/transfer_bare_repo_setup_problem/comment_1_3b6c15b81c5df09587bc75c5e358b39c._comment
@@ -0,0 +1,48 @@
+[[!comment format=mdwn
+ username="kolam"
+ avatar="http://cdn.libravatar.org/avatar/4d2741e5e0b47928bc599b00397b5e59"
+ subject="comment 1"
+ date="2023-12-07T20:08:52Z"
+ content="""
+Seems like I needed to manually run:
+
+```
+cd $DIR/client2
+git annex sync --content
+```
+
+which solves the file not found issue. Wonder why the git-annex assistant didn't just sync again...
+
+Then, `cat file.txt` still gives `/annex/objects/SHA256E-s14--42e950c34152a022a2ec82b2201a2287689e39d4d97bfcef67f8940b49d25d4b.txt`.
+
+I managed to solve it by running `git-annex restage` on client2.
+
+However, after running `git annex whereis`, I get:
+
+```
+whereis file.txt (3 copies) 
+  	1d6bad3e-6a1c-4371-af5f-794bf387480c -- kolam@xxx:~/git-annex-scenarios/share-between-clients/share [share]
+   	87ba86d9-4922-4a4a-8c8d-57c791e0f80e -- kolam@xxx:~/git-annex-scenarios/share-between-clients/client2 [here]
+   	f5b9efb1-e5b3-4573-aa6d-354180193c74 -- kolam@xxx:~/git-annex-scenarios/share-between-clients/client1
+ok
+```
+
+but I noticed that the assistant doesn't drop the file from the \"share\" repo.
+
+Running `git annex drop --auto --from share`, I get:
+
+```
+drop file.txt (from share...) (unsafe) 
+  Could only verify the existence of 1 out of 2 necessary copies
+
+  Maybe add some of these git remotes (git remote add ...):
+  	f5b9efb1-e5b3-4573-aa6d-354180193c74 -- kolam@nelson:~/git-annex-scenarios/share-between-clients/client1
+
+  (Use --force to override this check, or adjust numcopies.)
+failed
+drop: 1 failed
+```
+
+Of course, that also means that if I modify \"$DIR/client2/file.txt\", the assistant doesn't send the updates to client1. I need to do a manual `git annex sync --content` on `client1`.
+Then again, I can't drop the file from \"share\" because of that same error as before.
+"""]]
diff --git a/doc/forum/transfer_bare_repo_setup_problem/comment_2_c011e8fcda61b6ee0067020abbddf61a._comment b/doc/forum/transfer_bare_repo_setup_problem/comment_2_c011e8fcda61b6ee0067020abbddf61a._comment
new file mode 100644
index 0000000000..d25bc6fc92
--- /dev/null
+++ b/doc/forum/transfer_bare_repo_setup_problem/comment_2_c011e8fcda61b6ee0067020abbddf61a._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="kolam"
+ avatar="http://cdn.libravatar.org/avatar/4d2741e5e0b47928bc599b00397b5e59"
+ subject="comment 2"
+ date="2023-12-07T20:16:29Z"
+ content="""
+There's a similar setup page for running your own server with a bare git repo: <https://git-annex.branchable.com/tips/centralized_git_repository_tutorial/on_your_own_server/>
+
+However: 
+
+1. the instructions are not specific for a transfer repo and showcasing the git-annex-drop statement 
+2. out of date as the instructions don't work.
+
+I also found <https://git-annex.branchable.com/forum/How_to_work_with_transfer_repos_manually63/>. 
+But the solution doesn't seem to work here either. 
+Maybe something changed since then.
+
+"""]]
diff --git a/doc/forum/very_slow_on_exfat_drives.mdwn b/doc/forum/very_slow_on_exfat_drives.mdwn
new file mode 100644
index 0000000000..718b008a5b
--- /dev/null
+++ b/doc/forum/very_slow_on_exfat_drives.mdwn
@@ -0,0 +1,7 @@
+I want to use git-annex to keep track of and archive of large tarballs (on the order of 10 to 100GB each).
+One of the locations are a set of external HDDs that are formatted to exFAT.
+
+Unfortunately every git command takes hours to execute.
+e.g. every time I use `git status` the index is refreshed which takes about 3 hours, committing a single takes similarly long.
+
+Is there anything I can do to speed things up?
diff --git a/doc/forum/very_slow_on_exfat_drives/comment_10_5f67c870fc7e43ae75d8e1f8ced975c2._comment b/doc/forum/very_slow_on_exfat_drives/comment_10_5f67c870fc7e43ae75d8e1f8ced975c2._comment
new file mode 100644
index 0000000000..df1ebb3e7c
--- /dev/null
+++ b/doc/forum/very_slow_on_exfat_drives/comment_10_5f67c870fc7e43ae75d8e1f8ced975c2._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="imlew"
+ avatar="http://cdn.libravatar.org/avatar/23858c3eed3c3ea9e21522f4c999f1ed"
+ subject="thank you"
+ date="2023-11-17T11:43:39Z"
+ content="""
+Makes sense.
+Thanks again, you've helped me a lot.
+"""]]
diff --git a/doc/forum/very_slow_on_exfat_drives/comment_11_65460ed123404478ae59fed1b5cff627._comment b/doc/forum/very_slow_on_exfat_drives/comment_11_65460ed123404478ae59fed1b5cff627._comment
new file mode 100644
index 0000000000..5b4cac8ac2
--- /dev/null
+++ b/doc/forum/very_slow_on_exfat_drives/comment_11_65460ed123404478ae59fed1b5cff627._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 11"""
+ date="2023-11-29T17:36:28Z"
+ content="""
+What might be happening on the exfat drive is, every time that filesystem
+is mounted, it generates new inode numbers for all the files. So when you
+run `git status`, git sees the new inode and needs to do work to determine
+if it's changed. When the file is an annexed file that is unlocked (which
+all annexed files necessarily are on this filesystem since it doesn't
+support symlinks), git status needs to ask git-annex about it.
+And git-annex has to either re-hash the file (for SHA) or do a
+smaller amount of work (for WORM).
+
+A bare repository does get around that. But what I tend to use in these
+situations is a [[/special_remotes/directory]] special remote configured
+with `ignoreinodes=yes`.
+"""]]
diff --git a/doc/forum/very_slow_on_exfat_drives/comment_12_77cd7b20abd0cd1eadee0ebeb186b3cf._comment b/doc/forum/very_slow_on_exfat_drives/comment_12_77cd7b20abd0cd1eadee0ebeb186b3cf._comment
new file mode 100644
index 0000000000..e99a4e1093
--- /dev/null
+++ b/doc/forum/very_slow_on_exfat_drives/comment_12_77cd7b20abd0cd1eadee0ebeb186b3cf._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="imlew"
+ avatar="http://cdn.libravatar.org/avatar/23858c3eed3c3ea9e21522f4c999f1ed"
+ subject="comment 12"
+ date="2023-12-12T11:56:05Z"
+ content="""
+Hi joey,
+
+if it was slow because the inodes changed it should only be slow the first time `git status` etc are run.
+What I experienced was that it got slower the more files were in the repo whie the drive was continuously connected.
+
+
+Thanks for your suggestion to use a directory special remote, but it's not clear to me how that would be an improvement over a bare repo.
+The only drawback to using a bare repo is the lack of a working tree and special remotes don't seem to have that either.
+
+
+
+"""]]
diff --git a/doc/forum/very_slow_on_exfat_drives/comment_13_1d6588477082dd6de6eaad02fdc53463._comment b/doc/forum/very_slow_on_exfat_drives/comment_13_1d6588477082dd6de6eaad02fdc53463._comment
new file mode 100644
index 0000000000..d99d5b0a61
--- /dev/null
+++ b/doc/forum/very_slow_on_exfat_drives/comment_13_1d6588477082dd6de6eaad02fdc53463._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="comment 13"
+ date="2023-12-12T12:44:38Z"
+ content="""
+A directory special remote is just a bunch of files. A bare repo has the git history and all the metadata for the bunch of files. Git itself on slow and bad filesystems is not fun and git-annex having to comb through many git objects to extract metadata for the actual annexed files is most likely the bottleneck here. Best is to not run any git-annex commands directly on the bad filesystem, but elsewhere and operate the bad filesystem repo as a remote. Then you let git-annex gather its information on a fast filesysetm and hardware and let it do only the copying of real files to and from the bad filesystem. At least that's my experience.
+"""]]
diff --git a/doc/forum/very_slow_on_exfat_drives/comment_14_6487a9a170f7caa7c9db06ac3bee8c0e._comment b/doc/forum/very_slow_on_exfat_drives/comment_14_6487a9a170f7caa7c9db06ac3bee8c0e._comment
new file mode 100644
index 0000000000..47e1d68d76
--- /dev/null
+++ b/doc/forum/very_slow_on_exfat_drives/comment_14_6487a9a170f7caa7c9db06ac3bee8c0e._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="imlew"
+ avatar="http://cdn.libravatar.org/avatar/23858c3eed3c3ea9e21522f4c999f1ed"
+ subject="comment 14"
+ date="2023-12-12T13:36:20Z"
+ content="""
+The speeds reported by `get` and `copy` were similar to what `rsync` reports when I just copy files to and from the disks.
+It was really just the work tree (and I guess the clean/smudge filters) being slow.
+
+And bare repos have the advantage that they carry for metadata and possibly file history.
+"""]]
diff --git a/doc/forum/very_slow_on_exfat_drives/comment_15_1d7c3ff1eaeffe03fe7e3e25af822e4e._comment b/doc/forum/very_slow_on_exfat_drives/comment_15_1d7c3ff1eaeffe03fe7e3e25af822e4e._comment
new file mode 100644
index 0000000000..560beff222
--- /dev/null
+++ b/doc/forum/very_slow_on_exfat_drives/comment_15_1d7c3ff1eaeffe03fe7e3e25af822e4e._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="jkniiv"
+ avatar="http://cdn.libravatar.org/avatar/05fd8b33af7183342153e8013aa3713d"
+ subject="comment 15"
+ date="2023-12-12T19:25:16Z"
+ content="""
+The fact of the matter is that HDDs have gotten shittier during the past decade or so because most of them
+(except for sizes above 8TB and drives meant for the enterprise) already employ SMR (shingled magnetic recording)
+instead of conventional recording techniques. It seems SMR is poison to all sorts of workloads having
+small files and directories being rewritten (not that drives employing it have sequential speeds that are exactly
+stellar either) like what git and git-annex are doing under the hood. I bought an 6TB WD Elements desktop drive recently
+(knowingly an SMR unit because non-SMR HDDs are rather expensive here in Finland) as an git-annex archival drive and I was
+flabbergasted at how slow it turned out for merely syncing git metadata. I'm on Windows and have to use adjusted-unlocked
+branches so there's that but NTFS on Windows is not terribly slow -- maybe not as speedy as Linux filesystems but it caches
+metadata rather well, so it's ok. The problem is that while my 4TB Seagate IronWolf non-SMR drive (git-annex) syncs
+in a minute or two, my new 6TB takes a minimum of *ten* minutes or so to do that. At this point I have just resigned myself
+to the fact that all my future archival drives where I use regular git remotes will be slow as molasses. I'd love to own
+big, fast non-SMR enterprise drives but those will be outside my budget for years to come, I'm afraid. :/
+"""]]
diff --git a/doc/forum/very_slow_on_exfat_drives/comment_1_0f16c72ed7bcba01398b36cb8b3cee08._comment b/doc/forum/very_slow_on_exfat_drives/comment_1_0f16c72ed7bcba01398b36cb8b3cee08._comment
new file mode 100644
index 0000000000..8d94f7094a
--- /dev/null
+++ b/doc/forum/very_slow_on_exfat_drives/comment_1_0f16c72ed7bcba01398b36cb8b3cee08._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="imlew"
+ avatar="http://cdn.libravatar.org/avatar/23858c3eed3c3ea9e21522f4c999f1ed"
+ subject="I guess this question is about backends"
+ date="2023-11-16T12:51:59Z"
+ content="""
+As so often happens, search the docs for hours and don't find anything and right after I asking the question I found a (seemingly) relevant page I had somehow missed before.
+
+In this case [backends](https://git-annex.branchable.com/backends/).
+
+Creating a new annex repo on a drive that already had the data but no repo and changing the backend to `WORM` (with `git config --local --add annex.backend WORM`) seems to have made things a little bit faster. Adding the first 3.5GB file in just under 2 minutes and `git status` returning after 44 seconds when first run and thereafter returning instantaneously. However adding all ~3TB in the repo is shaping up to take multiple days anyway.
+
+Is there anything else I've missed? I don't see what could be taking so long if all that is being checked is mtime, name and size.
+"""]]
diff --git a/doc/forum/very_slow_on_exfat_drives/comment_2_769b971263b9124ec08079336d03869d._comment b/doc/forum/very_slow_on_exfat_drives/comment_2_769b971263b9124ec08079336d03869d._comment
new file mode 100644
index 0000000000..b2be94d837
--- /dev/null
+++ b/doc/forum/very_slow_on_exfat_drives/comment_2_769b971263b9124ec08079336d03869d._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="comment 2"
+ date="2023-11-16T15:54:08Z"
+ content="""
+Normally, using a bare repo helps on slow hardware/filesystems. This means making yokr repo on the HDD a bare repo, then adding your files somewhere else, where it's fast, and having git annex sync it over to the HDD. Uncool, but it seems lile exFAT or your HDD is rather on the shitty side. File size shouldn't matter much, the amount of files is often a problem.
+"""]]
diff --git a/doc/forum/very_slow_on_exfat_drives/comment_3_b6ee8c6384c73b44ae9ccc0ba32c3135._comment b/doc/forum/very_slow_on_exfat_drives/comment_3_b6ee8c6384c73b44ae9ccc0ba32c3135._comment
new file mode 100644
index 0000000000..5ecf5e9309
--- /dev/null
+++ b/doc/forum/very_slow_on_exfat_drives/comment_3_b6ee8c6384c73b44ae9ccc0ba32c3135._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="comment 3"
+ date="2023-11-16T15:55:49Z"
+ content="""
+Ah, and make sure you're not on an adjusted-unlocked branch. And only use locked files. If exFAT can't do symlinks properly, that might be the problem. Unlocked gigantic files are also a bottleneck.
+"""]]
diff --git a/doc/forum/very_slow_on_exfat_drives/comment_4_b1dbbf57a46c79e8e32885c2ee8f45d2._comment b/doc/forum/very_slow_on_exfat_drives/comment_4_b1dbbf57a46c79e8e32885c2ee8f45d2._comment
new file mode 100644
index 0000000000..27486cb3de
--- /dev/null
+++ b/doc/forum/very_slow_on_exfat_drives/comment_4_b1dbbf57a46c79e8e32885c2ee8f45d2._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="comment 4"
+ date="2023-11-16T16:34:12Z"
+ content="""
+Yep, apparently, [exFAT can't do symlinks](https://superuser.com/questions/232257/does-or-will-exfat-support-symlinks).
+
+In general, it is best to use git-annex on non-shitty filesystems or one will run into problems with their limitations -- something git-annex can't really do much about.
+
+But you can work around it by using a bare git repository on the HDD as I mentioned above (see [here](https://stackoverflow.com/a/2200662) for how to do that) or to use the HDD just as a directory/rsync special remote from other git-annex repos. In both cases, symlinks are not needed and no expensive slow piping through smudge/clean filters is done. The downside is that you can't work (i.e. add, read, modify files) directly on the HDD, but you only use it as a storage drive.
+"""]]
diff --git a/doc/forum/very_slow_on_exfat_drives/comment_5_ec5d339021f2a26942248962ea818a80._comment b/doc/forum/very_slow_on_exfat_drives/comment_5_ec5d339021f2a26942248962ea818a80._comment
new file mode 100644
index 0000000000..6c36556513
--- /dev/null
+++ b/doc/forum/very_slow_on_exfat_drives/comment_5_ec5d339021f2a26942248962ea818a80._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="imlew"
+ avatar="http://cdn.libravatar.org/avatar/23858c3eed3c3ea9e21522f4c999f1ed"
+ subject="not about backends after all"
+ date="2023-11-17T08:09:22Z"
+ content="""
+Thanks for your help. 
+I've created a bare repo on one of the drives that didn't have a repo yet and have been moving the files to my laptops internal drive to add and commit and then back with `git annex move`, this seems to be working much better. (I already have more than half the files in the repo after one night, previously I had left it running for days and couldn't get that far. (And yes, the drives are very slow.))
+
+re the filesystems, I thought so, but unfortunately these have to compatible with both mac and windoze...
+"""]]
diff --git a/doc/forum/very_slow_on_exfat_drives/comment_6_a6c5e8dc86ed6b4b5ff45f66f5bad50a._comment b/doc/forum/very_slow_on_exfat_drives/comment_6_a6c5e8dc86ed6b4b5ff45f66f5bad50a._comment
new file mode 100644
index 0000000000..4bb4cb5291
--- /dev/null
+++ b/doc/forum/very_slow_on_exfat_drives/comment_6_a6c5e8dc86ed6b4b5ff45f66f5bad50a._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="imlew"
+ avatar="http://cdn.libravatar.org/avatar/23858c3eed3c3ea9e21522f4c999f1ed"
+ subject="one more question"
+ date="2023-11-17T08:23:32Z"
+ content="""
+I'm not sure if this is caused by the disk's repo being bare, but after I have `git annex move`d a file there I still need to run `git annex sync` in the local source repo before I can find or copy the file to a third repo (in this case a second repo on my laptop).
+
+This is a little confusing because it seems the bare repo on the disk doesn't know it has the files that have been moved to it.
+
+Is this because it is bare, am I doing something wrong or why doesn't `git annex move` result in the target knowing that it has a given file?
+"""]]
diff --git a/doc/forum/very_slow_on_exfat_drives/comment_7_3e66ef8493d2839cb26418788c510463._comment b/doc/forum/very_slow_on_exfat_drives/comment_7_3e66ef8493d2839cb26418788c510463._comment
new file mode 100644
index 0000000000..d8f8111ed4
--- /dev/null
+++ b/doc/forum/very_slow_on_exfat_drives/comment_7_3e66ef8493d2839cb26418788c510463._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="comment 7"
+ date="2023-11-17T08:45:56Z"
+ content="""
+I'm not sure exactly what you mean. If you `git annex move`d a file from a local repo to the bare repo on the HDD, your local repo should know about this immediately. I'm not sure if an immediate `git annex move` *on the HDD* afterwards knows about this. Sounds like it should but maybe it doesn't for performance reasons. That would explain the need for a subsequent `sync`. In general, I would recommend setting up preferred content expressions for each repo, and then always just run `git annex assist` to have it sync everything. Slower than manual moving and copying though, but less worrying.
+"""]]
diff --git a/doc/forum/very_slow_on_exfat_drives/comment_8_ae7380ff84e4200985c22deb647853e3._comment b/doc/forum/very_slow_on_exfat_drives/comment_8_ae7380ff84e4200985c22deb647853e3._comment
new file mode 100644
index 0000000000..5cccc65493
--- /dev/null
+++ b/doc/forum/very_slow_on_exfat_drives/comment_8_ae7380ff84e4200985c22deb647853e3._comment
@@ -0,0 +1,18 @@
+[[!comment format=mdwn
+ username="imlew"
+ avatar="http://cdn.libravatar.org/avatar/23858c3eed3c3ea9e21522f4c999f1ed"
+ subject="comment 8"
+ date="2023-11-17T09:41:19Z"
+ content="""
+To try out accessing the file from another location I created a second repo on my laptop.
+So I have the \"real\" local repo (`L`), the repo on the disk (`D`) and another local repo for testing (`T`).
+In `L` I added and committed `$FILE` and then moved it to `D`.
+If I now run `git annex whereis $FILE` in all the repos
+`L` tells me it's in `D`, while both `D` and `T` tell me the file isn't know to git. 
+Only when I run `git annex sync` in `L` does `T` know and `D` still doesn't.
+
+Not a big issue, just a little suprising, but it's fine to have to remember to run `sync` in the local repo before disconnecting the disk.
+
+I have started looking into preferred content and groups and I will most likely use them. At least to begin with I want to try doing things manually and then later on move on to the more sophisticated tools.
+
+"""]]
diff --git a/doc/forum/very_slow_on_exfat_drives/comment_9_5b69d3cb7d3a51ce8a9cbdb608be676c._comment b/doc/forum/very_slow_on_exfat_drives/comment_9_5b69d3cb7d3a51ce8a9cbdb608be676c._comment
new file mode 100644
index 0000000000..f8a4c883ae
--- /dev/null
+++ b/doc/forum/very_slow_on_exfat_drives/comment_9_5b69d3cb7d3a51ce8a9cbdb608be676c._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="comment 9"
+ date="2023-11-17T09:49:02Z"
+ content="""
+Alright, that's how it's supposed to be. Copying files around and keeping track of the locations are two separate things. The location tracking, metadata, etc. are stored in the `git-annex`-branch, which is only synced with `git annex sync` or `git annex assist`. If you're on the manual route (i.e. no preferred content, no `git annex sync --content`, no `git annex assist`), then you are supposed to sync the git repos yourself, e.g. with `git annex sync`. It also makes sense from a performance standpoint. git syncing can be slow, especially on slow hardware. Maybe you don't want to sync the metadata after every copy/move/drop/etc., but you batch it up. And as long as the info where the files are is *somewhere* in a `git-annex`-branch (e.g. your local repo `L`), it's fine as it will eventually be synced around.
+"""]]
diff --git a/doc/git-annex-copy.mdwn b/doc/git-annex-copy.mdwn
index 57905b672c..add7a96f06 100644
--- a/doc/git-annex-copy.mdwn
+++ b/doc/git-annex-copy.mdwn
@@ -41,6 +41,11 @@ Paths of files or directories to operate on can be specified.
   then deleting the content from the local repository (if it was not present
   to start with).
 
+* `--from-anywhere --to=remote`
+
+  Copy to the remote files from the local repository as well as from any reachable
+  remotes.
+
 * `--jobs=N` `-JN`
 
   Enables parallel transfers with up to the specified number of jobs
diff --git a/doc/git-annex-diffdriver/comment_1_b45fd606668e2eee0a37bf1db3391f37._comment b/doc/git-annex-diffdriver/comment_1_b45fd606668e2eee0a37bf1db3391f37._comment
new file mode 100644
index 0000000000..a6b63ceed5
--- /dev/null
+++ b/doc/git-annex-diffdriver/comment_1_b45fd606668e2eee0a37bf1db3391f37._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="v@80d733f66c2e675c27ee39df565ebed8d0ea12de"
+ nickname="v"
+ avatar="http://cdn.libravatar.org/avatar/5d1f51d6e21cfcfb0da6dfe0aea2fc77"
+ subject="Outdated?"
+ date="2023-10-24T13:26:44Z"
+ content="""
+This article seems outdated. `--text` is not a valid option in version 8.20210223 and `GIT_EXTERNAL_DIFF=\"git annex diffdriver -- diff --\" git diff` fails with `diff: extra operand '<commit hash>'`.
+"""]]
diff --git a/doc/git-annex-diffdriver/comment_2_05c3a29d7827dd6fc5babe0ac1ff2a94._comment b/doc/git-annex-diffdriver/comment_2_05c3a29d7827dd6fc5babe0ac1ff2a94._comment
new file mode 100644
index 0000000000..5733e664e4
--- /dev/null
+++ b/doc/git-annex-diffdriver/comment_2_05c3a29d7827dd6fc5babe0ac1ff2a94._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="v@80d733f66c2e675c27ee39df565ebed8d0ea12de"
+ nickname="v"
+ avatar="http://cdn.libravatar.org/avatar/5d1f51d6e21cfcfb0da6dfe0aea2fc77"
+ subject="Minimum required version"
+ date="2023-10-24T14:27:24Z"
+ content="""
+It was me that was outdated. `--text` works as expected on `10.20230828`.
+"""]]
diff --git a/doc/git-annex-import.mdwn b/doc/git-annex-import.mdwn
index ee9b4937c1..b72fae5d5f 100644
--- a/doc/git-annex-import.mdwn
+++ b/doc/git-annex-import.mdwn
@@ -74,16 +74,17 @@ Any files that are gitignored will not be included in the import,
 but will be left on the remote.
 
 When the special remote has a preferred content expression set by
-[[git-annex-wanted]](1), it will be honored when importing from it.
-Files that are not preferred content of the remote will not be
+[[git-annex-wanted]](1), that is used to pick which files to import from
+it. Files that are not preferred content of the remote will not be
 imported from it, but will be left on the remote.
 
-However, preferred content expressions that relate to the key
-can't be matched when importing, because the content of the file is not
-known. Importing will fail when such a preferred content expression is
-set. This includes expressions containing "copies=", "metadata=", and other
-things that depend on the key. Preferred content expressions containing
-"include=", "exclude=" "smallerthan=", "largerthan=" will work.
+So for example, a preferred content expression like 
+`"include=*.jpeg or largerthan=100mb"` will make only jpegs and 
+large files be imported.
+
+Parts of a preferred content expression that relate to the key,
+such as "copies=" are ignored when importing, because the key
+is not known before importing.
 
 Things in the expression like "include=" match relative to the top of
 the tree of files on the remote, even when importing into a subdirectory.
@@ -101,6 +102,11 @@ the tree of files on the remote, even when importing into a subdirectory.
   `git-annex get` can later be used to download files, as desired.
   The --no-content option is not supported by all special remotes.
 
+* `--message=msg` `-m msg`
+
+  Use this option to specify a commit message for the changes that have
+  been made to the special remote since the last import from it.
+
 # IMPORTING FROM A DIRECTORY
 
 When run with a path, `git annex import` **moves** files from somewhere outside
diff --git a/doc/git-annex-info.mdwn b/doc/git-annex-info.mdwn
index aaba7ef182..9b0e9715df 100644
--- a/doc/git-annex-info.mdwn
+++ b/doc/git-annex-info.mdwn
@@ -8,18 +8,36 @@ git annex info `[directory|file|treeish|remote|description|uuid ...]`
 
 # DESCRIPTION
 
-Displays statistics and other information for the specified item,
-which can be a directory, or a file, or a treeish, or a remote,
-or the description or uuid of a repository.
-
-When no item is specified, displays statistics and information
-for the local repository and all annexed content.
+Displays statistics and other information for the specified item.
+
+When no item is specified, displays overall information. This includes a
+list of all known repositories, how much annexed data is present in the
+local repository, the total size of all annexed data in the working
+tree, the combined size of annexed data in all repositories, and the annex
+sizes of each repository.
+
+When a directory is specified, displays information
+about the annexed files in that directory (and subdirectories).
+This includes how much annexed data is present in the local repository,
+the total size of all annexed data in the directory, how many files
+have the specified numcopies or more (+1, +2 etc) or less (-1, -2 etc),
+and information about how much of the annexed data is stored in known
+repositories.
+
+When a treeish is specified, displays similar information
+as when a directory is specified, but about the annexed files in that
+treeish.
+
+When a remote, or description of a repository, or uuid is specified, 
+displays information about the specified repository, including the total
+amount of annexed data stored in it, and a variety of configuration
+information.
 
 # OPTIONS
 
 * `--fast`
 
-  Only show the data that can be gathered quickly.
+  Only show the information that can be gathered quickly.
 
 * `--json`
 
diff --git a/doc/git-annex-log.mdwn b/doc/git-annex-log.mdwn
index 0361dac306..68f749b241 100644
--- a/doc/git-annex-log.mdwn
+++ b/doc/git-annex-log.mdwn
@@ -1,6 +1,6 @@
 # NAME
 
-git-annex log - shows location log
+git-annex log - shows location log information
 
 # SYNOPSIS
 
@@ -8,57 +8,129 @@ git annex log `[path ...]`
 
 # DESCRIPTION
 
-Displays the location log for the specified file or files, showing each
-repository they were added to ("+") and removed from ("-"). Note that the
-location log is for the particular file contents currently at these paths,
-not for any different content that was there in earlier commits.
+This command displays information from the history of the git-annex branch. 
 
-This displays information from the history of the git-annex branch. Several
-things can prevent that information being available to display. When
-[[git-annex-dead]] and [[git-annex-forget]] are used, old historical
-data gets cleared from the branch. When annex.private or 
+Several things can prevent that information being available to display.
+When [[git-annex-forget]] is used, old historical
+data gets cleared from the branch. When annex.private or
 remote.name.annex-private is configured, git-annex does not write
 information to the branch at all. And when annex.alwayscommit is set to
 false, information may not have been committed to the branch yet.
 
 # OPTIONS
 
-* `--since=date`, `--after=date`, `--until=date`, `--before=date`, `--max-count=N`
+* `[path ...]`
 
-  These options are passed through to `git log`, and can be used to limit
-  how far back to search for location log changes.
+  Displays the location log for the specified file or files, showing each
+  repository they were added to ("+") and removed from ("-"). Note that
+  it displays information about the file content currently at these paths,
+  not for any different content that was there in earlier commits.
+
+* matching options
   
-  For example: `--since "1 month ago"`
+  The [[git-annex-matching-options]](1)
+  can be used to control what to act on when displaying the location log
+  for specified files.
+
+* `--all` `-A`
+
+  Shows location log changes to all content, with the most recent changes first.
+  In this mode, the names of files are not available and keys are displayed
+  instead.
+
+* `--sizesof=repository`
+
+  Displays a history of the total size of the annexed files in a repository
+  over time from the creation of the repository to the present.
+
+  The repository can be "here" for the current repository, or the name of a
+  remote, or a repository description or uuid.
+
+  Note that keys that do not have a known size are not included in the
+  total.
+
+* `--sizes`
+
+  This is like --sizesof, but rather than display the size of a single
+  repository, it displays the sizes of all known repositories.
+
+  The output is a CSV formatted table.
+
+* `--totalsizes`
+
+  This is like `--sizesof`, but it displays the total size of all
+  known repositories.
+
+* `--interval=time`
+
+  When using `--sizesof`, `--sizes`, and `--totalsizes`, this 
+  controls the minimum interval between displays of the size.
+  The default is to display each new recorded size.
+
+  The time is of the form "30d" or "1y".
+
+* `--received`
+
+  Combine this option with `--sizesof` or `--sizes` to display
+  the amount of data received into repositories since the last
+  line was output.
+
+* `--gnuplot`
+
+  Combine this option with `--sizesof` or `--sizes` or `--totalsizes`
+  to use gnuplot(1) to graph the data. The gnuplot file will be left on
+  disk for you to reuse.
+
+  For example, to graph the sizes of all repositories:
+
+  git-annex log --sizes --interval=1d --gnuplot
+
+  To graph the amount of new data received into each repository every 30
+  days:
+  
+  git-annex log --sizes --interval=30d --gnuplot --recieved
+
+* `--bytes`
+
+  Show sizes in bytes, disabling the default nicer units.
 
 * `--raw-date`
 
   Rather than the normal display of a date in the local time zone,
   displays seconds since the unix epoch.
 
+* `--since=date`, `--after=date`, `--until=date`, `--before=date`, `--max-count=N`
+
+  These options are passed through to `git log`, and can be used to limit
+  how far back to search for location log changes.
+  
+  For example: `--since "1 month ago"`
+
+  These options do not have an affect when using `--sizesof`, `--sizes`, 
+  and `--totalsizes`.
+
 * `--gource`
 
   Generates output suitable for the `gource` visualization program.
+  
+  This option does not have an affect when using `--sizesof`, `--sizes`, 
+  and `--totalsizes`.
 
 * `--json`
 
   Enable JSON output. This is intended to be parsed by programs that use
   git-annex. Each line of output is a JSON object.
+  
+  This option does not have an affect when using `--sizesof`, `--sizes`, 
+  and `--totalsizes`.
 
 * `--json-error-messages`
 
   Messages that would normally be output to standard error are included in
   the JSON instead.
-
-* matching options
   
-  The [[git-annex-matching-options]](1)
-  can be used to control what to act on.
-
-* `--all` `-A`
-
-  Shows location log changes to all content, with the most recent changes first.
-  In this mode, the names of files are not available and keys are displayed
-  instead.
+  This option does not have an affect when using `--sizesof`, `--sizes`, 
+  and `--totalsizes`.
 
 * Also the [[git-annex-common-options]](1) can be used.
 
diff --git a/doc/git-annex-matching-options.mdwn b/doc/git-annex-matching-options.mdwn
index 83e07105e7..ea29f98848 100644
--- a/doc/git-annex-matching-options.mdwn
+++ b/doc/git-annex-matching-options.mdwn
@@ -50,7 +50,7 @@ in either of two repositories.
 * `--includesamecontent=glob`
 
   Skips files when there is no other file with the same content
-  whose name matches the glob. (Same as `--not --includesamecontent`)
+  whose name matches the glob. (Same as `--not --excludesamecontent`)
 
   For example, if you have inbox and outbox directories, and want to find
   anything in the inbox that has the same content as something in the outbox:
@@ -60,12 +60,15 @@ in either of two repositories.
 * `--in=repository`
 
   Matches only when git-annex believes that the content is present in a
-  repository. Note that it does not check the repository to verify
-  that it still has the content.
+  repository.
 
   The repository should be specified using the name of a configured remote,
   or the UUID or description of a repository. For the current repository,
   use `--in=here`
+  
+  Note that this does not check remote repositories to verify that content
+  is still present on them. However, when checking the current repository,
+  it does verify that content is present in it.
 
 * `--in=repository@{date}`
 
@@ -80,6 +83,20 @@ in either of two repositories.
   free up disk space. The next day, you can get back the files you dropped
   using `git annex get . --in=here@{yesterday}`
 
+* `--expected-present`
+
+  Matches only when git-annex believes that the content is present 
+  in the local repository.
+
+  This is like `--in=here`, except it does not verify that the content
+  is actually present. So it can be used in situations where the location
+  tracking information is known to be out of date.
+
+  For example, if a repository is being restored from a backup
+  that did not include the git-annex objects, this could be used to get
+  back all files that were expected to be in it: 
+  `git-annex get --expected-present`
+
 * `--copies=number`
 
   Matches only when git-annex believes there are the specified number
@@ -103,8 +120,8 @@ in either of two repositories.
 
 * `--lackingcopies=number`
 
-  Matches only when git-annex beleives that the specified number or 
-  more additional copies to be made in order to satisfy numcopies
+  Matches only when git-annex believes that the specified number or 
+  more additional copies need to be made in order to satisfy numcopies
   settings.
 
 * `--approxlackingcopies=number`
diff --git a/doc/git-annex-matching-options/comment_1_09d1a7961a79dff47bef8797b3775766._comment b/doc/git-annex-matching-options/comment_1_09d1a7961a79dff47bef8797b3775766._comment
new file mode 100644
index 0000000000..83849ea16d
--- /dev/null
+++ b/doc/git-annex-matching-options/comment_1_09d1a7961a79dff47bef8797b3775766._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="mike@2d6d71f56ce2a992244350475251df87c26fe351"
+ nickname="mike"
+ avatar="http://cdn.libravatar.org/avatar/183fa439752e2f0c6f39ede658d81050"
+ subject="--includesamecontent same as  --not --includesamecontent?"
+ date="2023-12-06T22:12:47Z"
+ content="""
+The text for `--includesamecontent` states that is the \"same as `--not --includesamecontent`\", this is probably a bug in the documentation?
+"""]]
diff --git a/doc/git-annex-matching-options/comment_2_78a5e7090e5ac3e6710813bac600fde0._comment b/doc/git-annex-matching-options/comment_2_78a5e7090e5ac3e6710813bac600fde0._comment
new file mode 100644
index 0000000000..2156da9311
--- /dev/null
+++ b/doc/git-annex-matching-options/comment_2_78a5e7090e5ac3e6710813bac600fde0._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""Re: --includesamecontent same as  --not --includesamecontent?"""
+ date="2023-12-18T17:40:04Z"
+ content="""
+Fixed that, thanks.
+"""]]
diff --git a/doc/git-annex-migrate.mdwn b/doc/git-annex-migrate.mdwn
index 30d24f943f..235a32c5e3 100644
--- a/doc/git-annex-migrate.mdwn
+++ b/doc/git-annex-migrate.mdwn
@@ -6,24 +6,55 @@ git-annex migrate - switch data to different backend
 
 git annex migrate `[path ...]`
 
+git annex migrate --update
+
 # DESCRIPTION
 
 Changes the specified annexed files to use the default key-value backend
 (or the one specified with `--backend`). Only files whose content
-is currently available are migrated.
-
-Note that the content is also still available using the old key after
-migration. Use `git annex unused` to find and remove the old key.
+is currently present are migrated.
 
-Normally, nothing will be done to files already using the new backend.
-However, if a backend changes the information it uses to construct a key,
-this can also be used to migrate files to use the new key format.
+Note that the content is also still stored using the old keys after
+migration. When possible, hard links are used to avoid that taking up
+extra disk space. Use `git annex unused` to find and remove the old keys.
 
-When you have multiple repositories that each contain a copy of a file,
-it's best to run migrate in all of them.
+Normally, nothing will be done to specified files that are already using
+the new backend. However, if a backend changes the information it uses to
+construct a key, this can also be used to migrate files to use the new key
+format.
 
 # OPTIONS
 
+* `--update`
+
+  This updates the local repository for migrations that were performed
+  elsewhere. Only new migrations since the last time this was run will
+  be performed. 
+
+  This does not modify the working tree, but only hard links
+  (or in some cases copies) annex objects to their new keys.
+
+  `git-annex pull` and `git-annex sync --content` automatically do this,
+  unless the `annex.syncmigrations` config is set to false.
+
+  Note that older versions of git-annex did not record migrations in a
+  way that this can use. Migrations performed with those older versions
+  had to be manually run in each clone of the repository.
+
+* `--apply`
+
+  This applies all recorded migrations to the local repository. It is the
+  non-incremental form of `--update`.
+
+  One situation where this can be useful is when `git-annex migrate
+  --update` has been run, but since then un-migrated 
+  objects have entered the repository. Using this option ensures that
+  any such objects get migrated.
+
+  Note that older versions of git-annex did not record migrations in a
+  way that this can use. Migrations performed with those older versions
+  had to be manually run in each clone of the repository.
+
 * `--backend`
 
   Specify the new key-value backend to use for migrated data.
@@ -51,6 +82,10 @@ it's best to run migrate in all of them.
 
   	git-annex migrate --remove-size --backend=URL somefile
 
+  To add back the size to an URL key, use this:
+
+    git-annex migrate --backend=URL somefile
+
 * `--json`
 
   Enable JSON output. This is intended to be parsed by programs that use
diff --git a/doc/git-annex-move.mdwn b/doc/git-annex-move.mdwn
index b30163c891..c93953f985 100644
--- a/doc/git-annex-move.mdwn
+++ b/doc/git-annex-move.mdwn
@@ -38,6 +38,11 @@ Paths of files or directories to operate on can be specified.
   then deleting the content from the local repository (if it was not present
   to start with).
 
+* `--from-anywhere --to=remote`
+
+  Move to the remote files from the local repository and from all 
+  reachable remotes.
+
 * `--force`
 
   Override numcopies and required content checking, and always remove
diff --git a/doc/git-annex-pull.mdwn b/doc/git-annex-pull.mdwn
index 594bd8fa14..eabc599995 100644
--- a/doc/git-annex-pull.mdwn
+++ b/doc/git-annex-pull.mdwn
@@ -29,7 +29,10 @@ this command will also pull changes from the parent branch.
 When [[git-annex-view]](1) has been used to check out a view branch,
 this command will update the view branch to reflect any changes 
 to the parent branch or metadata.
-  
+
+When [[git-annex-migrate]](1) has been used in other repositories,
+this updates the content in the local repository for those migrations as well.
+
 Normally this tries to download the content of each annexed file,
 from any remote that it's pulling from that has a copy. 
 To control which files it downloads, configure the preferred
@@ -70,12 +73,17 @@ See [[git-annex-preferred-content]](1).
 * `--no-content, `-g`, `--content`
 
   Use `--no-content` or `-g` to avoid downloading (and dropping)
-  the content of annexed files.
+  the content of annexed files, and also prevent doing any migrations of
+  content.
 
   If you often use `--no-content`, you can set the `annex.synccontent`
   configuration to false to prevent downloading content by default.
   The `--content` option overrides that configuration.
 
+  To only prevent only migrations of content, you can set the
+  `annex.syncmigrations` configuration to false.
+  The `--content` option overrides that configuration as well.
+
 * `--content-of=path` `-C path`
 
   Only download (and drop) annexed files in the given path.
diff --git a/doc/git-annex-satisfy.mdwn b/doc/git-annex-satisfy.mdwn
index ddbec766ae..1ed4ec543d 100644
--- a/doc/git-annex-satisfy.mdwn
+++ b/doc/git-annex-satisfy.mdwn
@@ -15,6 +15,11 @@ It does the same thing as `git-annex sync --content` without the pulling
 and pushing of git repositories, and without changing the trees that are
 imported to or exported from special remotes.
 
+Note that it (like [[git-annex-sync]] or [[git-annex-assist]]) does not work
+specifically towards satisfying the [[git-annex-numcopies]] setting,
+unless the preferred content setting of the local repository is written to
+do so by using eg `approxlackingcopies=1`.
+
 # OPTIONS
 
 * `[remote]`
diff --git a/doc/git-annex-test.mdwn b/doc/git-annex-test.mdwn
index 18a219c6b2..83bdc5bd3e 100644
--- a/doc/git-annex-test.mdwn
+++ b/doc/git-annex-test.mdwn
@@ -44,6 +44,10 @@ framework. Pass --help for details about those.
   One valid use of this is to change a git configuration to a value that
   is planned to be the new default in a future version of git.
 
+  Also, some things can only be tested with a git configuration. For
+  example, annex.shared-sop-command has to be set for the test suite to
+  test using that command.
+
 * `--test-debug`
 
   Normally output of commands run by the test suite is hidden, so even
diff --git a/doc/git-annex-uninit/comment_4_3386d8419830354b4422d38448467e95._comment b/doc/git-annex-uninit/comment_4_3386d8419830354b4422d38448467e95._comment
new file mode 100644
index 0000000000..a99f5f4701
--- /dev/null
+++ b/doc/git-annex-uninit/comment_4_3386d8419830354b4422d38448467e95._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 4"""
+ date="2023-11-21T19:56:32Z"
+ content="""
+Moved some comments about a bug to 
+[[bugs/uninit_fails_when_I_symlink_your_symlink]]. 
+Please do not use this man page's comment section to file bug reports.
+"""]]
diff --git a/doc/git-annex-webapp.mdwn b/doc/git-annex-webapp.mdwn
index 47dfae504a..d53e76f8cb 100644
--- a/doc/git-annex-webapp.mdwn
+++ b/doc/git-annex-webapp.mdwn
@@ -10,7 +10,8 @@ git annex webapp
 
 Opens a web app, that allows easy setup of a git-annex repository,
 and control of the git-annex assistant. If the assistant is not
-already running, it will be started.
+already running, it will be started. This will cause new files to 
+be added and syncing operations to be performed.
 
 By default, the webapp can only be accessed from localhost, and running
 it opens a browser window.
@@ -20,13 +21,19 @@ it opens a browser window.
 * `--listen=address`
 
   Useful for using the webapp on a remote computer. This makes the webapp
-  listen on the specified address.
+  listen on the specified IP address. (Or on the address that a specified
+  hostname resolves to.)
 
   This disables running a local web browser, and outputs the url you
   can use to open the webapp.
 
   Set annex.listen in the git config to make the webapp always
-  listen on an address.
+  listen on an IP address.
+
+* `--port=number`
+
+  Use this option to specify a port for the webapp.
+  By default, the webapp picks an unused port.
 
 * Also the [[git-annex-common-options]](1) can be used.
 
diff --git a/doc/git-annex-webapp/comment_1_443c5595412a19ef9c6948c4224297a3._comment b/doc/git-annex-webapp/comment_1_443c5595412a19ef9c6948c4224297a3._comment
new file mode 100644
index 0000000000..aeba3edaad
--- /dev/null
+++ b/doc/git-annex-webapp/comment_1_443c5595412a19ef9c6948c4224297a3._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="NewUser"
+ nickname="dont.post.me"
+ avatar="http://cdn.libravatar.org/avatar/90f59ddc341eaf9b2657422206c06901"
+ subject="launching `git annex webapp` starts adding to the annex which is surprising"
+ date="2023-11-18T20:03:06Z"
+ content="""
+I have gotten as far as running `git init` and `git annex init` on my server. There&rsquo;s a bunch of stuff there and I figure I&rsquo;ll just add the things that I care about as I go.
+
+I read the page on the [webapp](https://git-annex.branchable.com/git-annex-webapp/) and it just says that it &ldquo;allows easy setup of a git-annex repository, and control of the git-annex assistant&rdquo;. I started the webapp on my server and I was alarmed to see a message in the webapp that it was adding things to git annex. Over on the page for the [git annex assistant](https://git-annex.branchable.com/git-annex-assistant/) it says &ldquo;By default, all new files in the directory will be added to the repository&rdquo; which is definitely not what I expected. I&rsquo;m a new user trying to learn how to use git annex and this was an unpleasant surprise. Can we amend the docs for the webapp to warn about this behavior?
+
+"""]]
diff --git a/doc/git-annex-webapp/comment_2_7b1c4c4356e801006081588b32075fb4._comment b/doc/git-annex-webapp/comment_2_7b1c4c4356e801006081588b32075fb4._comment
new file mode 100644
index 0000000000..ad79367d0f
--- /dev/null
+++ b/doc/git-annex-webapp/comment_2_7b1c4c4356e801006081588b32075fb4._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="NewUser"
+ nickname="dont.post.me"
+ avatar="http://cdn.libravatar.org/avatar/90f59ddc341eaf9b2657422206c06901"
+ subject="launching `git annex webapp` starts adding to the annex which is surprising"
+ date="2023-11-18T20:08:17Z"
+ content="""
+Since I panicked upon seeing a message about adding files to the annex, I promptly killed the webapp. I might just be misunderstanding what's going on. Thanks!
+"""]]
diff --git a/doc/git-annex-webapp/comment_3_aee70625f7cff6e7312f9bc2cbbb02d0._comment b/doc/git-annex-webapp/comment_3_aee70625f7cff6e7312f9bc2cbbb02d0._comment
new file mode 100644
index 0000000000..c96c374c1a
--- /dev/null
+++ b/doc/git-annex-webapp/comment_3_aee70625f7cff6e7312f9bc2cbbb02d0._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="comment 3"
+ date="2023-11-19T16:16:15Z"
+ content="""
+Welcome on board. `git annex webapp` launches the assistant. It might even do `git annex assistant --autostart` and launch the assistant in all configured repos. That is indeed an important info to know. Everyone can edit these pages (the Edit button above) btw. 
+"""]]
diff --git a/doc/git-annex-webapp/comment_4_c1754cdb4087ad278867ed8fddd99409._comment b/doc/git-annex-webapp/comment_4_c1754cdb4087ad278867ed8fddd99409._comment
new file mode 100644
index 0000000000..71371439bb
--- /dev/null
+++ b/doc/git-annex-webapp/comment_4_c1754cdb4087ad278867ed8fddd99409._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="comment 4"
+ date="2023-11-19T16:27:34Z"
+ content="""
+I added a note to make it more clear that the webapp will sync stuff via the assistant.
+"""]]
diff --git a/doc/git-annex-webapp/comment_5_31301895752f0dd81db72c463f0dc732._comment b/doc/git-annex-webapp/comment_5_31301895752f0dd81db72c463f0dc732._comment
new file mode 100644
index 0000000000..0041f9127e
--- /dev/null
+++ b/doc/git-annex-webapp/comment_5_31301895752f0dd81db72c463f0dc732._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="NewUser"
+ nickname="dont.post.me"
+ avatar="http://cdn.libravatar.org/avatar/90f59ddc341eaf9b2657422206c06901"
+ subject="Thanks Yann!"
+ date="2023-11-19T18:20:03Z"
+ content="""
+Thank you [nobodyinperson](https://git-annex.branchable.com/users/nobodyinperson/)! Your note looks perfect. Oh, wow; I didn&rsquo;t know that I could edit this page. Thanks again!
+"""]]
diff --git a/doc/git-annex.mdwn b/doc/git-annex.mdwn
index dfcd600c49..c4bd2b8df3 100644
--- a/doc/git-annex.mdwn
+++ b/doc/git-annex.mdwn
@@ -1222,6 +1222,12 @@ repository, using [[git-annex-config]]. See its man page for a list.)
   To configure the behavior in all clones of the repository,
   this can be set in [[git-annex-config]](1).
 
+* `annex.syncmigrations`
+
+  Set to false to prevent `git-annex sync` and `git-annex pull`
+  from scanning for migrations and updating the local 
+  repository for those migrations.
+
 * `annex.viewunsetdirectory`
 
   This configures the name of a directory that is used in a view to contain
@@ -1388,9 +1394,9 @@ Remotes are configured using these settings in `.git/config`.
 
 * `remote.<name>.annex-ignore`
 
-  If set to `true`, prevents git-annex
-  from storing annexed file contents on this remote by default.
-  (You can still request it be used by the `--from` and `--to` options.)
+  If set to `true`, prevents git-annex from storing or retrieving annexed
+  file contents on this remote by default. 
+  (You can still request it be used with the `--from` and `--to` options.)
 
   This is, for example, useful if the remote is located somewhere
   without git-annex-shell. (For example, if it's on GitHub).
@@ -1399,7 +1405,7 @@ Remotes are configured using these settings in `.git/config`.
 
   This does not prevent `git-annex sync`, `git-annex pull`, `git-annex push`,
   `git-annex assist` or the `git-annex assistant` from operating on the
-  git repository.
+  git repository. It only affects annexed content.
 
 * `remote.<name>.annex-ignore-command`
 
@@ -1515,25 +1521,37 @@ Remotes are configured using these settings in `.git/config`.
   for remotes where the transfer is run by a separate program than
   git-annex. 
 
-* `remote.<name>.annex-stalldetecton`, `annex.stalldetection`
+* `remote.<name>.annex-bwlimit-download`, `annex.bwlimit-download`
+
+  Limit bandwith for downloads from a remote.
+  
+  Overrides `remote.<name>.annex-bwlimit` and `annex.bwlimit`
+
+* `remote.<name>.annex-bwlimit-upload`, `annex.bwlimit-upload`
+  
+  Limit bandwith for uploads to a remote. 
+
+  Overrides `remote.<name>.annex-bwlimit` and `annex.bwlimit`
+
+* `remote.<name>.annex-stalldetection`, `annex.stalldetection`
 
   Configuring this lets stalled or too-slow transfers be detected, and
   dealt with, so rather than getting stuck, git-annex will cancel the
   stalled operation. The transfer will be considered to have failed, so
   settings like annex.retry will control what it does next.
 
-  By default, git-annex detects transfers that have probably stalled,
-  and suggests configuring this. If it is incorrectly detecting
-  stalls, setting this to "false" will avoid that.
-
-  Set to "true" to enable automatic stall detection. If a remote does not
-  update its progress consistently, no automatic stall detection will be
+  The default is to automatically detect when transfers that have probably
+  stalled, and suggest configuring this, but not cancel the stalled
+  operations. For this to work, a remote needs to update its progress
+  consistently. Remotes that do not will not have automatic stall detection
   done. And it may take a while for git-annex to decide a remote is really
   stalled when using automatic stall detection, since it needs to be
   conservative about what looks like a stall.
 
-  For more fine control over what constitutes a stall, set to a value in
-  the form "$amount/$timeperiod" to specify how much data git-annex should
+  Set to "false" to avoid all attempts at stall detection.
+
+  To detect and cancel stalled transfers, set this to a value in the form
+  "$amount/$timeperiod" which specifies how much data git-annex should
   expect to see flowing, minimum, over a given period of time.
 
   For example, to detect outright stalls where no data has been transferred
@@ -1544,11 +1562,40 @@ Remotes are configured using these settings in `.git/config`.
   stuck for a long time, you could use:
   `git config remote.usbdrive.annex-stalldetection "1MB/1m"`
 
-  This is not enabled by default, because it can make git-annex use
-  more resources. To be able to cancel stalls, git-annex has to run
-  transfers in separate processes (one per concurrent job). So it
-  may need to open more connections to a remote than usual, or
-  the communication with those processes may make it a bit slower.
+  Some remotes don't report transfer progress, and stalls cannot be
+  detected when using those.
+
+  Some remotes only report transfer progress occasionally, eg
+  after each chunk. To avoid false timeouts in such a situation, if the
+  first progress update takes longer to arrive than the configured time
+  period, the stall detection will be automically adjusted to use a longer
+  time period. For example, if the first progress update comes after 10
+  minutes, but annex.stalldetection is "1MB/1m", it will be treated as eg
+  "30MB/30m".
+
+  Configuring stall detection can make git-annex use more resources. To be
+  able to cancel stalls, git-annex has to run transfers in separate
+  processes (one per concurrent job). So it may need to open more
+  connections to a remote than usual, or the communication with those
+  processes may make it a bit slower.
+
+* `remote.<name>.annex-stalldetection-download`, `annex.stalldetection-download`
+
+  Stall detection for downloads from a remote.
+ 
+  For example, if a remote is often fast, but sometimes is very slow,
+  and there is another remote that is consistently medium speed
+  and that contains the same data, this could be set to treat the fast
+  remote as stalled when it's slow. Then a command like `git-annex get`
+  will fall back to downloading from the medium speed remote.
+
+  Overrides `remote.<name>.annex-stalldetection`, `annex.stalldetection`
+
+* `remote.<name>.annex-stalldetection-upload`, `annex.stalldetection-upload`
+
+  Stall detection for uploads to a remote.
+
+  Overrides `remote.<name>.annex-stalldetection`, `annex.stalldetection`
 
 * `remote.<name>.annex-checkuuid`
 
@@ -1649,10 +1696,29 @@ Remotes are configured using these settings in `.git/config`.
   precedence over the default GnuPG configuration, which is otherwise
   used.)
 
+* `remote.<name>.annex-shared-sop-command`
+
+  Use this command, which is an implementation of the Stateless OpenPGP
+  command line interface, rather than GnuPG for encrypting and decrypting
+  data. This is only used when a special remote is configured with
+  encryption=shared.
+
+  For example, to use Sequoia PGP's sqop command, set this to "sqop".
+
+* `remote.<name>.annex-shared-sop-profile`
+
+  When encrypting with a Stateless OpenPGP command, this can be used
+  to specify the profile to use, such as "rfc4880".
+
+  For a list of available profiles, run eg "sqop list-profiles encrypt"
+
+  sqop list-profiles encrypt
+
 * `annex.ssh-options`, `annex.rsync-options`,
   `annex.rsync-upload-options`, `annex.rsync-download-options`,
   `annex.bup-split-options`, `annex.gnupg-options`,
-  `annex.gnupg-decrypt-options`
+  `annex.gnupg-decrypt-options`,
+  `annex.shared-sop-command`, `annex.shared-sop-profile`
 
   Default options to use if a remote does not have more specific options
   as described above.
@@ -1970,9 +2036,14 @@ Remotes are configured using these settings in `.git/config`.
 
 * `annex.listen`
 
-  Configures which address the webapp listens on. The default is localhost.
-  Can be either an IP address, or a hostname that resolves to the desired
-  address.
+  Configures which IP address the webapp listens on. 
+  The default is localhost. Can be either an IP address, 
+  or a hostname that resolves to the desired address.
+
+* `annex.port`
+
+  Configures which port address the webapp listens on. 
+  The default is to pick an unused port.
 
 # CONFIGURATION VIA .gitattributes
 
diff --git a/doc/install/fromsource/comment_75_58dfd18135aa187d8b3977521ecff2d2._comment b/doc/install/fromsource/comment_75_58dfd18135aa187d8b3977521ecff2d2._comment
new file mode 100644
index 0000000000..9d07aef2ec
--- /dev/null
+++ b/doc/install/fromsource/comment_75_58dfd18135aa187d8b3977521ecff2d2._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="TTTTAAAx"
+ avatar="http://cdn.libravatar.org/avatar/9edd4b69b9f9fc9b8c1cb8ecd03902d5"
+ subject="comment 75"
+ date="2024-01-11T14:15:58Z"
+ content="""
+Can I make armv7 version for alpine Linux? Or WASM?
+`git-annex` can't run on iOS without jailbreak.
+The only way is to build it on iSH or A-shell.
+
+git-annex(aarch64) is already provided on alpine repo. But iSH supports only armv7. So I can't install git-annex. I tried to build ghc for armv7. But the Haskell foundation has dropped supporting the armv7 since ghc-9. 
+
+A-shell needs WASI binary. I tried ghc-wasm-meta to compile git-annex to WASI binary, But I couldn't handle package version conflicts inside git-annex.
+
+
+"""]]
diff --git a/doc/internals.mdwn b/doc/internals.mdwn
index 47fd0b55cb..67cfd42f04 100644
--- a/doc/internals.mdwn
+++ b/doc/internals.mdwn
@@ -346,3 +346,14 @@ Example:
 ## `multicast.log`
 
 Records uftp public key fingerprints, for use by [[git-annex-multicast]].
+
+## `migrate.tree/old` and `migrate.tree/new`
+
+These are used to record migrations done by `git-annex migrate`. By diffing
+between the two, the old and new keys can be determined. This lets
+migrations be recorded while using a minimum of space in the git
+repository. The filenames in these trees have no connection to the names
+of actual annexed files.
+
+These trees are recorded in history of the git-annex branch, but the
+head of the git-annex branch will never contain them.
diff --git a/doc/news/version_10.20230407.mdwn b/doc/news/version_10.20230407.mdwn
deleted file mode 100644
index 6f3883cb46..0000000000
--- a/doc/news/version_10.20230407.mdwn
+++ /dev/null
@@ -1,11 +0,0 @@
-git-annex 10.20230407 released with [[!toggle text="these changes"]]
-[[!toggleable text="""  * Fix laziness bug introduced in last release that breaks use
-    of --unlock-present and --hide-missing adjusted branches.
-  * Support user.useConfigOnly git config.
-  * registerurl, unregisterurl: Added --remote option.
-  * registerurl: When an url is claimed by a special remote other than the
-    web, update location tracking for that special remote.
-    (This was the behavior before version 6.20181011)
-  * Sped up sqlite inserts 2x when built with persistent 2.14.5.0
-  * git-annex.cabal: Prevent building with unix-compat 0.7 which
-    removed System.PosixCompat.User."""]]
\ No newline at end of file
diff --git a/doc/news/version_10.20230626.mdwn b/doc/news/version_10.20230626.mdwn
deleted file mode 100644
index 33e2594c3a..0000000000
--- a/doc/news/version_10.20230626.mdwn
+++ /dev/null
@@ -1,106 +0,0 @@
-News for git-annex 10.20230626:
-
-git-annex (10.20230626) upstream; urgency=medium
-.
-  Many commands now quote filenames that contain unusual characters the
-  same way that git does, to avoid exposing control characters to the
-  terminal. The core.quotePath config can be set to false to disable this
-  quoting.
-
-
-git-annex 10.20230626 released with [[!toggle text="these changes"]]
-[[!toggleable text="""  * Split out two new commands, git-annex pull and git-annex push.
-    Those plus a git commit are equivalent to git-annex sync.
-    (Note that the new commands default to syncing content, unless
-    annex.synccontent is explicitly set to false.)
-  * assist: New command, which is the same as git-annex sync but with
-    new files added and content transferred by default.
-  * sync: Started a transition to --content being enabled by default.
-    When used without --content or --no-content, warn about the upcoming
-    transition, and suggest using one of the options, or setting
-    annex.synccontent.
-  * sync: Added -g as a short option for --no-content.
-  * Many commands now quote filenames that contain unusual characters the
-    same way that git does, to avoid exposing control characters to the
-    terminal.
-  * Support core.quotePath, which can be set to false to display utf8
-    characters as-is in filenames.
-  * Control characters in non-filename data coming from the repository or
-    other possible untrusted sources are filtered out of the display of many
-    commands. When the command output is intended for use in scripting,
-    control characters are only filtered out when displaying to the
-    terminal.
-  * find, findkeys, examinekey: When outputting to a terminal and --format
-    is not used, quote control characters. Output to a pipe is unchanged.
-    (Similar to the behavior of GNU find.)
-  * addurl --preserve-filename now rejects filenames that contain other
-    control characters, besides the escape sequences it already rejected.
-  * init: Avoid autoenabling special remotes that have control characters
-    in their names.
-  * Support core.sharedRepository=0xxx at long last.
-  * Support --json and --json-error-messages in many more commands
-    (addunused, configremote, dead, describe, dropunused, enableremote,
-    expire, fix, importfeed, init, initremote, log, merge, migrate, reinit,
-    reinject, rekey, renameremote, rmurl, semitrust, setpresentkey, trust,
-    unannex, undo, uninit, untrust, unused, upgrade)
-  * importfeed: Support -J
-  * importfeed: Support --json-progress
-  * httpalso: Support being used with special remotes that use chunking.
-  * Several significant speedups to importing large trees from special
-    remotes. Imports that took over an hour now take only a few minutes.
-  * Cache negative lookups of global numcopies and mincopies.
-    Speeds up eg git-annex sync --content by up to 50%.
-  * Speed up sync in an adjusted branch by avoiding re-adjusting the branch
-    unnecessarily, particularly when it is adjusted with --hide-missing
-    or --unlock-present.
-  * config: Added the --show-origin and --for-file options.
-  * config: Support annex.numcopies and annex.mincopies.
-  * whereused: Fix display of branch:file when run in a subdirectory.
-  * enableremote: Support enableremote of a git remote (that was previously
-    set up with initremote) when additional parameters such as autoenable=
-    are passed.
-  * configremote: New command, currently limited to changing autoenable=
-    setting of a special remote.
-  * Honor --force option when operating on a local git remote.
-  * When a nonexistant file is passed to a command and
-    --json-error-messages is enabled, output a JSON object indicating the
-    problem. (But git ls-files --error-unmatch still displays errors about
-    such files in some situations.)
-  * Bug fix: Create .git/annex/, .git/annex/fsckdb,
-    .git/annex/sentinal, .git/annex/sentinal.cache, and
-    .git/annex/journal/* with permissions configured by core.sharedRepository.
-  * Bug fix: Lock files were created with wrong modes for some combinations
-    of core.sharedRepository and umask.
-  * initremote: Avoid creating a remote that is not encrypted when gpg is
-    broken.
-  * log: When --raw-date is used, display only seconds from the epoch, as
-    documented, omitting a trailing "s" that was included in the output
-    before.
-  * addunused: Displays the names of the files that it adds.
-  * reinject: Fix support for operating on multiple pairs of files and keys.
-  * sync: Fix buggy handling of --no-pull and --no-push when syncing
-    --content. With --no-pull, avoid downloading content, and with
-    --no-push avoid uploading content. This was done before, but
-    inconsistently.
-  * uninit: Avoid buffering the names of all annexed files in memory.
-  * Fix bug in -z handling of trailing NUL in input.
-  * version: Avoid error message when entire output is not read.
-  * Fix excessive CPU usage when parsing yt-dlp (or youtube-dl) progress
-    output fails.
-  * Use --progress-template with yt-dlp to fix a failure to parse
-    progress output when only an estimated total size is known.
-  * When yt-dlp is available, default to using it in preference to
-    youtube-dl. Using youtube-dl is now deprecated, and git-annex no longer
-    tries to parse its output to display download progress
-  * Improve resuming interrupted download when using yt-dlp or youtube-dl.
-  * assistant: Add dotfiles to git by default, unless annex.dotfiles
-    is configured, the same as git-annex add does.
-  * assistant --autostop: Avoid crashing when ~/.config/git-annex/autostart
-    lists a directory that it cannot chdir to.
-  * Fix display when run with -J1.
-  * assistant: Fix a crash when a small file is deleted immediately after
-    being created.
-  * repair: Fix handling of git ref names on Windows.
-  * repair: Fix a crash when .git/annex/journal/ does not exist.
-  * Support building with optparse-applicative 0.18.1
-    (Thanks, Peter Simons)"""]]
diff --git a/doc/news/version_10.20230802.mdwn b/doc/news/version_10.20230802.mdwn
deleted file mode 100644
index 619038805f..0000000000
--- a/doc/news/version_10.20230802.mdwn
+++ /dev/null
@@ -1,40 +0,0 @@
-git-annex 10.20230802 released with [[!toggle text="these changes"]]
-[[!toggleable text="""  * satisfy: New command that gets/sends/drops content to satisfy
-    preferred content settings. This is like to the --content
-    part of git-annex sync.
-  * --explain: New option to display explanations of what git-annex
-    takes into account when deciding what to do. Including explaining
-    matching of preferred content expressions, annex.largefiles, and
-    annex.addunlocked.
-  * reinject: Added --guesskeys option.
-  * diffdriver: Added --text option for easy diffing of the contents of
-    annexed text files.
-  * assist: With --jobs, parallelize transferring content to/from remotes.
-  * Bug fix: Re-running git-annex adjust or sync when in an adjusted branch
-    would overwrite the original branch, losing any commits that had been
-    made to it since the adjusted branch was created.
-  * Bug fix: Fix behavior when git is configured with
-    safe.bareRepository = explicit.
-  * importfeed bug fix: When -J was used with multiple feeds, some feeds
-    did not get their items downloaded.
-  * importfeed: Add feedurl to the metadata (and allow it to be used in the
-    --template)
-  * Improve resuming interrupted download when using yt-dlp.
-  * S3: Amazon S3 buckets created after April 2023 do not support ACLs,
-    so public=yes cannot be used with them. Existing buckets configured
-    with public=yes will keep working.
-  * S3: Allow setting publicurl=yes without public=yes, to support
-    buckets that are configured with a Bucket Policy that allows public
-    access.
-  * directory, gcrypt: Remove empty hash directories when dropping content.
-  * dropunused: Support --jobs
-  * Support "onlyingroup=" in preferred content expressions.
-  * Support --onlyingroup= matching option.
-  * Setup.hs: Stop installing man pages, desktop files, and the
-    git-annex-shell and git-remote-tor-annex symlinks.
-    Anything still relying on that, eg via cabal v1-install will need to
-    change to using make install-home.
-  * Support building with unix-compat 0.7
-  * Support building with unix-2.8.0.
-  * stack.yaml: Update to build with ghc-9.6.2 and aws-0.24.
-    (For windows, stack-lts-18.13.yaml has to be used instead for now.)"""]]
\ No newline at end of file
diff --git a/doc/news/version_10.20230926/comment_1_6edf64272975935ac4874d7e8dd44390._comment b/doc/news/version_10.20230926/comment_1_6edf64272975935ac4874d7e8dd44390._comment
new file mode 100644
index 0000000000..1017b4ea4c
--- /dev/null
+++ b/doc/news/version_10.20230926/comment_1_6edf64272975935ac4874d7e8dd44390._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="kdm9"
+ avatar="http://cdn.libravatar.org/avatar/b7b736335a0e9944a8169a582eb4c43d"
+ subject="Linux 64bit failing?"
+ date="2023-10-04T14:39:48Z"
+ content="""
+FYI Joey: seems like the x86_64 build failed? In any case the \"current\" Linux x86_64 standalone tarball points to the last release from the end of August.
+"""]]
diff --git a/doc/news/version_10.20230926/comment_2_b1bb446c48a4abc4df73354980f9f7cf._comment b/doc/news/version_10.20230926/comment_2_b1bb446c48a4abc4df73354980f9f7cf._comment
new file mode 100644
index 0000000000..aa0b3ce0b8
--- /dev/null
+++ b/doc/news/version_10.20230926/comment_2_b1bb446c48a4abc4df73354980f9f7cf._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2023-10-09T18:27:34Z"
+ content="""
+Publishing the new version had not happened due to a problem. I've fixed
+that now.
+"""]]
diff --git a/doc/news/version_10.20231129.mdwn b/doc/news/version_10.20231129.mdwn
new file mode 100644
index 0000000000..103697888d
--- /dev/null
+++ b/doc/news/version_10.20231129.mdwn
@@ -0,0 +1,22 @@
+git-annex 10.20231129 released with [[!toggle text="these changes"]]
+[[!toggleable text="""  * Fix bug in git-annex copy --from --to that skipped files that were
+    locally present.
+  * Make git-annex copy --from --to --fast actually fast.
+  * Fix crash of enableremote when the special remote has embedcreds=yes.
+  * Ignore directories and other unusual files in .git/annex/journal/
+  * info: Added calculation of combined annex size of all repositories.
+  * log: Added options --sizesof, --sizes and --totalsizes that
+    display how the size of repositories changed over time.
+  * log: Added options --interval, --bytes, --received, and --gnuplot
+    to tune the output of the above added options.
+  * findkeys: Support --largerthan and --smallerthan.
+  * importfeed: Use caching database to avoid needing to list urls
+    on every run, and avoid using too much memory.
+  * Improve memory use of --all when using annex.private.
+  * lookupkey: Sped up --batch.
+  * Windows: Consistently avoid ending standard output lines with CR.
+    This matches the behavior of git on Windows.
+  * Windows: Fix CRLF handling in some log files.
+  * Windows: When git-annex init is installing hook scripts, it will
+    avoid ending lines with CR for portability. Existing hook scripts
+    that do have CR line endings will not be changed."""]]
\ No newline at end of file
diff --git a/doc/news/version_10.20231227.mdwn b/doc/news/version_10.20231227.mdwn
new file mode 100644
index 0000000000..580da5cbfb
--- /dev/null
+++ b/doc/news/version_10.20231227.mdwn
@@ -0,0 +1,28 @@
+git-annex 10.20231227 released with [[!toggle text="these changes"]]
+[[!toggleable text="""  * migrate: Support distributed migrations by recording each migration,
+    and adding a --update option that updates the local repository
+    incrementally, hard linking annex objects to their new keys.
+  * pull, sync: When operating on content, automatically handle
+    distributed migrations.
+  * Added annex.syncmigrations config that can be set to false to prevent
+    pull and sync from migrating object content.
+  * migrate: Added --apply option that (re)applies all recorded
+    distributed migrations to the objects in repository.
+  * migrate: Support adding size to URL keys that were added with
+    --relaxed, by running eg: git-annex migrate --backend=URL foo
+  * When importing from a special remote, support preferred content
+    expressions that use terms that match on keys (eg "present", "copies=1").
+    Such terms are ignored when importing, since the key is not known yet.
+    Before, such expressions caused the import to fail.
+  * Support git-annex copy/move --from-anywhere --to remote.
+  * Make git-annex get/copy/move --from foo override configuration of
+    remote.foo.annex-ignore, as documented.
+  * Lower precision of timestamps in git-annex branch, which can reduce the
+    size of the branch by up to 8%.
+  * sync: Fix locking problems during merge when annex.pidlock is set.
+  * Avoid a problem with temp file names ending in "." on certian
+    filesystems that have problems with such filenames.
+  * sync, push: Avoid trying to send individual files to special remotes
+    configured with importtree=yes exporttree=no, which would always fail.
+  * Fix a crash opening sqlite databases when run in a non-unicode locale.
+    (Needs persistent-sqlite 2.13.3.)"""]]
\ No newline at end of file
diff --git a/doc/profiling/comment_9_7e1a06690b52d241b2d57be0864d6f26._comment b/doc/profiling/comment_9_7e1a06690b52d241b2d57be0864d6f26._comment
deleted file mode 100644
index 3212ef1fd3..0000000000
--- a/doc/profiling/comment_9_7e1a06690b52d241b2d57be0864d6f26._comment
+++ /dev/null
@@ -1,18 +0,0 @@
-[[!comment format=mdwn
- username="joey"
- subject="""comment 9"""
- date="2020-10-19T18:35:02Z"
- content="""
-Also, /usr/bin/time git-annex find:
-
-	1.70user 0.27system 0:01.55elapsed 126%CPU (0avgtext+0avgdata 97352maxresident)k
-	0inputs+0outputs (0major+9303minor)pagefaults 0swaps
-
-The maxresident seems high, but a stack profile does not show a memory
-leak, or such a large amount of memory use at all. Currently, I
-think that memory is being preallocated by the ghc runtime,
-or something like that. (See [[todo/memory_use_increase]].)
-
-ghc 8.8.4  
-Should keep an eye on this with newer ghc versions.
-"""]]
diff --git a/doc/projects/repronim/bugs-done/multiple_ssh_prompts__44___and_thread_blocked_indefinitely_in_an___63____63____63___transaction/comment_14_2d8cf0d8b020941d143abd424729c6bc/comment_1_73e63e87f26a47e465a8540eb13b28c6._comment b/doc/projects/repronim/bugs-done/multiple_ssh_prompts__44___and_thread_blocked_indefinitely_in_an___63____63____63___transaction/comment_14_2d8cf0d8b020941d143abd424729c6bc/comment_1_73e63e87f26a47e465a8540eb13b28c6._comment
deleted file mode 100644
index 4770e9e080..0000000000
--- a/doc/projects/repronim/bugs-done/multiple_ssh_prompts__44___and_thread_blocked_indefinitely_in_an___63____63____63___transaction/comment_14_2d8cf0d8b020941d143abd424729c6bc/comment_1_73e63e87f26a47e465a8540eb13b28c6._comment
+++ /dev/null
@@ -1,14 +0,0 @@
-[[!comment format=mdwn
- username="joey"
- subject="""comment 1"""
- date="2018-11-15T18:33:11Z"
- content="""
-The problems at least so far appear to be related; it should be enough
-to bisect the easier one to deal with.
-
-BTW, I noticed a couple of weird things that may or may not be related.
-Your repository is on NFS but annex.pidlock is not set, which I guess means
-that NFS is providing working real locking, or perhaps subtly broken real
-locking. And, the test suite failed with permission denied when writing a
-file, which is very strange.
-"""]]
diff --git a/doc/special_remotes.mdwn b/doc/special_remotes.mdwn
index 2ba9525605..f776605423 100644
--- a/doc/special_remotes.mdwn
+++ b/doc/special_remotes.mdwn
@@ -26,7 +26,6 @@ the git history is not stored in them.
 * [[git]]
 * [[httpalso]]
 * [[borg]]
-* [[xmpp]]
 
 The above special remotes are built into git-annex, and can be used
 to tie git-annex into many cloud services.
diff --git a/doc/special_remotes/comment_24_2c9eda62766c9d5000346a092fe5d0d8._comment b/doc/special_remotes/comment_24_2c9eda62766c9d5000346a092fe5d0d8._comment
index 905184f958..2d5f5b0b15 100644
--- a/doc/special_remotes/comment_24_2c9eda62766c9d5000346a092fe5d0d8._comment
+++ b/doc/special_remotes/comment_24_2c9eda62766c9d5000346a092fe5d0d8._comment
@@ -9,7 +9,7 @@ sufficient to back that up. You can run `git annex enableremote`
 in an clone to enable an existing special remote.
 
 The only catch is that, if you have chosen to initremote a special remote
-using a gpg key, with `encryption=shared keyid=whatever`, you'll of course
+using a gpg key, with `keyid=whatever`, you'll of course
 also need that gpg key to to use it. If you run `git annex info $myremote`
 it will tell you amoung other things, any gpg keys that are used by that
 remote.
diff --git a/doc/special_remotes/hook.mdwn b/doc/special_remotes/hook.mdwn
index 0bb76d98a0..1878aba704 100644
--- a/doc/special_remotes/hook.mdwn
+++ b/doc/special_remotes/hook.mdwn
@@ -73,17 +73,18 @@ The settings to use in git config for the hook commands are as follows:
 
 ## combined hook program
 
-This interface is deprecated -- it's better, and not much harder,
-to write an [[external_special_remote|external]]!
-
 Rather than setting all of the above hooks, you can write a single
 program that handles everything, and set a single hook to make it be used.
 
 	# git config annex.demo-hook /usr/local/bin/annexdemo
 	# git annex initremote mydemorepo type=hook hooktype=demo encryption=none
 
-The program just needs to look at the `ANNEX_ACTION` environment variable
-to see what it's being asked to do. For example:
+But, doing that is deprecated -- it's better, and not much harder,
+to write an [[external_special_remote|external]]!
+
+If you still want to do this, the program just needs to look at the
+`ANNEX_ACTION` environment variable to see what it's being asked to do.
+For example:
 
 [[!format sh """
 #!/bin/sh
diff --git a/doc/special_remotes/rsync.mdwn b/doc/special_remotes/rsync.mdwn
index e9f54dc684..04af848543 100644
--- a/doc/special_remotes/rsync.mdwn
+++ b/doc/special_remotes/rsync.mdwn
@@ -2,12 +2,12 @@ This special remote type rsyncs file contents to somewhere else.
 
 Setup example:
 
-	# git annex initremote myrsync type=rsync rsyncurl=rsync://rsync.example.com/myrsync keyid=id@joeyh.name encryption=shared
+	# git annex initremote myrsync type=rsync rsyncurl=rsync://rsync.example.com/myrsync encryption=shared
 	# git annex describe myrsync "rsync server"
 
 Or for using rsync over SSH
 
-	# git annex initremote myrsync type=rsync rsyncurl=ssh.example.com:/myrsync keyid=id@joeyh.name encryption=shared
+	# git annex initremote myrsync type=rsync rsyncurl=ssh.example.com:/myrsync encryption=shared
 	# git annex describe myrsync "rsync server"
 
 ## configuration
@@ -58,4 +58,4 @@ To pass parameters to ssh/rsh, include the parameters after "rsh" or
 "ssh".  For example, to configure ssh to use the private key at
 `/path/to/private/key`:
 
-	git config renote.name.annex-rsync-transport "ssh -i /path/to/private/key"
+	git config remote.name.annex-rsync-transport "ssh -i /path/to/private/key"
diff --git a/doc/submodules/comment_6_d1877da19346bbd6067ff95252974a9b._comment b/doc/submodules/comment_6_d1877da19346bbd6067ff95252974a9b._comment
new file mode 100644
index 0000000000..6a14a15ffe
--- /dev/null
+++ b/doc/submodules/comment_6_d1877da19346bbd6067ff95252974a9b._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="TTTTAAAx"
+ avatar="http://cdn.libravatar.org/avatar/9edd4b69b9f9fc9b8c1cb8ecd03902d5"
+ subject="comment 6"
+ date="2024-01-11T13:58:27Z"
+ content="""
+Since git 1.8.5, `git mv projects/2023/prj_1 archives/2023/prj_1` can update local path of submodule.
+Currently, `git-annex` doesn't detect submodule path changed and Just moving parent directories breaks git-modules thoroughly.
+The only way I found is to move all submodule to another tree structure one by one using `git mv ...`.
+
+If the parent directory name(e.g: projects->01_Projects) or its depth was changed by chance, all submodules inside the directory are broken.
+
+So I cannot use submodules to handle source code in my git-annex repo.
+Is there an easy (cool) way for this?
+"""]]
diff --git a/doc/submodules/comment_7_899cf2aa74cbb8160b1e249b314e7d60._comment b/doc/submodules/comment_7_899cf2aa74cbb8160b1e249b314e7d60._comment
new file mode 100644
index 0000000000..0ea3c9f4df
--- /dev/null
+++ b/doc/submodules/comment_7_899cf2aa74cbb8160b1e249b314e7d60._comment
@@ -0,0 +1,8 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="Try datalad"
+ date="2024-01-12T09:23:16Z"
+ content="""
+Sounds like you might want to use [datalad](https://datalad.org), which is built around git annex and where submodules are a first-class citizen.
+"""]]
diff --git a/doc/submodules/comment_8_8755b0d18c1b013ed0dc121ecfb83360._comment b/doc/submodules/comment_8_8755b0d18c1b013ed0dc121ecfb83360._comment
new file mode 100644
index 0000000000..8eb3679826
--- /dev/null
+++ b/doc/submodules/comment_8_8755b0d18c1b013ed0dc121ecfb83360._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="TTTTAAAx"
+ avatar="http://cdn.libravatar.org/avatar/9edd4b69b9f9fc9b8c1cb8ecd03902d5"
+ subject="comment 8"
+ date="2024-01-15T01:43:28Z"
+ content="""
+> Sounds like you might want to use datalad, which is built around git annex and where submodules are a first-class citizen. 
+
+Thank you for information. I'll try datalad. I considered it for audio and transcription dataset management few years ago. But I didn't use it because datalad using git-annex in it and I already used git-annex for data management. It was seemd to be redundant to me.
+
+I think submodule path detecting can be implemented still and be useful in `git-annex-assist` also.
+"""]]
diff --git a/doc/submodules/comment_9_5e3a7d5c81fb37efbfe22cbbe8b8a247._comment b/doc/submodules/comment_9_5e3a7d5c81fb37efbfe22cbbe8b8a247._comment
new file mode 100644
index 0000000000..7cae80016e
--- /dev/null
+++ b/doc/submodules/comment_9_5e3a7d5c81fb37efbfe22cbbe8b8a247._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2024-01-18T16:14:11Z"
+ content="""
+Datalad relies on git-annex's handling of submodules to work afaik so I
+don't see why using it would avoid this problem.
+
+If I understand correctly, the .git symlink in the submodule that git-annex
+sets up gets broken when the submodule is relocated. But I would have
+expected that moving from `a/b/` to `x/y/` would keep the symlink working,
+since it's at the same path depth.
+
+I'd need more information to fix this problem, so I recommend filing a
+[[bugs|bug]] report with details.
+"""]]
diff --git a/doc/thanks.mdwn b/doc/thanks.mdwn
index 3c1a2fdab1..6ab5cf9fff 100644
--- a/doc/thanks.mdwn
+++ b/doc/thanks.mdwn
@@ -35,7 +35,6 @@ contributed good bug reports and great ideas.
 
 <img alt="McGill logo" src="https://mcgill.ca/hbhl/sites/all/themes/moriarty/images/logo-red.svg" width=100>
 <img alt="Neurohub" src="https://joeyh.name/blog/pics/neurohub.png" width=100>
-<img alt="NSF logo" src="https://www.nsf.gov/images/logos/nsf1.gif">
 <img alt="Powderhouse logo" src="https://d33wubrfki0l68.cloudfront.net/e9285c9a8db37874efdadd61f2231774ce1d86cb/5de87/assets/phs-leftaligned-logo.svg" width=100>
 
 git-annex development was supported in large part by:
@@ -61,8 +60,6 @@ Thanks also to these folks for their support:
 
 ## financial support, 2014-2018
 
-<img alt="NSF logo" src="https://www.nsf.gov/images/logos/nsf1.gif">
-
 git-annex development was partially supported by the
 [Datalad project](http://datalad.org/), funded by a
 [NSF grant](https://www.nsf.gov/awardsearch/showAward?AWD_ID=1429999),
diff --git a/doc/thanks/list b/doc/thanks/list
index 77124cb6a8..a549f5fc64 100644
--- a/doc/thanks/list
+++ b/doc/thanks/list
@@ -139,3 +139,7 @@ Leon Schuermann,
 KDM, 
 vemek, 
 Luke T. Shumaker, 
+Nathaniel B, 
+Jaime Marquínez Ferrándiz, 
+Lukas Waymann, 
+kk, 
diff --git a/doc/tips/cloning_a_repository_privately/comment_1_f5490d034074ca80a712bdd41c307139._comment b/doc/tips/cloning_a_repository_privately/comment_1_f5490d034074ca80a712bdd41c307139._comment
new file mode 100644
index 0000000000..fb46c94c95
--- /dev/null
+++ b/doc/tips/cloning_a_repository_privately/comment_1_f5490d034074ca80a712bdd41c307139._comment
@@ -0,0 +1,39 @@
+[[!comment format=mdwn
+ username="mih"
+ avatar="http://cdn.libravatar.org/avatar/f881df265a423e4f24eff27c623148fd"
+ subject="What about temporary annex.private declaration?"
+ date="2023-11-07T15:49:47Z"
+ content="""
+The instructions indicate that `annex.private` should be set in the local repository configuration.
+
+However, the following approach is also a possibility:
+
+```
+❯ mkdir priv
+❯ cd priv
+❯ git init
+Initialized empty Git repository in /tmp/priv/.git/
+
+❯ git -c annex.private=1 annex init
+init  ok
+
+❯ ls .git/annex/journal-private
+uuid.log
+
+❯ cat .git/config
+[core]
+        repositoryformatversion = 0
+        filemode = true
+        bare = false
+        logallrefupdates = true
+[annex]
+        uuid = 955373ac-6044-493e-a696-1a706437b542
+        version = 10
+[filter \"annex\"]
+        smudge = git-annex smudge -- %f
+        clean = git-annex smudge --clean -- %f
+        process = git-annex filter-process
+```
+
+It seems this repository was in private mode when it was initialized (expected). What is the implication of the switch not being permanent in the config? And by extension: what are the implications of removing the switch later in the lifetime of a repository clone?
+"""]]
diff --git a/doc/tips/cloning_a_repository_privately/comment_2_0fb78b2183932da08809d60dfc5a7374._comment b/doc/tips/cloning_a_repository_privately/comment_2_0fb78b2183932da08809d60dfc5a7374._comment
new file mode 100644
index 0000000000..7ba12def9b
--- /dev/null
+++ b/doc/tips/cloning_a_repository_privately/comment_2_0fb78b2183932da08809d60dfc5a7374._comment
@@ -0,0 +1,37 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""Re: What about temporary annex.private declaration?"""
+ date="2023-11-29T16:50:05Z"
+ content="""
+I'm sure that the private information will not leak out from
+`.git/annex/journal-private/` into the git-annex branch
+after annex.private is unset. The design ensures this because, when
+making a change to the branch, it only reads the private journal file
+when the repository whose information is being changed is private.
+
+However, when git-annex does not have any private repositories configured,
+an optimisation makes it skip trying to read from the private journal. So
+information about those repositories, that were private, will no longer be
+read.
+
+This effect is easy to see, for example:
+
+	joey@darkstar:~/tmp/xxx>git-annex whereis
+	whereis foo (1 copy)
+	  	ff1f0bbd-7be6-45ff-8c90-fd322820b717 -- joey@darkstar:~/tmp/xxx [here]
+	ok
+	joey@darkstar:~/tmp/xxx>git config annex.private false
+	joey@darkstar:~/tmp/xxx>git-annex whereis
+	whereis foo (0 copies) failed
+	whereis: 1 failed
+
+I think this could be improved, eg it could check once if the private
+journal exists and if so read from it even when no private uuids are
+currently configured. A single stat to support this would be ok; the goal
+was to avoid checking nonexistany files on every branch read when private
+repositories are not used.
+
+Configuring any remote with annex-private can be used to work around that
+problem, that lets it read information about all previously-private repositories
+as well.
+"""]]
diff --git a/doc/tips/largefiles/comment_16_4c6f52958246757dfbedd0b410d43ffe._comment b/doc/tips/largefiles/comment_16_4c6f52958246757dfbedd0b410d43ffe._comment
new file mode 100644
index 0000000000..7575720540
--- /dev/null
+++ b/doc/tips/largefiles/comment_16_4c6f52958246757dfbedd0b410d43ffe._comment
@@ -0,0 +1,11 @@
+[[!comment format=mdwn
+ username="imlew"
+ avatar="http://cdn.libravatar.org/avatar/23858c3eed3c3ea9e21522f4c999f1ed"
+ subject="comment 16"
+ date="2023-12-12T13:32:17Z"
+ content="""
+I thought I had the same problem as lh (`git add` not respecting the largefiles config), but when I tried to make a minimal example I noticed that `git add` does add files to the annex, it just doesn't print the progress message that `git annex add` usually prints.
+
+Is there any way to get it to do that?
+It would help newbs like me know that largefiles is indeed working and for files that are actually large it can be helpful to see the progress.
+"""]]
diff --git a/doc/tips/largefiles/comment_17_066fdebc2655dd11077fda1ef6e102e4._comment b/doc/tips/largefiles/comment_17_066fdebc2655dd11077fda1ef6e102e4._comment
new file mode 100644
index 0000000000..7f862aa4b7
--- /dev/null
+++ b/doc/tips/largefiles/comment_17_066fdebc2655dd11077fda1ef6e102e4._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 17"""
+ date="2023-12-18T17:52:30Z"
+ content="""
+@imlew `git add` runs git-annex as a filter, so it cannot output progress
+displays or other messages on stdout at that point. It would have to output
+to stderr, but outputting non-errors to stderr is not great either.
+"""]]
diff --git a/doc/tips/migrating_data_to_a_new_backend.mdwn b/doc/tips/migrating_data_to_a_new_backend.mdwn
index b9acb8bd15..d81a42e0f9 100644
--- a/doc/tips/migrating_data_to_a_new_backend.mdwn
+++ b/doc/tips/migrating_data_to_a_new_backend.mdwn
@@ -1,14 +1,29 @@
-Maybe you started out using the WORM backend, and have now configured
-git-annex to use SHA1. But files you added to the annex before still
-use the WORM backend. There is a simple command that can migrate that
+Maybe you started out using the SHA1 backend, and have now configured
+git-annex to use SHA256. But files you added to the annex before still
+use the SHA1 backend. There is a simple command that can migrate that
 data:
 
 	# git annex migrate my_cool_big_file
 	migrate my_cool_big_file (checksum...) ok
 
+This stages a change to the file, which you can `git commit` like any other
+change.
+
 You can only migrate files whose content is currently available. Other
 files will be skipped.
 
+## distributed migration
+
+When you pull changes into your repository that include migration of files,
+your repository then needs to be updated to follow the migration.
+
+    # git-annex migrate --update
+	migrate my_cool_big_file (checksum...) ok
+
+This is done automatically by commands like `git-annex pull`.
+
+## unused old content
+
 After migrating a file to a new backend, the old content in the old backend
 will still be present. That is necessary because multiple files
 can point to the same content. The `git annex unused` subcommand can be
diff --git a/doc/tips/unlocked_files.mdwn b/doc/tips/unlocked_files.mdwn
index e530f4edf3..4f6d5901c6 100644
--- a/doc/tips/unlocked_files.mdwn
+++ b/doc/tips/unlocked_files.mdwn
@@ -50,7 +50,7 @@ an unlocked file, it will be replaced by a pointer file, which
 looks like "/annex/objects/...". So if you open a file and see
 that, you'll need to use `git annex get`.
 
-Under the hood, unlocked files use git's [[todo/smudge]] filter interface,
+Under the hood, unlocked files use git's smudge/clean filter interface,
 and git-annex converts between the content of the big file and a pointer
 file, which is what gets committed to git.
 
@@ -58,7 +58,8 @@ file, which is what gets committed to git.
 By default, git-annex commands will add files in locked mode,
 unless used on a filesystem that does not support symlinks, when unlocked
 mode is used. To make them always use unlocked mode, run:
-`git config annex.addunlocked true`
+`git config annex.addunlocked true`  
+`git add` always adds files in unlocked mode.
 """]]
 
 ## adjusted branches
diff --git a/doc/tips/using_Backblaze_B2/comment_1_b8b045d5f3d2bed1905c05b4fa00c5ca._comment b/doc/tips/using_Backblaze_B2/comment_1_b8b045d5f3d2bed1905c05b4fa00c5ca._comment
new file mode 100644
index 0000000000..48d60f7ffd
--- /dev/null
+++ b/doc/tips/using_Backblaze_B2/comment_1_b8b045d5f3d2bed1905c05b4fa00c5ca._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="emi@f69ad15b49bfecb4ea3836b6480e5ab8df9a5f7d"
+ nickname="emi"
+ avatar="http://cdn.libravatar.org/avatar/84bf8bb9fcad258f264f28740d91981e"
+ subject="Rclone B2 vs S3"
+ date="2023-10-15T15:03:18Z"
+ content="""
+ I've been using the rclone B2 special remote for a while now, and it seems to produce a very large number of Class C (the most expensive class) transactions while performing certain operations (uploads & dropunused, at least). I just started using the S3 special remote today, and while I've only uploaded so far, it seems to produce far fewer Class C transactions (after uploading a pretty large number of files, I'm only seeing 13). If cost is a concern, that might be one reason to prefer the S3 special remote. That said, this is mostly anecdotal. Perhaps someone else might chip in empirical data or check out the code to back up these observations. 
+"""]]
diff --git a/doc/tips/using_Backblaze_B2/comment_2_d7fd727cb38a0290de10f06539a23d27._comment b/doc/tips/using_Backblaze_B2/comment_2_d7fd727cb38a0290de10f06539a23d27._comment
new file mode 100644
index 0000000000..2cc81b7689
--- /dev/null
+++ b/doc/tips/using_Backblaze_B2/comment_2_d7fd727cb38a0290de10f06539a23d27._comment
@@ -0,0 +1,12 @@
+[[!comment format=mdwn
+ username="aaa"
+ avatar="http://cdn.libravatar.org/avatar/df34e736d4a63a5c29e8bd513ed719ca"
+ subject="Key permissions"
+ date="2023-12-12T22:29:24Z"
+ content="""
+I tried to use Backblaze B2 using the second method, and tried different application key permissions. Hopefully I can save your time if you’re going to use this special remote.
+
+In order to init a B2 remote, you should create a key with the ability to read & write all buckets, and you must create a new bucket using git-annex (by setting a unique name in `bucket=`). If you create an empty bucket using Backblaze's web UI, then use that bucket for `git annex initremote`, you will receive this error message: `(InternalException (HandshakeFailed (Error_Protocol (\"expecting server hello, got alert : [(AlertLevel_Fatal,IllegalParameter)]\",True,HandshakeFailure))))`.
+
+In order to enable a B2 remote, you need a key with read & write permission to the bucket you're using. If you created a key with only read permission, you can’t use `git annex get` on that remote.
+"""]]
diff --git a/doc/tips/using_nested_git_repositories/comment_3_029571d8331ba2dcf0b149d071fef75c._comment b/doc/tips/using_nested_git_repositories/comment_3_029571d8331ba2dcf0b149d071fef75c._comment
new file mode 100644
index 0000000000..669a1979bd
--- /dev/null
+++ b/doc/tips/using_nested_git_repositories/comment_3_029571d8331ba2dcf0b149d071fef75c._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="branch"
+ subject="comment 3"
+ date="2023-10-05T21:40:54Z"
+ content="""
+On a similar topic, I also have multiple git repositories that I want to backup (multiple copies...). These repositories belong to a parent repository that is properly set up with git-annex, and the necessary remotes. I want to be able to recover the entire state of the parent folder (including these children repositories) at any given time.
+
+I am quite unfamiliar with submodules, so feel free to correct me, but based on my experiments, using them makes each child repository/submodule independent/invisible to the parent one. If all these child repositories were submodules, I wouldn't be able to use the parent config to back them up, and I would have to repeat the same configuration on each submodule.
+
+If I were to leave the repositories as they are, the enclosing files seem to be annexed by the parent repository as I would want them up, but the .git repository is ignored. To achieve my goal, I can imagine one solution where every child .git folder would be zipped and annexed alongside, maybe on a pre-commit hook, to be restored in certain occasions.
+
+Is my understanding of the issue reasonable? Is there any other option?
+"""]]
diff --git a/doc/tips/using_nested_git_repositories/comment_4_3c701a4c6789fc6c73f71217e7ed8c65._comment b/doc/tips/using_nested_git_repositories/comment_4_3c701a4c6789fc6c73f71217e7ed8c65._comment
new file mode 100644
index 0000000000..abf5049da8
--- /dev/null
+++ b/doc/tips/using_nested_git_repositories/comment_4_3c701a4c6789fc6c73f71217e7ed8c65._comment
@@ -0,0 +1,14 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="comment 4"
+ date="2023-10-06T03:22:12Z"
+ content="""
+Submodules can feel a bit clunky, that's right. They're 'invisible to the parent repo' in that they indeed have separate configs (remotes, etc.), so one needs to manually set it up again when replicating.
+
+DataLad embraces this and provides e.g. `datalad save`, which commits recursively into all submodules and it will also initialize and handle submodules a bit more automatic. But it lacks the fully bidirectional `git annex sync|assist`.
+
+I use submodules extensively and am not entirely happy with it due to the fragile manual config necessary.
+
+Auto-enabled special remotes can help out a bit (those will be configured upon first submodule creation by git annex). 
+"""]]
diff --git a/doc/tips/what_to_do_when_a_repository_is_corrupted/comment_5_9b3738aa678015a58f30a22baa2012df._comment b/doc/tips/what_to_do_when_a_repository_is_corrupted/comment_5_9b3738aa678015a58f30a22baa2012df._comment
new file mode 100644
index 0000000000..b951df26c3
--- /dev/null
+++ b/doc/tips/what_to_do_when_a_repository_is_corrupted/comment_5_9b3738aa678015a58f30a22baa2012df._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="sng@353ca358075d9aa328f60a5439a3cee10f8301fe"
+ nickname="sng"
+ avatar="http://cdn.libravatar.org/avatar/d64f4854965b2b1c3ecafee4b2a66fac"
+ subject="corrupted bare repo"
+ date="2023-11-29T15:34:30Z"
+ content="""
+I've tried following these steps with a bare repo that became corrupted somehow, but at the cloning another repo step I'm stuck... how do I clone a repo when the bare repo is the one corrupted? (if it matters, this is a very large repo of photo files)
+"""]]
diff --git a/doc/tips/what_to_do_when_a_repository_is_corrupted/comment_6_0e3224af10362a10aa1c8786423960a9._comment b/doc/tips/what_to_do_when_a_repository_is_corrupted/comment_6_0e3224af10362a10aa1c8786423960a9._comment
new file mode 100644
index 0000000000..d31da3d014
--- /dev/null
+++ b/doc/tips/what_to_do_when_a_repository_is_corrupted/comment_6_0e3224af10362a10aa1c8786423960a9._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 6"""
+ date="2023-11-29T16:44:35Z"
+ content="""
+@sng you can clone any git repository, it does not need to be a bare one.
+"""]]
diff --git a/doc/todo/Allow_remote_sync__42___and_ignore__42___in_git_annex_config.mdwn b/doc/todo/Allow_remote_sync__42___and_ignore__42___in_git_annex_config.mdwn
new file mode 100644
index 0000000000..506dfe1fba
--- /dev/null
+++ b/doc/todo/Allow_remote_sync__42___and_ignore__42___in_git_annex_config.mdwn
@@ -0,0 +1,12 @@
+I've been using git annex in a star topology (one NAS always available to all repositories) for the longest time but now want to interconnect the repositories on the "outside" aswell. A hurdle to that is that not all remotes will be online all the time (in fact some of them will rarely be online) but git annex expects them to be reachable via ssh at all times.  
+
+There is an escape hatch here in the form of the `remote.<remote>.annex-sync-command` and `remote.<remote>.annex-ignore-command` git config flags where you can script a liveness check.  
+However, in order to actually use that, you must set these options on each and every instance of the repository which is not feasible. Usually for cases like this, there is `git annex config` but it doesn't allow this setting.
+
+Could these become git annex config flags aswell?
+
+Alternatives:
+
+* A built-in liveness check with configurable optionality (though you'd probably want this setting on the git-annex branch aswell)
+
+Am I missing another way to do this without manually setting the commands perhaps?
diff --git a/doc/todo/Allow_remote_sync__42___and_ignore__42___in_git_annex_config/comment_1_7a55c03691a6b6738d1a0e2555cb4cea._comment b/doc/todo/Allow_remote_sync__42___and_ignore__42___in_git_annex_config/comment_1_7a55c03691a6b6738d1a0e2555cb4cea._comment
new file mode 100644
index 0000000000..9be581fbdb
--- /dev/null
+++ b/doc/todo/Allow_remote_sync__42___and_ignore__42___in_git_annex_config/comment_1_7a55c03691a6b6738d1a0e2555cb4cea._comment
@@ -0,0 +1,15 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-10-09T18:39:21Z"
+ content="""
+We can't put commands in `git-annex config` because cloning
+a git repository and using git(-annex) in it should not
+expose you to executing arbitrary code.
+
+It's not clear to me what kind of liveness check git-annex could do. A ssh
+host is not necessarily pingable.
+
+Configuring ssh to use ConnectTimeout=1 or something like that can help
+with long timeouts.
+"""]]
diff --git a/doc/todo/Filter_a_tree_with_pattern/comment_3_ca3058fe82eff771f3664ab2f8aa78ae._comment b/doc/todo/Filter_a_tree_with_pattern/comment_3_ca3058fe82eff771f3664ab2f8aa78ae._comment
new file mode 100644
index 0000000000..f76d855997
--- /dev/null
+++ b/doc/todo/Filter_a_tree_with_pattern/comment_3_ca3058fe82eff771f3664ab2f8aa78ae._comment
@@ -0,0 +1,9 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2024-01-25T17:11:33Z"
+ content="""
+Well, ajusted branches and views can be composed now.
+
+Leaving this open for the glob patterns.
+"""]]
diff --git a/doc/todo/Incremental_git_annex_sync_--content_--all/comment_4_c232e1e1cfcc47f70079f2d32c2b4633._comment b/doc/todo/Incremental_git_annex_sync_--content_--all/comment_4_c232e1e1cfcc47f70079f2d32c2b4633._comment
index 78f0c942a9..bd167f075e 100644
--- a/doc/todo/Incremental_git_annex_sync_--content_--all/comment_4_c232e1e1cfcc47f70079f2d32c2b4633._comment
+++ b/doc/todo/Incremental_git_annex_sync_--content_--all/comment_4_c232e1e1cfcc47f70079f2d32c2b4633._comment
@@ -6,6 +6,6 @@
 My recent optimisations of `git-annex sync` with importtree remotes uses a
 similar diffing approach.
 
-A transition is underway to making `--content` be enabled by default, and
-faster syncing with it would be a nice thing to do before then.
+`git-annex satisfy` syncs `--content` by default, so this optimisation would
+be especially nice to have for it.
 """]]
diff --git a/doc/todo/Incremental_git_annex_sync_--content_--all/comment_5_e81719f23565579674249db5d0a883da._comment b/doc/todo/Incremental_git_annex_sync_--content_--all/comment_5_e81719f23565579674249db5d0a883da._comment
new file mode 100644
index 0000000000..d68d10d4a7
--- /dev/null
+++ b/doc/todo/Incremental_git_annex_sync_--content_--all/comment_5_e81719f23565579674249db5d0a883da._comment
@@ -0,0 +1,27 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2023-10-24T17:26:53Z"
+ content="""
+To implement this optimisation for a non-all sync, when 
+the tree being synced has changed, it ought to diff from the old
+tree to the current tree, and sync those files. Preferred
+content can vary depending on filename, and diffing like that will avoid
+scanning every file in the whole tree.
+
+And when there are location log changes, it needs to also sync files in the
+tree that use keys whose location log changed, using the git-annex branch
+diff to find those keys. (And presumably then using the keys database to get
+back to the filenames.)
+
+So, implementing an optimisation like this for a non-all sync has two
+separate diffs which would have to be combined together somehow.
+
+Doing that in constant memory would be hard. It seems that a bloom filter
+cannot be used to check if a file was processed in the first diff and avoid
+processing it again in the second diff. Because a false positive would
+avoid processing a file whose location log did change. I think it would
+need to use an on-disk structure maybe (eg sqlite)?
+
+None of which should prevent implementing this nice optimisation for --all.
+"""]]
diff --git a/doc/todo/Keeping_a_repo__39__s_description_up_to_date.mdwn b/doc/todo/Keeping_a_repo__39__s_description_up_to_date.mdwn
new file mode 100644
index 0000000000..199e4a9cac
--- /dev/null
+++ b/doc/todo/Keeping_a_repo__39__s_description_up_to_date.mdwn
@@ -0,0 +1,14 @@
+Hi joey,
+
+I was recently moving many of my git annex repos around. This caused all of the (partly auto-generated) repo descriptions (e.g. `user@host:path`) to be outdated, making it more difficult to re-add them later from another host. Updating all of them manually was really error prone and tedious.
+
+What do you think about having git annex update the repo description "from time to time"? `git annex sync|assist|...` could check if the current repo description matches what the auto-generated string would be and update it accordingly.
+
+I see the following problems with this:
+
+- Adding this check to each and every git annex command is probably not a good idea. Maybe `git annex sync` and `git annex assist`. The overhead might be negligible though.
+- How to detect if the current description was really auto-generated and not user-specified? git annex could parse it with a regex (e.g. `(?P<user>[^@]+@(?P<host>[^:]+:(?P<path>.*)$`) and if that matches could assume it was auto-generated. Feels a little fragile though.
+
+Maybe the whole auto-updating idea is not ideal, but a new command like `git annex redescribe` or `git annex autodescribe` or `git annex describe --auto` could be introduced, so users can run it periodically or on-demand. Following the discussion on '`git annex sync` defaulting to syncing content', I have a feeling that people wouldn't like git annex messing with their repo descriptions 😉. Ideally, auto-describing would (optionally) also be able to update remotes' descriptions properly.
+
+All of this could also be done by a third-party program, but having this functionality in git annex itself would be handy.
diff --git a/doc/todo/Keeping_a_repo__39__s_description_up_to_date/comment_1_2fcce6a8fcfec28a6edeab40291deaba._comment b/doc/todo/Keeping_a_repo__39__s_description_up_to_date/comment_1_2fcce6a8fcfec28a6edeab40291deaba._comment
new file mode 100644
index 0000000000..263dcb789d
--- /dev/null
+++ b/doc/todo/Keeping_a_repo__39__s_description_up_to_date/comment_1_2fcce6a8fcfec28a6edeab40291deaba._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-10-25T17:21:16Z"
+ content="""
+I think a good thing to do if you plan to be moving a repo around is to
+describe it with something that does not depend on its current location.
+
+For example, suppose I am making a repo on a removable USB drive. I'm gonna
+literally move that from place to place by plugging it into different
+computers. So the default user@host:/mntpoint description is not a good one
+for that repository. Instead I use something like "2 tb passport USB drive", 
+or even better I slap a sticker on that drive and give it a real 
+name and use that as the description.
+
+I guess that someone who was moving a USB drive back and forth between 2
+computers would not be enthused if git-annex started updating the
+description after each move. Even if that prevented the 
+description from being wrong half the time.
+
+I do think that `git-annex describe --auto` is a reasonable idea.
+"""]]
diff --git a/doc/todo/Keeping_a_repo__39__s_description_up_to_date/comment_2_1c73d94cc84bbfdeb65a71995265491f._comment b/doc/todo/Keeping_a_repo__39__s_description_up_to_date/comment_2_1c73d94cc84bbfdeb65a71995265491f._comment
new file mode 100644
index 0000000000..3f75359f85
--- /dev/null
+++ b/doc/todo/Keeping_a_repo__39__s_description_up_to_date/comment_2_1c73d94cc84bbfdeb65a71995265491f._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2023-10-25T17:57:50Z"
+ content="""
+Note that `git-annex describe remote --auto` has a small problem 
+when the remote is a git ssh remote, that there
+may be multiple hostnames, the one that happens to be used locally
+might not be as fully qualified as the "right" one. Or it may even be a ssh
+host alias, which can't be converted to a FQDN.
+
+For special remotes, `git-annex initremote` does not set a description at
+all, and whatever hostname might be used for one is hidden underneath an
+abstraction layer anyway. So it couldn't do anything useful for those.
+
+So that limits it to local git repositories..
+"""]]
diff --git a/doc/todo/Keeping_a_repo__39__s_description_up_to_date/comment_3_394c87782dd3318a6fa3705322fea4fb._comment b/doc/todo/Keeping_a_repo__39__s_description_up_to_date/comment_3_394c87782dd3318a6fa3705322fea4fb._comment
new file mode 100644
index 0000000000..bad3c205dd
--- /dev/null
+++ b/doc/todo/Keeping_a_repo__39__s_description_up_to_date/comment_3_394c87782dd3318a6fa3705322fea4fb._comment
@@ -0,0 +1,16 @@
+[[!comment format=mdwn
+ username="nobodyinperson"
+ avatar="http://cdn.libravatar.org/avatar/736a41cd4988ede057bae805d000f4f5"
+ subject="comment 3"
+ date="2023-10-26T17:46:27Z"
+ content="""
+> I think a good thing to do if you plan to be moving a repo around is to describe it with something that does not depend on its current location.
+> 
+> For example, suppose I am making a repo on a removable USB drive. I'm gonna literally move that from place to place by plugging it into different computers. So the default user@host:/mntpoint description is not a good one for that repository. Instead I use something like \"2 tb passport USB drive\", or even better I slap a sticker on that drive and give it a real name and use that as the description.
+
+I do that too (adding unique information about the storage medium, e.g. the HDD's manufacturer, serial number and human-readable description), but still it is important (to me) to have a (be it partly) copy-pastable link/path that simplifies re-adding the remote later elsewhere. In my case, I moved three HDDs (some of them internal, some connected via USB) to a different host and had to change the mountpoints for consistency. On all of these I stored many (>10) git annex repos, so that makes >30 remote descriptions I had to update manually -- probably forgot a lot of those. 
+
+Automating this would have helped a lot. A quick'n'dirty way to get FQDN(s) is this `nslookup \"$(curl -s icanhazip.com)\" | perl -ne 'print if s|^.*name\s+=\s+(.*)\.$|$1|g'` (doesn't work for DynDNS though...).
+
+I'd agree with you if you said, something like this is too specific to have git annex do it ('it' being updating descriptions of *remotes*). But if `git annex describe here --auto` would check if `here`'s description looks auto-generated and if yes, update it with the default auto-generated one, that would already help. Maybe just `git annex describe --auto` would suffice, as the `here` is kind of redundant -- any other remote wouldn't work like this.
+"""]]
diff --git a/doc/todo/Make_webapp_port_configurable.mdwn b/doc/todo/Make_webapp_port_configurable.mdwn
index fa9ee10f0f..2c94690293 100644
--- a/doc/todo/Make_webapp_port_configurable.mdwn
+++ b/doc/todo/Make_webapp_port_configurable.mdwn
@@ -39,3 +39,4 @@ dependency versions: aws-0.20 bloomfilter-2.0.1.0 cryptonite-0.25 DAV-1.3.3 feed
 ### Have you had any luck using git-annex before? (Sometimes we get tired of reading bug reports all day and a lil' positive end note does wonders)
 git annex seems awesome with the little bit of testing I've done. It seems like the perfect tool for what I want to accomplish. Thanks!
 
+> [[done]] via --port option --[[Joey]]
diff --git a/doc/todo/RawFilePath_conversion.mdwn b/doc/todo/RawFilePath_conversion.mdwn
new file mode 100644
index 0000000000..b1e1d3e9cd
--- /dev/null
+++ b/doc/todo/RawFilePath_conversion.mdwn
@@ -0,0 +1,75 @@
+For a long time (since 2019) git-annex has been progressively converting from
+FilePath to RawFilePath (aka ByteString).
+
+The reason is mostly performance, also a simpler representation of
+filepaths that doesn't need encoding hacks to support non-UTF8 values.
+
+Some commands like `git-annex find` use RawFilePath end-to-end. 
+But this conversion is not yet complete. This is a todo to keep track of the
+status.
+
+* The Abstract FilePath proposal (AFPP) has been implemented, and so a number of
+  libraries like unix and directory now have versions that operate on
+  OSPath. That could be used in git-annex eg for things like
+  getDirectoryContents, when built against those versions.
+  (But OSPath uses ShortByteString, while RawFilePath is ByteString, so
+  conversion still entails a copy.)
+* withFile remains to be converted, and is used in several important code
+  paths, including Annex.Journal and Annex.Link. 
+  There is a RawFilePath version in file-io library, but that is
+  not currently a git-annex dependency. (withFile is in base, and base is
+  unlikely to convert to AFPP soon)
+
+[[!tag confirmed]]
+
+----
+
+The following patch can be useful to find points where conversions are
+done. Especially useful to identify cases where a value is converted
+`FilePath -> RawFilePath -> FilePath`.
+
+    diff --git a/Utility/FileSystemEncoding.hs b/Utility/FileSystemEncoding.hs
+    index 2a1dc81bc1..03e6986f6e 100644
+    --- a/Utility/FileSystemEncoding.hs
+    +++ b/Utility/FileSystemEncoding.hs
+    @@ -84,6 +84,9 @@ encodeBL = L.fromStrict . encodeBS
+     encodeBL = L8.fromString
+     #endif
+     
+    +debugConversions :: String -> IO ()
+    +debugConversions s = hPutStrLn stderr ("conversion: " ++ s)
+    +
+     decodeBS :: S.ByteString -> FilePath
+     #ifndef mingw32_HOST_OS
+     -- This does the same thing as System.FilePath.ByteString.decodeFilePath,
+    @@ -92,6 +95,7 @@ decodeBS :: S.ByteString -> FilePath
+     -- something other than a unix filepath.
+     {-# NOINLINE decodeBS #-}
+     decodeBS b = unsafePerformIO $ do
+    +	debugConversions (show b)
+     	enc <- Encoding.getFileSystemEncoding
+     	S.useAsCStringLen b (GHC.peekCStringLen enc)
+     #else
+    @@ -106,17 +110,19 @@ encodeBS :: FilePath -> S.ByteString
+     -- something other than a unix filepath.
+     {-# NOINLINE encodeBS #-}
+     encodeBS f = unsafePerformIO $ do
+    +	debugConversions f
+     	enc <- Encoding.getFileSystemEncoding
+    -	GHC.newCStringLen enc f >>= unsafePackMallocCStringLen
+    +	b <- GHC.newCStringLen enc f >>= unsafePackMallocCStringLen
+    +	return b
+     #else
+     encodeBS = S8.fromString
+     #endif
+     
+     fromRawFilePath :: RawFilePath -> FilePath
+    -fromRawFilePath = decodeFilePath
+    +fromRawFilePath = decodeBS -- decodeFilePath
+     
+     toRawFilePath :: FilePath -> RawFilePath
+    -toRawFilePath = encodeFilePath
+    +toRawFilePath = encodeBS -- encodeFilePath
+     
+     {- Truncates a FilePath to the given number of bytes (or less),
+      - as represented on disk.
diff --git a/doc/todo/Split_autocommit_option.mdwn b/doc/todo/Split_autocommit_option.mdwn
index 8e5ebf24f7..76b611f25a 100644
--- a/doc/todo/Split_autocommit_option.mdwn
+++ b/doc/todo/Split_autocommit_option.mdwn
@@ -15,3 +15,6 @@ Additionally, there should be two more options to control the other two. `annex.
 Making the new options namespaced would also allow the old option to retain its current function; enabling all three.
 
 I've got multiple use-cases for git-annex where I cannot use the assistant because these finer-grained options don't exist.
+
+> Closing because remote.name.annex-sync seems to be sufficient. [[done]]
+> --[[Joey]] 
diff --git a/doc/todo/allow_configuring_assistant_to_add_files_locked.mdwn b/doc/todo/allow_configuring_assistant_to_add_files_locked.mdwn
new file mode 100644
index 0000000000..00c67762ea
--- /dev/null
+++ b/doc/todo/allow_configuring_assistant_to_add_files_locked.mdwn
@@ -0,0 +1,12 @@
+The assistant adds files unlocked, even when git-annex is otherwise
+configured to add them locked.
+
+There are good reasons for that different behavior to be the default,
+but it would be worth having a config to override that.
+
+Eg, when annex.assistant.honor.addunlocked is set, honor the
+annex.addunlocked configuration, which defaults to adding files locked.
+Or perhaps a better name would be annex.assistant.allowaddlocked.
+
+See here for some motivating use cases
+<https://git-annex.branchable.com/forum/Is_there_a_way_to_have_assistant_add_files_locked__63__/>
diff --git a/doc/todo/alternate_keys_for_same_content/comment_10_22ff867952875856b20339a8829c5944._comment b/doc/todo/alternate_keys_for_same_content/comment_10_22ff867952875856b20339a8829c5944._comment
new file mode 100644
index 0000000000..47ce3cdfbb
--- /dev/null
+++ b/doc/todo/alternate_keys_for_same_content/comment_10_22ff867952875856b20339a8829c5944._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""Re: simpler proposal"""
+ date="2023-12-01T18:00:20Z"
+ content="""
+About the idea of recording a checksum of the content of a URL or WORM key,
+without migrating to a SHA key, that does seem worth considering. (And
+maybe was the original idea of this todo really..)
+
+If that were implemented, it would be necessary for more than one checksum
+to be able to be recorded for a given URL key. Because different
+clones might get different content from the URL and each add its checksum.
+
+So, this would not be as strong an assurance as using a SHA key that you're
+referring to a specific peice of data. It would be useful to protect
+against bit rot, but not as a way to pin a file to a particular version.
+Which is often something one does want to do in a git repository!
+
+I do think that implementing that would be a lot simpler. And it would
+only affect performance when verifying the content of URL or WORM keys,
+when it would need to look up the checksum in the git-annex branch.
+"""]]
diff --git a/doc/todo/alternate_keys_for_same_content/comment_11_3323eff3d94d366595bf2b7e78c01dce._comment b/doc/todo/alternate_keys_for_same_content/comment_11_3323eff3d94d366595bf2b7e78c01dce._comment
new file mode 100644
index 0000000000..154fa5a8b5
--- /dev/null
+++ b/doc/todo/alternate_keys_for_same_content/comment_11_3323eff3d94d366595bf2b7e78c01dce._comment
@@ -0,0 +1,7 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 11"""
+ date="2023-12-01T18:41:30Z"
+ content="""
+See [[distributed_migration]]...
+"""]]
diff --git a/doc/todo/alternate_keys_for_same_content/comment_8_4b16c48a2d9f4926d63f6ab54fe801d3._comment b/doc/todo/alternate_keys_for_same_content/comment_8_4b16c48a2d9f4926d63f6ab54fe801d3._comment
new file mode 100644
index 0000000000..cea7fe2855
--- /dev/null
+++ b/doc/todo/alternate_keys_for_same_content/comment_8_4b16c48a2d9f4926d63f6ab54fe801d3._comment
@@ -0,0 +1,19 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2023-11-30T20:43:53Z"
+ content="""
+I think Ilya Shlyakhter gets to a fundamental problem in his comment above.
+Any way that git-annex stores data about an alternate key that is recorded
+in git, allows anyone to spoof bad data.
+
+For example, if I have a SHA256 key stored in git-annex, it would be a bad
+security hole if I fetched from Ilya's repository and suddenly git-annex
+was willing to accept some MD5 key as being the same content as my SHA256
+key. Even if the two keys had the same content currently, that MD5 key can
+be collision attacked later.
+
+So there would need to be a direction in which key upgrades were allowed.
+Which is fine for `WORM -> SHA256`, but less clear for `SHA1 -> SHA256`
+and much less clear for other pairs of modern hashes.
+"""]]
diff --git a/doc/todo/alternate_keys_for_same_content/comment_9_42d240bbfc6ab858219ffa0f873c3eb4._comment b/doc/todo/alternate_keys_for_same_content/comment_9_42d240bbfc6ab858219ffa0f873c3eb4._comment
new file mode 100644
index 0000000000..5b185f2ef6
--- /dev/null
+++ b/doc/todo/alternate_keys_for_same_content/comment_9_42d240bbfc6ab858219ffa0f873c3eb4._comment
@@ -0,0 +1,50 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""git-annex branch size when storing migration information"""
+ date="2023-12-01T16:10:11Z"
+ content="""
+I did a small experiment to gauge how much the git repo size would grow if
+migration were recorded in log files in the git-annex branch.
+
+In my experiment, I started with 1000 files using sha256. The size of the
+git objects (after repack by git gc --aggressive) was 0.5 mb. I then
+migrated them to sha512, which increased the size of git objects to 1.1 mb
+(after repacking).
+
+Then I recorded in the git-annex branch additional log files for each of
+the sha512 keys that contained the corresponding sha256 key. That grew the
+git objects to 1.4 mb after repacking.
+
+This was a little disappointing. I'd hoped that repacking would avoid
+duplication of the sha256 keys, which are both in the log files I wrote
+and are used as filenames. But the data I wrote to the logs is only 75 kb
+total, and git grew 4x that.
+
+I tried the same thing except instead of separate log files I added to git
+one log file that contained pairs of sha256 and sha512 keys. That log file
+was 213 kb and adding it to the git repo grew it by 102 kb. So there was
+some compression there, but less than I would have hoped, and not much
+better than just gzip -9 of the log file (113 kb). Of course putting all
+the migration information in a single file like this would add a lot of  
+complexity to accessing it.
+
+So adding this information to the git-annex branch would involve at best
+around a 16% overhead, which is a surprising amount. 
+
+(It would be possible to make `git-annex forget --drop-dead` remove the
+information about old migrated keys if they later get marked as dead, and
+so regain the space.)
+
+This is also rather redundant information to store in git, since most
+of the time when file foo has been migrated, the old key can be determined
+by looking at `git log foo`. Not always of course because foo might have
+been renamed after migration, for example.
+
+Another way to store migration information in the git-annex branch would to
+be graft in the pre-migration tree and the post-migration tree. Diffing
+those two trees would show what migrated, and most of the time this would
+use almost no additional space in git, because the user will have committed
+both those trees anyway, or something very close to them. But it would be
+more expensive to extract the migration information then, and this would
+need a local cache of migrations to be built up from examining those diffs..
+"""]]
diff --git a/doc/todo/distributed_migration.mdwn b/doc/todo/distributed_migration.mdwn
new file mode 100644
index 0000000000..45a6d14398
--- /dev/null
+++ b/doc/todo/distributed_migration.mdwn
@@ -0,0 +1,71 @@
+Currently `git-annex migrate` only hard links the objects in the local
+repo. This leaves other clones without the new keys' objects unless
+they re-download them, or unless the same migrate command is
+re-run, in the same tree, on each clone.
+
+It would be good to support distributed migration, so that whatever
+migration is done in one repo is reflected in the other repos.
+
+This needs some way to store, in the git repo, a mapping between the old
+key and the new key it has been migrated to. (I investigated
+how much space that would need in the git repo, in 
+[this comment](https://git-annex.branchable.com/todo/alternate_keys_for_same_content/#comment-917eba0b2d1637236c5d900ecb5d8da0).)
+The mapping might be communicated via the git branch but be locally stored
+in a sqlite database to make querying it fast.
+
+Once that mapping is available, one simple way to use it would be a
+git-annex command that updates the local repo to reflect migrations that
+have happened elsewhere. It would not touch the HEAD branch, but would 
+just hard link object files from the old to new key, and update the location
+log for the new key to indicate the content is present in the repo.
+This command could be something like `git-annex migrate --update`.
+
+Rather than a dedicated command that users need to remember to run,
+distributed migration could be done automatically when merging a git-annex
+branch that adds migration information. Just hardlink object files and
+update the location log.
+
+It would be possible to avoid updating the location log, but then all
+location log queries would have to check the migration mapping. It would be
+hard to make that fast enough. Consider `git-annex find --in foo`, which
+queries the location log for each file.
+
+--[[Joey]]
+
+> [[done]] --[[Joey]]
+
+# security
+
+It is possible for bad migration information to be recorded in the
+git-annex branch by someone malicious. To avoid bad or insecure behavior
+when bad migration information is recorded:
+
+* When updating the local repository with a migration, verify that
+  the object file hashes to the new key before hardlinking.
+
+> This was done.
+
+That leaves at these possible security problems:
+
+* DOS by flooding the git-annex branch with migrations, resulting in 
+  lots of hard links (or copies on filesystems not supporting hard links)
+  and hashing of large files.
+
+Note that a malicious person who can write to the git-annex branch
+can already set their own repo as trusted, wait for someone
+to drop their local copy, and then demand a ransom for the content.
+For that matter, someone hosting a git-annex remote on a server can wait
+for someone to rely on it to contain the only copy of content and ransom
+it then. 
+
+git-annex is probably not normally used in situations where we
+need to worry about this kind of attack; if we don't trust someone we
+shouldn't pull the git-annex branch from them, and should not trust their
+remote to contain the only copy.
+
+If we pull a git-annex branch from someone, they can already DOS disk space
+and CPU by checking a lot of junk into git. So maybe a DOS by migration is
+not really a concern.
+
+> If people are worried about this kind of thing, they can avoid using the
+> feature. --[[Joey]]
diff --git a/doc/todo/distributed_migration/comment_1_8734d30aa0c1cb27dce81a0277d24948._comment b/doc/todo/distributed_migration/comment_1_8734d30aa0c1cb27dce81a0277d24948._comment
new file mode 100644
index 0000000000..180e937c2d
--- /dev/null
+++ b/doc/todo/distributed_migration/comment_1_8734d30aa0c1cb27dce81a0277d24948._comment
@@ -0,0 +1,10 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-12-01T19:00:41Z"
+ content="""
+Even if the stuff with special remotes turned out to be too complicated to
+implement, `git-annex migrate --update` would be useful for some users.
+So it's worth implementing the mapping and then incrementally implementing
+these ideas.
+"""]]
diff --git a/doc/todo/distributed_migration/comment_2_fdaef5d870221d44c57fc3bd1501e7ee._comment b/doc/todo/distributed_migration/comment_2_fdaef5d870221d44c57fc3bd1501e7ee._comment
new file mode 100644
index 0000000000..40276efe9d
--- /dev/null
+++ b/doc/todo/distributed_migration/comment_2_fdaef5d870221d44c57fc3bd1501e7ee._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 2"""
+ date="2023-12-06T19:31:42Z"
+ content="""
+On the `distributedmigration` branch I have `git-annex migrate` recording
+migrations on the git-annex branch.
+
+Its method of grafting in 2 trees, one with the old keys and one with the
+new is quite efficient. In a migration of 1000 files from SHA256E to SHA1,
+the git objects only needs 52kb to record the migration trees. 
+Compared with 424 kb needed to update the location logs. 
+The total git repo grew from 508kb to 984k.
+
+Next up: Make `git-annex migrate --update` find new migrations started
+elsewhere and apply them to the local annex objects.
+"""]]
diff --git a/doc/todo/distributed_migration_for_special_remotes.mdwn b/doc/todo/distributed_migration_for_special_remotes.mdwn
new file mode 100644
index 0000000000..5794aed185
--- /dev/null
+++ b/doc/todo/distributed_migration_for_special_remotes.mdwn
@@ -0,0 +1,41 @@
+[[distributed_migration]] is implemented for local repositories via
+`git-annex migrate --update`.
+
+That leaves updating special remotes after a migration as the main pain
+point in doing migrations.
+
+One approach would be a command like `git-annex migrate
+--update-remote=foo` that uploads new keys and drops old keys.
+But that would double the data stored in the special remote and use a lot
+of bandwidth.
+
+Alternatively, the old key could be left on a special remote, but update
+the location log for the special remote to say it has the new key,
+and have git-annex request the old key when it wants to get (or checkpresent)
+the new key from the special remote.
+This would need the mapping to be cheap enough to query that it won't
+signficantly slow down accessing a special remote.
+
+Dropping the new key from the special remote would then need to drop the
+old key. But that could violate numcopies for the old key. Perhaps it could
+check numcopies for the old key and drop it, otherwise leave the old key on
+the special remote.
+
+--[[Joey]]
+
+# security
+
+
+When downloading content from a special remote by getting the old
+pre-migration key it's important to verify that download hashes to the new key.
+See [[distributed_migration]]'s security section for relevant background.
+
+Another problem to consider: checkpresent against the special remote has to
+trust that the content stored on it for the old key will hash to the new
+key. This could result in data loss when a bad migration is provided, and
+the special remote is trusted.
+
+Eg, if key A is locally present, and B is present on the special
+remote, and then wrong migration is recorded from B to A, 
+the special remote will be treated as containing a copy of A,
+allowing dropping the local copy of A, which was the only copy.
diff --git a/doc/todo/git-annex-unused_--history.mdwn b/doc/todo/git-annex-unused_--history.mdwn
index f30aa0de8f..515754d9c8 100644
--- a/doc/todo/git-annex-unused_--history.mdwn
+++ b/doc/todo/git-annex-unused_--history.mdwn
@@ -11,3 +11,5 @@ To traverse the whole history and get the list of changes, could use `git
 log --raw`. That is reasonably performant but then it needs to feed every
 sha into cat-file to find the annex objects, which would probably take a
 while. Alternatively perhaps git log --diff and parsing might be faster.
+
+[[!tag confirmed]]
diff --git a/doc/todo/import_symlinks_when_importing_from_directory.mdwn b/doc/todo/import_symlinks_when_importing_from_directory.mdwn
index 08b2c28593..f01a47bf96 100644
--- a/doc/todo/import_symlinks_when_importing_from_directory.mdwn
+++ b/doc/todo/import_symlinks_when_importing_from_directory.mdwn
@@ -10,3 +10,6 @@ Absolute symlinks pointing into the tree being imported should be turned into re
 
 Absolute symlinks pointing outside the tree are a harder case.  I would still suggest importing these as git symlinks, pointing to the same absolute target path.
 As long as the repo is checked out into a filesystem where the same absolute paths are mounted, the links will work.  That's a common scenario when all repo users use the same compute cluster with a shared filesystem.
+
+> I've decided to reject this idea, although I'm still open to discussion
+> about use cases for it. [[done]] --[[Joey]]
diff --git a/doc/todo/import_tree_preferred_content_expansions.mdwn b/doc/todo/import_tree_preferred_content_expansions.mdwn
new file mode 100644
index 0000000000..4ff04ffc82
--- /dev/null
+++ b/doc/todo/import_tree_preferred_content_expansions.mdwn
@@ -0,0 +1,17 @@
+Importing a tree from a special remote
+when it has a preferred content configured currently only works when the
+expression does not use any terms that operate on keys.
+(Eg things like copies=).
+
+[[!commit e06feb7316af35b1277b7159a74c216c8f2e7422]]
+implemented that. Could it be extended to also support
+using it with an expression that does operate on keys?
+
+This would probably entail making any term of the expression that needs a
+key evaluate to true. Then it would import all files
+(that match also terms that don't need a key). After the import, it may
+turn out that the special remote doesn't want to contain particular content
+that was imported from it, and it would make sense that an export to the
+special remote would remove those files. --[[Joey]]
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/import_tree_preferred_content_expansions/comment_1_df8ca8665e1dfc530a832b6d24d60ea4._comment b/doc/todo/import_tree_preferred_content_expansions/comment_1_df8ca8665e1dfc530a832b6d24d60ea4._comment
new file mode 100644
index 0000000000..7e9b4507a4
--- /dev/null
+++ b/doc/todo/import_tree_preferred_content_expansions/comment_1_df8ca8665e1dfc530a832b6d24d60ea4._comment
@@ -0,0 +1,29 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 1"""
+ date="2023-12-18T18:02:40Z"
+ content="""
+Hmm, consider for example a camera. If the user wants to import all jpeg
+files from it, and not export to it jpeg files that have a copy in an
+archive, they might use:
+
+	not copies=archive:1 and include=*.jpeg
+
+But on import, if "copies=archive:1" were made to evaluate to true
+as suggested here, then this expression would not match, and so 
+nothing would be imported.
+
+Seems that the approach needs to be instead to traverse the expression
+and prune terms that operate on keys. So convert the example
+above to "include=*.jpeg".
+
+How to prune in some other cases:
+
+	not (copies=archive:1 and copies=backup:1) => anything
+
+	not (copies=archive:1 and include=*.jpeg) => not (include=*.jpeg)
+
+	not (copies=archive:1 or include=*.jpeg) => not (include=*.jpeg)
+	
+	not ((not copies=archive:1) or include=*.jpeg) => not (include=*.jpeg)
+"""]]
diff --git a/doc/todo/importfeed_needs_more_memory_the_more_urls_there_are.mdwn b/doc/todo/importfeed_needs_more_memory_the_more_urls_there_are.mdwn
index 2515e755c9..eaca6becb1 100644
--- a/doc/todo/importfeed_needs_more_memory_the_more_urls_there_are.mdwn
+++ b/doc/todo/importfeed_needs_more_memory_the_more_urls_there_are.mdwn
@@ -15,3 +15,5 @@ branch is changed. --[[Joey]]
 > significantly slow in large repos. So I think worth doing.
 
 [[!tag confirmed]]
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/improve_memory_usage_of_--all.mdwn b/doc/todo/improve_memory_usage_of_--all.mdwn
index 3f940434f3..13dbe6250f 100644
--- a/doc/todo/improve_memory_usage_of_--all.mdwn
+++ b/doc/todo/improve_memory_usage_of_--all.mdwn
@@ -1,16 +1,23 @@
-Using --all, or running in a bare repo, as well as
-`git annex unused` and `git annex info` all end up buffering the list of
-all keys that have uncommitted journalled changes in memory.
-This is due to Annex.Branch.files's call to getJournalledFilesStale which
-reads all the files in the directory into a buffer.
+`git annex unused --from=$remote` and `git annex info $remote`
+buffer the list of keys that have uncommitted journalled changes
+in memory. This is due to Annex.Branch.files's which reads all the
+files in the journal into a buffer.
 
 Note that the list of keys in the branch *does* stream in, so this
 is only really a problem when using annex.alwayscommit=false to build
-up big git-annex branch commits via the journal.
+up big git-annex branch commits via the journal. Or using annex.private,
+since the private journal can build up a lot of keys in it.
 
 An attempt at making it stream via unsafeInterleaveIO failed miserably
 and that is not the right approach. This would be a good place to use
 ResourceT, but it might need some changes to the Annex monad to allow
 combining the two. --[[Joey]]
 
+> This used to also affect --all and using git-annex in a bare repo, but
+> that was avoided by using the overBranchFileContents interface. This
+> suggests that changing to that interface in unused and info would be a
+> solution.
+
 [[!tag confirmed]]
+
+[[!meta title="improve memory usage of unused and info when the journal contains a lot of files"]]
diff --git a/doc/todo/info_--size-history.mdwn b/doc/todo/info_--size-history.mdwn
new file mode 100644
index 0000000000..15c464ace8
--- /dev/null
+++ b/doc/todo/info_--size-history.mdwn
@@ -0,0 +1,62 @@
+Support eg `git-annex info --size-history=30d` which would display
+the combined size of all repositories every 30 days throughout the history
+of the git-annex branch. This would allow graphing, analysis, etc of repo
+growth patterns.
+
+Also, `git-annex info somerepo --size-history=30d` would display the size
+of only the selected repository.
+
+Maybe also a way to get the size of each repository plus total size in a
+single line of output?
+
+----
+
+Implementation of this is fairly subtle. My abandoned first try just went
+through `git log` and updated counters as the location logs were updated.
+That resulted in bad numbers. (The size went negative eventually in fact!)
+The problem is that the git-annex branch is often updated both locally and
+on a remote, eg when copying a file to a remote. And that results in 2
+changes to the git-annex branch that both record the same data. So it gets
+counted twice by my naive implementation.
+
+I think it is not possible for an accumulation based approach to work in
+constant memory and fast. In the worst case, there is a fork of the branch
+that diverges hugely over a long period of time. So that divergence either
+needs to be buffered in memory, or recalculated repeatedly.
+
+What I think needs to be done is use `git log --reverse --date-order git-annex`.
+Feed the changed annex log file refs into catObjectStream to get the log
+files. (Or use --patch and parse the diff to extract log file lines, 
+might be faster?) Parse the log files, and update a simple data structure:
+
+    Map Key [UUIDPointer]
+
+Where UUIDPointer is a number that points to the UUID in a Map. This
+avoids storing copies of the uuids in the map.
+
+This is essentially union merging all forks of the git-annex branch at
+each commit, but far faster and in memory. Since union merging a git-annex
+branch can be done at any point and always results in a consistent view of
+the data, this will be consistent as well.
+
+And when updating the data structure, then it can update a counter when
+something changed, and avoid updating it when a redundant log was logged.
+
+This approach will use an amount of memory that scales with
+the number of keys and numbers of copies. I mocked it up using my big
+repository. Storing every key in it in such a map, with 64 UUIDPointers
+in the list (many more than the usual number of copies) took 2 gb of
+memory. Which is a lot but also most users have that much if necessary.
+With a more usual 5 copies, memory use was only 0.5 gb. So I think this is
+an acceptable exception to git-annex's desire to use a constant amount of
+memory.
+
+(I considered a bloom filter, but a false positive would wreck the
+statistics. An in-memory sqlite db might be more efficient, but probably
+not worth the bother.)
+
+[[!tag confirmed]]
+
+--[[Joey]] 
+
+> [[done]]
diff --git a/doc/todo/info__58___allow_file_matching_options_for_all_keys/comment_2_2d0a6abd5be8afce677eeea575aae548._comment b/doc/todo/info__58___allow_file_matching_options_for_all_keys/comment_2_2d0a6abd5be8afce677eeea575aae548._comment
new file mode 100644
index 0000000000..5b91f004fa
--- /dev/null
+++ b/doc/todo/info__58___allow_file_matching_options_for_all_keys/comment_2_2d0a6abd5be8afce677eeea575aae548._comment
@@ -0,0 +1,25 @@
+[[!comment format=mdwn
+ username="Atemu"
+ avatar="http://cdn.libravatar.org/avatar/86b8c2d893dfdf2146e1bbb8ac4165fb"
+ subject="comment 2"
+ date="2023-12-25T13:37:58Z"
+ content="""
+Sorry for the late reply but this somehow didn't reach me.
+
+I just tried querying some basic stats such as `--not --copies 2` and running that with `git annex info .` works as expected:
+
+```
+directory: .
+local annex keys: 0
+local annex size: 0 bytes
+annexed files in working tree: 0
+size of annexed files in working tree: 0 bytes
+```
+
+
+`git annex info --not --copies 2` confusingly displays the repo overview and then starts calculating first local and then \"annexed files in working tree\" keys. It does not show the useful repo overview which displays how many matched keys are in each repo that has at least one however.
+
+With that filter, `git annex info .` takes 15s in my case while `git annex info` takes 20s, likely due to the local key lookup. Trying to avoid the local lookup using `--fast` unfortunately also avoids the working tree lookup; outputting the same info as `git annex info --fast` without any filters which doesn't seem very useful.
+
+Does the latter really query all keys however? It appears to me that it's the same as querying `.`. I have nearly 100 unused keys in that repo but both info commands show the same amount.
+"""]]
diff --git a/doc/todo/info__58___allow_file_matching_options_for_all_keys/comment_3_aae866dc71811074d69b91c2390ccb01._comment b/doc/todo/info__58___allow_file_matching_options_for_all_keys/comment_3_aae866dc71811074d69b91c2390ccb01._comment
new file mode 100644
index 0000000000..940d1e2801
--- /dev/null
+++ b/doc/todo/info__58___allow_file_matching_options_for_all_keys/comment_3_aae866dc71811074d69b91c2390ccb01._comment
@@ -0,0 +1,17 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 3"""
+ date="2023-12-29T14:24:45Z"
+ content="""
+`git annex info --not --copies 2` displays the global overview (because you
+have not specified a directory), and limits it to keys that have the
+specified number of copies.
+
+Unused keys are included in the "local annexed keys" count when using
+`git-annex info` without a directory.
+
+I agree it would make sense for `git-annex info` to display something
+similar to the "repositories containing these files". Although in
+the global overview it should show the total annex size of each repository.
+[[todo/info_show_total_annex_sizes_of_repositories]]
+"""]]
diff --git a/doc/todo/info__58___allow_file_matching_options_for_all_keys/comment_4_371d53b4539bee6e3d595154033a2471._comment b/doc/todo/info__58___allow_file_matching_options_for_all_keys/comment_4_371d53b4539bee6e3d595154033a2471._comment
new file mode 100644
index 0000000000..3dbdaf3a62
--- /dev/null
+++ b/doc/todo/info__58___allow_file_matching_options_for_all_keys/comment_4_371d53b4539bee6e3d595154033a2471._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="Atemu"
+ avatar="http://cdn.libravatar.org/avatar/86b8c2d893dfdf2146e1bbb8ac4165fb"
+ subject="comment 4"
+ date="2023-12-29T17:06:37Z"
+ content="""
+I may have been unclear in my wording but what I really want is `info` to be able to show me numbers on *all* keys of the git-annex branch, including keys not in the current working tree or repo.  
+The intent is that I want to preserve past versions and also want to be able to query copies, where they're located etc. of those together with the \"used\" keys. I don't care whether they're referenced in the current revision and want to be able to tell `git-annex info` to not care too.
+
+Ideally, this should skip any unused check; operating only on the git-annex branch's data and none of master's. (That might also yield a much needed performance benefit.)
+
+Total size of annex sounds amazing; I've been wanting that ever since I set up distributed cold storage. I'll give it a spin :)
+"""]]
diff --git a/doc/todo/info_show_total_annex_sizes_of_repositories.mdwn b/doc/todo/info_show_total_annex_sizes_of_repositories.mdwn
new file mode 100644
index 0000000000..d96e7e9b51
--- /dev/null
+++ b/doc/todo/info_show_total_annex_sizes_of_repositories.mdwn
@@ -0,0 +1,10 @@
+`git-annex info` when run with a path displays "repositories containing these files"
+with a list of repositories and sizes. It would be good for it, when not
+run with a path to display a corresponding "annex size of repositories".
+
+I think that this recently became possible to inplement cheaply.
+cachedAllRepoData is collected now, used for "combined annex size of all repositories".
+For this, it also needs to accumulate the size of each separate repository.
+--[[Joey]]
+
+> [[done]] --[[Joey]]
diff --git a/doc/todo/option_for___40__fast__41___compression_on_special_remotes_like___34__directory__34__/comment_5_a154a025d441db4e0140fee821d1791c._comment b/doc/todo/option_for___40__fast__41___compression_on_special_remotes_like___34__directory__34__/comment_5_a154a025d441db4e0140fee821d1791c._comment
new file mode 100644
index 0000000000..48ed25bbf9
--- /dev/null
+++ b/doc/todo/option_for___40__fast__41___compression_on_special_remotes_like___34__directory__34__/comment_5_a154a025d441db4e0140fee821d1791c._comment
@@ -0,0 +1,39 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 5"""
+ date="2024-01-23T16:25:28Z"
+ content="""
+There would need to be a way for git-annex to tell if an object on a remote
+was compressed or not, and with what compressor. The two reaonable ways to
+do it are to use different namespaces for objects, or to make compression
+an immutable part of a special remote's configuration that is set at
+initremote time. 
+
+(A less reasonable way (IMHO) would be to record in the remote's state
+for each object what compression was used for it; this would bloat the
+git-annex branch.)
+
+A problem with using different namespaces is that git-annex then will
+have to do extra work to check for each one when retriving or checking
+presence of an object.
+
+When chunking is enabled, git-annex already has to check for chunked and
+unchunked versions of an object. This can include several different chunk
+sizes that have been in use at different times.
+
+So, we have a bit of an expontential blowup when combining chunking with
+compression namespaces. Probably the number of chunk sizes tried is less
+than 4 (eg logged chunk size, currently configured chunk size (when
+different), unchunked), and the number of possible compressors is less than
+10. So 20x or so increase in overhead. So I'm cautious of the namespaces approach.
+
+It would be possible to configure a single compressor that may be used at
+initremote time, but let some objects be chunked and others not. That would
+only double the overhead, and only for remotes with a compressor
+configured.
+
+As far as enabling compression based on filename though, it seems to me
+that an easier way to do that would be to have one remote with compression
+enabled, and a second one without it, and use preferred content to control
+which files are stored in which.
+"""]]
diff --git a/doc/todo/option_to_limit_to_files_that_are_expected_to_be_present.mdwn b/doc/todo/option_to_limit_to_files_that_are_expected_to_be_present.mdwn
index 53917edd78..2ed8715fc5 100644
--- a/doc/todo/option_to_limit_to_files_that_are_expected_to_be_present.mdwn
+++ b/doc/todo/option_to_limit_to_files_that_are_expected_to_be_present.mdwn
@@ -3,3 +3,5 @@ situation where my repo was copied from elsewhere but missing the object
 files, and I wanted to get back the same objects. So I had to disable that
 check. So an option that checks for files expected to be here would be
 useful. --[[Joey]]
+
+> [[done]] as --expected-present --[[Joey]]
diff --git a/doc/todo/record_ETag_when_using_addurl_--fast.mdwn b/doc/todo/record_ETag_when_using_addurl_--fast.mdwn
index 252fd1ae97..b15a9a202d 100644
--- a/doc/todo/record_ETag_when_using_addurl_--fast.mdwn
+++ b/doc/todo/record_ETag_when_using_addurl_--fast.mdwn
@@ -1 +1,3 @@
 Many websites return an Etag in the http response header, indicating the version of the resource.  Could the etag (or a checksum of it) be recorded in the URL- key, the way size is now?  Then e.g. `fsck --from web` could do a stronger check that the same file is still downloadable from the web, and the situation where different remotes have different versions of a file with the same URL- key could be better prevented.
+
+> Closing as this does not seem like a useful idea. [[done]] --[[Joey]]
diff --git a/doc/todo/speed_up_import_tree_with_many_excluded_files.mdwn b/doc/todo/speed_up_import_tree_with_many_excluded_files.mdwn
new file mode 100644
index 0000000000..8c1971d67e
--- /dev/null
+++ b/doc/todo/speed_up_import_tree_with_many_excluded_files.mdwn
@@ -0,0 +1,89 @@
+git-annex import from a special remote has a stage where
+addBackExportExcluded is run, to add back files that were excluded from
+being exported to the special remote to the tree of files imported from it. 
+
+This is quite slow when there are a lot of files in the tree. 
+Eg, in my sound repo, exporting to my phone, I have 50k excluded files, and
+it takes about 2 minutes to run this step.
+
+Can it be optimised?
+
+## speed up adjustTree
+
+It currently uses adjustTree with a big list of files to add back 
+to the tree. That performs badly when the list is large.
+
+This seems to be the bottleneck; git-annex is using 100% cpu,
+and the two helper processes `git mktree` and `git ls-tree` are barely active.
+
+At each level of the tree, it partitions the list to find files present
+in that subtree. It also filters the list. So the list gets traversed
+many times. 
+
+What's needed is a data structure, eg a tree, that avoids needing to
+repeatedly traverse the list like that.
+
+Note that adjustTree is also used with a possibly big list of files to add
+in Annex.Import.buildImportTrees. No other calls of adjustTree pass files
+to add.
+
+> [[done]] --[[Joey]]
+
+## git merge-tree
+
+Would it be possible to use `git merge-tree` instead?
+
+That command needs commits, and one commit cannot be a parent of the other
+in this case, so it seems it would be used like this:
+
+* Make a commit of the tree imported from the remote. This is a temporary
+  commit, and should have no parent.
+* Make a commit of the full tree from the remote tracking branch. This is a
+  temporary commit and should have no parent.
+* `git merge-tree` with the two temporary commits.
+  (Needs `--allow-unrelated-histories`)
+
+The cases are:
+
+1. new file added to remote
+2. file is present in the full tree, but was excluded out of the tree
+   exported to the remote
+3. file is modified on the remote
+4. file is present in the full tree, was excluded out of the tree exported
+   to the remote, but then got created separately on the remote
+5. file is present in the full tree, and got deleted from the remote
+
+In case 1, the new file will just be included in the merged tree.
+
+In case 2, the excluded file will also be just included in the merged tree.
+This is the point of this exercise.
+
+In case 3, there will be a conflict output. The right resolution of this
+conflict is to force in the version of the file from the imported tree.
+
+In case 4, there will be a conflict output. The right resolution of this
+conflict is same as in case 3.
+
+In case 5, the deleted file will be included in the merged tree.
+Oops, this is not what we want. Seems that this won't work?
+
+--- 
+
+What if the commit of the tree imported from the remote has as its parent a
+commit of the previous tree that was on the remote?
+
+Then in case 5, the deleted file will be in conflict; it was deleted from
+the import, and added from the full tree. Resolving this import in favor of
+the deletion will work.
+
+In case 4, there will be an add/add conflict. Force in the file from the
+imported tree.
+
+In case 3, same.
+
+In case 2, the excluded file will also be just included in the merged tree.
+This is the point of this exercise.
+
+In case 1, the new file will just be included in the merged tree.
+
+If that analysis is correct, this would work.. --[[Joey]]
diff --git a/doc/todo/support_using_Stateless_OpenPGP_instead_of_gpg.mdwn b/doc/todo/support_using_Stateless_OpenPGP_instead_of_gpg.mdwn
new file mode 100644
index 0000000000..e6600e88a5
--- /dev/null
+++ b/doc/todo/support_using_Stateless_OpenPGP_instead_of_gpg.mdwn
@@ -0,0 +1,52 @@
+<https://datatracker.ietf.org/doc/draft-dkg-openpgp-stateless-cli/> or "sop" is
+a standard for a command-line interface for OpenPGP. There are several
+implementations available in Debian already, like sqop (using Sequoia), gosop,
+gpainless-cli, and hop from hopenpgp-tools (haskell).
+
+It's possible to use this in a way that interoperates with gpg. For example:
+
+	joey@darkstar:~>cat pw
+	hunter1
+	joey@darkstar:~>cat foo
+	Tue Jan  9 15:10:32 JEST 2024
+	joey@darkstar:~>sqop encrypt --with-password=pw < foo > bar
+	joey@darkstar:~>gpg --passphrase-file pw --decrypt bar
+	gpg: AES256.CFB encrypted session key
+	gpg: encrypted with 1 passphrase
+	Tue Jan  9 15:10:32 JEST 2024
+
+That example uses symmetric encryption, which is what git-annex uses
+for encryption=shared. So git-annex could use this or gpg to access the
+same encrypted special remote.
+
+Update: That's implemented now, when annex.shared-sop-command is configured
+it will be used for encryption=shared special remotes. It interoperates
+fine with using gpg, as long as the sop command uses a compatable profile
+(setting annex.shared-sop-profile = rfc4880 is probably a good idea).
+
+Leaving this todo open because there are other encryption schemes than 
+encryption=shared, for which using sop is not yet supported.
+
+For the public key encryption used by the other encryption= schemes,
+sop would be harder to use, because it does not define any location
+to store private keys. Though it is possible to export gpg private keys
+and use them with sop to encrypt/decrypt files, it wouldn't make sense
+for git-annex to do that for the user. So there would need to be some way
+to map from keyid= value to a file containing the key. Which could be as
+simple as files named `.git/annex/sop/$keyid.sec`, which would be installed
+by the user using whatever means they prefer.
+
+Since sop also supports hardware-backed secret keys, and using one avoids
+the problem of where to store the secret key file, it would be good to
+support using those. This could be something like `keyid=@HARDWARE:xxx`
+making `@HARDWARE:xxx` be passed to sop.
+
+It seems that git-annex would need to prompt for the secret key's password
+itself, since sop doesn't prompt, and feed it in via `--with-key-password`.
+It can detect if a password is needed by trying the sop operation without a
+password and checking for an exit code of 67.
+See [this issue on sop password prompting](https://gitlab.com/dkg/openpgp-stateless-cli/-/issues/64)
+
+See also: [[todo/whishlist__58___GPG_alternatives_like_AGE]]
+
+[[!meta title="support using Stateless OpenPGP instead of gpg for encryption methods other than encryption=shared"]]
diff --git a/doc/todo/the_same_path_looked_up_3_times_for_libpcre__42__.so.mdwn b/doc/todo/the_same_path_looked_up_3_times_for_libpcre__42__.so.mdwn
index 6b0c259c2c..c4584eaf55 100644
--- a/doc/todo/the_same_path_looked_up_3_times_for_libpcre__42__.so.mdwn
+++ b/doc/todo/the_same_path_looked_up_3_times_for_libpcre__42__.so.mdwn
@@ -37,3 +37,5 @@ where those 3 extra (identical) looks up are coming from now?
 PS Since not critical, I will workaround for now by raising max count to 7.
 
 [[!tag projects/datalad]]
+
+> [[done]] per my earlier comment --[[Joey]]
diff --git a/doc/todo/whishlist__58___GPG_alternatives_like_AGE/comment_8_d784886328bc00b2adaba2391776d2c8._comment b/doc/todo/whishlist__58___GPG_alternatives_like_AGE/comment_8_d784886328bc00b2adaba2391776d2c8._comment
new file mode 100644
index 0000000000..9d0e56feed
--- /dev/null
+++ b/doc/todo/whishlist__58___GPG_alternatives_like_AGE/comment_8_d784886328bc00b2adaba2391776d2c8._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 8"""
+ date="2024-01-09T20:28:58Z"
+ content="""
+<https://crates.io/crates/sequoia-chameleon-gnupg> should be possible to
+use as a drop-in replacement for gpg. Assuming they implemented enough of
+gpg's interface.
+
+To try using it with git-annex, it should be sufficient to run:
+
+	git config gpg.program gpg-sq
+"""]]
diff --git a/doc/todo/whishlist__58___GPG_alternatives_like_AGE/comment_9_18861f94b677a40c2d1c0014338231c6._comment b/doc/todo/whishlist__58___GPG_alternatives_like_AGE/comment_9_18861f94b677a40c2d1c0014338231c6._comment
new file mode 100644
index 0000000000..7635074db3
--- /dev/null
+++ b/doc/todo/whishlist__58___GPG_alternatives_like_AGE/comment_9_18861f94b677a40c2d1c0014338231c6._comment
@@ -0,0 +1,22 @@
+[[!comment format=mdwn
+ username="joey"
+ subject="""comment 9"""
+ date="2024-01-09T20:33:08Z"
+ content="""
+git-annex now supports using any program that implements the
+Stateless OpenPGP standard for remotes using encryption=shared
+
+Eg, to use Sequoia's sqpop:
+
+	git config annex.shared-sop-command sqop
+
+That fully interoperates with gpg, though it's probably a good idea
+to specify an enryption profile that is backwards compatable if interop
+is a concern. Eg:
+
+	git config annex.shared-sop-profile rfc4880
+
+For other encryption= schemes, Stateless OpenPGP is not yet supported,
+mostly because it's not clear how to manage the encryption keys.
+See this todo [[support_using_Stateless_OpenPGP_instead_of_gpg]]
+"""]]
diff --git a/doc/todo/windows_support/comment_26_99d0541d1c7f7c82a67d481c61209670._comment b/doc/todo/windows_support/comment_26_99d0541d1c7f7c82a67d481c61209670._comment
new file mode 100644
index 0000000000..2a586da213
--- /dev/null
+++ b/doc/todo/windows_support/comment_26_99d0541d1c7f7c82a67d481c61209670._comment
@@ -0,0 +1,13 @@
+[[!comment format=mdwn
+ username="NewUser"
+ nickname="dont.post.me"
+ avatar="http://cdn.libravatar.org/avatar/90f59ddc341eaf9b2657422206c06901"
+ subject="Is `annex.tune.objecthashlower=true` recommended for interop with windows?"
+ date="2023-11-20T04:24:35Z"
+ content="""
+I've been adding stuff to git annex on my linux server and now I'm thinking that I need to uninit and re-init with `annex.tune.objecthashlower=true`. I'll want to use git-annex from windows as well.
+
+Should I use `annex.tune.objecthashlower=true`? Also, what's the advice around the other tuning options `objecthash1` and `branchhash1`?
+
+Thanks!
+"""]]
diff --git a/doc/todo/worktree_overwrite_races.mdwn b/doc/todo/worktree_overwrite_races.mdwn
index 815e438a7d..d11bb3c7e8 100644
--- a/doc/todo/worktree_overwrite_races.mdwn
+++ b/doc/todo/worktree_overwrite_races.mdwn
@@ -32,3 +32,14 @@ place. It gets 3 appended to it. So the worktree file ends up containing
 `1,3`.
 
 So, perhaps there is really no good solution to this. --[[Joey]]
+
+> I suppose the balance of probabilities is that a worktree file being
+> overwritten in the current race window is fairly plausibly likely, 
+> while a file being repeatedly appended to in that way is very unlikely.
+> 
+> But the balance of ill effects is that in the current case you lose data
+> in a way that is easy to understand (and that git is also subject to),
+> and in the repeated append case, a file gets into a state it never ought
+> to possibly be in.
+> 
+> So I think it makes sense to abandon this idea. [[done]] --[[Joey]]
diff --git a/doc/users/kolam.mdwn b/doc/users/kolam.mdwn
new file mode 100644
index 0000000000..1333ed77b7
--- /dev/null
+++ b/doc/users/kolam.mdwn
@@ -0,0 +1 @@
+TODO
diff --git a/doc/users/nobodyinperson.mdwn b/doc/users/nobodyinperson.mdwn
index cf3f4a9d9d..18b8cfb8f1 100644
--- a/doc/users/nobodyinperson.mdwn
+++ b/doc/users/nobodyinperson.mdwn
@@ -1,9 +1,13 @@
 I use git-annex to:
 
-- manage my research data
+- manage my research data, partly also with [DataLad](https://datalad.org)
 - sync and backup personal documents
 - sync media files to and from my SailfishOS phone
 - sync, organise and backup a huge media collection
 - experiment doing inventory management
 
 I made a [Thunar plugin](https://gitlab.com/nobodyinperson/thunar-plugins) for git-annex, here's a [📹 screencast](https://fosstodon.org/@nobodyinperson/109836827575976439).
+
+In an attempt to [#gitAnnexAllTheThings](https://fosstodon.org/tags/gitAnnexAllTheThings), I used git annex as a backend for a cli time tracker [annextimelog](https://pypi.org/project/annextimelog/). It has similarities with timewarrior but adresses many of its inconveniences.
+
+At the [Tübix 2023](https://www.tuebix.org/) I gave a (German) git annex workshop, of which you can find a recording of the initial talk [📹 here in the fediverse](https://tube.tchncs.de/w/db1ec5ca-94ad-4f49-a507-2124fd699ff1) and [📹 here on Odysee](https://odysee.com/@nobodyinperson:6/T%C3%BCbix2023-Yann-B%C3%BCchau-git-annex:6). 
diff --git a/doc/users/unqueued.mdwn b/doc/users/unqueued.mdwn
new file mode 100644
index 0000000000..7ae93d8608
--- /dev/null
+++ b/doc/users/unqueued.mdwn
@@ -0,0 +1,7 @@
+Hello.
+
+I have been using git-annex since 2017, and it has been a huge boon.
+
+I mostly consider myself a hobbyist, but I have used git-annex professionally. Along the way, I've learned a lot about unix system programming, git, and automation.
+
+I personally find managing large sets of files to be very annoying. So being able to wrangle giant sets of files with the precision of git is awesome. I used to spend a lot of time checksumming files, making download queues, and trying to catalog where stuff was.
diff --git a/doc/videos/tuebix.mdwn b/doc/videos/tuebix.mdwn
index fe30992fb3..95e9fce42c 100644
--- a/doc/videos/tuebix.mdwn
+++ b/doc/videos/tuebix.mdwn
@@ -3,5 +3,9 @@ Yann Büchau's (German) talk about Git Annex on the [Tübix2023 Linux Day](https
 - [on PeerTube](https://tube.tchncs.de/w/t4mLqLM2YdFZf21SoW9gQn)
 - [on Odysee](https://odysee.com/@nobodyinperson:6/T%C3%BCbix2023-Yann-B%C3%BCchau-git-annex:6)
 
+Yann re-recorded the talk in English later, it is available here:
+
+- [on PeerTube](https://tube.tchncs.de/w/1U4vbTAhSEje3KQ1dGqvxh)
+
 [[!meta date="1 Jul 2023"]]
 [[!meta title="git-annex presentation by Yann Büchau at Tübix2023"]]
diff --git a/git-annex.cabal b/git-annex.cabal
index 3699145243..31e66f9f27 100644
--- a/git-annex.cabal
+++ b/git-annex.cabal
@@ -1,11 +1,11 @@
 Name: git-annex
-Version: 10.20230926
+Version: 10.20240129
 Cabal-Version: 1.12
 License: AGPL-3
 Maintainer: Joey Hess <id@joeyh.name>
 Author: Joey Hess
 Stability: Stable
-Copyright: 2010-2023 Joey Hess
+Copyright: 2010-2024 Joey Hess
 License-File: COPYRIGHT
 Homepage: http://git-annex.branchable.com/
 Build-type: Custom
@@ -159,6 +159,11 @@ Flag Pairing
 Flag Production
   Description: Enable production build (slower build; faster binary)
 
+Flag ParallelBuild
+  Description: Enable production build (slower build; faster binary)
+  Default: False
+  Manual: True
+
 Flag TorrentParser
   Description: Use haskell torrent library to parse torrent files
 
@@ -292,6 +297,9 @@ Executable git-annex
   else
     GHC-Options: -O0
 
+  if flag(ParallelBuild)
+    GHC-Options: -j
+
   -- Avoid linking with unused dynamic libraries.
   if os(linux) || os(freebsd)
     GHC-Options: -optl-Wl,--as-needed
@@ -568,6 +576,7 @@ Executable git-annex
     Annex.YoutubeDl
     Assistant.Install.AutoStart
     Assistant.Install.Menu
+    Author
     Backend
     Backend.External
     Backend.Hash
@@ -730,12 +739,14 @@ Executable git-annex
     Database.Export
     Database.Fsck
     Database.Handle
+    Database.ImportFeed
     Database.Init
     Database.Keys
     Database.Keys.Handle
     Database.Keys.Tables
     Database.Keys.SQL
     Database.Queue
+    Database.RawFilePath
     Database.Types
     Database.Utility
     Git
@@ -765,6 +776,7 @@ Executable git-annex
     Git.Hook
     Git.Index
     Git.LockFile
+    Git.Log
     Git.LsFiles
     Git.LsTree
     Git.Merge
@@ -810,6 +822,7 @@ Executable git-annex
     Logs.MapLog
     Logs.MetaData
     Logs.MetaData.Pure
+    Logs.Migrate
     Logs.Multicast
     Logs.NumCopies
     Logs.PreferredContent
@@ -1047,6 +1060,7 @@ Executable git-annex
     Utility.Split
     Utility.SshConfig
     Utility.SshHost
+    Utility.StatelessOpenPGP
     Utility.Su
     Utility.SystemDirectory
     Utility.Terminal
diff --git a/stack-lts-18.13.yaml b/stack-lts-18.13.yaml
index 4109d79747..5ee274f930 100644
--- a/stack-lts-18.13.yaml
+++ b/stack-lts-18.13.yaml
@@ -1,6 +1,7 @@
 flags:
   git-annex:
     production: true
+    parallelbuild: true
     assistant: true
     pairing: true
     torrentparser: true
diff --git a/stack.yaml b/stack.yaml
index 561eff1a4a..07ad3e5c12 100644
--- a/stack.yaml
+++ b/stack.yaml
@@ -1,6 +1,7 @@
 flags:
   git-annex:
     production: true
+    parallelbuild: true
     assistant: true
     pairing: true
     torrentparser: true
@@ -11,7 +12,7 @@ flags:
     crypton: false
 packages:
 - '.'
-resolver: nightly-2023-08-01
+resolver: lts-22.3
 extra-deps:
 - git-lfs-1.2.1
 - http-client-restricted-0.1.0
diff --git a/standalone/linux/stack-i386ancient.yaml b/standalone/linux/stack-i386ancient.yaml
index 67d6295903..4ad5330fc0 100644
--- a/standalone/linux/stack-i386ancient.yaml
+++ b/standalone/linux/stack-i386ancient.yaml
@@ -1,6 +1,7 @@
 flags:
   git-annex:
     production: true
+    parallelbuild: true
     assistant: true
     pairing: true
     torrentparser: true
diff --git a/templates/configurators/genkeymodal.hamlet b/templates/configurators/genkeymodal.hamlet
index 23c3863054..b8f47cb513 100644
--- a/templates/configurators/genkeymodal.hamlet
+++ b/templates/configurators/genkeymodal.hamlet
@@ -5,7 +5,7 @@
         <div .modal-header>
           <h3>
             <img src="@{StaticR activityicon_gif}" alt=""> #
-            Generating a #{maxRecommendedKeySize} bit GnuPg key.
+            Generating a GnuPg key.
         <div .modal-body>
         <p>
           Generating a GnuPg key can take a long time. To speed up the process, #