From 35e48b188329bf1048dfe154247b3259441de29f Mon Sep 17 00:00:00 2001 From: Chris Wilson Date: Sat, 28 Mar 2009 12:25:05 +0000 Subject: Reorganise docs in trunk to match distribution layout, which is cleaner, and makes Makefile work on distributions and trunk equally. --- distribution/boxbackup/DISTRIBUTION-MANIFEST.txt | 64 +- docs/Makefile | 91 +- docs/adminguide.xml | 1981 -------------------- docs/api-docs/backup/INDEX.txt | 61 - docs/api-docs/backup/Win32_Clients.txt | 13 - docs/api-docs/backup/backup_encryption.txt | 109 -- docs/api-docs/backup/bin_bbackupd.txt | 88 - docs/api-docs/backup/bin_bbstored.txt | 54 - docs/api-docs/backup/encrypt_rsync.txt | 66 - docs/api-docs/backup/lib_backupclient.txt | 46 - docs/api-docs/backup/lib_backupstore.txt | 30 - .../backup/win32_build_on_cygwin_using_mingw.txt | 53 - .../backup/win32_build_on_linux_using_mingw.txt | 108 -- docs/api-docs/backup/windows_porting.txt | 100 - docs/api-docs/common/lib_common.txt | 52 - docs/api-docs/common/lib_common/BoxTime.txt | 7 - .../common/lib_common/CollectInBufferStream.txt | 26 - docs/api-docs/common/lib_common/Configuration.txt | 102 - docs/api-docs/common/lib_common/Conversion.txt | 14 - docs/api-docs/common/lib_common/ExcludeList.txt | 21 - docs/api-docs/common/lib_common/FdGetLine.txt | 11 - docs/api-docs/common/lib_common/Guards.txt | 5 - docs/api-docs/common/lib_common/IOStream.txt | 89 - .../api-docs/common/lib_common/IOStreamGetLine.txt | 29 - docs/api-docs/common/lib_common/MainHelper.txt | 4 - docs/api-docs/common/lib_common/WaitForEvent.txt | 16 - docs/api-docs/common/lib_common/xStream.txt | 40 - docs/api-docs/common/lib_compress.txt | 8 - .../common/lib_compress/CompressStream.txt | 27 - docs/api-docs/common/lib_crypto.txt | 28 - docs/api-docs/common/lib_crypto/CipherContext.txt | 28 - .../api-docs/common/lib_crypto/RollingChecksum.txt | 36 - docs/api-docs/common/lib_server.txt | 9 - docs/api-docs/common/lib_server/Daemon.txt | 96 - docs/api-docs/common/lib_server/Protocol.txt | 120 -- docs/api-docs/common/lib_server/ServerStream.txt | 29 - docs/api-docs/common/lib_server/ServerTLS.txt | 6 - docs/api-docs/common/lib_server/SocketStream.txt | 8 - .../api-docs/common/lib_server/SocketStreamTLS.txt | 11 - docs/api-docs/common/lib_server/TLSContext.txt | 16 - docs/api-docs/common/memory_leaks.txt | 44 - docs/api-docs/raidfile/lib_raidfile.txt | 73 - .../raidfile/lib_raidfile/RaidFileRead.txt | 14 - .../raidfile/lib_raidfile/RaidFileWrite.txt | 36 - docs/api-notes/backup/INDEX.txt | 61 + docs/api-notes/backup/Win32_Clients.txt | 13 + docs/api-notes/backup/backup_encryption.txt | 109 ++ docs/api-notes/backup/bin_bbackupd.txt | 88 + docs/api-notes/backup/bin_bbstored.txt | 54 + docs/api-notes/backup/encrypt_rsync.txt | 66 + docs/api-notes/backup/lib_backupclient.txt | 46 + docs/api-notes/backup/lib_backupstore.txt | 30 + .../backup/win32_build_on_cygwin_using_mingw.txt | 53 + .../backup/win32_build_on_linux_using_mingw.txt | 108 ++ docs/api-notes/backup/windows_porting.txt | 100 + docs/api-notes/common/lib_common.txt | 52 + docs/api-notes/common/lib_common/BoxTime.txt | 7 + .../common/lib_common/CollectInBufferStream.txt | 26 + docs/api-notes/common/lib_common/Configuration.txt | 102 + docs/api-notes/common/lib_common/Conversion.txt | 14 + docs/api-notes/common/lib_common/ExcludeList.txt | 21 + docs/api-notes/common/lib_common/FdGetLine.txt | 11 + docs/api-notes/common/lib_common/Guards.txt | 5 + docs/api-notes/common/lib_common/IOStream.txt | 89 + .../common/lib_common/IOStreamGetLine.txt | 29 + docs/api-notes/common/lib_common/MainHelper.txt | 4 + docs/api-notes/common/lib_common/WaitForEvent.txt | 16 + docs/api-notes/common/lib_common/xStream.txt | 40 + docs/api-notes/common/lib_compress.txt | 8 + .../common/lib_compress/CompressStream.txt | 27 + docs/api-notes/common/lib_crypto.txt | 28 + docs/api-notes/common/lib_crypto/CipherContext.txt | 28 + .../common/lib_crypto/RollingChecksum.txt | 36 + docs/api-notes/common/lib_server.txt | 9 + docs/api-notes/common/lib_server/Daemon.txt | 96 + docs/api-notes/common/lib_server/Protocol.txt | 120 ++ docs/api-notes/common/lib_server/ServerStream.txt | 29 + docs/api-notes/common/lib_server/ServerTLS.txt | 6 + docs/api-notes/common/lib_server/SocketStream.txt | 8 + .../common/lib_server/SocketStreamTLS.txt | 11 + docs/api-notes/common/lib_server/TLSContext.txt | 16 + docs/api-notes/common/memory_leaks.txt | 44 + docs/api-notes/raidfile/lib_raidfile.txt | 73 + .../raidfile/lib_raidfile/RaidFileRead.txt | 14 + .../raidfile/lib_raidfile/RaidFileWrite.txt | 36 + docs/bb-book.xsl | 17 - docs/bb-man.xsl.tmpl | 9 - docs/bb-nochunk-book.xsl | 17 - docs/bbackupctl.xml | 205 -- docs/bbackupd-config.xml | 153 -- docs/bbackupd.conf.xml | 479 ----- docs/bbackupd.xml | 209 --- docs/bbackupquery.xml | 506 ----- docs/bblogo-alpha.xcf | Bin 93980 -> 0 bytes docs/bbstoreaccounts.xml | 386 ---- docs/bbstored-certs.xml | 180 -- docs/bbstored-config.xml | 148 -- docs/bbstored.conf.xml | 211 --- docs/bbstored.xml | 90 - docs/docbook/adminguide.xml | 1981 ++++++++++++++++++++ docs/docbook/bb-book.xsl | 17 + docs/docbook/bb-man.xsl.tmpl | 9 + docs/docbook/bb-nochunk-book.xsl | 17 + docs/docbook/bbackupctl.xml | 205 ++ docs/docbook/bbackupd-config.xml | 153 ++ docs/docbook/bbackupd.conf.xml | 479 +++++ docs/docbook/bbackupd.xml | 209 +++ docs/docbook/bbackupquery.xml | 506 +++++ docs/docbook/bbstoreaccounts.xml | 386 ++++ docs/docbook/bbstored-certs.xml | 180 ++ docs/docbook/bbstored-config.xml | 148 ++ docs/docbook/bbstored.conf.xml | 211 +++ docs/docbook/bbstored.xml | 90 + docs/docbook/html/bbdoc-man.css | 104 + docs/docbook/html/bbdoc.css | 112 ++ docs/docbook/html/favicon.ico | Bin 0 -> 67646 bytes docs/docbook/html/images/arrow.png | Bin 0 -> 197 bytes docs/docbook/html/images/bblogo.png | Bin 0 -> 5882 bytes docs/docbook/html/images/stepahead.png | Bin 0 -> 298 bytes docs/docbook/instguide.xml | 766 ++++++++ docs/docbook/raidfile-config.xml | 198 ++ docs/docbook/raidfile.conf.xml | 143 ++ docs/favicon.ico | Bin 67646 -> 0 bytes docs/generate_except_xml.pl | 74 - docs/html/bbdoc-man.css | 104 - docs/html/bbdoc.css | 112 -- docs/html/images/arrow.png | Bin 197 -> 0 bytes docs/html/images/bblogo.png | Bin 5882 -> 0 bytes docs/html/images/stepahead.png | Bin 298 -> 0 bytes docs/images/bblogo-alpha.xcf | Bin 0 -> 93980 bytes docs/instguide.xml | 766 -------- docs/raidfile-config.xml | 198 -- docs/raidfile.conf.xml | 143 -- docs/tools/generate_except_xml.pl | 74 + 134 files changed, 7803 insertions(+), 7794 deletions(-) delete mode 100644 docs/adminguide.xml delete mode 100644 docs/api-docs/backup/INDEX.txt delete mode 100644 docs/api-docs/backup/Win32_Clients.txt delete mode 100644 docs/api-docs/backup/backup_encryption.txt delete mode 100644 docs/api-docs/backup/bin_bbackupd.txt delete mode 100644 docs/api-docs/backup/bin_bbstored.txt delete mode 100644 docs/api-docs/backup/encrypt_rsync.txt delete mode 100644 docs/api-docs/backup/lib_backupclient.txt delete mode 100644 docs/api-docs/backup/lib_backupstore.txt delete mode 100644 docs/api-docs/backup/win32_build_on_cygwin_using_mingw.txt delete mode 100644 docs/api-docs/backup/win32_build_on_linux_using_mingw.txt delete mode 100644 docs/api-docs/backup/windows_porting.txt delete mode 100644 docs/api-docs/common/lib_common.txt delete mode 100644 docs/api-docs/common/lib_common/BoxTime.txt delete mode 100644 docs/api-docs/common/lib_common/CollectInBufferStream.txt delete mode 100644 docs/api-docs/common/lib_common/Configuration.txt delete mode 100644 docs/api-docs/common/lib_common/Conversion.txt delete mode 100644 docs/api-docs/common/lib_common/ExcludeList.txt delete mode 100644 docs/api-docs/common/lib_common/FdGetLine.txt delete mode 100644 docs/api-docs/common/lib_common/Guards.txt delete mode 100644 docs/api-docs/common/lib_common/IOStream.txt delete mode 100644 docs/api-docs/common/lib_common/IOStreamGetLine.txt delete mode 100644 docs/api-docs/common/lib_common/MainHelper.txt delete mode 100644 docs/api-docs/common/lib_common/WaitForEvent.txt delete mode 100644 docs/api-docs/common/lib_common/xStream.txt delete mode 100644 docs/api-docs/common/lib_compress.txt delete mode 100644 docs/api-docs/common/lib_compress/CompressStream.txt delete mode 100644 docs/api-docs/common/lib_crypto.txt delete mode 100644 docs/api-docs/common/lib_crypto/CipherContext.txt delete mode 100644 docs/api-docs/common/lib_crypto/RollingChecksum.txt delete mode 100644 docs/api-docs/common/lib_server.txt delete mode 100644 docs/api-docs/common/lib_server/Daemon.txt delete mode 100644 docs/api-docs/common/lib_server/Protocol.txt delete mode 100644 docs/api-docs/common/lib_server/ServerStream.txt delete mode 100644 docs/api-docs/common/lib_server/ServerTLS.txt delete mode 100644 docs/api-docs/common/lib_server/SocketStream.txt delete mode 100644 docs/api-docs/common/lib_server/SocketStreamTLS.txt delete mode 100644 docs/api-docs/common/lib_server/TLSContext.txt delete mode 100644 docs/api-docs/common/memory_leaks.txt delete mode 100644 docs/api-docs/raidfile/lib_raidfile.txt delete mode 100644 docs/api-docs/raidfile/lib_raidfile/RaidFileRead.txt delete mode 100644 docs/api-docs/raidfile/lib_raidfile/RaidFileWrite.txt create mode 100644 docs/api-notes/backup/INDEX.txt create mode 100644 docs/api-notes/backup/Win32_Clients.txt create mode 100644 docs/api-notes/backup/backup_encryption.txt create mode 100644 docs/api-notes/backup/bin_bbackupd.txt create mode 100644 docs/api-notes/backup/bin_bbstored.txt create mode 100644 docs/api-notes/backup/encrypt_rsync.txt create mode 100644 docs/api-notes/backup/lib_backupclient.txt create mode 100644 docs/api-notes/backup/lib_backupstore.txt create mode 100644 docs/api-notes/backup/win32_build_on_cygwin_using_mingw.txt create mode 100644 docs/api-notes/backup/win32_build_on_linux_using_mingw.txt create mode 100644 docs/api-notes/backup/windows_porting.txt create mode 100644 docs/api-notes/common/lib_common.txt create mode 100644 docs/api-notes/common/lib_common/BoxTime.txt create mode 100644 docs/api-notes/common/lib_common/CollectInBufferStream.txt create mode 100644 docs/api-notes/common/lib_common/Configuration.txt create mode 100644 docs/api-notes/common/lib_common/Conversion.txt create mode 100644 docs/api-notes/common/lib_common/ExcludeList.txt create mode 100644 docs/api-notes/common/lib_common/FdGetLine.txt create mode 100644 docs/api-notes/common/lib_common/Guards.txt create mode 100644 docs/api-notes/common/lib_common/IOStream.txt create mode 100644 docs/api-notes/common/lib_common/IOStreamGetLine.txt create mode 100644 docs/api-notes/common/lib_common/MainHelper.txt create mode 100644 docs/api-notes/common/lib_common/WaitForEvent.txt create mode 100644 docs/api-notes/common/lib_common/xStream.txt create mode 100644 docs/api-notes/common/lib_compress.txt create mode 100644 docs/api-notes/common/lib_compress/CompressStream.txt create mode 100644 docs/api-notes/common/lib_crypto.txt create mode 100644 docs/api-notes/common/lib_crypto/CipherContext.txt create mode 100644 docs/api-notes/common/lib_crypto/RollingChecksum.txt create mode 100644 docs/api-notes/common/lib_server.txt create mode 100644 docs/api-notes/common/lib_server/Daemon.txt create mode 100644 docs/api-notes/common/lib_server/Protocol.txt create mode 100644 docs/api-notes/common/lib_server/ServerStream.txt create mode 100644 docs/api-notes/common/lib_server/ServerTLS.txt create mode 100644 docs/api-notes/common/lib_server/SocketStream.txt create mode 100644 docs/api-notes/common/lib_server/SocketStreamTLS.txt create mode 100644 docs/api-notes/common/lib_server/TLSContext.txt create mode 100644 docs/api-notes/common/memory_leaks.txt create mode 100644 docs/api-notes/raidfile/lib_raidfile.txt create mode 100644 docs/api-notes/raidfile/lib_raidfile/RaidFileRead.txt create mode 100644 docs/api-notes/raidfile/lib_raidfile/RaidFileWrite.txt delete mode 100644 docs/bb-book.xsl delete mode 100644 docs/bb-man.xsl.tmpl delete mode 100644 docs/bb-nochunk-book.xsl delete mode 100644 docs/bbackupctl.xml delete mode 100644 docs/bbackupd-config.xml delete mode 100644 docs/bbackupd.conf.xml delete mode 100644 docs/bbackupd.xml delete mode 100644 docs/bbackupquery.xml delete mode 100644 docs/bblogo-alpha.xcf delete mode 100644 docs/bbstoreaccounts.xml delete mode 100644 docs/bbstored-certs.xml delete mode 100644 docs/bbstored-config.xml delete mode 100644 docs/bbstored.conf.xml delete mode 100644 docs/bbstored.xml create mode 100644 docs/docbook/adminguide.xml create mode 100644 docs/docbook/bb-book.xsl create mode 100644 docs/docbook/bb-man.xsl.tmpl create mode 100644 docs/docbook/bb-nochunk-book.xsl create mode 100644 docs/docbook/bbackupctl.xml create mode 100644 docs/docbook/bbackupd-config.xml create mode 100644 docs/docbook/bbackupd.conf.xml create mode 100644 docs/docbook/bbackupd.xml create mode 100644 docs/docbook/bbackupquery.xml create mode 100644 docs/docbook/bbstoreaccounts.xml create mode 100644 docs/docbook/bbstored-certs.xml create mode 100644 docs/docbook/bbstored-config.xml create mode 100644 docs/docbook/bbstored.conf.xml create mode 100644 docs/docbook/bbstored.xml create mode 100644 docs/docbook/html/bbdoc-man.css create mode 100644 docs/docbook/html/bbdoc.css create mode 100644 docs/docbook/html/favicon.ico create mode 100644 docs/docbook/html/images/arrow.png create mode 100644 docs/docbook/html/images/bblogo.png create mode 100644 docs/docbook/html/images/stepahead.png create mode 100644 docs/docbook/instguide.xml create mode 100644 docs/docbook/raidfile-config.xml create mode 100644 docs/docbook/raidfile.conf.xml delete mode 100644 docs/favicon.ico delete mode 100644 docs/generate_except_xml.pl delete mode 100644 docs/html/bbdoc-man.css delete mode 100644 docs/html/bbdoc.css delete mode 100644 docs/html/images/arrow.png delete mode 100644 docs/html/images/bblogo.png delete mode 100644 docs/html/images/stepahead.png create mode 100644 docs/images/bblogo-alpha.xcf delete mode 100644 docs/instguide.xml delete mode 100644 docs/raidfile-config.xml delete mode 100644 docs/raidfile.conf.xml create mode 100644 docs/tools/generate_except_xml.pl diff --git a/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt b/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt index 7afa2548..228b29b8 100644 --- a/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt +++ b/distribution/boxbackup/DISTRIBUTION-MANIFEST.txt @@ -19,39 +19,39 @@ test/backupstorepatch test/bbackupd test/bbackupd/testfiles test/backupdiff -docs/api-docs/raidfile docs/api-notes -docs/api-docs/raidfile/lib_raidfile docs/api-notes/lib_raidfile -docs/api-docs/backup docs/api-notes -docs/box-html docs/htmlguide -docs/box-html/adminguide docs/htmlguide/adminguide -docs/box-html/images docs/htmlguide/images -docs/box-html/instguide docs/htmlguide/instguide -docs/box-html/man-html docs/htmlguide/manpages +docs/Makefile +docs/tools +docs/api-notes +docs/api-notes/lib_raidfile +docs/htmlguide +docs/htmlguide/adminguide +docs/htmlguide/images +docs/htmlguide/instguide +docs/htmlguide/manpages docs/man -MKDIR docs/docbook -RUN cd docs; sed -i"" -e '/^ExceptionCodes/,/^$/s/\(.*\)/# &/' Makefile -docs/Makefile docs/docbook/Makefile -docs/ExceptionCodes.xml docs/docbook/ExceptionCodes.xml -docs/adminguide.xml docs/docbook/adminguide.xml -docs/bb-book.xsl docs/docbook/bb-book.xsl -docs/bb-man.xsl.tmpl docs/docbook/bb-man.xsl.tmpl -docs/bb-nochunk-book.xsl docs/docbook/bb-nochunk-book.xsl -docs/bbackupctl.xml docs/docbook/bbackupctl.xml -docs/bbackupd-config.xml docs/docbook/bbackupd-config.xml -docs/bbackupd.conf.xml docs/docbook/bbackupd.conf.xml -docs/bbackupd.xml docs/docbook/bbackupd.xml -docs/bbackupquery.xml docs/docbook/bbackupquery.xml -docs/bbstoreaccounts.xml docs/docbook/bbstoreaccounts.xml -docs/bbstored-certs.xml docs/docbook/bbstored-certs.xml -docs/bbstored-config.xml docs/docbook/bbstored-config.xml -docs/bbstored.conf.xml docs/docbook/bbstored.conf.xml -docs/bbstored.xml docs/docbook/bbstored.xml -docs/instguide.xml docs/docbook/instguide.xml -docs/raidfile-config.xml docs/docbook/raidfile-config.xml -docs/raidfile.conf.xml docs/docbook/raidfile.conf.xml -docs/html docs/docbook/html -docs/html/images docs/docbook/html/images -RUN svn revert docs/Makefile +# RUN cd docs; sed -i"" -e '/^ExceptionCodes/,/^$/s/\(.*\)/# &/' Makefile +docs/docbook/Makefile +docs/docbook/ExceptionCodes.xml +docs/docbook/adminguide.xml +docs/docbook/bb-book.xsl +docs/docbook/bb-man.xsl.tmpl +docs/docbook/bb-nochunk-book.xsl +docs/docbook/bbackupctl.xml +docs/docbook/bbackupd-config.xml +docs/docbook/bbackupd.conf.xml +docs/docbook/bbackupd.xml +docs/docbook/bbackupquery.xml +docs/docbook/bbstoreaccounts.xml +docs/docbook/bbstored-certs.xml +docs/docbook/bbstored-config.xml +docs/docbook/bbstored.conf.xml +docs/docbook/bbstored.xml +docs/docbook/instguide.xml +docs/docbook/raidfile-config.xml +docs/docbook/raidfile.conf.xml +docs/docbook/html +docs/docbook/html/images +# RUN svn revert docs/Makefile TODO.txt BUGS.txt contrib diff --git a/docs/Makefile b/docs/Makefile index 43a4eadb..af6bb9e6 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -3,33 +3,39 @@ # Process DocBook to HTML DBPROC=xsltproc -BOOKXSL=bb-book.xsl -NOCHUNKBOOKXSL=bb-nochunk-book.xsl -MANXSL=bb-man.xsl -HTMLPREFIX=box-html -VPATH= adminguide -.SUFFIXES: .html .xml .1 .5 .8 + +DOCBOOK_DIR = docbook +HTML_DIR = htmlguide +MAN_DIR = man + +BOOKXSL = $(DOCBOOK_DIR)/bb-book.xsl +NOCHUNKBOOKXSL = $(DOCBOOK_DIR)/bb-nochunk-book.xsl +MANXSL = $(DOCBOOK_DIR)/bb-man.xsl + +# VPATH= adminguide +# .SUFFIXES: .html .xml .1 .5 .8 all: docs docs: instguide adminguide manpages - @mkdir -p $(HTMLPREFIX)/images - @cp html/images/*.png $(HTMLPREFIX)/images/. - @cp html/*.css $(HTMLPREFIX)/. + @mkdir -p $(HTML_DIR)/images + @cp $(DOCBOOK_DIR)/html/images/*.png $(HTML_DIR)/images/. + @cp $(DOCBOOK_DIR)/html/*.css $(HTML_DIR)/. + @cp $(DOCBOOK_DIR)/html/*.ico $(HTML_DIR)/. -adminguide: $(HTMLPREFIX)/adminguide/index.html +adminguide: $(HTML_DIR)/adminguide/index.html -$(HTMLPREFIX)/adminguide/index.html: adminguide.xml ExceptionCodes.xml $(BOOKXSL) +$(HTML_DIR)/adminguide/index.html: $(DOCBOOK_DIR)/adminguide.xml $(DOCBOOK_DIR)/ExceptionCodes.xml $(BOOKXSL) # docname=`echo $@ | sed -e 's/\/index.html//'` - $(DBPROC) -o $(HTMLPREFIX)/adminguide/ $(BOOKXSL) adminguide.xml + $(DBPROC) -o $(HTML_DIR)/adminguide/ $(BOOKXSL) $< -instguide: $(HTMLPREFIX)/instguide/index.html +instguide: $(HTML_DIR)/instguide/index.html -$(HTMLPREFIX)/instguide/index.html: instguide.xml $(BOOKXSL) - $(DBPROC) -o $(HTMLPREFIX)/instguide/ $(BOOKXSL) instguide.xml +$(HTML_DIR)/instguide/index.html: $(DOCBOOK_DIR)/instguide.xml $(BOOKXSL) + $(DBPROC) -o $(HTML_DIR)/instguide/ $(BOOKXSL) $< -ExceptionCodes.xml: ../ExceptionCodes.txt - perl ./generate_except_xml.pl +$(DOCBOOK_DIR)/ExceptionCodes.xml: ../ExceptionCodes.txt + perl tools/generate_except_xml.pl $< $@ manpages: $(MANXSL) man-dirs man-nroff man-html @@ -45,46 +51,49 @@ $(MANXSL): $(MANXSL).tmpl fi; \ sed -e "s,%%DOCBOOK%%,$${DOCBOOK}," $(MANXSL).tmpl > $(MANXSL) -man-dirs: man/.there $(HTMLPREFIX)/man-html/.there +man-dirs: man/.there $(HTML_DIR)/man-html/.there -$(HTMLPREFIX)/man-html/.there: - if [ ! -d $(HTMLPREFIX)/man-html ]; then mkdir -p $(HTMLPREFIX)/man-html; touch $(HTMLPREFIX)/man-html/.there; fi +$(HTML_DIR)/man-html/.there: + mkdir -p $(HTML_DIR)/man-html + touch $(HTML_DIR)/man-html/.there man/.there: - if [ ! -d man ]; then mkdir man; touch man/.there; fi + then mkdir -p man + touch man/.there NROFF_PAGES = bbackupd.8 bbackupd-config.8 bbackupctl.8 bbackupquery.8 \ bbstored.8 bbstored-config.8 bbstoreaccounts.8 bbstored-certs.8 \ raidfile-config.8 \ bbackupd.conf.5 bbstored.conf.5 raidfile.conf.5 -man-nroff: $(NROFF_PAGES) +NROFF_FILES = $(NROFF_PAGES:%=$(MAN_DIR)/%.gz) + +man-nroff: $(NROFF_FILES) + +HTML_FILES_1 = $(NROFF_PAGES:%.5=%.html) +HTML_FILES_2 = $(HTML_FILES_1:%.8=%.html) +HTML_FILES = $(HTML_FILES_2:%=$(HTML_DIR)/man-html/%) -HTML_PAGES = bbackupd.html bbackupd-config.html bbackupctl.html \ - bbackupquery.html bbstored.html bbstored-config.html \ - bbstoreaccounts.html bbstored-certs.html raidfile-config.html \ - bbackupd.conf.html bbstored.conf.html raidfile.conf.html +man-html: $(HTML_FILES) -man-html: $(HTML_PAGES) +$(HTML_DIR)/man-html/%.html: $(DOCBOOK_DIR)/%.xml $(NOCHUNKBOOKXSL) + $(DBPROC) -o $@ $(NOCHUNKBOOKXSL) $< -.xml.html: - @$(DBPROC) -o $@ $(NOCHUNKBOOKXSL) $< - @cp $@ $(HTMLPREFIX)/man-html/. +$(MAN_DIR)/%.8.gz: $(DOCBOOK_DIR)/%.xml $(MANXSL) + $(DBPROC) -o $(@:.gz=) $(MANXSL) $< + gzip $(@:.gz=) -.xml.8 .xml.5: $(MANXSL) - @$(DBPROC) -o $@ $(MANXSL) $< - @cp $@ man/ - @rm -f man/$@.gz - @gzip -f -9 man/$@ +$(MAN_DIR)/%.5.gz: $(DOCBOOK_DIR)/%.xml $(MANXSL) + $(DBPROC) -o $(@:.gz=) $(MANXSL) $< + gzip $(@:.gz=) dockit: clean docs - tar zcf documentation-kit-0.10.tar.gz $(HTMLPREFIX)/ + tar zcf documentation-kit-0.10.tar.gz $(HTML_DIR)/ clean: - if [ -d ./$(HTMLPREFIX) ]; then rm -rf $(HTMLPREFIX) ; fi - if [ -d ./man ]; then rm -rf ./man/; fi - if [ -f ExceptionCodes.xml ]; then rm ExceptionCodes.xml; fi - rm -f $(NROFF_PAGES) $(HTML_PAGES) - if [ -f documentation-kit-0.10.tar.gz ]; then rm documentation-kit-0.10.tar.gz; fi + rm -f $(HTML_FILES) + rm -f $(NROFF_FILES) + rm -f $(DOCBOOK_DIR)/ExceptionCodes.xml + rm -f documentation-kit-0.10.tar.gz diff --git a/docs/adminguide.xml b/docs/adminguide.xml deleted file mode 100644 index edb0a58c..00000000 --- a/docs/adminguide.xml +++ /dev/null @@ -1,1981 +0,0 @@ - - -]> - - Box Backup administrator's guide - - - License - - Copyright © 2003 - 2007, Ben Summers and contributors. All rights - reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - - - Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - - - Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - - - All use of this software and associated advertising materials - must display the following acknowledgement: This product includes - software developed by Ben Summers and contributors. - - - - The names of the Authors may not be used to endorse or promote - products derived from this software without specific prior written - permission. - - - - [Where legally impermissible the Authors do not disclaim liability - for direct physical injury or death caused solely by defects in the - software unless it is modified by a third party.] - - THIS SOFTWARE IS PROVIDED BY THE AUTHORS "AS IS" AND ANY EXPRESS OR - IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES - OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN - NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, - SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED - TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR - PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF - LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING - NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - - - Configuration - -
- System configuration - -
- Server - - After you've downloaded and compiled the programs you need to - install the programs on your server. As root do the following: - - make install-backup-server - - This assumes that you are installing on the same server that you - compiled the software on. If not, copy the - boxbackup-x.xx-backup-server-OSNAME.tgz file to the server you want to - run on, and install there. For example (on Mac OS X): - - tar zxvf boxbackup-0.10-server-darwin8.5.0.tgz -cd boxbackup-0.10-server-darwin8.5.0 -./install-backup-server - - Then create the user for the backup daemon on the server: - - useradd _bbstored - - Box Backup has a built-in software RAID facility (redundant - array of inexpensive disks) for the backup store. This allows you to - spread the store data over three disks, and recover from the loss of - any one disk without losing data. However, this is now deprecated, and - you are recommended to use the software or hardware RAID facilities of - your operating system instead. Use the following command if you want - to create a simple server without Box Backup RAID: - - mkdir /tmp/boxbackupRepository # Create the directory -chown _bbstored /tmp/boxbackupRepository/ # Change the owner to the new boxbackup daemon user - -/usr/local/sbin/raidfile-config /etc/box/ 1024 /tmp/boxbackupRepository - -#substitute 1024 with the desired blocksize -#substitute /tmp/boxbackupRepository with a directory that exists where you want the backup store located -#/usr/local/sbin/raidfile-config --help shows you the options - - Then create the configuration file /etc/box/bbstored.conf The - hostname is tricky as it is used for two things: The name of the - server in the certificate and the address the server is listening on. - Since you might be using NAT, might move the server around or the - domain name might change, choose a name that describes the server. - When the network address of the server changes, you need to update the - ListenAddresses directive in the - /etc/box/bbstored.conf file. - - /usr/local/sbin/bbstored-config /etc/box hostname _bbstored - - This last step outputs 5 instructions that you must execute to - the letter. A lot of questions are raised on the mailing list because - these steps have not been followed properly. - - TODO: Expand on this. Explain the 5 steps in detail. - - If you want to run the server as a non-root user, look here. -
- -
- Certificate Management - - There are two steps involved to create an account. You need to - create the account on the server, and sign a certificate to give the - client permission to connect to the server. - - Running a Certification Authority for TLS (SSL) connections is - not trivial. However, a script to does most of the work in a way which - should be good enough for most deployments. - - - The certificate authority directory is intended to be stored - on another server. It should not be kept on the backup server, in - order to limit the impact of a server compromise. The instructions - and the script assume that it will be kept elsewhere, so will ask - you to copy files to and from the CA. - - - - SSL certificates contain validity dates, including a "valid - from" time. If the clock on the machine which signs the certificates - is not syncronised to the clocks of the machines using these - certificates, you will probably get strange errors until the start - time is reached on all machines. If you get strange errors when - attempting to use new certificates, check the clocks on all machines - (client, store and CA). You will probably just need to wait a while - until the certificates become valid, rather than having to - regenerate them. - - -
- Set up a Certificate Authority - - It is recommended that you keep your Certificate Authority on - a separate machine than either the client or the server, preferably - without direct network access. The contents of this directory - control who can access your backup store server. - - To setup the basic key structure, do the following: - - /usr/local/sbin/bbstored-certs ca init - - (See OpenSSL notes if you - get an OpenSSL error) - - This creates the directory called ca in - the current directory, and initialises it with basic keys. -
- -
- Sign a server certificate - - When you use the bbstored-config script to - set up a config file for a server, it will generate a certificate - request (CSR) for you. Transfer it to the machine with your CA, then - do: - - /usr/local/sbin/bbstored-certs ca sign-server hostname-csr.pem - - This signs the certificate for the server. Follow the - instructions in the output on which files to install on the server. - The CSR file is now no longer needed. Make sure you run this command - from the directory above the directory 'ca'. - - TODO: Explain instructions in output. -
- -
- Set up an account - - Choose an account number for the user. This must be unique on - the server, and is presented as a 31 bit number in hex greater than - 0, for example, 1 or 75AB23C. Then on the backup store server, - create the account with: - - /usr/local/sbin/bbstoreaccounts create 75AB23C 0 4096M 4505M - - This looks complicated. The numbers are, in order: - - - - The account number allocated (hex) - - - - The RAID disc set (0 if you use raidfile-config and don't - add a new set) - - - - Soft limit (size) - - - - Hard limit (size) - - - - The sizes are are specified in Mb, Gb, or blocks, depending on - the suffix. 1M specifies 1 Mb, 1G specifies 1 Gb, and 1B specifies 1 - block, the size of which depends on how you have configured the - raidfile system with raidfile-config. - - In this example, I have allocated 4Gb (assuming you use 2048 - byte blocks as per my example) as the soft limit, and 4Gb + 10% as - the hard limit. - - NOTE The sizes specified here are pre-RAID. So if you are - using userland RAID, you are actually allocating two-thirds of this - amount. This means that, when you take compression into account, - that if you allocate 2Gb on the server, it'll probably hold about - 2Gb of backed up files (depending on the compressability of those - files). - - The backup client will (voluntarily) try not to upload more - data than is allowed by the soft limit. The store server will refuse - to accept a file if it would take it over the hard limit, and when - doing housekeeping for this account, try and delete old versions and - deleted files to reduce the space taken to below the soft - limit. - - This command will create some files on disc in the raid file - directories (if you run as root, the utility will change to the user - specified in the bbstored.conf file to write them) and update the - accounts file. A server restart is not required. - - NOTE If you get a message saying 'Exception: RaidFile (2/8)', - the directories you specified in the raidfile.conf are not writable - by the _bbstored user -- fix it, and try again. - - Finally, tell the user their account number, and the hostname - of your server. They will use this to set up the backup client, and - send you a CSR. This has the account number embedded in it, and you - should be sure that it has the right account number in it. - - Sign this CSR with this command: - - /usr/local/sbin/bbstored-certs ca sign 75AB23C-csr.pem - - Don't forget to check that the embedded account number is - correct! Then send the two files back to the user, as instructed by - the script. - - Please read the Troubleshooting page if you have - problems. - - TODO: Link to troubleshooting... -
-
- -
- Log Files - - You may wish to see what's going on with the server. Edit - /etc/syslog.conf, and add: - - local6.info /var/log/box -local5.info /var/log/raidfile - - Note: Separators must be tabs, - otherwise these entries will be ignored. - - touch /var/log/box -touch /var/log/raidfile - - Set up log rotation for these new log files. For example, if you - have /etc/newsyslog.conf, add the following lines - to it: - - /var/log/box 644 7 2000 * Z -/var/log/raidfile 644 7 2000 * Z - - If you have /etc/logrotate.d, create a new - file in there (for example - /etc/logrotate.d/boxbackup) containing the - following: - - /var/log/box /var/log/raidfile { - weekly - create - compress - rotate 52 -} - - Then restart syslogd, for example: - - /etc/init.d/syslogd restart -
- -
- Configuring a client - - Before you can do any configuration, you need to know the - hostname of the server you will be using, and your account number on - that server. - - Later in the process, you will need to send a certificate - request to the administrator of that server for it to be - signed. - - Installation is covered in the compiling and installing section. - You only need the backup-client parcel. - - It is important that you read all the output of the config - scripts. See the end of this page for an example. - - The backup client has to be run as root, because it needs to - read all your files to back them up, although it is possible to back - up a single user's files by running it as that user. (Tip: specify a - directory other than /etc/box, and then give the - alternate config file as the first argument to - bbackupd). However, it will fall over if you don't - give yourself read access to one of your files. - -
- Basic configuration - - Run the bbackupd-config script to generate - the configuration files and generate a private key and certificate - request. - - /usr/local/sbin/bbackupd-config /etc/box lazy 999 hostname /var/bbackupd /home - - (See OpenSSL notes if you - get an OpenSSL error) - - The items in bold need to be changed. In order, they are the - account number, the hostname of the server you're using, and - finally, the directories you want backed up. You can include as many - you want here. - - However, the directories you specify must not contain other - mounted file systems within them at any depth. Specify them - separately, one per mount point. No checks are currently made to - catch bad configuration of this nature! - - You may also want to consider changing the mode from lazy to - snapshot, depending on what your system is used for: - - - - Lazy Mode - - - This mode regularly scans the files, with only a rough - schedule. It uploads files as and when they are changed, if - the latest version is more than a set age. This is good for - backing up user's documents stored on a server, and spreads - the load out over the day. - - - - - Snapshot Mode - - - This mode emulates the traditional backup behaviour of - taking a snapshot of the filesystem. The backup daemon does - absolutely nothing until it is instructed to make a backup - using the bbackupctl utility (probably as a cron job), at - which point it uploads all files which have been changed since - the last time it uploaded. - - - - - When you run the config script, it will tell you what you need - to do next. Don't forget to read all the output. An example is shown - at the end of this page, but the instructions for your installation - may be different. -
- -
- Certificates - - After you have sent your certificate request off to the server - administrator and received your certificate and CA root back, - install them where instructed by the bbackupd-config script during - basic bbackupd configuration. - - You can then run the daemon (as root) by running - /usr/local/sbin/bbackupd, and of course, adding it - to your system's startup scripts. The first time it's run it will - upload everything. Interrupting it and restarting it will only - upload files which were not uploaded before - it's very - tolerant. - - If you run in snapshot mode, you will need to add a cron job - to schedule backups. The config script will tell you the exact - command to use for your system. - - Please read the Troubleshooting page if you have - problems. - - Remember to make a traditional backup of the keys file, as - instructed. You cannot restore files without it. - - It is recommended that you backup up all of /etc/box as it - will make things easier if you need to restore files. But only the - keys are absolutely essential. - - If you want to see what it's doing in more detail (probably a - good idea), follow the instructions in the server setup to create - new log files with syslog. -
- -
- Adding and removing backed up locations - - By editing the /etc/box/bbackupd.conf file, you can add and - remove directories to back up - see comments in this file for help. - Send bbackupd a HUP signal after you modify it. - - When you remove a location, it will not be marked as deleted - immediately. Instead, bbackupd waits about two days before doing so, - just in case you change your mind. After this, it will be eventually - removed from the store by the housekeeping process. Run as - root. - - The backup client is designed to be run as root. It is - possible to run without root, but this is not recommended. Clock - synchronisation for file servers. - - If you are using the backup client to backup a filesystem - served from a fileserver, you should ideally ensure that the - fileserver clocks are synchronised with the fileserver. - - bbackupd will cope perfectly well if the clocks are not - synchronised. Errors up to about half an hour cause no problems. - Larger discrepancies cause a loss of efficiency and the potential to - back up a file during a write process. - - There is a configuration parameter MaxFileTimeInFuture, which - specifies how far in the future a file must be for it to be uploaded - as soon as it is seen. You should not need to adjust this (default - is 2 days). Instead, get those clocks synchronised. Excluding files - and directories from the backup. - - Within the bbackupd.conf file, there is a section named - BackupLocations which specifies which locations on disc should be - backed up. It has subsections, each of which is in the - format: - - name - { - Path = /path/of/directory - (optional exclude directives) - } - - name is derived from the Path - by the config script, but should merely be unique. - - The exclude directives are of the form: - - [Exclude|AlwaysInclude][File|Dir][|sRegex] = regex or full pathname - - (The regex suffix is shown as 'sRegex' to make File or Dir - plural) - - For example: - - ExcludeDir = /home/guest-user - ExcludeFilesRegex = *.(mp3|MP3)\$ - AlwaysIncludeFile = /home/username/veryimportant.mp3 - - This excludes the directory /home/guest-user from the backup - along with all mp3 files, except one MP3 file in particular. - - In general, Exclude excludes a file or directory, unless the - directory is explicitly mentioned in a AlwaysInclude - directive. - - If a directive ends in Regex, then it is a regular expression - rather than a explicit full pathname. See - - man 7 re_format - - for the regex syntax on your platform. -
- -
- Example configuration output - - This is an example of output from the bbstored-config - script. - - - Follow the instructions output by your script, not the ones - here -- they may be different for your system. - - - /usr/local/sbin/bbackupd-config /etc/box lazy 51 server.example.com /var/bbackupd /home /etc/samba - -Setup bbackupd config utility. - -Configuration: - Writing configuration file: /etc/box/bbackupd.conf - Account: 51 - Server hostname: server.example.com - Directories to back up: - /home - /etc/samba - -Note: If other file systems are mounted inside these directories, then problems may occur -with files on the store server being renamed incorrectly. This will cause efficiency -problems, but not affect the integrity of the backups. - -WARNING: Directories not checked against mountpoints. Check mounted filesystems manually. - -Creating /etc/box... -Creating /etc/box/bbackupd -Generating private key... - [OpenSSL output omitted] - -Generating keys for file backup -Writing notify script /etc/box/bbackupd/NotifyStoreFull.sh -Writing configuration file /etc/box/bbackupd.conf - -=================================================================== - -bbackupd basic configuration complete. - -What you need to do now... - -1) Make a backup of /etc/box/bbackupd/51-FileEncKeys.raw - This should be a secure offsite backup. - Without it, you cannot restore backups. Everything else can - be replaced. But this cannot. - KEEP IT IN A SAFE PLACE, OTHERWISE YOUR BACKUPS ARE USELESS. - -2) Send /etc/box/bbackupd/51-csr.pem - to the administrator of the backup server, and ask for it to - be signed. - -3) The administrator will send you two files. Install them as - /etc/box/bbackupd/51-cert.pem - /etc/box/bbackupd/serverCA.pem - after checking their authenticity. - -4) You may wish to read the configuration file - /etc/box/bbackupd.conf - and adjust as appropraite. - - There are some notes in it on excluding files you do not - wish to be backed up. - -5) Review the script - /etc/box/bbackupd/NotifyStoreFull.sh - and check that it will email the right person when the store - becomes full. This is important -- when the store is full, no - more files will be backed up. You want to know about this. - -6) Start the backup daemon with the command - /usr/local/sbin/bbackupd - in /etc/rc.local, or your local equivalent. - Note that bbackupd must run as root. - -=================================================================== - - Remember to make a secure, offsite backup of your backup keys, - as described in Basic - configuration above. If you do not, and that key is lost, you - have no backups. -
-
- -
- Configuration Options - - Box Backup has many options in its configuration file. We will - try to list them all here. - - First of all, here is an example configuration file, for - reference: - - - Example Configuration File - - StoreHostname = localhost -AccountNumber = 0x2 - -KeysFile = /etc/box/2-FileEncKeys.raw -CertificateFile = /etc/box/2-cert.pem -PrivateKeyFile = /etc/box/2-key.pem -TrustedCAsFile = /etc/box/serverCA.pem -DataDirectory = /var/run/boxbackup -NotifyScript = /etc/box/NotifySysadmin.sh -CommandSocket = /var/run/box/bbackupd.sock - -UpdateStoreInterval = 86400 -MinimumFileAge = 3600 -MaxUploadWait = 7200 -FileTrackingSizeThreshold = 65536 -DiffingUploadSizeThreshold = 65536 -MaximumDiffingTime = 20 -ExtendedLogging = no -LogAllFileAccess = yes - -Server -{ - PidFile = /var/run/bbackupd.pid -} -BackupLocations -{ - etc - { - Path = /etc - } - home - { - Path = /home - ExcludeDir = /home/shared - ExcludeDir = /home/chris/.ccache - ExcludeDir = /home/chris/.mozilla/firefox/vvvkq3vp.default/Cache - } -} - - - As you can see from the example above, the configuration file - has a number of subsections, enclosed in curly braces {}. Some options - appear outside of any subsection, and we will refer to these as root options. The available options in - each section are described below. - - Every option has the form name = value. Names are - not case-sensitive, but values are. Depending on the option, the value - may be: - - - - a path (to a file or directory); - - - - a number (usually in seconds or bytes); - - - - a boolean (the word Yes or No); - - - - a hostname (or IP address). - - - - Paths are specified in native format, i.e. a full Windows path - with drive letter on Windows clients, or a full Unix path on Unix - clients. - - - Example: - - StoreObjectInfoFile = - /var/state/boxbackup/bbackupd.dat - - StoreObjectInfoFile = C:\Program Files\Box - Backup\data\bbackupd.dat - The use of relative paths (which do not start with a - forward slash on Unix, or a drive specification on Windows) is - possible but not recommended, since they are interpreted relative to - the current working directory when bbackupd was started, which is - liable to change unexpectedly over time. - - Numbers which start with "0x" are interpreted as hexadecimal. - Numbers which do not start with "0x" are interpreted as - decimal. - -
- Root Options - - These options appear outside of any subsection. By convention - they are at the beginning of the configuration file. - - Some options are required, and some are optional. - - - - StoreHostname (required) - - - The Internet host name (DNS name) or IP address of the - server. This is only used to connect to the server. - - - - - AccountNumber (required) - - - The number of the client's account on the server. This - must be provided by the server operator, and must match the - account number in the client's certificate, otherwise the - client will not be able to log into the server. - - The account number may be specified in hexadecimal - (starting with 0x, as in the example above) or in decimal, but - since the server operator works in hexadecimal, that format is - highly recommended and is the default. - - - - - KeysFile (required) - - - The path to the file containing the encryption key used - for data encryption of client file data and filenames. This is - the most important file to keep safe, since without it your - backups cannot be decrypted and are useless. Likewise, if an - attacker gets access to this key and to your encrypted - backups, he can decrypt them and read all your data. - - Do not change the encryption key without deleting all - files from the account on the server first. None of your old - files on the store will be readable if you do so, and if you - change it back, none of the files uploaded with the new key - will be readable. - - - - - CertificateFile (required) - - - The path to the OpenSSL client certificate in PEM - format. This is supplied by the server operator in response to - the certificate request which you send to them. Together with - the PrivateKeyFile, this provides access to the store server - and the encrypted data stored there. - - It is not critical to protect this file or to back it up - safely, since it can be regenerated by creating a new - certificate request, and asking the server operator to sign - it. You may wish to back it up, together with the - PrivateKeyFile, to avoid this inconvenience if you lose all - your data and need quick access to your backups. - - If you do back them up, you should keep them in a - separate location to the KeysFile, since any person holding - the KeysFile and the PrivateKeyFile can gain access to your - encrypted data and decrypt it. - - - - - PrivateKeyFile (required) - - - The path to the OpenSSL private key in PEM format. This - is generated at the same time as the certificate request, but - there is no need to send it to the server operator, and you - should not do so, in case the communication is intercepted by - an attacker. Together with the CertificateFile, this provides - access to the store server and the encrypted data stored - there. - - See the notes under CertificateFile for information - about backing up this file. - - - - - TrustedCAsFile (required) - - - The path to the OpenSSL certificate of the Client - Certificate Authority (CCA), in PEM format. This is supplied - by the server operator along with your account details, or - along with your signed client certificate. This is used to - verify that the server which you are connecting to is - authorised by the person who signed your certificate. It - protects you against DNS and ARP poisoning and IP spoofing - attacks. - - - - - DataDirectory (required) - - - The path to a directory where bbackupd will keep local - state information. This consists of timestamp files which - identify the last backup start and end times, used by - bbackupquery to determine whether files - have changed, and optionally a database of inode numbers, - which are used to check for files being renamed. The database - is only saved if Box Backup is built with Berkeley Database - (BDB) support. - - - - - NotifyScript (optional) - - - The path to the script or command to run when the Box - Backup client detects an error during the backup process. This - is normally used to notify the client system administrator by - e-mail when a backup fails for any reason. - - The script or command is called with one of the - following additional arguments to identify the cause of the - problem: - - - - store-full - - - The backup store is full. No new files are being - uploaded. If some files are marked as deleted, they - should be removed in due course by the server's - housekeeping process. Otherwise, you need to remove some - files from your backup set, or ask the store operator - for more space. - - - - - read-error - - - One or more files which were supposed to be backed - up could not be read. This could be due to: - - running the server as a non-root user; - - - - backing up a mounted filesystem such as - NFS; - - - - access control lists being applied to some - files; - - - - SELinux being enabled; - - - - trying to back up open files under - Windows; - - - - strange directory permissions such as 0000 or - 0400. - - Check the client logs, e.g. - /var/log/bbackupd on Unix, or the Windows Event Viewer - in Control Panel > Administrative Tools, for more - information about which files are not being backed up - and why. - - - - - backup-error - - - There was a communications error with the server, - or an unexpected exception was encountered during a - backup run. Check the client logs, e.g. - /var/log/box on Unix, or the - Windows Event Viewer in Control Panel > - Administrative Tools, for more information about the - problem. - - You may wish to check your Internet access to the - server, check that the server is running, and ask your - server operator to check your account on the - server. - - - - - - - - CommandSocket (optional) - - - The path to the Unix socket which - bbackupd creates when running, and which - bbackupctl uses to communicate with it, for - example to force a sync or a configuration reload. If this - option is omitted, no socket will be created, and - bbackupctl will not function. - - Unix sockets appear within the filesystem on Unix, as a - special type of file, and must be created in a directory which - exists and to which bbackupd has write access, and bbackupctl - has read access. - - On Windows, the path is ignored, and a named - pipe is created instead. This does not currently - have any security attached, so it can be accessed by any user. - Unlike a Unix socket it can also be accessed remotely. Please - use this option with extreme caution on Windows, and only on - fully trusted networks. - - - - - AutomaticBackup (optional) - - - Enable or disable the client from connecting - automatically to the store every - UpdateStoreInterval seconds. When - enabled (set to Yes), the client is in - Lazy Mode. When disabled (set to - No), it is in Snapshot - Mode. This setting is optional, and the default - value is Yes. - - - - - UpdateStoreInterval (required) - - - The approximate time between successive connections to - the server, in seconds, when the client is in Lazy - Mode. The actual time is randomised slightly to - prevent "rush hour" traffic jams on the server, where many - clients try to connect at the same time. - - This value is ignored if the client is in - Snapshot Mode. However, it is still - required. It can be set to zero in this case. - - You will probably need to experiment with the value of - this option. A good value to start with is probably 86400 - seconds, which is one day. - - - - - MinimumFileAge (required) - - - The number of seconds since a file was last modified - before it will be backed up. The reason for this is to avoid - repeatedly backing up files which are repeatedly changing. A - good value is about 3600 seconds (one hour). If set to zero, - files which have changed will always be backed up on the next - backup run. - - The MaxUploadWait option - overrides this option in some circumstances. - - - - - MaxUploadWait (required) - - - The number of seconds since a file was last uploaded - before it will be uploaded again, even if it keeps changing. - The reason for this is to ensure that files which are - continuously modified are eventually uploaded anyway. This - should be no less than the value of - MinimumFileAge. A good value is about - 14400 seconds (4 hours). - - - - - MaxFileTimeInFuture (optional) - - - The maximum time that a file's timestamp can be in the - future, before it will be backed up anyway. Due to clock - synchronisation problems, it is inevitable that you will - occasionally see files timestamped in the future. Normally, - for files which are dated only slightly in the future, you - will want to wait until after the file's date before backing - it up. However, for files whose dates are very wrong (more - than a few hours) you will normally prefer to back them up - immediately. - - A good value is about 7200 seconds (2 hours) to cope - with potential problems when moving in and out of daylight - saving time, if applicable in your timezone. The default - value, if this setting is not provided, is 172800 seconds (2 - days). - - - - - FileTrackingSizeThreshold (required) - - - The minimum size of files which will be tracked by inode - number to detect renames. It is not worth detecting renames of - small files, since they are quick to upload again in full, and - keeping their inode numbers in memory increases the client's - memory usage and slows down searches. Larger files should be - tracked to avoid wasting space on the store and long - uploads. - - A good value is about 65536 bytes (64 kilobytes). - - - - - DiffingUploadSizeThreshold (required) - - - The minimum size of files which will be compared to the - old file on the server, and for which only changes will be - uploaded. It is not worth comparing small files, since they - are quick to upload again in full, and sending the entire file - reduces the risk of data loss if the store is accidentally - corrupted. Larger files should have only their differences - uploaded to avoid wasting space on the store and long - uploads. - - A good value is about 65536 bytes (64 kilobytes). - - - - - MaximumDiffingTime (optional) - - - The maximum time for which the client will attempt to - find differences between the current version and the old - version in the store, before giving up and uploading the - entire file again. Very large files (several gigabytes) may - take a very long time to scan for changes, but would also take - a very long time to upload again and use a lot of space on the - store, so it is normally worth omitting this value. - - Use this option only if, for some bizarre reason, you - prefer to upload really large files in full rather than spend - a long time scanning them for changes. - - - - - KeepAliveTime (optional) - - - The interval (in seconds) between sending Keep-Alive - messages to the server while performing long operations such - as finding differences in large files, or scanning large - directories. - - These messages ensure that the SSL connection is not - closed by the server, or an intervening firewall, due to lack - of activity. - - The server will normally wait up to 15 minutes (900 - seconds) before disconnecting the client, so the value should - be given and should be less than 900. Some firewalls may time - out inactive connections after 10 or 5 minutes. - - A good value is 300 seconds (5 minutes). You may need to - reduce this if you frequently see TLSReadFailed or - TLSWriteFailed errors on the client. - - - - - StoreObjectInfoFile (optional) - - - Enables the use of a state file, which stores the - client's internal state when the client is not running. This - is useful on clients machines which are frequently shut down, - for example desktop and laptop computers, because it removes - the need for the client to recontact the store and rescan all - directories on the first backup run, which may take some time. - This feature is somewhat experimental and not well tested. - - - This is option is disabled by default, in which case the - state is stored in memory only. The value is the path to the - state file. - - - - - ExtendedLogging (optional) - - - Enables the connection debugging mode of the client, - which writes all commands sent to or received from the server - to the system logs. This generates a lot - of output, so it should only be used when instructed, or when - you suspect a connection problem or client-server protocol - error (and you know how to interpret the output). - - This is a boolean value, which may be set to - Yes or No. The default is of - course No. - - - - - ExtendedLogFile (optional, new in 0.11) - - - Enables the same debugging output as - ExtendedLogging, but written to a file - instead of the system logs. This is useful if you need - extended logging, but do not have access to the system logs, - for example if you are not the administrator of the - computer. - - The value is the path to the file where these logs will - be written. If omitted, extended logs will not be written to a - file. This is entirely independent of the - ExtendedLogging option. It does not - make much sense to use both at the same time. - - - - - LogAllFileAccess (optional, new in 0.11) - - - Enables logging of all local file and directory access, - file uploads (full and differential), and excluded files. This - may be useful if the client is failing to upload a particular - file, or crashing while trying to upload it. The logs will be - sent to the system log or Windows Event Viewer. - - This generates a lot - of output, so it should only be used when instructed, or when - you suspect that bbackupd is skipping some files and want to - know why. Because it is verbose, the messages are hidden by - default even if the option is enabled. To see them, you must - run bbackupd with at least one -v option. - - This is a boolean value, which may be set to - Yes or No. The default is of - course No. - - - - - SyncAllowScript (optional) - - - The path to the script or command to run when the client - is about to start an automatic backup run, and wishes to know - whether it is safe to do so. This is useful for clients which - do not always have access to the server, for example laptops - and computers on dial-up Internet connections. - - The script should either output the word - now if the backup should proceed, or else a - number, in seconds, which indicates how long the client should - wait before trying to connect again. Any other output will - result in an error on the client, and the backup will not - run. - - This value is optional, and by default no such script is - used. - - - -
- -
- Server Section - - These options appear within the Server subsection, which is at - the root level. - - - - PidFile - - - This option enables the client to write its processs - identifier (PID) to the specified file after starting. The - file will be deleted when the client daemon exits for any - reason. This is disabled by default, but is recommended - whenever you run the client daemon as a daemon (in the - background), which is usually the case. This file can be used - by scripts to determine whether the daemon is still running, - and to send it messages to reload its configuration or to - terminate. - - - Example Server Section - - Server -{ - PidFile = /var/state/boxbackup/bbackupd.pid -} - - - - -
- -
- Backup Locations Section - - This section serves only as a container for all defined backup - locations. - - - Example Backup Locations Section - - BackupLocations -{ - etc - { - Path = /etc - } - home - { - Path = /home - ExcludeDir = /home/shared - ExcludeDir = /home/chris/.ccache - ExcludeDir = /home/chris/.mozilla/firefox/vvvkq3vp.default/Cache - } -} - - - Each subsection is a backup location. The name of the - subsection is the name that will be used on the server. The root - directory of the account on the server contains one subdirectory per - location. The name should be simple, not containing any spaces or - special characters. - - If you do not define any locations, the client will not back - up any files! - - It is currently not recommended to back up the root directory - of the filesystem on Unix. Box Backup is designed to back up - important data and configuration files, not full systems. - Nevertheless, nothing prevents you from doing so if you - desire. - - On Windows, it is currently not possible to back up files - which are open (currently in use), such as open documents in - Microsoft Office, and system files such as the registry and the - paging file. You will get an error for each open file which the - client attempts to back up. Once the file has been closed, it will - be backed up normally. System files will always be open, and should - be excluded from your backups. -
-
-
- - - - Administration - - This chapter deals with the dauily running and management of the Box - Backup system. It explains most day-to-day tasks. - -
- Regular Maintenance - - The steps involved in maintaining and keeping the backup sets - healthy are outlined in this section. - -
- Controlling a backup client - - The bbackupctl program sends control commands to the bbackupd - daemon. It must be run as the same user as the daemon, and there is no - exception for root. - - The command line syntax is: - - /usr/local/sbin/bbackupctl [-q] [-c config-file] command - - The -q option reduces the amount of output the program emits, - and -c allows an alternative configuration file to be - specified. - - Valid commands are: - - - - terminate - - Stop the bbackupd daemon now (equivalent to kill) - - - - reload - - Reload the configuration file (equivalent to kill - -HUP) - - - - sync - - Connect to the server and synchronise files now - - - - bbackupctl communicates with - the server via a UNIX domain socket, specified in bbackupd.conf with - the CommandSocket directive. This does not need to be specified, and - bbackupd will run without the command - socket, but in this case bbackupctl will not be able to communicate - with the daemon. - - Some platforms cannot check the user id of the connecting - process, so this command socket becomes a denial of service security - risk. bbackupd will warn you when it - starts up if this is the case on your platform, and you should - consider removing the CommandSocket directive on these - platforms. -
- -
- Using bbackupctl to perform snapshots - - bbackupctl's main purpose is to - implement snapshot based backups, emulating the behaviour of - traditional backup software. - - Use bbackupd-config to write a configuration file in snapshot - mode, and then run the following command as a cron job. - - /usr/local/sbin/bbackupctl -q sync - - This will cause the backup daemon to upload all changed files - immediately. bbackupctl will exit - almost immediately, and will not output anything unless there is an - error. -
- -
- Checking storage space used on the server - -
- From the client machine - - bbackupquery can tell you how much space is used on the server - for this account. Either use the usage command in interactive mode, - or type: - - /usr/local/sbin/bbackupquery -q usage quit - - to show the space used as a single command. -
- -
- On the server - - bbstoreaccounts allows you to query the space used, and change - the limits. To display the space used on the server for an account, - use: - - /usr/local/sbin/bbstoreaccounts info 75AB23C - - To adjust the soft and hard limits on an account, use: - - /usr/local/sbin/bbstoreaccounts setlimit 75AB23C new-soft-limit new-hard-limit - - You do not need to restart the server. -
-
- -
- Verify and restore files - - Backups are no use unless you can restore them. The bbackupquery - utility does this and more. - - You don't provide any login information to it, as it just picks - up the data it needs from /etc/box/bbackupd.conf. You should run it as - root so it can find everything it needs. - - Full documentation can be found in the bbackupquery manual page. It follows - the model of a command line sftp client quite closely. - - TODO: Link to bbackupquery man-page here. - - On systems where GNU readline is available (by default) it uses - that for command line history and editing. Otherwise it falls back to - very basic UNIX text entry. - - TODO: Did the readline dependency change to editline? - -
- Using bbackupquery - - bbackupquery is the tool you use to verify, restore and - investigate your backup files with. When invoked, it simply logs - into the server using the certificates you have listed in - bbackupd.conf. - - After you run bbackupquery, you will see a prompt, allowing - you to execute commands. The list (or ls) command lets you view - files in the store. It works much like unix ls, but with different - options. An example: - - [pthomsen@host bbackupquery]$ bbackupquery -Box Backup Query Tool v0.10, (c) Ben Summers and contributors 2003-2006 -Using configuration file /etc/box/bbackupd.conf -Connecting to store... -Handshake with store... -Login to store... -Login complete. - -Type "help" for a list of commands. - -query > ls -00000002 -d---- mp3 -00000003 -d---- video -00000004 -d---- home-pthomsen -00000005 -d---- root -query > - - The ls commands shows the directories that are backed up. Now - we'll take a closer look at the home-pthomsen directory: - - query > cd home-pthomsen -query > ls -00002809 f----- sample.tiff -0000280a f----- s3.tiff -0000280b f----- s4.tiff -0000280d f----- s2.tiff -0000280e f----- foo.pdf -0000286c f----- core.28720 -0000339a -d---- .emacs.d -0000339d -d---- bbackup-contrib -00003437 f----- calnut.compare.txt -0000345d f----- DSCN1783.jpg -0000345e f----- DSCN1782.jpg -query > - - The ls command takes the following options; - - - - -r -- recursively list - all files - - - - -d -- list deleted - files/directories - - - - -o -- list old versions - of files/directories - - - - -I -- don't display - object ID - - - - -F -- don't display - flags - - - - -t -- show file - modification time (and attr mod time if has the object has - attributes, ~ separated) - - - - -s -- show file size in - blocks used on server (only very approximate indication of size - locally) - - - - The flags displayed from the ls command are as follows: - - - f = file - - d = directory - - X = deleted - - o = old version - - R = remove from server as soon as marked deleted or - old - - a = has attributes stored in directory record which - override attributes in backup file - -
- -
- Verify backups - - As with any backup system, you should frequently check that - your backups are working properly by comparing them. Box Backup - makes this very easy and completely automatic. All you have to do is - schedule the bbackupquery compare command to run - regularly, and check its output. You can run the command manually as - follows: - - /usr/local/sbin/bbackupquery "compare -a" quit - - This command will report all the differences found between the - store and the files on disc. It will download everything, so may - take a while. You should expect to see some differences on a typical - compare, because files which have recently changed are unlikely to - have been uploaded yet. It will also tell you how many files have - been modified since the last backup run, since these will normally - have changed, and such failures are expected. - - You are strongly recommended to add this command as a - cron job, at least once a month, and to check the - output for anything suspicious, particularly a large number of - compare failures, failures on files that have not been modified, or - any error (anything except a compare mismatch) that occurs during - the compare operation. - - Consider keeping a record of these messages and comparing them - with a future verification. - - If you would like to do a "quick" check which just downloads - file checksums and compares against that, then run: - - /usr/local/sbin/bbackupquery "compare -aq" quit - - However, this does not check that the file attributes are - correct, and since the checksums are generated on the client they - may not reflect the data on the server if there is a problem -- the - server cannot check the encrypted contents. View this as a quick - indication, rather than a definite check that your backup verifies - correctly. -
- -
- Restore backups - - You will need the keys file created when you configured the - server. Without it, you cannot restore the files; this is the - downside of encrypted backups. However, by keeping the small keys - file safe, you indirectly keep your entire backup safe. - - The first step is to recreate the configuration of the backup - client. It's probably best to have stored the /etc/box directory - with your keys. But if you're recreating it, all you really need is - to have got the login infomation correct (ie the certs and - keys). - - Don't run bbackupd yet! It will mark all your files as deleted - if you do, which is not hugely bad in terms of losing data, just a - major inconvenience. (This assumes that you are working from a blank - slate. If you want to restore some files to a different location, - it's fine to restore while bbackupd is running, just do it outside a - backed up directory to make sure it doesn't start uploading the - restored files.) - - Type: - - /usr/local/sbin/bbackupquery - - to run it in interactive mode. - - Type: - - list - - to see a list of the locations stored on the server. - - For each location you want to restore, type: - - restore name-on-server local-dir-name - - The directory specified by local-dir-name must not exist yet. - If the restore is interrupted for any reason, repeat the above - steps, but add the -r flag to the - restore command to tell it to resume. -
- -
- Retrieving deleted and old files - - Box Backup makes old versions of files and files you have - deleted available, subject to there being enough disc space on the - server to hold them. - - This is how to retrieve them using bbackupquery. Future - versions will make this far more user-friendly. - - Firstly, run bbackupquery in interactive mode. It behaves in a - similar manner to a command line sftp client. - - /usr/local/sbin/bbackupquery - - Then navigate to the directory containing the file you want, - using list, cd and pwd. - - query > cd home/profiles/USERNAME - - List the directory, using the "o" option to list the files - available without filtering out everything apart from the current - version. (if you want to see deleted files as well, use list - -odt) - - query > list -ot -00000078 f--o- 2004-01-21T20:17:48 NTUSER.DAT -00000079 f--o- 2004-01-21T20:17:48 ntuser.dat.LOG -0000007a f--o- 2004-01-21T17:55:12 ntuser.ini -0000007b f---- 2004-01-12T15:32:00 ntuser.pol -0000007c -d--- 1970-01-01T00:00:00 Templates -00000089 -d--- 1970-01-01T00:00:00 Start Menu -000000a0 -d--- 1970-01-01T00:00:00 SendTo -000000a6 -d--- 1970-01-01T00:00:00 Recent -00000151 -d--- 1970-01-01T00:00:00 PrintHood -00000152 -d--- 1970-01-01T00:00:00 NetHood -00000156 -d--- 1970-01-01T00:00:00 My Documents -0000018d -d--- 1970-01-01T00:00:00 Favorites -00000215 -d--- 1970-01-01T00:00:00 Desktop -00000219 -d--- 1970-01-01T00:00:00 Cookies -0000048b -d--- 1970-01-01T00:00:00 Application Data -000005da -d--- 1970-01-01T00:00:00 UserData -0000437e f--o- 2004-01-24T02:45:43 NTUSER.DAT -0000437f f--o- 2004-01-24T02:45:43 ntuser.dat.LOG -00004380 f--o- 2004-01-23T17:01:29 ntuser.ini -00004446 f--o- 2004-01-24T02:45:43 NTUSER.DAT -00004447 f--o- 2004-01-24T02:45:43 ntuser.dat.LOG -000045f4 f---- 2004-01-26T15:54:16 NTUSER.DAT -000045f5 f---- 2004-01-26T15:54:16 ntuser.dat.LOG -000045f6 f---- 2004-01-26T16:54:31 ntuser.ini - - (this is a listing from a server which is used as a Samba - server for a network of Windows clients.) You now need to fetch the - file using it's ID, rather than it's name. The ID is the hex number - in the first column. Fetch it like this: - - query > get -i 0000437e NTUSER.DAT -Object ID 0000437e fetched successfully. - - The object is now available on your local machine. You can use - lcd to move around, and sh ls to list directories on your local - machine. -
-
-
- -
- Fixing corruptions of store data - - This section gives help on what to do if your server has suffered - corruption, for example, after an unclean shutdown or other operating - system or hardware problem. - - In general, as updates to the store are made in an atomic manner, - the most likely result is wasted disc space. However, if really bad - things happen, or you believe that there is a lot of wasted space, then - these instructions will help to restore your data. - - You know you will need to do something if you get strange errors, - and bbackupd attempts to contact the server every 100 seconds or so. Or - if one of the discs in your RAID disc set has failed. - - After following these instructions, the end result will be that - bbackupquery will be able to see all the files which were stored on your - server, and retrieve them. Some of them may be in lost+found directories - in the root of the store (or in their original position if they have - been moved) but they will all be able to be retrieved. - - After you have retrieved the files you want, bbackupd will upload - new versions where necessary, and after about two days, mark any - lost+found directories as deleted. Finally, those directories will be - removed by the housekeeping process on the server. - - These instructions assume you're working on account 1234. Replace - this with the account number that you actually want to check (the one - that is experiencing errors). These steps will need to be repeated for - all affected accounts. - -
- Stop bbackupd - - First, make sure that bbackupd is not running on the client - machine for the account you are going to recover. Use - bbackupctl terminate to stop it. This step is not - strictly necessary, but is recommended. During any checks on the - account, bbackupd will be unable to log in, and after they are - complete, the account is marked as changed on the server so bbackupd - will perform a complete scan. -
- -
- Are you using RAID on the server? - - The raidfile recovery tools have not been written, and probably - will not be, since Box Backup RAID is deprecated. However, when two - out of three files are available, the server will successfully allow - access to your data, even if it complains a lot in the logs. The best - thing to do is to fix the accounts, if necessary, and retrieve any - files you need. Then move the old store directories aside (in case you - need them) and start afresh with new accounts, and let the clients - upload all their data again. -
- -
- Check and fix the account - - First, run the check utility, and see what errors it - reports. - - /usr/local/sbin/bbstoreaccounts check 1234 - - This will take some time, and use a fair bit of memory (about 16 - bytes per file and directory). If the output looks plausible and - reports errors which need fixing, run it again but with the fix - flag: - - /usr/local/sbin/bbstoreaccounts check 1234 fix - - This will fix any errors, and remove unrecoverable files. - Directories will be recreated if necessary. - - NOTE: The utility may adjust - the soft and hard limits on the account to make sure that housekeeping - will not remove anything -- check these afterwards. -
- -
- Grab any files you need with bbackupquery - - At this point, you will have a working store. Every file which - was on the server, and wasn't corrupt, will be available. - - On the client, use bbackupquery to log in and examine the store. - (type help at the prompt for instructions). Retrieve any files you - need, paying attention to any lost+found directories in the root - directory of the store. - - You can skip this step if you are sure that the client machine - is fine -- in this case, bbackupd will bring the store up to - date. -
- -
- Restart bbackupd - - Restart bbackupd on the client machine. The store account will - be brought up to date, and files in the wrong place will be marked for - eventual deletion. -
-
- -
- Troubleshooting - - If you are trying to fix a store after your disc has been - corrupted, see Fixing corruptions of - store data. - - Unfortunately, the error messages are not particularly helpful at - the moment. This page lists some of the common errors, and the most - likely causes of them. - - When an error occurs, you will see a message like 'Exception: - RaidFile/OSFileError (2/8)' either on the screen or in your log files. - (it is recommended you set up another log file as recommended in the - server setup instructions.) - - This error may not be particularly helpful, although some do have - extra information about probable causes. To get further information, - check the ExceptionCodes.txt file in the root of the distribution. This - file is generated by the ./configure script, so you will need to have - run that first. - - Some common causes of exceptions are listed below. - - Please email me with any other codes you get, and I will let you - know what they mean, and add notes here. - -
- RaidFile (2/8) - - This is found either when running bbstoreaccounts or in the - bbstored logs. - - Problem: The directories you - specified in the raidfile.conf are not writable by the _bbstored - user. - - Resolution: Change permissions - appropriately. -
- -
- Common (1/2) - - This usually occurs when the configuration files can't be - opened. - - Problem: You created your - configurations in non-standard locations, and the programs cannot find - them. - - Resolution: Explicitly specify - configuration file locations to daemons and programs. For - example - - /usr/local/sbin/bbstored /some/other/dir/bbstored.config /usr/local/sbin/bbackupquery -c /some/other/dir/bbackupd.config - - (daemons specify the name as the first argument, utility - programs with the -c option). - - Problem: bbstored can't find - the raidfile.conf file specified in bbstored.conf. - - Resolution: Edit bbstored.conf - to point to the correct location of this additional configuration - file. -
- -
- Server (3/16) - - The server can't listen for connections on the IP address - specified when you configured it. - - Problem: This probably means - you've specified the wrong hostname to bbstored-config -- maybe your - server is behind a NAT firewall? - - Resolution: Edit bbstored.conf - and correct the ListenAddresses line. You should replace the server - address with the IP address of your machine. -
- -
- Connection (7/x) - - These errors all relate to connections failing -- you may see - them during operation if there are network failures or other problems - between the client and server. The backup system will recover from - them automatically. - -
- Connection (7/30) - SSL problems - - Log snippet from client side: - - bbackupd[1904]: Opening connection to server xxxx.xxx... -bbackupd[1904]: SSL err during Connect: error:xxxxxxxx:rsa routines:RSA_padding_check_PKCS1_type_1:block type is not 01 -bbackupd[1904]: SSL err during Connect: error:xxxxxxxx:rsa routines:RSA_EAY_PUBLIC_DECRYPT:padding check failed -bbackupd[1904]: SSL err during Connect: error:xxxxxxxx:asn1 encoding routines:ASN1_verify:EVP lib -bbackupd[1904]: SSL err during Connect: error:xxxxxxxx:SSL routines:SSL3_GET_SERVER_CERTIFICATE:certificate verify failed -bbackupd[1904]: TRACE: Exception thrown: ConnectionException(Conn_TLSHandshakeFailed) at SocketStreamTLS.cpp(237) -bbackupd[1904]: Exception caught (7/30), reset state and waiting to retry... - - And from the server: - - bbstored[19291]: Incoming connection from xx.xxx.xx.xxx port xxxxx (handling in child xxxxx) -bbstored[21588]: SSL err during Accept: error:xxxxxxxx:SSL routines:SSL3_READ_BYTES:tlsv1 alert decrypt error -bbstored[21588]: in server child, exception Connection TLSHandshakeFailed (7/30) -- terminating child - - Solution: Create a new CA on - the server side and re-generate the client certificate. Re-creating - the client certificate request is not necessary. -
-
- -
- Advanced troubleshooting - - If this really doesn't help, then using the DEBUG builds of the - system will give you much more information -- a more descriptive - exception message and the file and line number where the error - occurred. - - For example, if you are having problems with bbstoreaccounts, - build the debug version with: - - cd boxbackup-0.0 -cd bin/bbstoreaccounts -make - - Within the module directories, make defaults to building the - debug version. At the top level, it defaults to release. - - This will build an executable in debug/bin/bbstoreaccounts which - you can then use instead of the release version. It will give far more - useful error messages. - - When you get an error message, use the file and line number to - locate where the error occurs in the code. There will be comments - around that line to explain why the exception happened. - - If you are using a debug version of a daemon, these extended - messages are found in the log files. -
-
- - - &__ExceptionCodes__elfjz3fu; - - - Running without root - - It is possible to run both the server and client without root - privileges. - -
- Server - - The server, by default, runs as a non-root user. However, it - expects to be run as root and changes user to a specified user as soon - as it can, simply for administrative convenience. The server uses a port - greater than 1024, so it doesn't need root to start. - - To run it entirely as a non-root user, edit the bbstored.conf - file, and remove the User directive from the Server section. Then simply - run the server as your desired user. -
- -
- Client - - The client requires root for normal operation, since it must be - able to access all files to back them up. However, it is possible to run - the client as a non-root user, with certain limitations. - - Follow the installation instructions, but install the executable - files manually to somewhere in your home directory. Then use - bbackupd-config to configure the daemon, but use a directory other than - /etc/box, probably somewhere in your home directory. - - All directories you specify to be backed up must be readable, and - all files must be owned by the user and readable to that user. - - Important: If any file or directory is not readable by this user, - the backup process will skip that file or directory. Keep an eye on the - logs for reports of this failure. - - Non-root operation of the backup client is recommended only for - testing, and should not be relied on in a production environment. -
- - diff --git a/docs/api-docs/backup/INDEX.txt b/docs/api-docs/backup/INDEX.txt deleted file mode 100644 index 50406cac..00000000 --- a/docs/api-docs/backup/INDEX.txt +++ /dev/null @@ -1,61 +0,0 @@ -TITLE Programmers Notes for Box Backup - -This directory contains the programmers notes for the Box Backup system. They will be of interest only if you want to review or modify the code. - -These notes are intended to be run through a script to produce HTML at some later stage, hence the marks such as 'TITLE'. - - -SUBTITLE Organisation - -The project is split up into several modules. The modules within 'lib' are building blocks for creation of the actual executable programs within 'bin'. 'test' contains unit tests for lib and bin modules. - -The file modules.txt lists the modules which are to be built, and their dependencies. It also allows for platform differences. - - -SUBTITLE Documentation Organisation - -In this directory, the files correspond to modules or areas of interest. Sub directories of the same name contain files documenting specific classes within that module. - - -SUBTITLE Suggested reading order - -* lib_common -* lib_server -* bin_bbackupd -* backup_encryption.txt -* bin_bstored -* lib_raidfile - -and refer to other sections as required. - - -SUBTITLE Building - -The makefiles are generated by makebuildenv.pl. (The top level makefile is generated by makeparcels.pl, but this is for the end user to use, not a programmer.) - -To build a module, cd to it and type make. If the -DRELEASE option is specified (RELEASE=1 with GNU make) the release version will be built. The object files and exes are placed in a directory structure under 'release' or 'debug'. - -It is intended that a test will be written for everything, so in general make commands will be issued only within the test/* modules. Once it has been built, cd to debug/test/ and run the test with ./t . - - -SUBTITLE Programming style - -The code is written to be easy to write. Ease of programming is the primary concern, as this should lead to fewer bugs. Efficiency improvements can be made later when the system as a whole works. - -Much use is made of the STL. - -There is no common base class. - -All errors are reported using exceptions. - -Some of the boring code is generated by perl scripts from description files. - -There are a lot of modules and classes which can easily be used to build other projects in the future -- there is a lot of "framework" code. - - -SUBTITLE Lots more documentation - -The files are extensively commented. Consider this notes as an overview, and then read the source files for detailed and definitive information. - -Each function and class has a very brief decsription of it's purpose in a standard header, and extensive efforts have been maed to comment the code itself. - diff --git a/docs/api-docs/backup/Win32_Clients.txt b/docs/api-docs/backup/Win32_Clients.txt deleted file mode 100644 index fad6c4af..00000000 --- a/docs/api-docs/backup/Win32_Clients.txt +++ /dev/null @@ -1,13 +0,0 @@ -The basic client tools now run on Win32 natively. -The port was done by nick@omniis.com. - -* bbackupd -* bbackupquery -* bbackupctl - -Have been ported. bbackupd runs as a NT style service. - -Known limitations: - -* File attributes and permissions are not backed up. - diff --git a/docs/api-docs/backup/backup_encryption.txt b/docs/api-docs/backup/backup_encryption.txt deleted file mode 100644 index 36580581..00000000 --- a/docs/api-docs/backup/backup_encryption.txt +++ /dev/null @@ -1,109 +0,0 @@ -TITLE Encryption in the backup system - -This document explains how everything is encrypted in the backup system, and points to the various functions which need reviewing to ensure they do actually follow this scheme. - - -SUBTITLE Security objectives - -The crpyto system is designed to keep the following things secret from an attacker who has full access to the server. - -* The names of the files and directories -* The contents of files and directories -* The exact size of files - -Things which are not secret are - -* Directory heirarchy and number of files in each directory -* How the files change over time -* Approximate size of files - - -SUBTITLE Keys - -There are four separate keys used: - -* Filename -* File attributes -* File block index -* File data - -and an additional secret for file attribute hashes. - -The Cipher is Blowfish in CBC mode in most cases, except for the file data. All keys are maximum length 448 bit keys, since the key size only affects the setup time and this is done very infrequently. - -The file data is encrypted with AES in CBC mode, with a 256 bit key (max length). Blowfish is used elsewhere because the larger block size of AES, while more secure, would be terribly space inefficient. Note that Blowfish may also be used when older versions of OpenSSL are in use, and for backwards compatibility with older versions. - -The keys are generated using "openssl rand", and a 1k file of key material is stored in /etc/box/bbackupd. The configuration scripts make this readable only by root. - -Code for review: BackupClientCryptoKeys_Setup() -in lib/backupclient/BackupClientCryptoKeys.cpp - - -SUBTITLE Filenames - -Filenames need to be secret from the attacker, but they need to be compared on the server so it can determine whether or not is it a new version of an old file. - -So, the same Initialisation Vector is used for every single filename, so the same filename encrypted twice will have the same binary representation. - -Filenames use standard PKCS padding implemented by OpenSSL. They are proceeded by two bytes of header which describe the length, and the encoding. - -Code for review: BackupStoreFilenameClear::EncryptClear() -in lib/backupclient/BackupStoreFilenameClear.cpp - - -SUBTITLE File attributes - -These are kept secret as well, since they reveal information. Especially as they contain the target name of symbolic links. - -To encrypt, a random Initialisation Vector is choosen. This is stored first, followed by the attribute data encrypted with PKCS padding. - -Code for review: BackupClientFileAttributes::EncryptAttr() -in lib/backupclient/BackupClientFileAttributes.cpp - - -SUBTITLE File attribute hashes - -To detect and update file attributes efficiently, the file status change time is not used, as this would give suprious results and result in unnecessary updates to the server. Instead, a hash of user id, group id, and mode is used. - -To avoid revealing details about attributes - -1) The filename is added to the hash, so that an attacker cannot determine whether or not two files have identical attributes - -2) A secret is added to the hash, so that an attacker cannot compare attributes between accounts. - -The hash used is the first 64 bits of an MD5 hash. - - -SUBTITLE File block index - -Files are encoded in blocks, so that the rsync algorithm can be used on them. The data is compressed first before encryption. These small blocks don't give the best possible compression, but there is no alternative because the server can't see their contents. - -The file contains a number of blocks, which contain among other things - -* Size of the block when it's not compressed -* MD5 checksum of the block -* RollingChecksum of the block - -We don't want the attacker to know the size, so the first is bad. (Because of compression and padding, there's uncertainty on the size.) - -When the block is only a few bytes long, the latter two reveal it's contents with only a moderate amount of work. So these need to be encrypted. - -In the header of the index, a 64 bit number is chosen. The sensitive parts of the block are then encrypted, without padding, with an Initialisation Vector of this 64 bit number + the block index. - -If a block from an previous file is included in a new version of a file, the same checksum data will be encrypted again, but with a different IV. An eavesdropper will be able to easily find out which data has been re-encrypted, but the plaintext is not revealed. - -Code for review: BackupStoreFileEncodeStream::Read() (IV base choosen about half-way through) -BackupStoreFileEncodeStream::EncodeCurrentBlock() (encrypt index entry) -in lib/backupclient/BackupStoreFileEncodeStream.cpp - - -SUBTITLE File data - -As above, the first is split into chunks and compressed. - -Then, a random initialisation vector is chosen, stored first, followed by the compressed file data encrypted using PKCS padding. - -Code for review: BackupStoreFileEncodeStream::EncodeCurrentBlock() -in lib/backupclient/BackupStoreFileEncodeStream.cpp - - diff --git a/docs/api-docs/backup/bin_bbackupd.txt b/docs/api-docs/backup/bin_bbackupd.txt deleted file mode 100644 index 67ad9267..00000000 --- a/docs/api-docs/backup/bin_bbackupd.txt +++ /dev/null @@ -1,88 +0,0 @@ -TITLE bin/bbackupd - -The backup client daemon. - -This aims to maintain as little information as possible to record which files have been uploaded to the server, while minimising the amount of queries which have to be made to the server. - - -SUBTITLE Scanning - -The daemon is given a length of time, t, which files over this age should be uploaded to the server. This is to stop recently updated files being uploaded immediately to avoid uploading something repeatedly (on the assumption that if a file has been written, it is likely to be modified again shortly). - -It will scan the files at a configured interval, and connect to the server if it needs to upload files or make queries about files and directories. - -The scan interval is actually varied slightly between each run by adding a random number up to a 64th of the configured time. This is to reduce cyclic patterns of load on the backup servers -- otherwise if all the boxes are turned on at about 9am, every morning at 9am there will be a huge spike in load on the server. - -Each scan chooses a time interval, which ends at the current time - t. This will be from 0 to current time - t on the first run, then the next run takes the start time as the end time of the previous run. The scan is only performed if the difference between the start and end times is greater or equal to t. - -For each configured location, the client scans the directories on disc recursively. - -For each directory - -* If the directory has never been scanned before (in this invocation of the daemon) or the modified time on the directory is not that recorded, the listing on the server is downloaded. - -* For each file, if it's modified time is within the time period, it is uploaded. If the directory has been downloaded, it is compared against that, and only uploaded if it's changed. - -* Find all the new files, and upload them if they lie within the time interval. - -* Recurse to sub directories, creating them on the server if necessary. - -Hence, the first time it runs, it will download and compare the entries on the disc to those on the server, but in future runs it will use the file and directory modification times to work out if there is anything which needs uploading. - -If there aren't any changes, it won't even need to connect to the server. - -There are some extra details which allow this to work reliably, but they are documented in the source. - - -SUBTITLE File attributes - -The backup client will update the file attributes on files as soon as it notices they are changed. It records most of the details from stat(), but only a few can be restored. Attributes will only be considered changed if the user id, group id or mode is changed. Detection is by a 64 bit hash, so detection is strictly speaking probablistic. - - -SUBTITLE Encryption - -All the user data is encrypted. There is a separate file, backup_encryption.txt which describes this, and where in the code to look to verify it works as described. - - -SUBTITLE Tracking files and directories - -Renaming files is a difficult problem under this minimal data scanning scheme, because you don't really know whether a file has been renamed, or another file deleted and new one created. - -The solution is to keep (on disc) a map of inode numbers to server object IDs for all directories and files over a certain user configurable threshold. Then, when a new file is discovered, it is first checked to see if it's in this map. If so, a rename is considered, which will take place if the local object corresponding to the name of the tracked object doesn't exist any more. - -Because of the renaming requirement, deletions of objects from the server are recorded and delayed until the end of the scan. - - -SUBTITLE Running out of space - -If the store server indicates on login to the backup client, it will scan, but not upload anything nor adjust it's internal stored details of the local objects. However, deletions and renames happen. - -This is to allow deletions to still work and reduce the amount of storage space used on the server, in the hope that in the future there will be enough space. - -Just not doing anything would mean that one big file created and then deleted at the wrong time would stall the whole backup process. - - -SUBTITLE BackupDaemon - -This is the daemon class for the backup daemon. It handles setting up of all the objects, and implements calulcation of the time intervals for the scanning. - - -SUBTITLE BackupClientContext - -State information for the scans, including maintaining a connection to the store server if required. - - -SUBTITLE BackupClientDirectoryRecord - -A record of state of a directory on the local filesystem. Containing the recursive scanning function, which is long and entertaining, but very necessary. It contains lots of comments which explain the exact details of what's going on. - - -SUBTITLE BackupClientInodeToIDMap - -A implementation of a map of inode number to object ID on the server. If Berkeley DB is available on the platform, it is stored on disc, otherwise there is an in memory version which isn't so good. - - - - - - diff --git a/docs/api-docs/backup/bin_bbstored.txt b/docs/api-docs/backup/bin_bbstored.txt deleted file mode 100644 index c9c4e229..00000000 --- a/docs/api-docs/backup/bin_bbstored.txt +++ /dev/null @@ -1,54 +0,0 @@ -TITLE bin/bbstored - -The backup store daemon. - -Maintains a store of encrypted files, and every so often goes through deleting unnecessary data. - -Uses an implementation of Protocol to communicate with the backup client daemon. See bin/bbstored/backupprotocol.txt for details. - - -SUBTITLE Data storage - -The data is arranged as a set of objects within a RaidFile disc set. Each object has a 64 bit object ID, which is turned into a filename in a mildly complex manner which ensure that directories don't have too many objects in them, but there is a minimal number of nested directories. See StoreStructure::MakeObjectFilename in lib/backupstore/StoreStructure.cpp for more details. - -An object can be a directory or a file. Directories contain files and other directories. - -Files in directories are supersceded by new versions when uploaded, but the old versions are flagged as such. A new version has a different object ID to the old version. - -Every so often, a housekeeping process works out what can be deleted, and deletes unnecessary files to take them below the storage limits set in the store info file. - - -SUBTITLE Note about file storage and downloading - -There's one slight entertainment to file storage, in that the format of the file streamed depends on whether it's being downloaded or uploaded. - -The problem is that it contains an index of all the blocks. For efficiency in managing these blocks, they all need to be in the same place. - -Files are encoded and decoded as they are streamed to and from the server. With encoding, the index is only completely known at the end of the process, so it's sent last, and lives in the filesystem last. - -When it's downloaded, it can't be decoded without knowing the index. So the index is sent first, followed by the data. - - -SUBTITLE BackupContext - -The context of the current connection, and the object which modifies the store. - -Maintains a cache of directories, to avoid reading them continuously, and keeps a track of a BackupStoreInfo object which is written back periodiocally. - - -SUBTITLE BackupStoreDaemon - -A ServerTLS daemon which forks off a separate housekeeping process as it starts up. - -Handling connections is delegated to a Protocol implementation. - - -SUBTITLE BackupCommands.cpp - -Implementation of all the commands. Work which requires writing is handled in the context, read only commands mainly in this file. - - -SUBTITLE HousekeepStoreAccount - -A class which performs housekeeping on a single account. - diff --git a/docs/api-docs/backup/encrypt_rsync.txt b/docs/api-docs/backup/encrypt_rsync.txt deleted file mode 100644 index a5db2df2..00000000 --- a/docs/api-docs/backup/encrypt_rsync.txt +++ /dev/null @@ -1,66 +0,0 @@ -TITLE Encrypted rsync algorithm - -The backup system uses a modified version of the rsync algorithm. A description of the plain algorithm can be found here: - - http://samba.anu.edu.au/rsync/tech_report/ - -The algorithm is modified to allow the server side to be encrypted, yet still benefit from the reduced bandwidth usage. For a single file transfer, the result will be only slightly less efficient than plain rsync. For a backup of a large directory, the overall bandwidth may be less due to the way the backup client daemon detects changes. - -This document assumes you have read the rsync document. - -The code is in lib/backupclient/BackupStoreFile*.*. - - -SUBTITLE Blocks - -Each file is broken up into small blocks. These are individually compressed and encrypted, and have an entry in an index which contains, encrypted, it's weak and strong checksums and decoded plaintext size. This is all done on the client. - -Why not just encrypt the file, and use the standard rsync algorithm? - -1) Compression cannot be used, since encryption turns the file into essentially random data. This is not very compressible. - -2) Any modification to the file will result in all data after that in the file having different ciphertext (in any cipher mode we might want to use). Therefore the rsync algorithm will only be able to detect "same" blocks up until the first modification. This significantly reduces the effectiveness of the process. - -Note that blocks are not all the same size. The last block in the file is unlikely to be a full block, and if data is inserted which is not a integral multiple of the block size, odd sized blocks need to be created. This is because the server cannot reassemble the blocks, because the contents are opaque to the server. - - -SUBTITLE Modifed algorithm - -To produce a list of the changes to send the new version, the client requests the block index of the file. This is the same step as requesting the weak and strong checksums from the remote side with rsync. - -The client then decrypts the index, and builds a list of the 8 most used block sizes above a certain threshold size. - -The new version of the file is then scanned in exactly the same way as rsync for these 8 block sizes. If a block is found, then it is added to a list of found blocks, sorted by position in the file. If a block has already been found at that position, then the old entry is only replaced by the new entry if the new entry is a "better" (bigger) match. - -The block size covering the biggest file area is searched first, so that most of the file can be skipped over after the first pass without expensive checksumming. - -A "recipe" is then built from the found list, by trivially discarding overlapping blocks. Each entry consists of a number of bytes of "new" data, a block start number, and a number of blocks from the old file. The data is stored like this as a memory optimisation, assuming that files mostly stay the same rather than having all their blocks reordered. - -The file is then encoded, with new data being sent as blocks of data, and references to blocks in the old file. The new index is built completely, as the checksums and size need to be rencrypted to match their position in the index. - - -SUBTITLE Combination on server - -The "diff" which is sent from the client is assembled into a full file on the server, simply by adding in blocks from the old file where they are specified in the block index. - - -SUBTITLE Storage on server - -Given that the server will in general store several versions of a file, combining old and new files to form a new file is not terribly efficient on storage space. Particularly for large multi-Gb database files. - -An alternative scheme is outlined below, however, it is significantly more complex to implement, and so is not implemented in this version. - -1) In the block index of the files, store the file ID of the file which each block is source from. This allows a single file to reference blocks from many files. - -2) When the file is downloaded, the server combines the blocks from all the files into a new file as it is streamed to the client. (This is not particuarly complicated to do.) - -This all sounds fine, until housekeeping is considered. Old versions need to be deleted, without losing any blocks necessary for future versions. - -Instead of just deleting a file, the server works out which blocks are still required, and rebuilds the file omitting those blocks which aren't required. - -This complicates working out how much space a file will release when it is "deleted", and indeed, adds a whole new level of complexity to the housekeeping process. (And the tests!) - -The directory structure will need an additional flag, "Partial file", which specifies that the entry cannot be built as previous blocks are no longer available. Entries with this flag should never be sent to the client. - - - diff --git a/docs/api-docs/backup/lib_backupclient.txt b/docs/api-docs/backup/lib_backupclient.txt deleted file mode 100644 index 3e4a079b..00000000 --- a/docs/api-docs/backup/lib_backupclient.txt +++ /dev/null @@ -1,46 +0,0 @@ -TITLE lib/backupclient - -Classes used on the store and on the server. - -See documentation in the files for more details. - - -SUBTITLE BackupStoreDirectory - -The directory listing class, containing a number of entries, representing files. - - -SUBTITLE BackupStoreFile - -Handles compressing and encrypting files, and decoding files downloaded from the server. - - -SUBTITLE BackupStoreFilename - -An encrypted filename. - - -SUBTITLE BackupStoreFilenameClear - -Derived from BackupStoreFilename, but with the ability to encrypt and decrypt filenames. Client side only. - - -SUBTITLE BackupClientFileAttributes - -Only used on the client -- the server treats attributes as blocks of opaque data. - -This reads attributes from files on discs, stores them, encrypts them, and applies them to new files. - -Also has a static function to generate filename attribute hashes given a struct stat and the filename. - - -SUBTITLE BackupClientRestore - -Routines to restore files from the server onto the client filesystem. - - -SUBTITLE BackupClientCryptoKeys - -This reads the key material from disc, and sets up the crypto for storing files, attributes and directories. - - diff --git a/docs/api-docs/backup/lib_backupstore.txt b/docs/api-docs/backup/lib_backupstore.txt deleted file mode 100644 index 8f24eb7b..00000000 --- a/docs/api-docs/backup/lib_backupstore.txt +++ /dev/null @@ -1,30 +0,0 @@ -TITLE lib/backupstore - -Classes which are shared amongst the server side store utilities, bbstored and bbstoreaccounts. Note also depends on lib/backupclient, as a lot of code is shared between the client and server. - - -SUBTITLE BackupStoreAccountDatabase - -A simple implementation of an account database. This will be replaced with a more suitable implementation. - - -SUBTITLE BackupStoreAccounts - -An interface to the account database, and knowledge of how to initialise an account on disc. - - -SUBTITLE BackupStoreConfigVerify - -The same configuration file is used by all the utilities. This is the Configuration verification structure for this file. - - -SUBTITLE BackupStoreInfo - -The "header" information about an account, specifying current disc usage, space limits, etc. - - -SUBTITLE StoreStructure - -Functions specifying how the files are laid out on disc in the store. - - diff --git a/docs/api-docs/backup/win32_build_on_cygwin_using_mingw.txt b/docs/api-docs/backup/win32_build_on_cygwin_using_mingw.txt deleted file mode 100644 index 1caaf4ca..00000000 --- a/docs/api-docs/backup/win32_build_on_cygwin_using_mingw.txt +++ /dev/null @@ -1,53 +0,0 @@ -How to build Box Backup on Win32 using Cygwin and MinGW -By Chris Wilson, 2007-05-26 - -(To read this document online with better formatting, browse to: -http://www.boxbackup.org/trac/wiki/CompileWithMinGW) - -Start by installing Cygwin on your Windows machine [http://www.cygwin.org]. -Make sure to select the following packages during installation: - -* Devel/gcc-mingw -* Devel/gcc-mingw-core -* Devel/gcc-mingw-g++ -* Mingw/mingw-zlib - -If you already have Cygwin installed, please re-run the installer and -ensure that those packages are installed. - -Download OpenSSL from -[http://www.openssl.org/source/openssl-0.9.7i.tar.gz] - -Open a Cygwin shell, and unpack OpenSSL: - - tar xzvf openssl-0.9.7i.tar.gz - -Configure OpenSSL for MinGW compilation, and build and install it: - - cd openssl-0.9.7i - ./Configure --prefix=/usr/i686-pc-mingw32/ mingw - make - make install - -Download PCRE from -[http://prdownloads.sourceforge.net/pcre/pcre-6.3.tar.bz2?download] - -Open a Cygwin shell, and unpack PCRE: - - tar xjvf pcre-6.3.tar.bz2 - -Configure PCRE for MinGW compilation, and build and install it: - - cd pcre-6.3 - export CFLAGS="-mno-cygwin" - ./configure - make winshared - cp .libs/libpcre.a .libs/libpcreposix.a /usr/lib/mingw - cp pcreposix.h /usr/include/mingw - -Now unpack the Box Backup sources, enter the source directory, -and configure like this: - - ./infrastructure/mingw/configure.sh - make - diff --git a/docs/api-docs/backup/win32_build_on_linux_using_mingw.txt b/docs/api-docs/backup/win32_build_on_linux_using_mingw.txt deleted file mode 100644 index 2c3e4fe9..00000000 --- a/docs/api-docs/backup/win32_build_on_linux_using_mingw.txt +++ /dev/null @@ -1,108 +0,0 @@ -How to build Box Backup for Windows (Native) on Linux using MinGW -By Chris Wilson, 2005-12-07 - -Install the MinGW cross-compiler for Windows: - -- Debian and Ubuntu users can "apt-get install mingw32" -- Fedora and SuSE users can download RPM packages from - [http://mirzam.it.vu.nl/mingw/] - -You will need to know the prefix used by the cross-compiler executables. -It will usually be something like "ix86-mingw32*-". All the binaries in the -cross-compiler package will start with this prefix. The documentation below -assumes that it is "i386-mingw32-". Adjust to taste. - -You will also need to install Wine and the Linux kernel "binary formats" -(binfmt) support, so that you can run Windows executables on Linux, -otherwise the configure scripts will not work properly with a cross-compiler. -On Ubuntu, run: - - apt-get install wine binfmt-support - /etc/init.d/binfmt-support start - -Start by downloading Zlib from [http://www.zlib.net/], unpack and enter -source directory: - - export CC=i386-mingw32-gcc - export AR="i386-mingw32-ar rc" - export RANLIB="i386-mingw32-ranlib" - ./configure - make - make install prefix=/usr/local/i386-mingw32 - -Download OpenSSL 0.9.8b from -[http://www.openssl.org/source/openssl-0.9.8b.tar.gz] - -Unpack and configure: - - tar xzvf openssl-0.9.8b.tar.gz - cd openssl-0.9.8b - ./Configure --prefix=/usr/local/i386-mingw32 mingw - make makefile.one - wget http://www.boxbackup.org/svn/box/chris/win32/support/openssl-0.9.8b-mingw-cross.patch - patch -p1 < openssl-0.9.8b-mingw-cross.patch - make -f makefile.one - make -f makefile.one install - -Download PCRE from -[http://prdownloads.sourceforge.net/pcre/pcre-6.3.tar.bz2?download] - -Unpack: - - tar xjvf pcre-6.3.tar.bz2 - cd pcre-6.3 - -Configure and make: - - export AR=i386-mingw32-ar - ./configure --host=i386-mingw32 --prefix=/usr/local/i386-mingw32/ - make winshared - -If you get this error: - - ./dftables.exe pcre_chartables.c - /bin/bash: ./dftables.exe: cannot execute binary file - make: *** [pcre_chartables.c] Error 126 - -then run: - - wine ./dftables.exe pcre_chartables.c - make winshared - -to complete the build. Finally: - - cp .libs/libpcre.a /usr/local/i386-pc-mingw32/lib - cp .libs/libpcreposix.a /usr/local/i386-pc-mingw32/lib - cp pcreposix.h /usr/local/i386-pc-mingw32/include - -You will need to find a copy of mingwm10.dll that matches your cross-compiler. -Most MinGW distributions should come with it. On Debian and Ubuntu, for some -bizarre reason, you'll find it compressed as -/usr/share/doc/mingw32-runtime/mingwm10.dll.gz, in which case you'll -have to un-gzip it with "gzip -d". Copy it to a known location, e.g. -/usr/local/i386-mingw32/bin. - -Download and extract Box Backup, and change into the base directory, -e.g. boxbackup-0.11rc2. Change the path to mingwm10.dll in parcels.txt to -match where you found or installed it. - -Now configure Box with: - - ./configure --host=i386-mingw32 \ - CXXFLAGS="-mthreads -I/usr/local/i386-mingw32/include" \ - LDFLAGS=" -mthreads -L/usr/local/i386-mingw32/lib" \ - LIBS="-lcrypto -lws2_32 -lgdi32" - make - -or, if that fails, try this: - - export CXX="i386-mingw32-g++" - export AR=i386-mingw32-ar - export RANLIB=i386-mingw32-ranlib - export CFLAGS="-mthreads" - export CXXFLAGS="-mthreads" - export LDFLAGS="-mthreads" - export LIBS="-lcrypto -lws2_32 -lgdi32" - (if you don't have a "configure" file, run "./bootstrap") - ./configure --target=i386-mingw32 - make CXX="$CXX" AR="$AR" RANLIB="$RANLIB" WINDRES="i386-mingw32-windres" diff --git a/docs/api-docs/backup/windows_porting.txt b/docs/api-docs/backup/windows_porting.txt deleted file mode 100644 index ada3b857..00000000 --- a/docs/api-docs/backup/windows_porting.txt +++ /dev/null @@ -1,100 +0,0 @@ -TITLE Notes on porting the backup client to Windows - -It should be relatively easy to port the backup client (bbackupd) to Windows. However, the server relies on unlink() behaviour of the UNIX filesystem which is different on the Windows platform, so it will not be so easy to port. - -An installation of perl is required to build the system. The ActiveState port is the easiest to install. - - -SUBTITLE Build environment - -The build environment generation script, makebuildenv.pl, uses perl to scan all the files and generate makefiles. It's rather orientated towards UNIX systems with gcc. - -Probably the easiest way to handle this is to detect the Windows platform, set up a few variables appropriately (in BoxPlatform.pm) and then post-process the generated Makefiles to mould them into something more handy for the MS Windows compiler and toolchain. - -The script itself is a bit messy. It was fine at first, but then the multi-platform thing got in the way. I do intend to rewrite it at some point in the future. - -Make sure your new version defines PLATFORM_WIN32 on the compile line. - -All files #include "Box.h" as the first include file. Use this for pre-compiled headers. Edit BoxPlatform.h to include the Windows headers required, and include a PLATFORM_WIN32 section. The easiest start will be to leave this blank, apart from typedefs for the basic types and any "not supported" #defines you can find. - -Boring bits of the code, such as exceptions and protocol definitions, are autogenerated using perl scripts. This code should be portable without modification. - -I have tried to avoid the things I know won't work with the MS compiler, so hopefully the code will be fairly clean. However, it might be a little easier to use the MinGW compiler [ http://www.mingw.org/ ] just to be consistent with the UNIX version. But I hope this won't be necessary. - -You'll need the latest version of OpenSSL. This was slightly difficult to get to compile last time I tried -- especially if you're determined to use the optimised assembler version. The main difficulty was getting a version which would link properly with the options used in my project, the default libraries selected got in the way. - - -SUBTITLE Porting as UNIX emulation - -Since the daemon uses so few UNIX system calls and with a limited set of options, it seems to make sense to port it by writing emulations of these functions. It's probably nicest to create a lib/win32 directory, and populate this with header files corresponding to the UNIX header files used. These just contain inline functions which map the UNIX calls to Win32 calls. - -File/socket handles may have to be translated. -1 is used as a failure return value and by the code internally to mark an invalid socket handle. (0 is a valid socket handle) - -Of course, some bits of code aren't relevant, so will just be #ifdefed out, or replaced. But this should be minimal. (Only perhaps the small bit relating to filesystem structure -- there aren't really mount points as such.) - - -SUBTITLE File tracking - -The daemon uses the inode number of a file to keep track of files and directories, so when they're renamed they can be moved efficiently on the store. Some unique (per filesystem) number will have to be found and used instead. - -It uses the Berkeley DB to store these on disc. It's likely another storage system will need to be used. (It just has to map the file's unique number into to a 8 byte struct.) - -There is a in-memory implementation for platforms which don't support Berkeley DB, but this isn't so good when the daemon has to be restarted as all the tracking is lost. But it's an easy start. - - -SUBTITLE Expected filesystem behaviour - -File and directories have (at least) two modification times, for contents and attributes. - -For files, the contents modification time must change when the contents change, and the attributes time when the attributes change (and may change when the contents change too.) - -For directories, the contents modification time must change when files or directories are deleted or added. If it changes any more frequently than this, then the client will be slightly less efficient -- it will download the store's directory listing whenever this time changes. The attributes modification time is less important, as the actual attributes are compared and only uploaded if different. - - -SUBTITLE Attributes - -Attributes means file modification times, flags, and filesystem permissions. - -The BackupClientFileAttribute class will need to be extended. Allocate another "attribute type" for the Win32 attributes, and then serialise it in a compatible way -- put your new attribute type in the header, and then a serialised network byte order structure in the rest. The different size of block is handled for you, and the server never looks inside. - -Add code so that under UNIX, Win32 attributes are ignored, and UNIX attributes under Win32. - -It's probably not necessary to worry too much about these for the first version. Not many people seem to use these attributes anyway. - - -SUBTITLE Times - -The system uses it's own 64 bit time type -- see BoxTime.h. Everything is translated to this from the various different system time types, and calculated and stored internally in this form. - - -SUBTITLE Daemon as a Service - -The client is derived from the Daemon class, which implements a daemon. The interface is simple, and it shouldn't be hard to write a compatible class which implements a Windows Service instead. - -Or cheat and run it as a Win32 application. - -Note that the daemon expects to be able to read every file it wants, and will abort a scan and upload run if it gets an error. The daemon must therefore be run with sufficient privileges. It runs as root under UNIX. - - -SUBTITLE Command Socket - -The backup daemon accepts commands from bbackupctl through a UNIX domain socket. When a connection is made, the user ID of the connecting process is checked to see if it's the same user ID as the daemon is running under. - -This may not have any exact analogue under Win32, so another communications scheme may have to be devised. - -This is only actually necessary if the client is to be run in snapshot mode. It can be safely left unimplemented if snapshot mode is not required, or the prompts for it to sync with the server are implemented some other way. - - -SUBTITLE NTFS streams - -If you want to back up NTFS streams, then a generic solution should probably be defined, so that the Mac OS X resource forks can be backed up with the same mechanism. - - -SUBTITLE Source code - -I work on a slightly different version of the source files. A make distribution script adds the license header and removes private sections of code. This means submitted diffs need a slight bit of translation. - - - - - diff --git a/docs/api-docs/common/lib_common.txt b/docs/api-docs/common/lib_common.txt deleted file mode 100644 index 11d7b02d..00000000 --- a/docs/api-docs/common/lib_common.txt +++ /dev/null @@ -1,52 +0,0 @@ -TITLE lib/common - -This module is the basic building block of the project. It is included implicitly as a dependency of all other modules. - -It provides basic services such as file and stream functionality, platform information, and various bits of utility code which doesn't fit naturally anywhere else but is useful to many other applications. - - -SUBTITLE Box.h - -The main include file for the project. It should be the first file included in every cpp file, followed by any system headers, then other project headers. This order has been chosen so that eventually it can be used as a target for pre-compiled headers. (This will be important for the Windows port) - - -SUBTITLE BoxPlatform.h - -This contains various bits of information on platform differences. The build scripts include a compile command line definition like PLATFORM_OPENBSD, which is then used to select the required options. - -Some of the stuff is not particularly pleasant, but it aims to produce a consistent compilation environment, and to have a abstracted way of setting other defines to tell the other files what things are available on this platform. - -GNU configure is not used, as it would be just another dependency. Although something does have to be done about compilation on Linux, which is just too different on each distribution to be caught by a common configuration here. - - -SUBTITLE Streams - -Much use is made of streams -- files, connections, buffers, all sorts of things are implemented using streams. - -The abstract base class IOStream defines the interface, see the class file for more details. - -Streams are lib/common's basic contribution to the project. A standard interface for handling data, and a number of utility classes which make performing common stream functions easy. - - -SUBTITLE Serialisation - -Note there is no "serialisable" class. Instead, objects which need to be serialised into Streams simply have two functions, ReadFromStream and WriteToStream. These perform as expected. - -As it turns out, there's no real need for a specific serialisation base class. If you do need to be able to serialise arbitary objects, there are two approaches. - -1) If you're serialising an arbitary object, you're going to know roughly what type it is. So it's likely to be a subclass of a known base class... in which case, you simply use virtual functions. - -2) If it really is an arbitary type, then you just write your function which accepts any type of object to serialise as a template. - - -SUBTITLE BoxException - -The exception base class for the project. All exceptions thrown by this code are dervied from BoxException. In general, each module which has unique errors has it's own exception class. - -CommonException errors are thrown throughout the project. - - -SUBTITLE Other bits - -There are some other extra useful bits which are documented only in the source files. - diff --git a/docs/api-docs/common/lib_common/BoxTime.txt b/docs/api-docs/common/lib_common/BoxTime.txt deleted file mode 100644 index 18bef5a5..00000000 --- a/docs/api-docs/common/lib_common/BoxTime.txt +++ /dev/null @@ -1,7 +0,0 @@ -TITLE BoxTime - -Not strictly a class, but time is presented in the project as 64 bit integers as microseconds since the UNIX Epoch. - -There are a number of utility functions in the BoxTime*.h files, which are useful. - -To get BoxTime values from a struct stat, use the FileModificationTime.h functions. \ No newline at end of file diff --git a/docs/api-docs/common/lib_common/CollectInBufferStream.txt b/docs/api-docs/common/lib_common/CollectInBufferStream.txt deleted file mode 100644 index 5f9556a3..00000000 --- a/docs/api-docs/common/lib_common/CollectInBufferStream.txt +++ /dev/null @@ -1,26 +0,0 @@ -CLASS CollectInBufferStream - -This class is essentially a buffer. Data is written to it, and stored in memory. Then when all data is written which will be written, it can be read out again. - -Useful for building streams in memory. - - -FUNCTION CollectInBufferStream::SetForReading() - -After you've written all the data, call this to set it to read mode. - - -FUNCTION CollectInBufferStream::Reset() - -Reset the stream to the state where is has no data, and is about to have data written. - - -FUNCTION CollectInBufferStream::GetSize() - -Get the size of the buffered data -- the entire data. Use the interface function BytesLeftToRead() in most cases. - - -FUNCTION CollectInBufferStream::GetBuffer() - -Get the buffer, so you can do bad things with it if you're feeling naughty. - diff --git a/docs/api-docs/common/lib_common/Configuration.txt b/docs/api-docs/common/lib_common/Configuration.txt deleted file mode 100644 index ef5f38f6..00000000 --- a/docs/api-docs/common/lib_common/Configuration.txt +++ /dev/null @@ -1,102 +0,0 @@ -CLASS Configuration - -Implements the reading of multi-level configuration files. This is intended to be a generic way of representing configuration implementation. - -Basic validation is performed on files as they are read, specified by a data structure. - -test/common has some examples of usage. - - -SUBTITLE Configuration file format - -The format is simple, a list of Key = Value pairs. For example - -Key1 = Value1 -Key2 = Value2 - -Optionally, multiple values for the same key are allowed. - -Lists of configurations can be nested to any level. These are called Sub-configurations: - -SubConfig -{ - SubKey1 = ValueX - SubKey2 = ValueY -} - - -SUBTITLE Verification - -The verification structure specifies what keys are required, what are optional, and what sub-configurations are expected. Default values can be specified. - -See Configuration.h for the structures. - -RaidFileController::Initialise has a good example of a simple verification layout. - -Wildcards can be used for SubConfigurations, by specifying a name of "*". This allows you to use multiple sections to configure any number of items. - -Each item has a number of flags, which are combined to say whether an item is required, should be an integer or boolean, and rather importantly, whether it's the last item in the list. - -Verification is limited, so you may wish to do more verification the file. There are unimplemented hooks for custom verification functions to be included in the verification definitions. Should be done at some point. - -Boolean keys have possible values "true", "yes", "false", "no" (case insensitive). - - -FUNCTION Configuration::LoadAndVerify() - -Loads the configuration from disc, and verifies it. If there are problems with the verification, it returns some text which can be used to tell the user the problems with the file. These are fairly basic error messages, but do say what should be done to get it to parse properly. - -This returns a Configuration object, which can then be queries for Keys and Values. Sub-configurations are implemented through an interface which returns a reference to another Configuration object. - - - -FUNCTION Configuration::KeyExists() - -Does a specified key exist? - -Use this for optional key values only -- specify them in the verification structure if they are required. - - -FUNCTION Configuration::GetKeyValue() - -Get the value of a key as a string. - -If ConfigTest_MultiValueAllowed is set in the relevant entry in the verify structure, this string may contain multiple values, separated by a single byte with value 0x01 (use Configuration::MultiValueSeparator in code). Use SplitString() defined in Utils.h to split it into components. - -This representation was chosen as multi-values are probably rare, and unlikely to justify writing a nicer (but more memory intensive and complex) solution. - - -FUNCTION Configuration::GetKeyValueInt() - -Get the value of a key as an integer. Make sure the AsInt property is requried in the verification structure. - - -FUNCTION Configuration::GetKeyValueBool() - -Get the value of a key as a boolean value. Make sure the AsBool property is requried in the verification structure. - -Default to "false", should this verification not be performed and an unknown value is specified. - - -FUNCTION Configuration::GetKeyNames() - -Return a list of all the keys in this configuration. - - -FUNCTION Configuration::SubConfigurationExists() - -Does a specified sub-configuration exist? - - -FUNCTION Configuration::GetSubConfiguration() - -Return another Configuration object representing the sub section. - - -FUNCTION Configuration::GetSubConfigurationNames() - -Get a list of all the sub configurations. - -As there isn't a particularly neat way that configurations can be iterated over, mSubConfigurations is public. (BAD!) - - diff --git a/docs/api-docs/common/lib_common/Conversion.txt b/docs/api-docs/common/lib_common/Conversion.txt deleted file mode 100644 index 62d1967a..00000000 --- a/docs/api-docs/common/lib_common/Conversion.txt +++ /dev/null @@ -1,14 +0,0 @@ -TITLE Generic type conversion - -Conversion.h provides generic type conversion. Within the BoxConvert namespace, it defines the templated function - - ToType Convert(FromType From) - -which converts the data type as expected. In general, from and to types have to be specified explicitly. - -Templates rather than overloaded functions are used, firstly to be absolutely explicit about the conversion required, and secondly because overloaded functions can't have differing return types for the same argument type. - -The function is specialised for various types, and the generic version uses C++ type conversion. - -Exceptions may be thrown if the conversion is not possible. These are all of the ConversionException type. - diff --git a/docs/api-docs/common/lib_common/ExcludeList.txt b/docs/api-docs/common/lib_common/ExcludeList.txt deleted file mode 100644 index 8a5bf36c..00000000 --- a/docs/api-docs/common/lib_common/ExcludeList.txt +++ /dev/null @@ -1,21 +0,0 @@ -TITLE ExcludeList - -A class implementing a list of strings which are to be excluded in some way. Entries can be added as strings to be matched exactly, or as regular expressions. - -Multiple entries can be added in a single function call in a manner designed to work with the multi-value entries store in a Configuation object. - - -FUNCTION ExcludeList::AddDefiniteEntries() - -Definite entries are exact strings to match. - - -FUNCTION ExcludeList::AddRegexEntries() - -Regular expressions as defined by POSIX, and supported by the host platform. Will throw an exception if regular expressions are not supported by the platform. - - -FUNCTION ExcludeList::IsExcluded() - -Test a string for being excluded by definite or regex entries. - diff --git a/docs/api-docs/common/lib_common/FdGetLine.txt b/docs/api-docs/common/lib_common/FdGetLine.txt deleted file mode 100644 index d92ff94d..00000000 --- a/docs/api-docs/common/lib_common/FdGetLine.txt +++ /dev/null @@ -1,11 +0,0 @@ -CLASS FdGetLine - -See IOStreamGetLine, difference is that it works on basic UNIX file descriptors rather than full blown streams. Follows basic interface, except... - - -FUNCTION FdGetLine::GetLine - -Returns a string containing the optionally preprocessed line. - -Do not call if IsEOF if true. - diff --git a/docs/api-docs/common/lib_common/Guards.txt b/docs/api-docs/common/lib_common/Guards.txt deleted file mode 100644 index 6174fea6..00000000 --- a/docs/api-docs/common/lib_common/Guards.txt +++ /dev/null @@ -1,5 +0,0 @@ -TITLE Guards.h - -This file contains a couple of classes for using file and memory. When the class goes out of scope, the underlying object is closed or freed, respectively. - - diff --git a/docs/api-docs/common/lib_common/IOStream.txt b/docs/api-docs/common/lib_common/IOStream.txt deleted file mode 100644 index 09460656..00000000 --- a/docs/api-docs/common/lib_common/IOStream.txt +++ /dev/null @@ -1,89 +0,0 @@ -CLASS IOStream - -The base class for streams of data. See IOStream.h for specific details on functions, but the interface is described here. - -Most streams only implement one direction, but some both. - - -SUBTITLE Reading - - -FUNCTION IOStream::Read() - -Reads data from the stream. Returns 0 if there is no data available within the timeout requested. - -Unlike the UNIX API, a return of 0 does not mean that there is no more data to read. Use IOStream::StreamDataLeft() to find this out. - - -FUNCTION IOStream::ReadFullBuffer() - -If you want to read a specific sized block of data from a stream, it is annoying ot use the Read function, because it might return less, so you have to loop. - -This implements that loop. Note that the timeout is the timeout for each individual read -- it might take a lot longer to complete. - -You must check the return value, which is whether or not it managed to get all that data. - - -FUNCTION IOStream::BytesLeftToRead() - -How many bytes are left to read in the stream treated as a whole. May return IOStream::SizeOfStreamUnknown if it is something like a socket which doesn't know how much data is in the stream. - - -FUNCTION IOStream::StreamDataLeft() - -Returns true if more data can be read. Or more specifically, if more data might be able to be read some time in the future. - - -SUBTITLE Writing - - -FUNCTION IOStream::Write() - -Write data to a stream. Writes the entire block, or exceptions. - - -FUNCTION IOStream::WriteAllBuffered() - -Writes any buffered data to the underlying object. - -Call at the end of a series of writes to make sure the data is actually written. (Buffering is not yet implemented anywhere, but it should be soon.) - - -FUNCTION IOStream::StreamClosed() - -Is the stream closed, and writing no longer possible? - - -FUNCTION IOStream::CopyStreamTo() - -A utility function to copy the contents of one stream to another, until the reading stream has no data left. - - -SUBTITLE Other interfaces - -These are slightly nasty interfaces, because they have to match the UNIX API to some extent. - - -FUNCTION IOStream::Close() - -Close the stream -- intended to indicate that nothing more will be written. - -However, also closes reading in most cases. Stream dependent. Probably best just to delete the object and let it sort itself out. - - -FUNCTION IOStream::Seek() - -Seeks within the stream -- mainly for using files within a stream interface, although some of the utility stream classes support it too. - -This is actually a bad interface, because it does not specify whether it applies to reading or writing. This matches the interface provided by files with a single file pointer, but does not map well onto other stream types. - -Should it be changed? Perhaps. But then it means that files would either have to have an inconsitent interface, or the class keep track of two separate file pointers. Which isn't nice. - -So use carefully, and remember whether you're using a file stream, a reading stream, or a writing stream. - - -FUNCTION IOStream::GetPosition() - -Get the current file pointer. Subject to same problems as Seek with regard to semantics. - - diff --git a/docs/api-docs/common/lib_common/IOStreamGetLine.txt b/docs/api-docs/common/lib_common/IOStreamGetLine.txt deleted file mode 100644 index 04c56b57..00000000 --- a/docs/api-docs/common/lib_common/IOStreamGetLine.txt +++ /dev/null @@ -1,29 +0,0 @@ -CLASS IOStreamGetLine - -This class provides a convenient way to read text from a file, line by line. It also can preprocess the line to remove leading and trailing whitespace and comments. Comments are started by the character # and run to the end of the line. - -Create an instance by passing a reference to a stream into the constructor. - -Note the class does internal buffering, so you can only detach it later if the stream supports seeking backwards. - - -FUNCTION IOStreamGetLine::GetLine() - -Returns true if a line could be retreieved without a read timing out. - - -FUNCTION IOStreamGetLine::IsEOF() - -Whether the end of the stream has been reached. Do not call GetLine if this is true. - - -FUNCTION IOStreamGetLine::GetLineNumber() - -Returns the line number. - - -FUNCTION IOStreamGetLine::DetachFile() - -Detaches the stream from the GetLine class. Will seek backwards to "replace" data it's buffered back in the stream. - - diff --git a/docs/api-docs/common/lib_common/MainHelper.txt b/docs/api-docs/common/lib_common/MainHelper.txt deleted file mode 100644 index eb5b07f0..00000000 --- a/docs/api-docs/common/lib_common/MainHelper.txt +++ /dev/null @@ -1,4 +0,0 @@ -TITLE MainHelper.h - -This header contains a couple of macros which add exception handling and reporting to main() functions for executable programs. - diff --git a/docs/api-docs/common/lib_common/WaitForEvent.txt b/docs/api-docs/common/lib_common/WaitForEvent.txt deleted file mode 100644 index 0bc55726..00000000 --- a/docs/api-docs/common/lib_common/WaitForEvent.txt +++ /dev/null @@ -1,16 +0,0 @@ -CLASS WaitForEvent - -This class implements a way to efficiently wait on one or more system objects, for example sockets and file descriptors. Where kqueue() is available, this is used, otherwise poll(). The poll() implementation is less comprehensive and rather more limited. - -To add a compatible object, call Add(). To remove it, call Remove(). To wait for an object to become ready, call Wait(), which returns a pointer to the first ready object, or 0 for a timeout. - -The timeout is set either in the constructor, or using the SetTimout() method. It is specified in milliseconds. - -The kqueue based implementation will automatically remove objects when they are closed (actually, the OS does this), but the poll implementation requires that Remove() be called. - -The flags passed to Add() or Remove() are passed to the object, and are ignored by this class. - -For an object to be able to be added to the list, it should implement FillInKEvent() and FillInPoll() for kqueue and poll implementations respectively. Use the PLATFORM_KQUEUE_NOT_SUPPORTED define to test which is necessary for the platform. - -It does not have to be derived off any particular class, as it uses templates to be compatible with any class. - diff --git a/docs/api-docs/common/lib_common/xStream.txt b/docs/api-docs/common/lib_common/xStream.txt deleted file mode 100644 index 91e9c0ea..00000000 --- a/docs/api-docs/common/lib_common/xStream.txt +++ /dev/null @@ -1,40 +0,0 @@ -TITLE Various stream types - -Some useful streams are implemented in lib/common. - - -SUBTITLE CollectInBufferStream - -Described in it's own page. - - -SUBTITLE FileStream - -Implements a stream from a file, allowing both reading and writing. - - -SUBTITLE MemBlockStream - -Turns a memory block into a readable stream. - -Can also be constructed using StreamableMemBlock, CollectInBufferStream and MemBlockStream as sources of the buffer. - - -SUBTITLE PartialReadStream - -Create a readable stream which will read a set number of bytes from another stream, and then declare itself as closed. - -Useful for extracting a small chunk of another stream to present to a function which expects to consume all of a stream. - - -SUBTITLE ReadGatherStream - -Create a readable stream out of blocks from many streams -- so various sections of any number of streams are composed into a single stream. - -To use, register each stream with AddComponent, then use the returned 'handle' with AddBlock to add blocks to the composed stream. - -Optionally, the object will take ownership of the streams added, and delete them when itself is deleted. - -See the comments in the function blocks in the cpp file for more info. - - diff --git a/docs/api-docs/common/lib_compress.txt b/docs/api-docs/common/lib_compress.txt deleted file mode 100644 index f3e26f20..00000000 --- a/docs/api-docs/common/lib_compress.txt +++ /dev/null @@ -1,8 +0,0 @@ -TITLE lib/compress - -This is a horrid C++ interface to zlib. It is written this way to avoid having to buffer any data, and so be unnecessarily inefficient. But this does end up just being a wrapper to the zlib interface. - -See test/compress for an example of how to use it. But it should be very familiar. - -For an easier interface, use CompressStream. - diff --git a/docs/api-docs/common/lib_compress/CompressStream.txt b/docs/api-docs/common/lib_compress/CompressStream.txt deleted file mode 100644 index eca43eb6..00000000 --- a/docs/api-docs/common/lib_compress/CompressStream.txt +++ /dev/null @@ -1,27 +0,0 @@ -TITLE CompressStream - -This implements a compressing stream class, which compresses and decompresses data sent to an underlying stream object. Data is likely to be buffered. - - -WARNING: Use WriteAllBuffered() (after objects which need to be sent in their entirity) and Close() (after you have finished writing to the stream) otherwise the compression buffering will lose the last bit of data for you. - - -The class works as a filter to an existing stream. Ownership can optionally be taken so that the source stream is deleted when this object is deleted. - -It can operate in one or two directions at any time. Data written to the stream is compressed, data read from the stream is decompressed. If a direction is not being (de)compressed, then it can act as a pass-through without touching the data. - -The constructor specifies the actions to be taken: - -CompressStream(IOStream *pStream, bool TakeOwnership, - bool DecompressRead, bool CompressWrite, - bool PassThroughWhenNotCompressed = false); - -pStream - stream to filter -TakeOwnership - if true, delete the stream when this object is deleted. -DecompressRead - reads pass through a decompress filter -CompressWrite - writes pass through a compression filter -PassThroughWhenNotCompressed - if not filtering a direction, pass through the data unaltered, otherwise exception if an attempt is made on that direction. - - -Note that the buffering and compression will result in different behaviour than the underlying stream: data may not be written in one go, and data being read may come out in different sized chunks. - diff --git a/docs/api-docs/common/lib_crypto.txt b/docs/api-docs/common/lib_crypto.txt deleted file mode 100644 index 9ddafe69..00000000 --- a/docs/api-docs/common/lib_crypto.txt +++ /dev/null @@ -1,28 +0,0 @@ -TITLE lib/crypto - -Provides cryptographic primatives using the OpenSSL implementation. - - -SUBTITLE CipherContexts and CipherDescriptions - -A CipherContext is the interface to encryption and decryption, providing a generic interface. - -It is constructed with a decrypt or encrypt flag, and a CipherDescription. The CipherDescription specifies which cipher is to be used, the key, and all other cipher specific paramteres. - -The CipherContext has support for padding and initialisation vectors. - - -SUBTITLE Random numbers - -An interface to the OpenSSL psuedo random number generater is provided. - - -SUBTITLE Digests - -An interface to the MD5 digest is provided. - - -SUBTITLE RollingChecksum - -The rsync rolling checksum is implemented. The interface is a little tricky to be efficient as the caller must track the position and provide relevant bytes to advance the checksum. - diff --git a/docs/api-docs/common/lib_crypto/CipherContext.txt b/docs/api-docs/common/lib_crypto/CipherContext.txt deleted file mode 100644 index 30fb4608..00000000 --- a/docs/api-docs/common/lib_crypto/CipherContext.txt +++ /dev/null @@ -1,28 +0,0 @@ -CLASS CipherContext - -Encryption and decryption using OpenSSL EVP interface. - -See the cpp file for more documentation in the function headers, and test/crypto for examples. - -General notes below. - - -SUBTITLE Construction - -Construct with the encryption direction, and a CipherDescription of the cipher and key required. - - -SUBTITLE Encrypting or decrypting - -Begin() and Transform() allow piece by piece transformation of a block. - -TransformBlock() transforms an entire block. - - -SUBTITLE Buffering - -All transforms expect to have enough space in the buffers for their entire output. Because of block boundaries and padding, it is necessary that the output buffer should be bigger than the input buffer. The amount of space depends on the cipher. - -InSizeForOutBufferSize() and MaxOutSizeForInBufferSize() perform these space calculations, returning the maximuim in size for a specified out size, and the reverse, respectively. - - diff --git a/docs/api-docs/common/lib_crypto/RollingChecksum.txt b/docs/api-docs/common/lib_crypto/RollingChecksum.txt deleted file mode 100644 index d871b3f2..00000000 --- a/docs/api-docs/common/lib_crypto/RollingChecksum.txt +++ /dev/null @@ -1,36 +0,0 @@ -CLASS RollingChecksum - -Implementing the rsync rolling checksum algorithm. Read it's description first: - -http://samba.anu.edu.au/rsync/tech_report/node3.html - - -SUBTITLE Construction and initial checksum calculation - -The constructor takes a pointer to a block of data and a size, and calculates the checksum of this block. It can now be "rolled forward" to find the checksum of the block of the same size, one byte forward, with minimal calculation. - - -FUNCTION RollingChecksum::GetChecksum() - -Returns the checksum for the current block. - - -FUNCTION RollingChecksum::RollForward() - -This function takes the byte at the start of the current block, and the last byte of the block it's rolling forward to, and moves the checksum on. - -If the block is - - char *pBlock = ; - -with size s, then it should be called with - - RollForward(pBlock[0], pBlock[s]) - -and now GetChecksum will return the checksum of the block (pBlock+1) of size s. - - -FUNCTION RollingChecksum::RollForwardSeveral() - -Similar to RollForward(), but is more efficient for skipping several bytes at once. Takes pointers to the data buffer rather than the actual data values. - diff --git a/docs/api-docs/common/lib_server.txt b/docs/api-docs/common/lib_server.txt deleted file mode 100644 index 392f331d..00000000 --- a/docs/api-docs/common/lib_server.txt +++ /dev/null @@ -1,9 +0,0 @@ -TITLE lib/server - -This provides various classes for implementing client/server applications (and UNIX daemons). - -The stars of the show are ServerTLS and Protocol, which allow client server applications which communicate using TLS (SSL successor) to be built with surprisingly few lines of code. - -All details in the respective class files. - -Look at test/basicserver for examples. diff --git a/docs/api-docs/common/lib_server/Daemon.txt b/docs/api-docs/common/lib_server/Daemon.txt deleted file mode 100644 index 6956ec2b..00000000 --- a/docs/api-docs/common/lib_server/Daemon.txt +++ /dev/null @@ -1,96 +0,0 @@ -CLASS Daemon - -Implement a UNIX daemon which - -* Daemonises (detach from console, etc) -* Sets up signal handlers, and does useful things with them -* Reads configuration files -* Writes PID file -* Opens syslog - -all the usual UNIX basic daemon behaviour. - -The daemon exe takes optional arguments: The first is a configuration file filename which overrides the default. If the second argument is 'SINGLEPROCESS' the daemon will not daemonise. - -The configuration file must have a section like this - -Server -{ - PidFile = /var/run/daemon.pid - User = username -} - -Use DAEMON_VERIFY_SERVER_KEYS (defined in Daemon.h) to include the necessary keys in your configuration file's verify structure. - -The "User" line is optional, and if it is present the Daemon will change user to this username just before it daemonises. Note that unless the directory the PID file is written to has write permission for this user, the PID file will not be deleted on exit. - - -To implement a daemon, derive a class from Daemon, and override Run(). - -Then in your main file, do something like - -int main(int argc, const char *argv[]) -{ - MAINHELPER_START - - BackupDaemon daemon; - return daemon.Main(BOX_FILE_BBACKUPD_DEFAULT_CONFIG, argc, argv); - - MAINHELPER_END -} - -and that's it. Obviously it's worth doing a few more things as well, but that's it. - - -FUNCTION Daemon::Run() - -Override with the main daemon code. It should behave like this - -void SomethingDaemon::Run() -{ - // Read configuration file - - // Do lots of work until StopRun() returns true - while(!StopRun()) - { - ::sleep(10); - } - - // Clean up nicely -} - -which allows the base class to implement the standard start, terminate and -HUP behaviours correctly. - - -FUNCTION Daemon::DaemonName() - -Returns the name of the daemon, for use in syslog. - - -FUNCTION Daemon::DaemonBanner() - -Returns the banner to be displayed on startup, or 0 for no banner. - - -FUNCTION Daemon::GetConfigVerify() - -Returns a configuration verify structure for verifying the config file. Note that this configuration structure should include a sub-configuration called "Server, and have entries defined by DAEMON_VERIFY_SERVER_KEYS. See one of the bin/bbackupd for an example. - - -FUNCTION Daemon::StopRun() - -Returns true when the daemon needs to be terminated or restarted. Use IsReloadConfigWanted() and IsTerminateWanted() to find out which one, if you need to know. - - -FUNCTION Daemon::SetupInInitialProcess() - -Override to perform additional functions in the initial process, before forking and detachment happens. - - -FUNCTION Daemon::EnterChild() - -Called when a child is entered. If you override it, remember to call the base class. - - - - diff --git a/docs/api-docs/common/lib_server/Protocol.txt b/docs/api-docs/common/lib_server/Protocol.txt deleted file mode 100644 index 09d3c1f1..00000000 --- a/docs/api-docs/common/lib_server/Protocol.txt +++ /dev/null @@ -1,120 +0,0 @@ -CLASS Protocol - -Protocol - -* serialises and deserialises data objects -* sends arbitary streams - -through a bi-directional IOStream object, usually a socket. - -These data objects are auto-generated by a perl script, along with the logic to implement a simple protocol where one end is a client, and the other a server. The client sends a command object (and optional streams) and the server returns a reply object (and optional streams). - -The perl script uses a description file which specifies the data inside these objects (which can be extended to include any type which implements the standard serialisation functions), the required responses to the commands, and whether streams are involved. - -It then implements a server object, which given a stream and a user defined context object, waits for commands, processes them, and sends the results back. All you need to do is implement a DoCommand() function for each command object you define. - -On the client side, a Query() function is implemented which takes objects as parameters, and returns the right type of reply object (or throws an exception if it doesn't get it.) Short cut Query() functions are also implemented which don't require objects to be created manually. - -Thus, implementing a server is as simple as deriving a daemon off ServerStream or ServerTLS, and implementing the Connection() function as - - void TestProtocolServer::Connection(SocketStream &rStream) - { - TestProtocolServer server(rStream); - TestContext context; - server.DoServer(context); - } - -and that's it. TestContext is a user defined class which keeps track of all the state of the connection, and is passed to all the DoCommand() functions, which look like this: - - std::auto_ptr - TestProtocolServerSimple::DoCommand(TestProtocolServer &rProtocol, - TestContext &rContext) - { - return std::auto_ptr - (new TestProtocolServerSimpleReply(mValue+1)); - } - -(taken from test/basicserver) - -The client code looks like this - - SocketStream conn; - conn.Open(Socket::TypeUNIX, "testfiles/srv4.sock"); - - TestProtocolClient protocol(conn); - - // Query - { - std::auto_ptr - reply(protocol.QuerySimple(41)); - TEST_THAT(reply->GetValuePlusOne() == 42); - } - - -Finally, debug logging can be generated which allows a list of all commands and their parameters to be logged to syslog or a file. - - -SUBTITLE Protocol Description File - -This file is passed to the lib/server/makeprotocol.pl script, which generates a h and cpp file to implement the protocol. - -It is in two sections, separated by a 'BEGIN_OBJECTS' on a line of it's own. - -In the top half, the following statements must be made. - -Name - The name of the protocol, used in naming classes. - -IdentString - The idenfitifaction string sent over the IOStream to confirm it it - is talking to another Protocol object speaking the same Protocol. - -ServerContextClass - The user defined context class used for the server, and the header - file it is defined in. - -Additionally, the following optional commands can be made. - -ClientType -ServerType (similarly) - Extends the types used in the objects below. Server and client - can use different types for the same object type. - -ImplementLog (Client|Server) (syslog|file) - Implement command logging for client or server into syslog or a file. - -LogTypeToText (Client|Server) - - For extended types, optionally define how to convert them into printf - elements and the code to run to get the argument. Within the evaluate - parameter, VAR is replaced by the name of the variable to display. - If this is not specified for a given type, OPAQUE is output instead. - - -In the object section, an object is defined by a line - - - -followed by lines beginning with whitespace defining the data transmitted in the object. The type may be list, which specifies a list (implemented as a std::vector) of entries of that type. - -The attributes specify exactly how that object is used in the defined protocol. - -Reply - The object is a reply object, sent from the server to the client. - -Command(Reply-Type) - The object is a command, send from the client to the server, and the server - will send back an object of type Reply-Type. - -IsError(Type-Field,SubType-Field) - The command is an error object, and the two files specify the data member - which describes the error type and sub type. - -EndsConversation - When this command is received, the connection is to be terminated. - (ie a logout command) - -StreamWithCommand - When this command is sent as a command, a stream follows it. - - diff --git a/docs/api-docs/common/lib_server/ServerStream.txt b/docs/api-docs/common/lib_server/ServerStream.txt deleted file mode 100644 index 6c5932a0..00000000 --- a/docs/api-docs/common/lib_server/ServerStream.txt +++ /dev/null @@ -1,29 +0,0 @@ -CLASS ServerStream - -ServerStream implementes a Daemon which accepts stream connections over sockets, and forks into a child process to handle them. - -To implement a daemon, derive from - - ServerStream - -The type SocketStream specifies that incoming connections should be treated as normal sockets, and SERVER_LISTEN_PORT is the port to listen to (if it's a inet socket, and if not overridden in the config file). The actual addresses (or names) to bind to are specified in the configuration file by the user. - -Make sure SERVERSTREAM_VERIFY_SERVER_KEYS(0) is included in the configuration verification structure in the "Server" sub configuration. 0 could be replaced with a default address, for example "unix:/var/run/server.sock" to specific a default UNIX socket in the filesystem. - -See test/basicserver for a simple example. - -The ListenAddresses key in the Server subconfiguration is a comma separated list of addresses, specified as family:name. Internet sockets are family 'inet', for example 'inet:localhost' (or 'inet:localhost:1080' to specify a port number as well), and unix domain sockets are 'unix', example above. - - -Override Connection to handle the connection. - -Remember to override Daemon functions like the server name, and start it up, just like a generic Daemon. - - -FUNCTION ServerStream::Connection - -This function takes a connected stream as it's argument. It should then proceed to do whatever it needs to do to talk to the client. - -Using IOStreamGetLine or a Protocol class to communicate may be quick ways of implementing this functionality. - - diff --git a/docs/api-docs/common/lib_server/ServerTLS.txt b/docs/api-docs/common/lib_server/ServerTLS.txt deleted file mode 100644 index dbde500f..00000000 --- a/docs/api-docs/common/lib_server/ServerTLS.txt +++ /dev/null @@ -1,6 +0,0 @@ -CLASS ServerTLS - -Implements a server which uses TLS (SSL) to encrypt and authenticate connections. - -Very similar to ServerStream, except it reads the certificate files for the TLSContext out of the Server sub-configuration to set up a TLSContext ("CertificateFile", "PrivateKeyFile" and "TrustedCAsFile"). Otherwise works exactly the same. - diff --git a/docs/api-docs/common/lib_server/SocketStream.txt b/docs/api-docs/common/lib_server/SocketStream.txt deleted file mode 100644 index 82813279..00000000 --- a/docs/api-docs/common/lib_server/SocketStream.txt +++ /dev/null @@ -1,8 +0,0 @@ -CLASS SocketStream - -A implementation of IOStream which wraps a socket connection. - -It can either be created by attaching to an existing object, or use the Open() function to open a connection to a named host on a specific port (or a local UNIX socket in the filesystem). - -Follows stream interface. - diff --git a/docs/api-docs/common/lib_server/SocketStreamTLS.txt b/docs/api-docs/common/lib_server/SocketStreamTLS.txt deleted file mode 100644 index ebb3f233..00000000 --- a/docs/api-docs/common/lib_server/SocketStreamTLS.txt +++ /dev/null @@ -1,11 +0,0 @@ -CLASS SocketStreamTLS - -An implementation of IOStream which wraps a TLS (SSL) connection over a socket. - -The Open function takes a TLSContext reference which specifies the parameters for the connection. - - -FUNCTION GetPeerCommonName() - -Returns the common name of the certificate presented by the remote end. - diff --git a/docs/api-docs/common/lib_server/TLSContext.txt b/docs/api-docs/common/lib_server/TLSContext.txt deleted file mode 100644 index ff50d3e6..00000000 --- a/docs/api-docs/common/lib_server/TLSContext.txt +++ /dev/null @@ -1,16 +0,0 @@ -CLASS TLSContext - -A wrapper over the OpenSSL context object. - -Note: you need to call SSLLib::Initialise at the beginning of your program to use these functions. - - -SUBTITLE Construction - -The constuctor takes the following parameters - -* Boolean for whether this is acting as a server or a client -* The .pem file containing the certificate which will be used -* The .pem file containing the private key for this certificate -* The .pem file containing the certificates which will certify the other end of the connection. - diff --git a/docs/api-docs/common/memory_leaks.txt b/docs/api-docs/common/memory_leaks.txt deleted file mode 100644 index 9a9764ea..00000000 --- a/docs/api-docs/common/memory_leaks.txt +++ /dev/null @@ -1,44 +0,0 @@ -TITLE Memory leak detection - -The build system contains a primative memory leak detection system in debug builds. - -It works by using #defines to replace calls to malloc, free, realloc, new and delete with debug versions which record their use. When a process ends, it dumps a list of all the blocks or objects which were allocated by not freed, and the file and line of the source where they are originally allocated. - -It's not perfect, but should catch most things and work on most platforms. - -If it doesn't work on your platform, define PLATFORM_DISABLE_MEM_LEAK_TESTING in BoxPlatform.h within the relevant section. - - -SUBTITLE Requirements in source - -It does require some extra lines in the source file. The last included file in each .cpp file must be - - #include "MemLeakFindOn.h" - -and if a .h file uses any of these functions, the contents of the .h file should be bounded with - - #include "MemLeakFindOn.h" - - // ... some code, but absolutely no #includes - - #include "MemLeakFindOff.h" - -The cleanupforcvs.pl script checks for correct usage. - - -SUBTITLE Use in tests - -Tests will automatically dump memory leaks and regard them as a failure for anything executing in the main test process. - -To test for memory leaks in programs run from the test, or daemons, use something like - - TestRemoteProcessMemLeaks("bbackupd.memleaks"); - -If memory leak detection is being used, it will check the specified file for memory leak reports (deleting it afterwards). Any leak is an error. - -The Daemon class automactically arranges for the memory leak reports to be written. Other exes should set this up themselves, preferably using the define in MainHelper.h, as the first thing in their main() function. - - MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT(file, name) - -File is the filename to write the report to, conventionally called ".memleaks", and name is the name of the exe. - diff --git a/docs/api-docs/raidfile/lib_raidfile.txt b/docs/api-docs/raidfile/lib_raidfile.txt deleted file mode 100644 index 0c4244ce..00000000 --- a/docs/api-docs/raidfile/lib_raidfile.txt +++ /dev/null @@ -1,73 +0,0 @@ -TITLE lib/raidfile - -Implements RAID type abilities in files in userland, along with simple transaction like semantics for writing and reading files. - -IOStream interfaces are used to read and write files. - -Eventually a raid file daemon will be written to "raidify" files in the background. This will allow fast streaming of data to a single disc, followed by splitting into separate raid files later. - -There is an option to disable raidification for use on systems with only one hard disc, or a hardware RAID array. - - -SUBTITLE Controller - -The raid disc sets are managed by RaidFileController. This reads the configuration file, /etc/box/raidfile.conf, and then stores the sets of discs for use by the other classes. - -In the code, files are referenced using an integer set number, and a filename within that set. For example, (0, "dir/subdir/filename.ext"). The RAID file controller turns this into actual physcial filenames transparently. - -The files are devided into blocks, which should match the fragment/block size of the underlying filesystem for maximum space efficiency. - -If the directories specified in the configuration files are all equal, then userland RAID is disabled. The files will not be processed into the redundant parts. - - -SUBTITLE Writing - -Files are written (and modified) using the RaidFileWrite class. This behaves as a seekable IOStream for writing. - -When all data has been written, the file is committed. This makes it available to be read. Processing to split it into the three files can be delayed. - -If the data is not commited, the temporary write file will be deleted. - -RaidFileWrite will optionally refuse to overwrite another file. If it is being overwritten, a read to that file will get the old file until it has been committed. - - -SUBTITLE Reading - -File are opened and read with RaidFileRead::Open. This returns an IOStream (with some extras) which reads a file. - -If the file has been turned into the RAID representation, then if an EIO error occurs on any operation, it will switch transparently into recovery mode where the file will be regenerated from the remaining two files using the parity file. (and log an error to syslog) - - -SUBTITLE Layout on disc - -The directory structure is replicated onto each of the three directories within the disc set. - -Given a filename x, within the disc set, one of the three discs is chosen as the 0th disc for this file. This is done by hashing the filename. This ensures that an equal load is placed on each disc -- otherwise one disc would only be accessed when a RAID file was being written. - -When the file is being written, the actual physical file is "x.rfwX". This just a striaght normal file -- nothing is done to the data. - -When it is commited, it is renamed to "x.rfw". - -Then, when it is turned into the raid representation, it is split across three discs, each file named "x.rf". - -When trying to open a file, the first attempt is for "x.rfw". If this exists, it is used as is. - -If this fails, two of the "x.rf" files are attempted to be opened. If this succeeds, the file is opened in RAID mode. - -This procedure guarenettes that a file will be opened either as a normal file, or as the correct three RAID file elements, even if a new version is being commited at the same time, or another process is working to turn it into a raid file. - - -SUBTITLE Raid file representation - -The file is split into three files, stripe 1, stripe 2 and parity. - -Considering the file as a set of blocks of the size specified in the configuration, odd numbered blocks go in stripe 1, even blocks in stripe 2, and a XOR of the corresponding blocks in parity. - -Thus only two files are needed to recreate the original. - -The complexity comes because the size of the file is not stored inside the files at any point. This means if stripe 2 is missing, it is ambiguous as to how big the file is. - -Size is stored as a 64 bit integer. This is appended to the parity file, unless it can be stored by XORing it into the last 8 bytes of the parity file. - -This scheme is complex to implement (see the RaidFileRead.cpp!) but minimises the disc spaced wasted. - diff --git a/docs/api-docs/raidfile/lib_raidfile/RaidFileRead.txt b/docs/api-docs/raidfile/lib_raidfile/RaidFileRead.txt deleted file mode 100644 index 0a5efbf7..00000000 --- a/docs/api-docs/raidfile/lib_raidfile/RaidFileRead.txt +++ /dev/null @@ -1,14 +0,0 @@ -CLASS RaidFileRead - -Read a raid file. - -IOStream interface, plus a few extras, including reading directories and checking that files exist. - - -FUNCTION RaidFileRead::Open - -Open a given raid file -- returns a pointer to a new RaidFileRead object. - -Note that one of two types could be returned, depending on the representation of the file. - - diff --git a/docs/api-docs/raidfile/lib_raidfile/RaidFileWrite.txt b/docs/api-docs/raidfile/lib_raidfile/RaidFileWrite.txt deleted file mode 100644 index 89500c37..00000000 --- a/docs/api-docs/raidfile/lib_raidfile/RaidFileWrite.txt +++ /dev/null @@ -1,36 +0,0 @@ -CLASS RaidFileWrite - -Interface to writing raidfiles. - -See IOStream interface. - -Some other useful functions are available, see h and cpp files. - - -FUNCTION RaidFileWrite::RaidFileWrite() - -The constructor takes the disc set number and filename of the file you're interested. - - -FUNCTION RaidFileWrite::Open() - -Open() opens the file for writing, and takes an "allow overwrite" flag. - - -FUNCTION RaidFileWrite::Commit() - -Commmit the file, and make it visible to RaidFileRead. If ConvertToRaidNow == true, it will be converted to raid file representation immediately. - -Setting it to false is not a good idea. Later on, it will tell a daemon to convert it in the background, but for now it simply won't be converted. - - -FUNCTION RaidFileWrite::Discard() - -Abort the creation/update. Equivalent to just deleting the object without calling Commit(). - - -FUNCTION RaidFileWrite::Delete() - -Delete a file -- don't need to Open() it first. - - diff --git a/docs/api-notes/backup/INDEX.txt b/docs/api-notes/backup/INDEX.txt new file mode 100644 index 00000000..50406cac --- /dev/null +++ b/docs/api-notes/backup/INDEX.txt @@ -0,0 +1,61 @@ +TITLE Programmers Notes for Box Backup + +This directory contains the programmers notes for the Box Backup system. They will be of interest only if you want to review or modify the code. + +These notes are intended to be run through a script to produce HTML at some later stage, hence the marks such as 'TITLE'. + + +SUBTITLE Organisation + +The project is split up into several modules. The modules within 'lib' are building blocks for creation of the actual executable programs within 'bin'. 'test' contains unit tests for lib and bin modules. + +The file modules.txt lists the modules which are to be built, and their dependencies. It also allows for platform differences. + + +SUBTITLE Documentation Organisation + +In this directory, the files correspond to modules or areas of interest. Sub directories of the same name contain files documenting specific classes within that module. + + +SUBTITLE Suggested reading order + +* lib_common +* lib_server +* bin_bbackupd +* backup_encryption.txt +* bin_bstored +* lib_raidfile + +and refer to other sections as required. + + +SUBTITLE Building + +The makefiles are generated by makebuildenv.pl. (The top level makefile is generated by makeparcels.pl, but this is for the end user to use, not a programmer.) + +To build a module, cd to it and type make. If the -DRELEASE option is specified (RELEASE=1 with GNU make) the release version will be built. The object files and exes are placed in a directory structure under 'release' or 'debug'. + +It is intended that a test will be written for everything, so in general make commands will be issued only within the test/* modules. Once it has been built, cd to debug/test/ and run the test with ./t . + + +SUBTITLE Programming style + +The code is written to be easy to write. Ease of programming is the primary concern, as this should lead to fewer bugs. Efficiency improvements can be made later when the system as a whole works. + +Much use is made of the STL. + +There is no common base class. + +All errors are reported using exceptions. + +Some of the boring code is generated by perl scripts from description files. + +There are a lot of modules and classes which can easily be used to build other projects in the future -- there is a lot of "framework" code. + + +SUBTITLE Lots more documentation + +The files are extensively commented. Consider this notes as an overview, and then read the source files for detailed and definitive information. + +Each function and class has a very brief decsription of it's purpose in a standard header, and extensive efforts have been maed to comment the code itself. + diff --git a/docs/api-notes/backup/Win32_Clients.txt b/docs/api-notes/backup/Win32_Clients.txt new file mode 100644 index 00000000..fad6c4af --- /dev/null +++ b/docs/api-notes/backup/Win32_Clients.txt @@ -0,0 +1,13 @@ +The basic client tools now run on Win32 natively. +The port was done by nick@omniis.com. + +* bbackupd +* bbackupquery +* bbackupctl + +Have been ported. bbackupd runs as a NT style service. + +Known limitations: + +* File attributes and permissions are not backed up. + diff --git a/docs/api-notes/backup/backup_encryption.txt b/docs/api-notes/backup/backup_encryption.txt new file mode 100644 index 00000000..36580581 --- /dev/null +++ b/docs/api-notes/backup/backup_encryption.txt @@ -0,0 +1,109 @@ +TITLE Encryption in the backup system + +This document explains how everything is encrypted in the backup system, and points to the various functions which need reviewing to ensure they do actually follow this scheme. + + +SUBTITLE Security objectives + +The crpyto system is designed to keep the following things secret from an attacker who has full access to the server. + +* The names of the files and directories +* The contents of files and directories +* The exact size of files + +Things which are not secret are + +* Directory heirarchy and number of files in each directory +* How the files change over time +* Approximate size of files + + +SUBTITLE Keys + +There are four separate keys used: + +* Filename +* File attributes +* File block index +* File data + +and an additional secret for file attribute hashes. + +The Cipher is Blowfish in CBC mode in most cases, except for the file data. All keys are maximum length 448 bit keys, since the key size only affects the setup time and this is done very infrequently. + +The file data is encrypted with AES in CBC mode, with a 256 bit key (max length). Blowfish is used elsewhere because the larger block size of AES, while more secure, would be terribly space inefficient. Note that Blowfish may also be used when older versions of OpenSSL are in use, and for backwards compatibility with older versions. + +The keys are generated using "openssl rand", and a 1k file of key material is stored in /etc/box/bbackupd. The configuration scripts make this readable only by root. + +Code for review: BackupClientCryptoKeys_Setup() +in lib/backupclient/BackupClientCryptoKeys.cpp + + +SUBTITLE Filenames + +Filenames need to be secret from the attacker, but they need to be compared on the server so it can determine whether or not is it a new version of an old file. + +So, the same Initialisation Vector is used for every single filename, so the same filename encrypted twice will have the same binary representation. + +Filenames use standard PKCS padding implemented by OpenSSL. They are proceeded by two bytes of header which describe the length, and the encoding. + +Code for review: BackupStoreFilenameClear::EncryptClear() +in lib/backupclient/BackupStoreFilenameClear.cpp + + +SUBTITLE File attributes + +These are kept secret as well, since they reveal information. Especially as they contain the target name of symbolic links. + +To encrypt, a random Initialisation Vector is choosen. This is stored first, followed by the attribute data encrypted with PKCS padding. + +Code for review: BackupClientFileAttributes::EncryptAttr() +in lib/backupclient/BackupClientFileAttributes.cpp + + +SUBTITLE File attribute hashes + +To detect and update file attributes efficiently, the file status change time is not used, as this would give suprious results and result in unnecessary updates to the server. Instead, a hash of user id, group id, and mode is used. + +To avoid revealing details about attributes + +1) The filename is added to the hash, so that an attacker cannot determine whether or not two files have identical attributes + +2) A secret is added to the hash, so that an attacker cannot compare attributes between accounts. + +The hash used is the first 64 bits of an MD5 hash. + + +SUBTITLE File block index + +Files are encoded in blocks, so that the rsync algorithm can be used on them. The data is compressed first before encryption. These small blocks don't give the best possible compression, but there is no alternative because the server can't see their contents. + +The file contains a number of blocks, which contain among other things + +* Size of the block when it's not compressed +* MD5 checksum of the block +* RollingChecksum of the block + +We don't want the attacker to know the size, so the first is bad. (Because of compression and padding, there's uncertainty on the size.) + +When the block is only a few bytes long, the latter two reveal it's contents with only a moderate amount of work. So these need to be encrypted. + +In the header of the index, a 64 bit number is chosen. The sensitive parts of the block are then encrypted, without padding, with an Initialisation Vector of this 64 bit number + the block index. + +If a block from an previous file is included in a new version of a file, the same checksum data will be encrypted again, but with a different IV. An eavesdropper will be able to easily find out which data has been re-encrypted, but the plaintext is not revealed. + +Code for review: BackupStoreFileEncodeStream::Read() (IV base choosen about half-way through) +BackupStoreFileEncodeStream::EncodeCurrentBlock() (encrypt index entry) +in lib/backupclient/BackupStoreFileEncodeStream.cpp + + +SUBTITLE File data + +As above, the first is split into chunks and compressed. + +Then, a random initialisation vector is chosen, stored first, followed by the compressed file data encrypted using PKCS padding. + +Code for review: BackupStoreFileEncodeStream::EncodeCurrentBlock() +in lib/backupclient/BackupStoreFileEncodeStream.cpp + + diff --git a/docs/api-notes/backup/bin_bbackupd.txt b/docs/api-notes/backup/bin_bbackupd.txt new file mode 100644 index 00000000..67ad9267 --- /dev/null +++ b/docs/api-notes/backup/bin_bbackupd.txt @@ -0,0 +1,88 @@ +TITLE bin/bbackupd + +The backup client daemon. + +This aims to maintain as little information as possible to record which files have been uploaded to the server, while minimising the amount of queries which have to be made to the server. + + +SUBTITLE Scanning + +The daemon is given a length of time, t, which files over this age should be uploaded to the server. This is to stop recently updated files being uploaded immediately to avoid uploading something repeatedly (on the assumption that if a file has been written, it is likely to be modified again shortly). + +It will scan the files at a configured interval, and connect to the server if it needs to upload files or make queries about files and directories. + +The scan interval is actually varied slightly between each run by adding a random number up to a 64th of the configured time. This is to reduce cyclic patterns of load on the backup servers -- otherwise if all the boxes are turned on at about 9am, every morning at 9am there will be a huge spike in load on the server. + +Each scan chooses a time interval, which ends at the current time - t. This will be from 0 to current time - t on the first run, then the next run takes the start time as the end time of the previous run. The scan is only performed if the difference between the start and end times is greater or equal to t. + +For each configured location, the client scans the directories on disc recursively. + +For each directory + +* If the directory has never been scanned before (in this invocation of the daemon) or the modified time on the directory is not that recorded, the listing on the server is downloaded. + +* For each file, if it's modified time is within the time period, it is uploaded. If the directory has been downloaded, it is compared against that, and only uploaded if it's changed. + +* Find all the new files, and upload them if they lie within the time interval. + +* Recurse to sub directories, creating them on the server if necessary. + +Hence, the first time it runs, it will download and compare the entries on the disc to those on the server, but in future runs it will use the file and directory modification times to work out if there is anything which needs uploading. + +If there aren't any changes, it won't even need to connect to the server. + +There are some extra details which allow this to work reliably, but they are documented in the source. + + +SUBTITLE File attributes + +The backup client will update the file attributes on files as soon as it notices they are changed. It records most of the details from stat(), but only a few can be restored. Attributes will only be considered changed if the user id, group id or mode is changed. Detection is by a 64 bit hash, so detection is strictly speaking probablistic. + + +SUBTITLE Encryption + +All the user data is encrypted. There is a separate file, backup_encryption.txt which describes this, and where in the code to look to verify it works as described. + + +SUBTITLE Tracking files and directories + +Renaming files is a difficult problem under this minimal data scanning scheme, because you don't really know whether a file has been renamed, or another file deleted and new one created. + +The solution is to keep (on disc) a map of inode numbers to server object IDs for all directories and files over a certain user configurable threshold. Then, when a new file is discovered, it is first checked to see if it's in this map. If so, a rename is considered, which will take place if the local object corresponding to the name of the tracked object doesn't exist any more. + +Because of the renaming requirement, deletions of objects from the server are recorded and delayed until the end of the scan. + + +SUBTITLE Running out of space + +If the store server indicates on login to the backup client, it will scan, but not upload anything nor adjust it's internal stored details of the local objects. However, deletions and renames happen. + +This is to allow deletions to still work and reduce the amount of storage space used on the server, in the hope that in the future there will be enough space. + +Just not doing anything would mean that one big file created and then deleted at the wrong time would stall the whole backup process. + + +SUBTITLE BackupDaemon + +This is the daemon class for the backup daemon. It handles setting up of all the objects, and implements calulcation of the time intervals for the scanning. + + +SUBTITLE BackupClientContext + +State information for the scans, including maintaining a connection to the store server if required. + + +SUBTITLE BackupClientDirectoryRecord + +A record of state of a directory on the local filesystem. Containing the recursive scanning function, which is long and entertaining, but very necessary. It contains lots of comments which explain the exact details of what's going on. + + +SUBTITLE BackupClientInodeToIDMap + +A implementation of a map of inode number to object ID on the server. If Berkeley DB is available on the platform, it is stored on disc, otherwise there is an in memory version which isn't so good. + + + + + + diff --git a/docs/api-notes/backup/bin_bbstored.txt b/docs/api-notes/backup/bin_bbstored.txt new file mode 100644 index 00000000..c9c4e229 --- /dev/null +++ b/docs/api-notes/backup/bin_bbstored.txt @@ -0,0 +1,54 @@ +TITLE bin/bbstored + +The backup store daemon. + +Maintains a store of encrypted files, and every so often goes through deleting unnecessary data. + +Uses an implementation of Protocol to communicate with the backup client daemon. See bin/bbstored/backupprotocol.txt for details. + + +SUBTITLE Data storage + +The data is arranged as a set of objects within a RaidFile disc set. Each object has a 64 bit object ID, which is turned into a filename in a mildly complex manner which ensure that directories don't have too many objects in them, but there is a minimal number of nested directories. See StoreStructure::MakeObjectFilename in lib/backupstore/StoreStructure.cpp for more details. + +An object can be a directory or a file. Directories contain files and other directories. + +Files in directories are supersceded by new versions when uploaded, but the old versions are flagged as such. A new version has a different object ID to the old version. + +Every so often, a housekeeping process works out what can be deleted, and deletes unnecessary files to take them below the storage limits set in the store info file. + + +SUBTITLE Note about file storage and downloading + +There's one slight entertainment to file storage, in that the format of the file streamed depends on whether it's being downloaded or uploaded. + +The problem is that it contains an index of all the blocks. For efficiency in managing these blocks, they all need to be in the same place. + +Files are encoded and decoded as they are streamed to and from the server. With encoding, the index is only completely known at the end of the process, so it's sent last, and lives in the filesystem last. + +When it's downloaded, it can't be decoded without knowing the index. So the index is sent first, followed by the data. + + +SUBTITLE BackupContext + +The context of the current connection, and the object which modifies the store. + +Maintains a cache of directories, to avoid reading them continuously, and keeps a track of a BackupStoreInfo object which is written back periodiocally. + + +SUBTITLE BackupStoreDaemon + +A ServerTLS daemon which forks off a separate housekeeping process as it starts up. + +Handling connections is delegated to a Protocol implementation. + + +SUBTITLE BackupCommands.cpp + +Implementation of all the commands. Work which requires writing is handled in the context, read only commands mainly in this file. + + +SUBTITLE HousekeepStoreAccount + +A class which performs housekeeping on a single account. + diff --git a/docs/api-notes/backup/encrypt_rsync.txt b/docs/api-notes/backup/encrypt_rsync.txt new file mode 100644 index 00000000..a5db2df2 --- /dev/null +++ b/docs/api-notes/backup/encrypt_rsync.txt @@ -0,0 +1,66 @@ +TITLE Encrypted rsync algorithm + +The backup system uses a modified version of the rsync algorithm. A description of the plain algorithm can be found here: + + http://samba.anu.edu.au/rsync/tech_report/ + +The algorithm is modified to allow the server side to be encrypted, yet still benefit from the reduced bandwidth usage. For a single file transfer, the result will be only slightly less efficient than plain rsync. For a backup of a large directory, the overall bandwidth may be less due to the way the backup client daemon detects changes. + +This document assumes you have read the rsync document. + +The code is in lib/backupclient/BackupStoreFile*.*. + + +SUBTITLE Blocks + +Each file is broken up into small blocks. These are individually compressed and encrypted, and have an entry in an index which contains, encrypted, it's weak and strong checksums and decoded plaintext size. This is all done on the client. + +Why not just encrypt the file, and use the standard rsync algorithm? + +1) Compression cannot be used, since encryption turns the file into essentially random data. This is not very compressible. + +2) Any modification to the file will result in all data after that in the file having different ciphertext (in any cipher mode we might want to use). Therefore the rsync algorithm will only be able to detect "same" blocks up until the first modification. This significantly reduces the effectiveness of the process. + +Note that blocks are not all the same size. The last block in the file is unlikely to be a full block, and if data is inserted which is not a integral multiple of the block size, odd sized blocks need to be created. This is because the server cannot reassemble the blocks, because the contents are opaque to the server. + + +SUBTITLE Modifed algorithm + +To produce a list of the changes to send the new version, the client requests the block index of the file. This is the same step as requesting the weak and strong checksums from the remote side with rsync. + +The client then decrypts the index, and builds a list of the 8 most used block sizes above a certain threshold size. + +The new version of the file is then scanned in exactly the same way as rsync for these 8 block sizes. If a block is found, then it is added to a list of found blocks, sorted by position in the file. If a block has already been found at that position, then the old entry is only replaced by the new entry if the new entry is a "better" (bigger) match. + +The block size covering the biggest file area is searched first, so that most of the file can be skipped over after the first pass without expensive checksumming. + +A "recipe" is then built from the found list, by trivially discarding overlapping blocks. Each entry consists of a number of bytes of "new" data, a block start number, and a number of blocks from the old file. The data is stored like this as a memory optimisation, assuming that files mostly stay the same rather than having all their blocks reordered. + +The file is then encoded, with new data being sent as blocks of data, and references to blocks in the old file. The new index is built completely, as the checksums and size need to be rencrypted to match their position in the index. + + +SUBTITLE Combination on server + +The "diff" which is sent from the client is assembled into a full file on the server, simply by adding in blocks from the old file where they are specified in the block index. + + +SUBTITLE Storage on server + +Given that the server will in general store several versions of a file, combining old and new files to form a new file is not terribly efficient on storage space. Particularly for large multi-Gb database files. + +An alternative scheme is outlined below, however, it is significantly more complex to implement, and so is not implemented in this version. + +1) In the block index of the files, store the file ID of the file which each block is source from. This allows a single file to reference blocks from many files. + +2) When the file is downloaded, the server combines the blocks from all the files into a new file as it is streamed to the client. (This is not particuarly complicated to do.) + +This all sounds fine, until housekeeping is considered. Old versions need to be deleted, without losing any blocks necessary for future versions. + +Instead of just deleting a file, the server works out which blocks are still required, and rebuilds the file omitting those blocks which aren't required. + +This complicates working out how much space a file will release when it is "deleted", and indeed, adds a whole new level of complexity to the housekeeping process. (And the tests!) + +The directory structure will need an additional flag, "Partial file", which specifies that the entry cannot be built as previous blocks are no longer available. Entries with this flag should never be sent to the client. + + + diff --git a/docs/api-notes/backup/lib_backupclient.txt b/docs/api-notes/backup/lib_backupclient.txt new file mode 100644 index 00000000..3e4a079b --- /dev/null +++ b/docs/api-notes/backup/lib_backupclient.txt @@ -0,0 +1,46 @@ +TITLE lib/backupclient + +Classes used on the store and on the server. + +See documentation in the files for more details. + + +SUBTITLE BackupStoreDirectory + +The directory listing class, containing a number of entries, representing files. + + +SUBTITLE BackupStoreFile + +Handles compressing and encrypting files, and decoding files downloaded from the server. + + +SUBTITLE BackupStoreFilename + +An encrypted filename. + + +SUBTITLE BackupStoreFilenameClear + +Derived from BackupStoreFilename, but with the ability to encrypt and decrypt filenames. Client side only. + + +SUBTITLE BackupClientFileAttributes + +Only used on the client -- the server treats attributes as blocks of opaque data. + +This reads attributes from files on discs, stores them, encrypts them, and applies them to new files. + +Also has a static function to generate filename attribute hashes given a struct stat and the filename. + + +SUBTITLE BackupClientRestore + +Routines to restore files from the server onto the client filesystem. + + +SUBTITLE BackupClientCryptoKeys + +This reads the key material from disc, and sets up the crypto for storing files, attributes and directories. + + diff --git a/docs/api-notes/backup/lib_backupstore.txt b/docs/api-notes/backup/lib_backupstore.txt new file mode 100644 index 00000000..8f24eb7b --- /dev/null +++ b/docs/api-notes/backup/lib_backupstore.txt @@ -0,0 +1,30 @@ +TITLE lib/backupstore + +Classes which are shared amongst the server side store utilities, bbstored and bbstoreaccounts. Note also depends on lib/backupclient, as a lot of code is shared between the client and server. + + +SUBTITLE BackupStoreAccountDatabase + +A simple implementation of an account database. This will be replaced with a more suitable implementation. + + +SUBTITLE BackupStoreAccounts + +An interface to the account database, and knowledge of how to initialise an account on disc. + + +SUBTITLE BackupStoreConfigVerify + +The same configuration file is used by all the utilities. This is the Configuration verification structure for this file. + + +SUBTITLE BackupStoreInfo + +The "header" information about an account, specifying current disc usage, space limits, etc. + + +SUBTITLE StoreStructure + +Functions specifying how the files are laid out on disc in the store. + + diff --git a/docs/api-notes/backup/win32_build_on_cygwin_using_mingw.txt b/docs/api-notes/backup/win32_build_on_cygwin_using_mingw.txt new file mode 100644 index 00000000..1caaf4ca --- /dev/null +++ b/docs/api-notes/backup/win32_build_on_cygwin_using_mingw.txt @@ -0,0 +1,53 @@ +How to build Box Backup on Win32 using Cygwin and MinGW +By Chris Wilson, 2007-05-26 + +(To read this document online with better formatting, browse to: +http://www.boxbackup.org/trac/wiki/CompileWithMinGW) + +Start by installing Cygwin on your Windows machine [http://www.cygwin.org]. +Make sure to select the following packages during installation: + +* Devel/gcc-mingw +* Devel/gcc-mingw-core +* Devel/gcc-mingw-g++ +* Mingw/mingw-zlib + +If you already have Cygwin installed, please re-run the installer and +ensure that those packages are installed. + +Download OpenSSL from +[http://www.openssl.org/source/openssl-0.9.7i.tar.gz] + +Open a Cygwin shell, and unpack OpenSSL: + + tar xzvf openssl-0.9.7i.tar.gz + +Configure OpenSSL for MinGW compilation, and build and install it: + + cd openssl-0.9.7i + ./Configure --prefix=/usr/i686-pc-mingw32/ mingw + make + make install + +Download PCRE from +[http://prdownloads.sourceforge.net/pcre/pcre-6.3.tar.bz2?download] + +Open a Cygwin shell, and unpack PCRE: + + tar xjvf pcre-6.3.tar.bz2 + +Configure PCRE for MinGW compilation, and build and install it: + + cd pcre-6.3 + export CFLAGS="-mno-cygwin" + ./configure + make winshared + cp .libs/libpcre.a .libs/libpcreposix.a /usr/lib/mingw + cp pcreposix.h /usr/include/mingw + +Now unpack the Box Backup sources, enter the source directory, +and configure like this: + + ./infrastructure/mingw/configure.sh + make + diff --git a/docs/api-notes/backup/win32_build_on_linux_using_mingw.txt b/docs/api-notes/backup/win32_build_on_linux_using_mingw.txt new file mode 100644 index 00000000..2c3e4fe9 --- /dev/null +++ b/docs/api-notes/backup/win32_build_on_linux_using_mingw.txt @@ -0,0 +1,108 @@ +How to build Box Backup for Windows (Native) on Linux using MinGW +By Chris Wilson, 2005-12-07 + +Install the MinGW cross-compiler for Windows: + +- Debian and Ubuntu users can "apt-get install mingw32" +- Fedora and SuSE users can download RPM packages from + [http://mirzam.it.vu.nl/mingw/] + +You will need to know the prefix used by the cross-compiler executables. +It will usually be something like "ix86-mingw32*-". All the binaries in the +cross-compiler package will start with this prefix. The documentation below +assumes that it is "i386-mingw32-". Adjust to taste. + +You will also need to install Wine and the Linux kernel "binary formats" +(binfmt) support, so that you can run Windows executables on Linux, +otherwise the configure scripts will not work properly with a cross-compiler. +On Ubuntu, run: + + apt-get install wine binfmt-support + /etc/init.d/binfmt-support start + +Start by downloading Zlib from [http://www.zlib.net/], unpack and enter +source directory: + + export CC=i386-mingw32-gcc + export AR="i386-mingw32-ar rc" + export RANLIB="i386-mingw32-ranlib" + ./configure + make + make install prefix=/usr/local/i386-mingw32 + +Download OpenSSL 0.9.8b from +[http://www.openssl.org/source/openssl-0.9.8b.tar.gz] + +Unpack and configure: + + tar xzvf openssl-0.9.8b.tar.gz + cd openssl-0.9.8b + ./Configure --prefix=/usr/local/i386-mingw32 mingw + make makefile.one + wget http://www.boxbackup.org/svn/box/chris/win32/support/openssl-0.9.8b-mingw-cross.patch + patch -p1 < openssl-0.9.8b-mingw-cross.patch + make -f makefile.one + make -f makefile.one install + +Download PCRE from +[http://prdownloads.sourceforge.net/pcre/pcre-6.3.tar.bz2?download] + +Unpack: + + tar xjvf pcre-6.3.tar.bz2 + cd pcre-6.3 + +Configure and make: + + export AR=i386-mingw32-ar + ./configure --host=i386-mingw32 --prefix=/usr/local/i386-mingw32/ + make winshared + +If you get this error: + + ./dftables.exe pcre_chartables.c + /bin/bash: ./dftables.exe: cannot execute binary file + make: *** [pcre_chartables.c] Error 126 + +then run: + + wine ./dftables.exe pcre_chartables.c + make winshared + +to complete the build. Finally: + + cp .libs/libpcre.a /usr/local/i386-pc-mingw32/lib + cp .libs/libpcreposix.a /usr/local/i386-pc-mingw32/lib + cp pcreposix.h /usr/local/i386-pc-mingw32/include + +You will need to find a copy of mingwm10.dll that matches your cross-compiler. +Most MinGW distributions should come with it. On Debian and Ubuntu, for some +bizarre reason, you'll find it compressed as +/usr/share/doc/mingw32-runtime/mingwm10.dll.gz, in which case you'll +have to un-gzip it with "gzip -d". Copy it to a known location, e.g. +/usr/local/i386-mingw32/bin. + +Download and extract Box Backup, and change into the base directory, +e.g. boxbackup-0.11rc2. Change the path to mingwm10.dll in parcels.txt to +match where you found or installed it. + +Now configure Box with: + + ./configure --host=i386-mingw32 \ + CXXFLAGS="-mthreads -I/usr/local/i386-mingw32/include" \ + LDFLAGS=" -mthreads -L/usr/local/i386-mingw32/lib" \ + LIBS="-lcrypto -lws2_32 -lgdi32" + make + +or, if that fails, try this: + + export CXX="i386-mingw32-g++" + export AR=i386-mingw32-ar + export RANLIB=i386-mingw32-ranlib + export CFLAGS="-mthreads" + export CXXFLAGS="-mthreads" + export LDFLAGS="-mthreads" + export LIBS="-lcrypto -lws2_32 -lgdi32" + (if you don't have a "configure" file, run "./bootstrap") + ./configure --target=i386-mingw32 + make CXX="$CXX" AR="$AR" RANLIB="$RANLIB" WINDRES="i386-mingw32-windres" diff --git a/docs/api-notes/backup/windows_porting.txt b/docs/api-notes/backup/windows_porting.txt new file mode 100644 index 00000000..ada3b857 --- /dev/null +++ b/docs/api-notes/backup/windows_porting.txt @@ -0,0 +1,100 @@ +TITLE Notes on porting the backup client to Windows + +It should be relatively easy to port the backup client (bbackupd) to Windows. However, the server relies on unlink() behaviour of the UNIX filesystem which is different on the Windows platform, so it will not be so easy to port. + +An installation of perl is required to build the system. The ActiveState port is the easiest to install. + + +SUBTITLE Build environment + +The build environment generation script, makebuildenv.pl, uses perl to scan all the files and generate makefiles. It's rather orientated towards UNIX systems with gcc. + +Probably the easiest way to handle this is to detect the Windows platform, set up a few variables appropriately (in BoxPlatform.pm) and then post-process the generated Makefiles to mould them into something more handy for the MS Windows compiler and toolchain. + +The script itself is a bit messy. It was fine at first, but then the multi-platform thing got in the way. I do intend to rewrite it at some point in the future. + +Make sure your new version defines PLATFORM_WIN32 on the compile line. + +All files #include "Box.h" as the first include file. Use this for pre-compiled headers. Edit BoxPlatform.h to include the Windows headers required, and include a PLATFORM_WIN32 section. The easiest start will be to leave this blank, apart from typedefs for the basic types and any "not supported" #defines you can find. + +Boring bits of the code, such as exceptions and protocol definitions, are autogenerated using perl scripts. This code should be portable without modification. + +I have tried to avoid the things I know won't work with the MS compiler, so hopefully the code will be fairly clean. However, it might be a little easier to use the MinGW compiler [ http://www.mingw.org/ ] just to be consistent with the UNIX version. But I hope this won't be necessary. + +You'll need the latest version of OpenSSL. This was slightly difficult to get to compile last time I tried -- especially if you're determined to use the optimised assembler version. The main difficulty was getting a version which would link properly with the options used in my project, the default libraries selected got in the way. + + +SUBTITLE Porting as UNIX emulation + +Since the daemon uses so few UNIX system calls and with a limited set of options, it seems to make sense to port it by writing emulations of these functions. It's probably nicest to create a lib/win32 directory, and populate this with header files corresponding to the UNIX header files used. These just contain inline functions which map the UNIX calls to Win32 calls. + +File/socket handles may have to be translated. -1 is used as a failure return value and by the code internally to mark an invalid socket handle. (0 is a valid socket handle) + +Of course, some bits of code aren't relevant, so will just be #ifdefed out, or replaced. But this should be minimal. (Only perhaps the small bit relating to filesystem structure -- there aren't really mount points as such.) + + +SUBTITLE File tracking + +The daemon uses the inode number of a file to keep track of files and directories, so when they're renamed they can be moved efficiently on the store. Some unique (per filesystem) number will have to be found and used instead. + +It uses the Berkeley DB to store these on disc. It's likely another storage system will need to be used. (It just has to map the file's unique number into to a 8 byte struct.) + +There is a in-memory implementation for platforms which don't support Berkeley DB, but this isn't so good when the daemon has to be restarted as all the tracking is lost. But it's an easy start. + + +SUBTITLE Expected filesystem behaviour + +File and directories have (at least) two modification times, for contents and attributes. + +For files, the contents modification time must change when the contents change, and the attributes time when the attributes change (and may change when the contents change too.) + +For directories, the contents modification time must change when files or directories are deleted or added. If it changes any more frequently than this, then the client will be slightly less efficient -- it will download the store's directory listing whenever this time changes. The attributes modification time is less important, as the actual attributes are compared and only uploaded if different. + + +SUBTITLE Attributes + +Attributes means file modification times, flags, and filesystem permissions. + +The BackupClientFileAttribute class will need to be extended. Allocate another "attribute type" for the Win32 attributes, and then serialise it in a compatible way -- put your new attribute type in the header, and then a serialised network byte order structure in the rest. The different size of block is handled for you, and the server never looks inside. + +Add code so that under UNIX, Win32 attributes are ignored, and UNIX attributes under Win32. + +It's probably not necessary to worry too much about these for the first version. Not many people seem to use these attributes anyway. + + +SUBTITLE Times + +The system uses it's own 64 bit time type -- see BoxTime.h. Everything is translated to this from the various different system time types, and calculated and stored internally in this form. + + +SUBTITLE Daemon as a Service + +The client is derived from the Daemon class, which implements a daemon. The interface is simple, and it shouldn't be hard to write a compatible class which implements a Windows Service instead. + +Or cheat and run it as a Win32 application. + +Note that the daemon expects to be able to read every file it wants, and will abort a scan and upload run if it gets an error. The daemon must therefore be run with sufficient privileges. It runs as root under UNIX. + + +SUBTITLE Command Socket + +The backup daemon accepts commands from bbackupctl through a UNIX domain socket. When a connection is made, the user ID of the connecting process is checked to see if it's the same user ID as the daemon is running under. + +This may not have any exact analogue under Win32, so another communications scheme may have to be devised. + +This is only actually necessary if the client is to be run in snapshot mode. It can be safely left unimplemented if snapshot mode is not required, or the prompts for it to sync with the server are implemented some other way. + + +SUBTITLE NTFS streams + +If you want to back up NTFS streams, then a generic solution should probably be defined, so that the Mac OS X resource forks can be backed up with the same mechanism. + + +SUBTITLE Source code + +I work on a slightly different version of the source files. A make distribution script adds the license header and removes private sections of code. This means submitted diffs need a slight bit of translation. + + + + + diff --git a/docs/api-notes/common/lib_common.txt b/docs/api-notes/common/lib_common.txt new file mode 100644 index 00000000..11d7b02d --- /dev/null +++ b/docs/api-notes/common/lib_common.txt @@ -0,0 +1,52 @@ +TITLE lib/common + +This module is the basic building block of the project. It is included implicitly as a dependency of all other modules. + +It provides basic services such as file and stream functionality, platform information, and various bits of utility code which doesn't fit naturally anywhere else but is useful to many other applications. + + +SUBTITLE Box.h + +The main include file for the project. It should be the first file included in every cpp file, followed by any system headers, then other project headers. This order has been chosen so that eventually it can be used as a target for pre-compiled headers. (This will be important for the Windows port) + + +SUBTITLE BoxPlatform.h + +This contains various bits of information on platform differences. The build scripts include a compile command line definition like PLATFORM_OPENBSD, which is then used to select the required options. + +Some of the stuff is not particularly pleasant, but it aims to produce a consistent compilation environment, and to have a abstracted way of setting other defines to tell the other files what things are available on this platform. + +GNU configure is not used, as it would be just another dependency. Although something does have to be done about compilation on Linux, which is just too different on each distribution to be caught by a common configuration here. + + +SUBTITLE Streams + +Much use is made of streams -- files, connections, buffers, all sorts of things are implemented using streams. + +The abstract base class IOStream defines the interface, see the class file for more details. + +Streams are lib/common's basic contribution to the project. A standard interface for handling data, and a number of utility classes which make performing common stream functions easy. + + +SUBTITLE Serialisation + +Note there is no "serialisable" class. Instead, objects which need to be serialised into Streams simply have two functions, ReadFromStream and WriteToStream. These perform as expected. + +As it turns out, there's no real need for a specific serialisation base class. If you do need to be able to serialise arbitary objects, there are two approaches. + +1) If you're serialising an arbitary object, you're going to know roughly what type it is. So it's likely to be a subclass of a known base class... in which case, you simply use virtual functions. + +2) If it really is an arbitary type, then you just write your function which accepts any type of object to serialise as a template. + + +SUBTITLE BoxException + +The exception base class for the project. All exceptions thrown by this code are dervied from BoxException. In general, each module which has unique errors has it's own exception class. + +CommonException errors are thrown throughout the project. + + +SUBTITLE Other bits + +There are some other extra useful bits which are documented only in the source files. + diff --git a/docs/api-notes/common/lib_common/BoxTime.txt b/docs/api-notes/common/lib_common/BoxTime.txt new file mode 100644 index 00000000..18bef5a5 --- /dev/null +++ b/docs/api-notes/common/lib_common/BoxTime.txt @@ -0,0 +1,7 @@ +TITLE BoxTime + +Not strictly a class, but time is presented in the project as 64 bit integers as microseconds since the UNIX Epoch. + +There are a number of utility functions in the BoxTime*.h files, which are useful. + +To get BoxTime values from a struct stat, use the FileModificationTime.h functions. \ No newline at end of file diff --git a/docs/api-notes/common/lib_common/CollectInBufferStream.txt b/docs/api-notes/common/lib_common/CollectInBufferStream.txt new file mode 100644 index 00000000..5f9556a3 --- /dev/null +++ b/docs/api-notes/common/lib_common/CollectInBufferStream.txt @@ -0,0 +1,26 @@ +CLASS CollectInBufferStream + +This class is essentially a buffer. Data is written to it, and stored in memory. Then when all data is written which will be written, it can be read out again. + +Useful for building streams in memory. + + +FUNCTION CollectInBufferStream::SetForReading() + +After you've written all the data, call this to set it to read mode. + + +FUNCTION CollectInBufferStream::Reset() + +Reset the stream to the state where is has no data, and is about to have data written. + + +FUNCTION CollectInBufferStream::GetSize() + +Get the size of the buffered data -- the entire data. Use the interface function BytesLeftToRead() in most cases. + + +FUNCTION CollectInBufferStream::GetBuffer() + +Get the buffer, so you can do bad things with it if you're feeling naughty. + diff --git a/docs/api-notes/common/lib_common/Configuration.txt b/docs/api-notes/common/lib_common/Configuration.txt new file mode 100644 index 00000000..ef5f38f6 --- /dev/null +++ b/docs/api-notes/common/lib_common/Configuration.txt @@ -0,0 +1,102 @@ +CLASS Configuration + +Implements the reading of multi-level configuration files. This is intended to be a generic way of representing configuration implementation. + +Basic validation is performed on files as they are read, specified by a data structure. + +test/common has some examples of usage. + + +SUBTITLE Configuration file format + +The format is simple, a list of Key = Value pairs. For example + +Key1 = Value1 +Key2 = Value2 + +Optionally, multiple values for the same key are allowed. + +Lists of configurations can be nested to any level. These are called Sub-configurations: + +SubConfig +{ + SubKey1 = ValueX + SubKey2 = ValueY +} + + +SUBTITLE Verification + +The verification structure specifies what keys are required, what are optional, and what sub-configurations are expected. Default values can be specified. + +See Configuration.h for the structures. + +RaidFileController::Initialise has a good example of a simple verification layout. + +Wildcards can be used for SubConfigurations, by specifying a name of "*". This allows you to use multiple sections to configure any number of items. + +Each item has a number of flags, which are combined to say whether an item is required, should be an integer or boolean, and rather importantly, whether it's the last item in the list. + +Verification is limited, so you may wish to do more verification the file. There are unimplemented hooks for custom verification functions to be included in the verification definitions. Should be done at some point. + +Boolean keys have possible values "true", "yes", "false", "no" (case insensitive). + + +FUNCTION Configuration::LoadAndVerify() + +Loads the configuration from disc, and verifies it. If there are problems with the verification, it returns some text which can be used to tell the user the problems with the file. These are fairly basic error messages, but do say what should be done to get it to parse properly. + +This returns a Configuration object, which can then be queries for Keys and Values. Sub-configurations are implemented through an interface which returns a reference to another Configuration object. + + + +FUNCTION Configuration::KeyExists() + +Does a specified key exist? + +Use this for optional key values only -- specify them in the verification structure if they are required. + + +FUNCTION Configuration::GetKeyValue() + +Get the value of a key as a string. + +If ConfigTest_MultiValueAllowed is set in the relevant entry in the verify structure, this string may contain multiple values, separated by a single byte with value 0x01 (use Configuration::MultiValueSeparator in code). Use SplitString() defined in Utils.h to split it into components. + +This representation was chosen as multi-values are probably rare, and unlikely to justify writing a nicer (but more memory intensive and complex) solution. + + +FUNCTION Configuration::GetKeyValueInt() + +Get the value of a key as an integer. Make sure the AsInt property is requried in the verification structure. + + +FUNCTION Configuration::GetKeyValueBool() + +Get the value of a key as a boolean value. Make sure the AsBool property is requried in the verification structure. + +Default to "false", should this verification not be performed and an unknown value is specified. + + +FUNCTION Configuration::GetKeyNames() + +Return a list of all the keys in this configuration. + + +FUNCTION Configuration::SubConfigurationExists() + +Does a specified sub-configuration exist? + + +FUNCTION Configuration::GetSubConfiguration() + +Return another Configuration object representing the sub section. + + +FUNCTION Configuration::GetSubConfigurationNames() + +Get a list of all the sub configurations. + +As there isn't a particularly neat way that configurations can be iterated over, mSubConfigurations is public. (BAD!) + + diff --git a/docs/api-notes/common/lib_common/Conversion.txt b/docs/api-notes/common/lib_common/Conversion.txt new file mode 100644 index 00000000..62d1967a --- /dev/null +++ b/docs/api-notes/common/lib_common/Conversion.txt @@ -0,0 +1,14 @@ +TITLE Generic type conversion + +Conversion.h provides generic type conversion. Within the BoxConvert namespace, it defines the templated function + + ToType Convert(FromType From) + +which converts the data type as expected. In general, from and to types have to be specified explicitly. + +Templates rather than overloaded functions are used, firstly to be absolutely explicit about the conversion required, and secondly because overloaded functions can't have differing return types for the same argument type. + +The function is specialised for various types, and the generic version uses C++ type conversion. + +Exceptions may be thrown if the conversion is not possible. These are all of the ConversionException type. + diff --git a/docs/api-notes/common/lib_common/ExcludeList.txt b/docs/api-notes/common/lib_common/ExcludeList.txt new file mode 100644 index 00000000..8a5bf36c --- /dev/null +++ b/docs/api-notes/common/lib_common/ExcludeList.txt @@ -0,0 +1,21 @@ +TITLE ExcludeList + +A class implementing a list of strings which are to be excluded in some way. Entries can be added as strings to be matched exactly, or as regular expressions. + +Multiple entries can be added in a single function call in a manner designed to work with the multi-value entries store in a Configuation object. + + +FUNCTION ExcludeList::AddDefiniteEntries() + +Definite entries are exact strings to match. + + +FUNCTION ExcludeList::AddRegexEntries() + +Regular expressions as defined by POSIX, and supported by the host platform. Will throw an exception if regular expressions are not supported by the platform. + + +FUNCTION ExcludeList::IsExcluded() + +Test a string for being excluded by definite or regex entries. + diff --git a/docs/api-notes/common/lib_common/FdGetLine.txt b/docs/api-notes/common/lib_common/FdGetLine.txt new file mode 100644 index 00000000..d92ff94d --- /dev/null +++ b/docs/api-notes/common/lib_common/FdGetLine.txt @@ -0,0 +1,11 @@ +CLASS FdGetLine + +See IOStreamGetLine, difference is that it works on basic UNIX file descriptors rather than full blown streams. Follows basic interface, except... + + +FUNCTION FdGetLine::GetLine + +Returns a string containing the optionally preprocessed line. + +Do not call if IsEOF if true. + diff --git a/docs/api-notes/common/lib_common/Guards.txt b/docs/api-notes/common/lib_common/Guards.txt new file mode 100644 index 00000000..6174fea6 --- /dev/null +++ b/docs/api-notes/common/lib_common/Guards.txt @@ -0,0 +1,5 @@ +TITLE Guards.h + +This file contains a couple of classes for using file and memory. When the class goes out of scope, the underlying object is closed or freed, respectively. + + diff --git a/docs/api-notes/common/lib_common/IOStream.txt b/docs/api-notes/common/lib_common/IOStream.txt new file mode 100644 index 00000000..09460656 --- /dev/null +++ b/docs/api-notes/common/lib_common/IOStream.txt @@ -0,0 +1,89 @@ +CLASS IOStream + +The base class for streams of data. See IOStream.h for specific details on functions, but the interface is described here. + +Most streams only implement one direction, but some both. + + +SUBTITLE Reading + + +FUNCTION IOStream::Read() + +Reads data from the stream. Returns 0 if there is no data available within the timeout requested. + +Unlike the UNIX API, a return of 0 does not mean that there is no more data to read. Use IOStream::StreamDataLeft() to find this out. + + +FUNCTION IOStream::ReadFullBuffer() + +If you want to read a specific sized block of data from a stream, it is annoying ot use the Read function, because it might return less, so you have to loop. + +This implements that loop. Note that the timeout is the timeout for each individual read -- it might take a lot longer to complete. + +You must check the return value, which is whether or not it managed to get all that data. + + +FUNCTION IOStream::BytesLeftToRead() + +How many bytes are left to read in the stream treated as a whole. May return IOStream::SizeOfStreamUnknown if it is something like a socket which doesn't know how much data is in the stream. + + +FUNCTION IOStream::StreamDataLeft() + +Returns true if more data can be read. Or more specifically, if more data might be able to be read some time in the future. + + +SUBTITLE Writing + + +FUNCTION IOStream::Write() + +Write data to a stream. Writes the entire block, or exceptions. + + +FUNCTION IOStream::WriteAllBuffered() + +Writes any buffered data to the underlying object. + +Call at the end of a series of writes to make sure the data is actually written. (Buffering is not yet implemented anywhere, but it should be soon.) + + +FUNCTION IOStream::StreamClosed() + +Is the stream closed, and writing no longer possible? + + +FUNCTION IOStream::CopyStreamTo() + +A utility function to copy the contents of one stream to another, until the reading stream has no data left. + + +SUBTITLE Other interfaces + +These are slightly nasty interfaces, because they have to match the UNIX API to some extent. + + +FUNCTION IOStream::Close() + +Close the stream -- intended to indicate that nothing more will be written. + +However, also closes reading in most cases. Stream dependent. Probably best just to delete the object and let it sort itself out. + + +FUNCTION IOStream::Seek() + +Seeks within the stream -- mainly for using files within a stream interface, although some of the utility stream classes support it too. + +This is actually a bad interface, because it does not specify whether it applies to reading or writing. This matches the interface provided by files with a single file pointer, but does not map well onto other stream types. + +Should it be changed? Perhaps. But then it means that files would either have to have an inconsitent interface, or the class keep track of two separate file pointers. Which isn't nice. + +So use carefully, and remember whether you're using a file stream, a reading stream, or a writing stream. + + +FUNCTION IOStream::GetPosition() + +Get the current file pointer. Subject to same problems as Seek with regard to semantics. + + diff --git a/docs/api-notes/common/lib_common/IOStreamGetLine.txt b/docs/api-notes/common/lib_common/IOStreamGetLine.txt new file mode 100644 index 00000000..04c56b57 --- /dev/null +++ b/docs/api-notes/common/lib_common/IOStreamGetLine.txt @@ -0,0 +1,29 @@ +CLASS IOStreamGetLine + +This class provides a convenient way to read text from a file, line by line. It also can preprocess the line to remove leading and trailing whitespace and comments. Comments are started by the character # and run to the end of the line. + +Create an instance by passing a reference to a stream into the constructor. + +Note the class does internal buffering, so you can only detach it later if the stream supports seeking backwards. + + +FUNCTION IOStreamGetLine::GetLine() + +Returns true if a line could be retreieved without a read timing out. + + +FUNCTION IOStreamGetLine::IsEOF() + +Whether the end of the stream has been reached. Do not call GetLine if this is true. + + +FUNCTION IOStreamGetLine::GetLineNumber() + +Returns the line number. + + +FUNCTION IOStreamGetLine::DetachFile() + +Detaches the stream from the GetLine class. Will seek backwards to "replace" data it's buffered back in the stream. + + diff --git a/docs/api-notes/common/lib_common/MainHelper.txt b/docs/api-notes/common/lib_common/MainHelper.txt new file mode 100644 index 00000000..eb5b07f0 --- /dev/null +++ b/docs/api-notes/common/lib_common/MainHelper.txt @@ -0,0 +1,4 @@ +TITLE MainHelper.h + +This header contains a couple of macros which add exception handling and reporting to main() functions for executable programs. + diff --git a/docs/api-notes/common/lib_common/WaitForEvent.txt b/docs/api-notes/common/lib_common/WaitForEvent.txt new file mode 100644 index 00000000..0bc55726 --- /dev/null +++ b/docs/api-notes/common/lib_common/WaitForEvent.txt @@ -0,0 +1,16 @@ +CLASS WaitForEvent + +This class implements a way to efficiently wait on one or more system objects, for example sockets and file descriptors. Where kqueue() is available, this is used, otherwise poll(). The poll() implementation is less comprehensive and rather more limited. + +To add a compatible object, call Add(). To remove it, call Remove(). To wait for an object to become ready, call Wait(), which returns a pointer to the first ready object, or 0 for a timeout. + +The timeout is set either in the constructor, or using the SetTimout() method. It is specified in milliseconds. + +The kqueue based implementation will automatically remove objects when they are closed (actually, the OS does this), but the poll implementation requires that Remove() be called. + +The flags passed to Add() or Remove() are passed to the object, and are ignored by this class. + +For an object to be able to be added to the list, it should implement FillInKEvent() and FillInPoll() for kqueue and poll implementations respectively. Use the PLATFORM_KQUEUE_NOT_SUPPORTED define to test which is necessary for the platform. + +It does not have to be derived off any particular class, as it uses templates to be compatible with any class. + diff --git a/docs/api-notes/common/lib_common/xStream.txt b/docs/api-notes/common/lib_common/xStream.txt new file mode 100644 index 00000000..91e9c0ea --- /dev/null +++ b/docs/api-notes/common/lib_common/xStream.txt @@ -0,0 +1,40 @@ +TITLE Various stream types + +Some useful streams are implemented in lib/common. + + +SUBTITLE CollectInBufferStream + +Described in it's own page. + + +SUBTITLE FileStream + +Implements a stream from a file, allowing both reading and writing. + + +SUBTITLE MemBlockStream + +Turns a memory block into a readable stream. + +Can also be constructed using StreamableMemBlock, CollectInBufferStream and MemBlockStream as sources of the buffer. + + +SUBTITLE PartialReadStream + +Create a readable stream which will read a set number of bytes from another stream, and then declare itself as closed. + +Useful for extracting a small chunk of another stream to present to a function which expects to consume all of a stream. + + +SUBTITLE ReadGatherStream + +Create a readable stream out of blocks from many streams -- so various sections of any number of streams are composed into a single stream. + +To use, register each stream with AddComponent, then use the returned 'handle' with AddBlock to add blocks to the composed stream. + +Optionally, the object will take ownership of the streams added, and delete them when itself is deleted. + +See the comments in the function blocks in the cpp file for more info. + + diff --git a/docs/api-notes/common/lib_compress.txt b/docs/api-notes/common/lib_compress.txt new file mode 100644 index 00000000..f3e26f20 --- /dev/null +++ b/docs/api-notes/common/lib_compress.txt @@ -0,0 +1,8 @@ +TITLE lib/compress + +This is a horrid C++ interface to zlib. It is written this way to avoid having to buffer any data, and so be unnecessarily inefficient. But this does end up just being a wrapper to the zlib interface. + +See test/compress for an example of how to use it. But it should be very familiar. + +For an easier interface, use CompressStream. + diff --git a/docs/api-notes/common/lib_compress/CompressStream.txt b/docs/api-notes/common/lib_compress/CompressStream.txt new file mode 100644 index 00000000..eca43eb6 --- /dev/null +++ b/docs/api-notes/common/lib_compress/CompressStream.txt @@ -0,0 +1,27 @@ +TITLE CompressStream + +This implements a compressing stream class, which compresses and decompresses data sent to an underlying stream object. Data is likely to be buffered. + + +WARNING: Use WriteAllBuffered() (after objects which need to be sent in their entirity) and Close() (after you have finished writing to the stream) otherwise the compression buffering will lose the last bit of data for you. + + +The class works as a filter to an existing stream. Ownership can optionally be taken so that the source stream is deleted when this object is deleted. + +It can operate in one or two directions at any time. Data written to the stream is compressed, data read from the stream is decompressed. If a direction is not being (de)compressed, then it can act as a pass-through without touching the data. + +The constructor specifies the actions to be taken: + +CompressStream(IOStream *pStream, bool TakeOwnership, + bool DecompressRead, bool CompressWrite, + bool PassThroughWhenNotCompressed = false); + +pStream - stream to filter +TakeOwnership - if true, delete the stream when this object is deleted. +DecompressRead - reads pass through a decompress filter +CompressWrite - writes pass through a compression filter +PassThroughWhenNotCompressed - if not filtering a direction, pass through the data unaltered, otherwise exception if an attempt is made on that direction. + + +Note that the buffering and compression will result in different behaviour than the underlying stream: data may not be written in one go, and data being read may come out in different sized chunks. + diff --git a/docs/api-notes/common/lib_crypto.txt b/docs/api-notes/common/lib_crypto.txt new file mode 100644 index 00000000..9ddafe69 --- /dev/null +++ b/docs/api-notes/common/lib_crypto.txt @@ -0,0 +1,28 @@ +TITLE lib/crypto + +Provides cryptographic primatives using the OpenSSL implementation. + + +SUBTITLE CipherContexts and CipherDescriptions + +A CipherContext is the interface to encryption and decryption, providing a generic interface. + +It is constructed with a decrypt or encrypt flag, and a CipherDescription. The CipherDescription specifies which cipher is to be used, the key, and all other cipher specific paramteres. + +The CipherContext has support for padding and initialisation vectors. + + +SUBTITLE Random numbers + +An interface to the OpenSSL psuedo random number generater is provided. + + +SUBTITLE Digests + +An interface to the MD5 digest is provided. + + +SUBTITLE RollingChecksum + +The rsync rolling checksum is implemented. The interface is a little tricky to be efficient as the caller must track the position and provide relevant bytes to advance the checksum. + diff --git a/docs/api-notes/common/lib_crypto/CipherContext.txt b/docs/api-notes/common/lib_crypto/CipherContext.txt new file mode 100644 index 00000000..30fb4608 --- /dev/null +++ b/docs/api-notes/common/lib_crypto/CipherContext.txt @@ -0,0 +1,28 @@ +CLASS CipherContext + +Encryption and decryption using OpenSSL EVP interface. + +See the cpp file for more documentation in the function headers, and test/crypto for examples. + +General notes below. + + +SUBTITLE Construction + +Construct with the encryption direction, and a CipherDescription of the cipher and key required. + + +SUBTITLE Encrypting or decrypting + +Begin() and Transform() allow piece by piece transformation of a block. + +TransformBlock() transforms an entire block. + + +SUBTITLE Buffering + +All transforms expect to have enough space in the buffers for their entire output. Because of block boundaries and padding, it is necessary that the output buffer should be bigger than the input buffer. The amount of space depends on the cipher. + +InSizeForOutBufferSize() and MaxOutSizeForInBufferSize() perform these space calculations, returning the maximuim in size for a specified out size, and the reverse, respectively. + + diff --git a/docs/api-notes/common/lib_crypto/RollingChecksum.txt b/docs/api-notes/common/lib_crypto/RollingChecksum.txt new file mode 100644 index 00000000..d871b3f2 --- /dev/null +++ b/docs/api-notes/common/lib_crypto/RollingChecksum.txt @@ -0,0 +1,36 @@ +CLASS RollingChecksum + +Implementing the rsync rolling checksum algorithm. Read it's description first: + +http://samba.anu.edu.au/rsync/tech_report/node3.html + + +SUBTITLE Construction and initial checksum calculation + +The constructor takes a pointer to a block of data and a size, and calculates the checksum of this block. It can now be "rolled forward" to find the checksum of the block of the same size, one byte forward, with minimal calculation. + + +FUNCTION RollingChecksum::GetChecksum() + +Returns the checksum for the current block. + + +FUNCTION RollingChecksum::RollForward() + +This function takes the byte at the start of the current block, and the last byte of the block it's rolling forward to, and moves the checksum on. + +If the block is + + char *pBlock = ; + +with size s, then it should be called with + + RollForward(pBlock[0], pBlock[s]) + +and now GetChecksum will return the checksum of the block (pBlock+1) of size s. + + +FUNCTION RollingChecksum::RollForwardSeveral() + +Similar to RollForward(), but is more efficient for skipping several bytes at once. Takes pointers to the data buffer rather than the actual data values. + diff --git a/docs/api-notes/common/lib_server.txt b/docs/api-notes/common/lib_server.txt new file mode 100644 index 00000000..392f331d --- /dev/null +++ b/docs/api-notes/common/lib_server.txt @@ -0,0 +1,9 @@ +TITLE lib/server + +This provides various classes for implementing client/server applications (and UNIX daemons). + +The stars of the show are ServerTLS and Protocol, which allow client server applications which communicate using TLS (SSL successor) to be built with surprisingly few lines of code. + +All details in the respective class files. + +Look at test/basicserver for examples. diff --git a/docs/api-notes/common/lib_server/Daemon.txt b/docs/api-notes/common/lib_server/Daemon.txt new file mode 100644 index 00000000..6956ec2b --- /dev/null +++ b/docs/api-notes/common/lib_server/Daemon.txt @@ -0,0 +1,96 @@ +CLASS Daemon + +Implement a UNIX daemon which + +* Daemonises (detach from console, etc) +* Sets up signal handlers, and does useful things with them +* Reads configuration files +* Writes PID file +* Opens syslog + +all the usual UNIX basic daemon behaviour. + +The daemon exe takes optional arguments: The first is a configuration file filename which overrides the default. If the second argument is 'SINGLEPROCESS' the daemon will not daemonise. + +The configuration file must have a section like this + +Server +{ + PidFile = /var/run/daemon.pid + User = username +} + +Use DAEMON_VERIFY_SERVER_KEYS (defined in Daemon.h) to include the necessary keys in your configuration file's verify structure. + +The "User" line is optional, and if it is present the Daemon will change user to this username just before it daemonises. Note that unless the directory the PID file is written to has write permission for this user, the PID file will not be deleted on exit. + + +To implement a daemon, derive a class from Daemon, and override Run(). + +Then in your main file, do something like + +int main(int argc, const char *argv[]) +{ + MAINHELPER_START + + BackupDaemon daemon; + return daemon.Main(BOX_FILE_BBACKUPD_DEFAULT_CONFIG, argc, argv); + + MAINHELPER_END +} + +and that's it. Obviously it's worth doing a few more things as well, but that's it. + + +FUNCTION Daemon::Run() + +Override with the main daemon code. It should behave like this + +void SomethingDaemon::Run() +{ + // Read configuration file + + // Do lots of work until StopRun() returns true + while(!StopRun()) + { + ::sleep(10); + } + + // Clean up nicely +} + +which allows the base class to implement the standard start, terminate and -HUP behaviours correctly. + + +FUNCTION Daemon::DaemonName() + +Returns the name of the daemon, for use in syslog. + + +FUNCTION Daemon::DaemonBanner() + +Returns the banner to be displayed on startup, or 0 for no banner. + + +FUNCTION Daemon::GetConfigVerify() + +Returns a configuration verify structure for verifying the config file. Note that this configuration structure should include a sub-configuration called "Server, and have entries defined by DAEMON_VERIFY_SERVER_KEYS. See one of the bin/bbackupd for an example. + + +FUNCTION Daemon::StopRun() + +Returns true when the daemon needs to be terminated or restarted. Use IsReloadConfigWanted() and IsTerminateWanted() to find out which one, if you need to know. + + +FUNCTION Daemon::SetupInInitialProcess() + +Override to perform additional functions in the initial process, before forking and detachment happens. + + +FUNCTION Daemon::EnterChild() + +Called when a child is entered. If you override it, remember to call the base class. + + + + diff --git a/docs/api-notes/common/lib_server/Protocol.txt b/docs/api-notes/common/lib_server/Protocol.txt new file mode 100644 index 00000000..09d3c1f1 --- /dev/null +++ b/docs/api-notes/common/lib_server/Protocol.txt @@ -0,0 +1,120 @@ +CLASS Protocol + +Protocol + +* serialises and deserialises data objects +* sends arbitary streams + +through a bi-directional IOStream object, usually a socket. + +These data objects are auto-generated by a perl script, along with the logic to implement a simple protocol where one end is a client, and the other a server. The client sends a command object (and optional streams) and the server returns a reply object (and optional streams). + +The perl script uses a description file which specifies the data inside these objects (which can be extended to include any type which implements the standard serialisation functions), the required responses to the commands, and whether streams are involved. + +It then implements a server object, which given a stream and a user defined context object, waits for commands, processes them, and sends the results back. All you need to do is implement a DoCommand() function for each command object you define. + +On the client side, a Query() function is implemented which takes objects as parameters, and returns the right type of reply object (or throws an exception if it doesn't get it.) Short cut Query() functions are also implemented which don't require objects to be created manually. + +Thus, implementing a server is as simple as deriving a daemon off ServerStream or ServerTLS, and implementing the Connection() function as + + void TestProtocolServer::Connection(SocketStream &rStream) + { + TestProtocolServer server(rStream); + TestContext context; + server.DoServer(context); + } + +and that's it. TestContext is a user defined class which keeps track of all the state of the connection, and is passed to all the DoCommand() functions, which look like this: + + std::auto_ptr + TestProtocolServerSimple::DoCommand(TestProtocolServer &rProtocol, + TestContext &rContext) + { + return std::auto_ptr + (new TestProtocolServerSimpleReply(mValue+1)); + } + +(taken from test/basicserver) + +The client code looks like this + + SocketStream conn; + conn.Open(Socket::TypeUNIX, "testfiles/srv4.sock"); + + TestProtocolClient protocol(conn); + + // Query + { + std::auto_ptr + reply(protocol.QuerySimple(41)); + TEST_THAT(reply->GetValuePlusOne() == 42); + } + + +Finally, debug logging can be generated which allows a list of all commands and their parameters to be logged to syslog or a file. + + +SUBTITLE Protocol Description File + +This file is passed to the lib/server/makeprotocol.pl script, which generates a h and cpp file to implement the protocol. + +It is in two sections, separated by a 'BEGIN_OBJECTS' on a line of it's own. + +In the top half, the following statements must be made. + +Name + The name of the protocol, used in naming classes. + +IdentString + The idenfitifaction string sent over the IOStream to confirm it it + is talking to another Protocol object speaking the same Protocol. + +ServerContextClass + The user defined context class used for the server, and the header + file it is defined in. + +Additionally, the following optional commands can be made. + +ClientType +ServerType (similarly) + Extends the types used in the objects below. Server and client + can use different types for the same object type. + +ImplementLog (Client|Server) (syslog|file) + Implement command logging for client or server into syslog or a file. + +LogTypeToText (Client|Server) + + For extended types, optionally define how to convert them into printf + elements and the code to run to get the argument. Within the evaluate + parameter, VAR is replaced by the name of the variable to display. + If this is not specified for a given type, OPAQUE is output instead. + + +In the object section, an object is defined by a line + + + +followed by lines beginning with whitespace defining the data transmitted in the object. The type may be list, which specifies a list (implemented as a std::vector) of entries of that type. + +The attributes specify exactly how that object is used in the defined protocol. + +Reply + The object is a reply object, sent from the server to the client. + +Command(Reply-Type) + The object is a command, send from the client to the server, and the server + will send back an object of type Reply-Type. + +IsError(Type-Field,SubType-Field) + The command is an error object, and the two files specify the data member + which describes the error type and sub type. + +EndsConversation + When this command is received, the connection is to be terminated. + (ie a logout command) + +StreamWithCommand + When this command is sent as a command, a stream follows it. + + diff --git a/docs/api-notes/common/lib_server/ServerStream.txt b/docs/api-notes/common/lib_server/ServerStream.txt new file mode 100644 index 00000000..6c5932a0 --- /dev/null +++ b/docs/api-notes/common/lib_server/ServerStream.txt @@ -0,0 +1,29 @@ +CLASS ServerStream + +ServerStream implementes a Daemon which accepts stream connections over sockets, and forks into a child process to handle them. + +To implement a daemon, derive from + + ServerStream + +The type SocketStream specifies that incoming connections should be treated as normal sockets, and SERVER_LISTEN_PORT is the port to listen to (if it's a inet socket, and if not overridden in the config file). The actual addresses (or names) to bind to are specified in the configuration file by the user. + +Make sure SERVERSTREAM_VERIFY_SERVER_KEYS(0) is included in the configuration verification structure in the "Server" sub configuration. 0 could be replaced with a default address, for example "unix:/var/run/server.sock" to specific a default UNIX socket in the filesystem. + +See test/basicserver for a simple example. + +The ListenAddresses key in the Server subconfiguration is a comma separated list of addresses, specified as family:name. Internet sockets are family 'inet', for example 'inet:localhost' (or 'inet:localhost:1080' to specify a port number as well), and unix domain sockets are 'unix', example above. + + +Override Connection to handle the connection. + +Remember to override Daemon functions like the server name, and start it up, just like a generic Daemon. + + +FUNCTION ServerStream::Connection + +This function takes a connected stream as it's argument. It should then proceed to do whatever it needs to do to talk to the client. + +Using IOStreamGetLine or a Protocol class to communicate may be quick ways of implementing this functionality. + + diff --git a/docs/api-notes/common/lib_server/ServerTLS.txt b/docs/api-notes/common/lib_server/ServerTLS.txt new file mode 100644 index 00000000..dbde500f --- /dev/null +++ b/docs/api-notes/common/lib_server/ServerTLS.txt @@ -0,0 +1,6 @@ +CLASS ServerTLS + +Implements a server which uses TLS (SSL) to encrypt and authenticate connections. + +Very similar to ServerStream, except it reads the certificate files for the TLSContext out of the Server sub-configuration to set up a TLSContext ("CertificateFile", "PrivateKeyFile" and "TrustedCAsFile"). Otherwise works exactly the same. + diff --git a/docs/api-notes/common/lib_server/SocketStream.txt b/docs/api-notes/common/lib_server/SocketStream.txt new file mode 100644 index 00000000..82813279 --- /dev/null +++ b/docs/api-notes/common/lib_server/SocketStream.txt @@ -0,0 +1,8 @@ +CLASS SocketStream + +A implementation of IOStream which wraps a socket connection. + +It can either be created by attaching to an existing object, or use the Open() function to open a connection to a named host on a specific port (or a local UNIX socket in the filesystem). + +Follows stream interface. + diff --git a/docs/api-notes/common/lib_server/SocketStreamTLS.txt b/docs/api-notes/common/lib_server/SocketStreamTLS.txt new file mode 100644 index 00000000..ebb3f233 --- /dev/null +++ b/docs/api-notes/common/lib_server/SocketStreamTLS.txt @@ -0,0 +1,11 @@ +CLASS SocketStreamTLS + +An implementation of IOStream which wraps a TLS (SSL) connection over a socket. + +The Open function takes a TLSContext reference which specifies the parameters for the connection. + + +FUNCTION GetPeerCommonName() + +Returns the common name of the certificate presented by the remote end. + diff --git a/docs/api-notes/common/lib_server/TLSContext.txt b/docs/api-notes/common/lib_server/TLSContext.txt new file mode 100644 index 00000000..ff50d3e6 --- /dev/null +++ b/docs/api-notes/common/lib_server/TLSContext.txt @@ -0,0 +1,16 @@ +CLASS TLSContext + +A wrapper over the OpenSSL context object. + +Note: you need to call SSLLib::Initialise at the beginning of your program to use these functions. + + +SUBTITLE Construction + +The constuctor takes the following parameters + +* Boolean for whether this is acting as a server or a client +* The .pem file containing the certificate which will be used +* The .pem file containing the private key for this certificate +* The .pem file containing the certificates which will certify the other end of the connection. + diff --git a/docs/api-notes/common/memory_leaks.txt b/docs/api-notes/common/memory_leaks.txt new file mode 100644 index 00000000..9a9764ea --- /dev/null +++ b/docs/api-notes/common/memory_leaks.txt @@ -0,0 +1,44 @@ +TITLE Memory leak detection + +The build system contains a primative memory leak detection system in debug builds. + +It works by using #defines to replace calls to malloc, free, realloc, new and delete with debug versions which record their use. When a process ends, it dumps a list of all the blocks or objects which were allocated by not freed, and the file and line of the source where they are originally allocated. + +It's not perfect, but should catch most things and work on most platforms. + +If it doesn't work on your platform, define PLATFORM_DISABLE_MEM_LEAK_TESTING in BoxPlatform.h within the relevant section. + + +SUBTITLE Requirements in source + +It does require some extra lines in the source file. The last included file in each .cpp file must be + + #include "MemLeakFindOn.h" + +and if a .h file uses any of these functions, the contents of the .h file should be bounded with + + #include "MemLeakFindOn.h" + + // ... some code, but absolutely no #includes + + #include "MemLeakFindOff.h" + +The cleanupforcvs.pl script checks for correct usage. + + +SUBTITLE Use in tests + +Tests will automatically dump memory leaks and regard them as a failure for anything executing in the main test process. + +To test for memory leaks in programs run from the test, or daemons, use something like + + TestRemoteProcessMemLeaks("bbackupd.memleaks"); + +If memory leak detection is being used, it will check the specified file for memory leak reports (deleting it afterwards). Any leak is an error. + +The Daemon class automactically arranges for the memory leak reports to be written. Other exes should set this up themselves, preferably using the define in MainHelper.h, as the first thing in their main() function. + + MAINHELPER_SETUP_MEMORY_LEAK_EXIT_REPORT(file, name) + +File is the filename to write the report to, conventionally called ".memleaks", and name is the name of the exe. + diff --git a/docs/api-notes/raidfile/lib_raidfile.txt b/docs/api-notes/raidfile/lib_raidfile.txt new file mode 100644 index 00000000..0c4244ce --- /dev/null +++ b/docs/api-notes/raidfile/lib_raidfile.txt @@ -0,0 +1,73 @@ +TITLE lib/raidfile + +Implements RAID type abilities in files in userland, along with simple transaction like semantics for writing and reading files. + +IOStream interfaces are used to read and write files. + +Eventually a raid file daemon will be written to "raidify" files in the background. This will allow fast streaming of data to a single disc, followed by splitting into separate raid files later. + +There is an option to disable raidification for use on systems with only one hard disc, or a hardware RAID array. + + +SUBTITLE Controller + +The raid disc sets are managed by RaidFileController. This reads the configuration file, /etc/box/raidfile.conf, and then stores the sets of discs for use by the other classes. + +In the code, files are referenced using an integer set number, and a filename within that set. For example, (0, "dir/subdir/filename.ext"). The RAID file controller turns this into actual physcial filenames transparently. + +The files are devided into blocks, which should match the fragment/block size of the underlying filesystem for maximum space efficiency. + +If the directories specified in the configuration files are all equal, then userland RAID is disabled. The files will not be processed into the redundant parts. + + +SUBTITLE Writing + +Files are written (and modified) using the RaidFileWrite class. This behaves as a seekable IOStream for writing. + +When all data has been written, the file is committed. This makes it available to be read. Processing to split it into the three files can be delayed. + +If the data is not commited, the temporary write file will be deleted. + +RaidFileWrite will optionally refuse to overwrite another file. If it is being overwritten, a read to that file will get the old file until it has been committed. + + +SUBTITLE Reading + +File are opened and read with RaidFileRead::Open. This returns an IOStream (with some extras) which reads a file. + +If the file has been turned into the RAID representation, then if an EIO error occurs on any operation, it will switch transparently into recovery mode where the file will be regenerated from the remaining two files using the parity file. (and log an error to syslog) + + +SUBTITLE Layout on disc + +The directory structure is replicated onto each of the three directories within the disc set. + +Given a filename x, within the disc set, one of the three discs is chosen as the 0th disc for this file. This is done by hashing the filename. This ensures that an equal load is placed on each disc -- otherwise one disc would only be accessed when a RAID file was being written. + +When the file is being written, the actual physical file is "x.rfwX". This just a striaght normal file -- nothing is done to the data. + +When it is commited, it is renamed to "x.rfw". + +Then, when it is turned into the raid representation, it is split across three discs, each file named "x.rf". + +When trying to open a file, the first attempt is for "x.rfw". If this exists, it is used as is. + +If this fails, two of the "x.rf" files are attempted to be opened. If this succeeds, the file is opened in RAID mode. + +This procedure guarenettes that a file will be opened either as a normal file, or as the correct three RAID file elements, even if a new version is being commited at the same time, or another process is working to turn it into a raid file. + + +SUBTITLE Raid file representation + +The file is split into three files, stripe 1, stripe 2 and parity. + +Considering the file as a set of blocks of the size specified in the configuration, odd numbered blocks go in stripe 1, even blocks in stripe 2, and a XOR of the corresponding blocks in parity. + +Thus only two files are needed to recreate the original. + +The complexity comes because the size of the file is not stored inside the files at any point. This means if stripe 2 is missing, it is ambiguous as to how big the file is. + +Size is stored as a 64 bit integer. This is appended to the parity file, unless it can be stored by XORing it into the last 8 bytes of the parity file. + +This scheme is complex to implement (see the RaidFileRead.cpp!) but minimises the disc spaced wasted. + diff --git a/docs/api-notes/raidfile/lib_raidfile/RaidFileRead.txt b/docs/api-notes/raidfile/lib_raidfile/RaidFileRead.txt new file mode 100644 index 00000000..0a5efbf7 --- /dev/null +++ b/docs/api-notes/raidfile/lib_raidfile/RaidFileRead.txt @@ -0,0 +1,14 @@ +CLASS RaidFileRead + +Read a raid file. + +IOStream interface, plus a few extras, including reading directories and checking that files exist. + + +FUNCTION RaidFileRead::Open + +Open a given raid file -- returns a pointer to a new RaidFileRead object. + +Note that one of two types could be returned, depending on the representation of the file. + + diff --git a/docs/api-notes/raidfile/lib_raidfile/RaidFileWrite.txt b/docs/api-notes/raidfile/lib_raidfile/RaidFileWrite.txt new file mode 100644 index 00000000..89500c37 --- /dev/null +++ b/docs/api-notes/raidfile/lib_raidfile/RaidFileWrite.txt @@ -0,0 +1,36 @@ +CLASS RaidFileWrite + +Interface to writing raidfiles. + +See IOStream interface. + +Some other useful functions are available, see h and cpp files. + + +FUNCTION RaidFileWrite::RaidFileWrite() + +The constructor takes the disc set number and filename of the file you're interested. + + +FUNCTION RaidFileWrite::Open() + +Open() opens the file for writing, and takes an "allow overwrite" flag. + + +FUNCTION RaidFileWrite::Commit() + +Commmit the file, and make it visible to RaidFileRead. If ConvertToRaidNow == true, it will be converted to raid file representation immediately. + +Setting it to false is not a good idea. Later on, it will tell a daemon to convert it in the background, but for now it simply won't be converted. + + +FUNCTION RaidFileWrite::Discard() + +Abort the creation/update. Equivalent to just deleting the object without calling Commit(). + + +FUNCTION RaidFileWrite::Delete() + +Delete a file -- don't need to Open() it first. + + diff --git a/docs/bb-book.xsl b/docs/bb-book.xsl deleted file mode 100644 index a4f05fdb..00000000 --- a/docs/bb-book.xsl +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - diff --git a/docs/bb-man.xsl.tmpl b/docs/bb-man.xsl.tmpl deleted file mode 100644 index e20eedd5..00000000 --- a/docs/bb-man.xsl.tmpl +++ /dev/null @@ -1,9 +0,0 @@ - - - - - - - - diff --git a/docs/bb-nochunk-book.xsl b/docs/bb-nochunk-book.xsl deleted file mode 100644 index 86574122..00000000 --- a/docs/bb-nochunk-book.xsl +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - diff --git a/docs/bbackupctl.xml b/docs/bbackupctl.xml deleted file mode 100644 index b4880bfd..00000000 --- a/docs/bbackupctl.xml +++ /dev/null @@ -1,205 +0,0 @@ - - - - bbackupctl - - 8 - - Box Backup - - Box Backup - - 0.11 - - - - bbackupctl - - Control the Box Backup client daemon - - - - - bbackupctl - - -q - - -c config-file - - command - - - - - Description - - bbackupctl sends commands to a running - bbackupd daemon on a client machine. It can be used to - force an immediate backup, tell the daemon to reload its configuration - files or stop the daemon. If bbackupd is configured in - snapshot mode, it will not back up automatically, and the - bbackupctl must be used to tell it when to start a - backup. - - Communication with the bbackupd daemon takes place over a local - socket (not over the network). Some platforms (notably Windows) can't - determine if the user connecting on this socket has the correct - credentials to execute the commands. On these platforms, ANY local user - can interfere with bbackupd. To avoid this, remove the CommandSocket - option from bbackupd.conf, which will also disable bbackupctl. See the - Client Configuration page for more information. - - bbackupctl needs to read the - bbackupd configuration file to find out the name of the - CommandSocket. If you have to tell bbackupd where to - find the configuration file, you will have to tell - bbackupctl as well. The default on Unix systems is - usually /etc/box/bbackupd.conf. On Windows systems, - it is bbackupd.conf in the same directory where - bbackupd.exe is located. If - bbackupctl cannot find or read the configuration file, - it will log an error message and exit. - - bbackupctl usually writes error messages to the - console and the system logs. If it is not doing what you expect, please - check these outputs first of all. - - - - - - - Run in quiet mode. - - - - - config-file - - - Specify configuration file. - - - - - - Commands - - The following commands are available in bbackupctl: - - - - terminate - - - This command cleanly shuts down bbackupd. - This is better than killing or terminating it any other - way. - - - - - reload - - - Causes the bbackupd daemon to re-read all - its configuration files. Equivalent to kill - -HUP. - - - - - sync - - - Initiates a backup. If no files need to be backed up, no - connection will be made to the server. - - - - - force-sync - - - Initiates a backup, even if the - SyncAllowScript says that no backup should run - now. - - - - - wait-for-sync - - - Passively waits until the next backup starts of its own - accord, and then terminates. - - - - - wait-for-end - - - Passively waits until the next backup starts of its own - accord and finishes, and then terminates. - - - - - sync-and-wait - - - Initiates a backup, waits for it to finish, and then - terminates. - - - - - - - - Files - - /etc/box/bbackupd.conf - - - - See Also - - - bbackupd.conf - - 5 - , - bbackupd-config - - 8 - , - bbackupctl - - 8 - - - - - Authors - - - Ben Summers - - - - Per Thomsen - - - - James O'Gorman - - - diff --git a/docs/bbackupd-config.xml b/docs/bbackupd-config.xml deleted file mode 100644 index 41bb6587..00000000 --- a/docs/bbackupd-config.xml +++ /dev/null @@ -1,153 +0,0 @@ - - - - bbackupd-config - - 8 - - Box Backup - - Box Backup - - 0.11 - - - - bbackupd-config - - Box Backup client daemon configuration file - generator - - - - - bbackupd-config - - config-dir - - backup-mode - - account-num - - server-hostname - - working-dir - - backup-dir - - backup-dir ... - - - - - Description - - The bbackupd-config script creates configuration files and client - certificates. It takes at least six parameters: - - - - config-dir - - - Configuration directory. Usually - /etc/box. - - - - - backup-mode - - - Either lazy or snapshot. - - - - - account-num - - - The client account number. This is set by the bbstored - administrator. - - - - - server-hostname - - - The hostname or IP address of the bbstored server. - - - - - working-dir - - - A directory to keep temporary state files. This is usually - something like /var/bbackupd. This can be - changed in bbackupd.conf later on if - required. - - - - - backup-dir - - - A space-separated list of directories to be backed up. Note - that this does not traverse mount - points. - - - - - - - Files - - /etc/box/bbackupd.conf - - /etc/box/bbackupd/NotifySysAdmin.sh - - - - See Also - - - bbackupd.conf - - 5 - , - bbackupd - - 8 - , - bbackupctl - - 8 - - - - - Authors - - - Ben Summers - - - - Per Thomsen - - - - James O'Gorman - - - diff --git a/docs/bbackupd.conf.xml b/docs/bbackupd.conf.xml deleted file mode 100644 index 43ae2bed..00000000 --- a/docs/bbackupd.conf.xml +++ /dev/null @@ -1,479 +0,0 @@ - - - - bbackupd.conf - - 5 - - Box Backup - - Box Backup - - 0.11 - - - - bbackupd.conf - - Box Backup client daemon configuration file - - - - - /etc/box/bbackupd.conf - - - - - Description - - - - AccountNumber - - - The account number of this client. This is set by the admin of - the store server. - - - - - UpdateStoreInterval - - - Specifies the interval between scanning of the local discs. To - avoid cycles of load on the server, this time is randomly adjusted - by a small percentage as the daemon runs. Defaults to 1 hour. - - - - - MinimumFileAge - - - Specifies how long since a file was last modified before it - will be uploaded. Defaults to 6 hours. - - - - - MaxUploadWait - - - If a file is repeatedly modified it won't be uploaded - immediately in case it's modified again. However it should be - uploaded eventually. This is how long we should wait after first - noticing a change. Defaults to 1 day. - - - - - MaxFileTimeInFuture - - - - - - - - AutomaticBackup - - - - - - - - SyncAllowScript - - - Use this to temporarily stop bbackupd from syncronising or - connecting to the store. This specifies a program or script script - which is run just before each sync, and ideally the full path to the - interpreter. It will be run as the same user bbackupd is running as, - usually root. - - The script prints either "now" or a number to STDOUT (and a - terminating newline, no quotes). If the result was "now", then the - sync will happen. If it's a number, then the script will be asked - again in that number of seconds. - - For example, you could use this on a laptop to only backup - when on a specific network. - - - - - MaximumDiffingTime - - - How much time should be spent on diffing files. - - - - - DeleteRedundantLocationsAfter - - - - - - - - FileTrackingSizeThreshold - - - - - - - - DiffingUploadSizeThreshold - - - - - - - - StoreHostname - - - The hostname or IP address of the - bbstored - - 8 - server. - - - - - StorePort - - - The port used by the server. Defaults to 2201. - - - - - ExtendedLogging - - - Logs everything that happens between the client and server. - The - bbackupd - - 8 - client must also be started with - . - - - - - ExtendedLogFile - - - - - - - - LogAllFileAccess - - - - - - - - LogFile - - - - - - - - LogFileLevel - - - - - - - - CommandSocket - - - Where the command socket is created in the filesystem. - - - - - KeepAliveTime - - - - - - - - StoreObjectInfoFile - - - - - - - - NotifyScript - - - The location of the script which runs at certain events. This - script is generated by - bbackupd-config - - 8 - . Defaults to - /etc/box/bbackupd/NotifySysAdmin.sh. - - - - - NotifyAlways - - - - - - - - CertificateFile - - - The path to the client's public certificate. - - - - - PrivateKeyFile - - - The path to the client's private key. This should only be - readable by root. - - - - - TrustedCAsFile - - - The Certificate Authority created by - bbstored-certs - - 8 - . - - - - - KeysFile - - - The data encryption key. This must be kept safe at all costs, your data is - useless without it! - - - - - DataDirectory - - - A directory to keep temporary state files. This is usually - something like /var/bbackupd. - - - - - Server - - - This section relates to the running daemon. - - - - PidFile - - - The location of the process ID file. Defaults to - /var/run/bbackupd.pid. - - - - - - - - BackupLocations - - - This section defines each directory to be backed up. Each - entry must have at least a Path entry and, optionally, include and - exclude directives. - - Multiple include and exclude directives may appear. - - - - Path - - - The path to back up. - - - - - ExcludeFile - - - Exclude a single file. - - - - - ExcludeFilesRegex - - - Exclude multiple files based on a regular expression. - See - re_format - - 7 - . - - - - - ExcludeDir - - - Exclude a single directory. - - - - - ExcludeDirsRegex - - - Exclude multiple directories based on a regular - expression. See - re_format - - 7 - . - - - - - AlwaysIncludeFile - - - Include a single file from a directory which has been - excluded. - - - - - AlwaysIncludeFilesRegex - - - Include multiple files from an excluded directory, - based on a regular expression. - - - - - AlwaysIncludeDir - - - Include a single directory from a directory which has - been excluded. - - - - - AlwaysIncludeDirsRegex - - - Include multiple directories from an excluded - directory, based on a regular expression. - - - - - - - - - - Examples - - The following is an example of a backup location: - - home -{ - Path = /home - ExcludeDir = /home/guest - ExcludeDir = /home/[^/]+/tmp - ExcludeFilesRegex = .*\.(mp3|MP3)$ - AlwaysIncludeFile = /home/someuser/importantspeech.mp3 -} - - - - Files - - /etc/box/bbackupd.conf - - - - See Also - - - bbackupd - - 8 - , - bbackupd-config - - 8 - , - bbackupctl - - 8 - - - - - Authors - - - Ben Summers - - - - Per Thomsen - - - - James O'Gorman - - - diff --git a/docs/bbackupd.xml b/docs/bbackupd.xml deleted file mode 100644 index 11de0f3f..00000000 --- a/docs/bbackupd.xml +++ /dev/null @@ -1,209 +0,0 @@ - - - - bbackupd - - 8 - - Box Backup - - Box Backup - - 0.11 - - - - bbackupd - - Box Backup client daemon - - - - - bbackupd - - -DFkqvVT - - -c config-file - - -t tag - - - - - Description - - bbackupd runs on client computers in the - background, finding new files to back up. When it is time for a backup, - bbackupd will connect to the server - (bbstored) to upload the files. - - A running bbackupd daemon can be controlled with - the bbackupctl command, to make it shut down, reload - its configuration, or start an immediate backup. - - bbackupd needs to be configured to tell it which - files to back up, how often, and to which server (running - bbstored). See the Client Configuration page for more - information. For this, you must write a configuration file. You must - either place it in the default location, or tell - bbackupd where to find it. - - You can check the default location with the - option. The default on Unix systems is usually - /etc/box/bbackupd.conf. On Windows systems, it is - bbackupd.conf in the same directory where - bbackupd.exe is located. If bbackupd cannot find or - read the configuration file, it will log an error message and exit. - - bbackupd usually writes log messages to the - system logs, using the facility local5, which you can - use to filter them to send them to a separate file. It can also write them - to the console, see options below. If bbackupd is not - doing what you expect, please check the logs first of all. - - - Options - - - - config-file - - - Use the specified configuration file. If - is omitted, the last argument is the configuration file. If none - is specified, the default is used (see above). - - - - - - - - Debugging mode. Do not fork into the background (do not run - as a daemon). Not available on Windows. - - - - - - - - No-fork mode. Same as for bbackupd. Not - available on Windows. - - - - - - - - Keep console open after fork, keep writing log messages to - it. Not available on Windows. - - - - - - - - Run more quietly. Reduce verbosity level by one. Available - levels are NOTHING, FATAL, - ERROR, WARNING, - NOTICE, INFO, - TRACE, EVERYTHING. Default - level is NOTICE in non-debugging builds. Use - once to drop to WARNING level, twice for - ERROR level, four times for no logging at - all. - - - - - -v - - - Run more verbosely. Increase verbosity level by one. Use - once to raise to INFO level, twice for - TRACE level, three times for - EVERYTHING (currently the same as - TRACE). - - - - - - - - Run at maximum verbosity (EVERYTHING - level). - - - - - tag - - - Tag each console message with specified marker. Mainly - useful in testing when running multiple daemons on the same - console. - - - - - - - - Timestamp each line of console output. - - - - - - - - Files - - /etc/box/bbackupd.conf - - - - See Also - - - bbackupd.conf - - 5 - , - bbackupd-config - - 8 - , - bbackupctl - - 8 - - - - - Authors - - - Ben Summers - - - - Per Thomsen - - - - James O'Gorman - - - diff --git a/docs/bbackupquery.xml b/docs/bbackupquery.xml deleted file mode 100644 index 016d5c7d..00000000 --- a/docs/bbackupquery.xml +++ /dev/null @@ -1,506 +0,0 @@ - - - - bbackupquery - - 8 - - Box Backup - - Box Backup - - 0.11 - - - - bbackupquery - - Box Backup store query and file retrieval - - - - - bbackupquery - - -q - - -c configfile - - command ... - - - - - Description - - bbackupquery is the main way of interacting with - the backup store from a Box Backup client machine. It supports both - interactive and batch modes of operation. - - It can be used to reviewing the status of a client machine's backup - store, getting status from the store server. The main use is to retrieve - files and directories when needed. - - bbackupquery supports interactive and batch modes - of operation. Interactive mode allows for interaction with the server much - like an interactive FTP client. - - Batch mode is invoked by putting commands into the invocation of - bbackupquery. Example: - - bbackupquery "list home-dirs" quit - - Note that commands that contain spaces are enclosed in double - quotes. If the quit command is omitted, after the - preceding commands are completed, bbackupquery will - enter interactive mode. - - - - Options - - - - - - - Quiet. Suppresses status output while running. - - - - - - - - Use configfile instead of the default bbackupd.conf file. - Can be a relative or full path. - - - - - - - Commands - - The commands that can be used in bbackupquery are listed - below. - - - - help - - - Displays the basic help message, which gives information about - the commands available in bbackupquery. Use the - form help command to get help on a specific - command. - - - - - quit - - - End the session with the store server, and quit - bbackupquery. - - - - - cd options - directory-name - - - Change directory. Options: - - - - - consider deleted directories for traversal - - - - - - - - consider old versions of directories for traversal. - This option should never be useful in a correctly formed - store. - - - - - - - - lcd - local-directory-name - - - Change directory on the client machine. To list the contents - of the local directory, type sh ls (on Unix-like - machines). - - - - - list options - directory-name - - - The list (or its synonym ls) command lists - the content of the current, or specified, directory. The options are - as follows: - - - - - - - recursively list all files - - - - - - - - list deleted files and directories - - - - - - - - list old versions of files and directories - - - - - - - - don't display object IDs - - - - - - - - don't display flags - - - - - - - - show file modification time (and attr mod time, if the - object has attributes). - - - - - - - - show file size in blocks used on server. Note that - this is only a very approximate indication of local file - size. - - - - - - - - ls options - directory-name - - - Synonym for list. - - - - - pwd - - - Print current directory, always relative to the backup store - root. - - - - - sh shell-command - - - Everything after the sh is passed to a shell and run. All - output from the command is displayed in the client. - - Example: to list the contents of the current directory on the - client machine type sh ls. - - - - - compare -a - - - - - - - - compare -l - location-name - - - - - - - - compare store-dir-name - local-dir-name - - - Compare the current data in the store with the data on the - disc. Please note that all the data will be downloaded from the - store, so this can be a very lengthy process depending on the size - of the store, and the size of the part you are comparing. - - Options: - - - - - - - compare all locations. - - - - - - - - compare one backup location as specified in the - configuration file. This compares one of the top level store - directories. - - - - - - - - set return code. The return code is set to the - following values, if quit is the next command. So, if - another command is run after the compare, the return code - will not refer to the compare. This option is very useful - for automating compares. Return code values: - - -- no differences were - found - - - - -- differences were - found - - - - -- an error occured - - - - - - - - - - get object-filename - local-filename - - - - - - - - get -i object-id - local-filename - - - Gets a file from the store. Object is specified as the - filename within the current directory. Local filename is optional. - Ignores old and deleted files when searching the directory for the - file to retrieve. - - To get an old or deleted file, use the - option and select the object as a hex object ID (first column in - listing). The local filename must be specified. - - - - - getobject object-id - local-filename - - - Gets the object specified by the object id (in hex) and stores - the raw contents in the local file specified. Note: This is only - useful for debugging as it does not decode files from the stored - format, which is encrypted and compressed. - - - - - restore -d - directory-name - local-directory-name - - - - - - - - restore -r - - - Restores a directory to the local disc. The local directory - specified must not exist (unless a previous restore is being - restarted). The root cannot be restored -- restore locations - individually. - - Options: - - - - - - - restore a deleted directory - - - - - - - - resume an interrupted restore - - - If a restore operation is interrupted for any - reason, it can be restarted using the switch. - Restore progress information is saved in a file at regular intervals - during the restore operation to allow restarts. - - - - - usage -m - - - Show space used on the server for this account. Display - fields: - - Used: Total amount of space used on - the server - - - - Old files: Space used by old - files - - - - Deleted files: Space used by - deleted files - - - - Directories: Space used by the - directory structure - - - - When Used exceeds the soft limit, the - server will start to remove old and deleted files until the usage - drops below the soft limit. After a while, you should expect to see - the usage stay at just below the soft limit. You only need more - space if the space used by old and deleted files is near - zero. - - The option displays output in - machine-readable form. - - - - - - - Bugs - - If you find a bug in Box Backup and you want to let us know about - it, join the mailing - list and send us a description of the problem there. - - To report a bug, give us at least the following information: - - - - The version of Box Backup you are running - - - - The platform you are running on (hardware and OS), for both - client and server. - - - - If possible attach your config files (bbstored.conf, - bbackupd.conf) to the bug report. - - - - Also attach any log file output that helps shed light on the - problem you are seeing. - - - - And last but certainly not least, a description of what you are - seeing, in as much detail as possible. - - - - - - Authors - - - Ben Summers - - - - Per Thomsen - - - - James O'Gorman - - - diff --git a/docs/bblogo-alpha.xcf b/docs/bblogo-alpha.xcf deleted file mode 100644 index 18f494f9..00000000 Binary files a/docs/bblogo-alpha.xcf and /dev/null differ diff --git a/docs/bbstoreaccounts.xml b/docs/bbstoreaccounts.xml deleted file mode 100644 index 08092acf..00000000 --- a/docs/bbstoreaccounts.xml +++ /dev/null @@ -1,386 +0,0 @@ - - - - bbstoreaccounts - - 8 - - Box Backup - - Box Backup - - 0.11 - - - - bbstoreaccounts - - Box Backup store accounts manager - - - - - bbstoreaccounts - - -c config-file - - command - - account-id - - command-specific arguments - - - - - Description - - bbstoreaccounts is the tool for managing accounts - on the store server. It can be used to view information related to - accounts, as well as create, change and delete accounts on the store - server. - - bbstoreaccounts always takes at least 2 - parameters: the command name and the account ID. Some commands require - additional parameters, and some commands have optional parameters. - - - Options - - - - - - - The configfile to use for connecting to the store. Default - is /etc/box/bbstored.conf. - - - - - - - Commands - - The commands tells bbstoreaccounts what action to perform. - - - - check account-id - fix - - - The check command verifies the - integrity of the store account given, and optionally fixes any - corruptions. Note: It is - recommended to run the 'simple' check command (without - fix) before using the fix - option. This gives an overview of the extent of any problems, - before attempting to fix them. - - - - - create account-id - disc-set soft-limit - hard-limit - - - Creates a new store account with the parameters given. The - parameters are as follows: - - - - - - - The ID of the new account to be created. A 32-bit - hexadecimal number. Cannot already exist on the - server. - - - - - - - - The disc set from - raidfile.conf - - 5 - where the backups for this client will - be stored. A number. Each RAID-file set has a number in - raidfile.conf. This number is what's used. - - - - - - - - The soft limit is the amount of storage that the - server will guarantee to be available for - storage. - - - - - - - - The amount of storage that the the server will - allow, before rejecting uploads, and starting to - eliminate old and deleted files to get back down to - soft-limit. - - - - - - - - delete account-id - yes - - - Deletes the account from the store server completely. - Removes all backups and deletes all references to the account in - the config files. - - delete will ask for confirmation from - the user, when called. Using the flag, - eliminates that need. This is useful when deleting accounts from - within a script or some other automated means. 0 - - - - - info account-id - - - Display information about the given account. - Example:[root]# bbstoreaccounts info 1 - Account ID: 00000001 - Last object ID: 58757 - Blocks used: 9864063 (38531.50Mb) - Blocks used by old files: 62058 (242.41Mb) -Blocks used by deleted files: 34025 (132.91Mb) - Blocks used by directories: 6679 (26.09Mb) - Block soft limit: 11796480 (46080.00Mb) - Block hard limit: 13107200 (51200.00Mb) - Client store marker: 1139559852000000 - - Explanation: - - - - Account ID - - - The account ID being displayed. - - - - - Last Object ID - - - A counter that keeps track of the objects that - have been backed up. This number refers to the last file - that was written to the store. The ID is displayed as a - decimal number, and the object ID can be converted to a - path name to a file as follows: convert the number to - hex (e.g.: 58757 => 0xE585); The last backed up file - will be (relative from the client's store root): - e5/o85.rfw. Longer numbers infer - more directories in the structure, so as an example - 3952697264 as the last object ID gives 0xEB995FB0, which - translates to a backup pathname of - eb/99/5f/ob0.rfw. - - - - - Blocks used - - - The number of blocks used by the store. The size - in Mb depends on the number of blocks, as well as the - block size for the disc set given in - raidfile.conf - - 5 - . In this case the block size is - 4096. - - - - - Blocks used by old files - - - The number of blocks occupied by files that have - newer versions in the store. This data is at risk for - being removed during housekeeping. - - - - - Blocks used by deleted files - - - The number of blocks used by files that have been - deleted on the client. This data is at risk for being - removed during housekeeping. - - - - - Blocks used by directories - - - The number of blocks used by directories in the - store. - - - - - Block soft limit - - - The soft limit in blocks. The soft limit is the - maximum guaranteed storage space available to the - account. When housekeeping starts, and the old and - deleted files are removed, they are removed in - chronological order (oldest first), until the data used - is less than the soft limit. - - - - - Block hard limit - - - The hard limit in blocks. The hard limit is the - most amount of storage the server will allow in an - account. Any data above this amount will be rejected. - Housekeeping will reduce the storage use, so more data - can be uploaded. - - - - - Client store marker - - - - bbstored - - 8 - uses this number to determine if it - needs to rescan the entire store. If this number is - different from the last time it checked, a rescan will - take place. - - - - - - - - setlimit account-id - soft-limit hard-limit - - - Changes the storage space allocation for the given - account. No server restart is needed. - - Parameters: - - - - - - - The ID of the account to be modified. - - - - - - - - The soft limit is the amount of storage that the - server will guarantee to be available for - storage. - - - - - - - - The amount of storage that the the server will - allow before rejecting uploads and starting to eliminate - old and deleted files to get back down to - . - - - - - - - - - - - Examples - - Create an account with ID 3af on disc set 0, with a 20GB soft-limit - and a 22GB hard-limit:bbstoreaccounts create 3af 0 20G 22GAlter - existing account ID 20 to have a 50GB soft-limit and a 55GB - hard-limit:bbstoreaccounts setlimit 20 50G 55G - - - - Files - - /etc/box/bbstored/accounts.txt - - - - See Also - - - bbstored - - 8 - , - bbstored-config - - 8 - - - - - Authors - - - Ben Summers - - - - Per Thomsen - - - - James O'Gorman - - - diff --git a/docs/bbstored-certs.xml b/docs/bbstored-certs.xml deleted file mode 100644 index 79308089..00000000 --- a/docs/bbstored-certs.xml +++ /dev/null @@ -1,180 +0,0 @@ - - - - bbstored-certs - - 8 - - Box Backup - - Box Backup - - 0.11 - - - - bbstored-certs - - Manage certificates for the Box Backup system - - - - - bbstored-certs - - certs-dir - - command - - arguments - - - - - Description - - bbstored-certs creates and signs certificates for - use in Box Backup. It allows the user to create and sign the server keys, - as well as signing client keys. - - All commands must be followed by the certs-dir, - which is the directory in which the certificates are stored. - - - Commands - - There are 3 commands: - - - - init - - - Create the certs-dir, and generate the - server keys for bbstored. certs-dir cannot - exist before running the command. - - - - - sign-server - servercsrfile - - - Sign the server certificate. The - servercsrfile is the file generated by the - init command. - - - - - sign - clientcsrfile - - - Sign a client certificate. The - clientcsrfile is generated during client setup. - See - bbackupd-config - - 8 - . Send the signed certificate back to the client, - and install according to the instructions given by - bbackupd-config. - - - - - - - - Files - - - raidfile-config - - 8 - generates the - raidfile.conf - - 5 - file. - - - - Bugs - - If you find a bug in Box Backup, and you want to let us know about - it, join the mailing - list, and send a description of the problem there. - - To report a bug, give us at least the following information: - - - - The version of Box Backup you are running - - - - The platform you are running on (hardware and OS), for both - client and server. - - - - If possible attach your config files (bbstored.conf, - bbackupd.conf) to the bug report. - - - - Also attach any log file output that helps shed light on the - problem you are seeing. - - - - And last but certainly not least, a description of what you are - seeing, in as much detail as possible. - - - - - - See Also - - - bbstored-config - - 8 - , - bbstored.conf - - 5 - , - bbstoreaccounts - - 8 - - - - - Authors - - - Ben Summers - - - - Per Thomsen - - - - James O'Gorman - - - diff --git a/docs/bbstored-config.xml b/docs/bbstored-config.xml deleted file mode 100644 index 6d0113c1..00000000 --- a/docs/bbstored-config.xml +++ /dev/null @@ -1,148 +0,0 @@ - - - - bbstored-config - - 8 - - Box Backup - - Box Backup - - 0.11 - - - - bbstored-config - - Box Backup store daemon configuration file - generator - - - - - bbstored-config - - configdir - - servername - - username - - - - - Description - - The bbstored-config script creates configuration files and server - certificates for a bbstored instance. It takes three parameters: - - - - configdir - - - The directory where config files will reside. A - bbstored subdirectory will be created where - several config files will reside. The - bbstored.conf file will be created in - configdir. - - - - - servername - - - The name of the server that is being configured. Usually the - fully qualified domain name of the machine in question. - - - - - username - - - The name of the user that should be running the - bbstored process. Recommended name: - _bbstored. - - - - - A valid - raidfile.conf - - 5 - must be found in configdir. Several steps are taken - during the run of bbstored-config: - - - - Server certificates are created. This requires interaction from - the operator. - - - - The RAID volumes are checked to ensure that the configuration is - consistent and will work. - - - - Instructions for next steps to take are shown. These steps may - be different for different OS platforms, so pay close attention to - these instructions. - - - - - - Files - - /etc/box/bbstored.conf - - - - See Also - - - bbstored.conf - - 5 - , - bbstored - - 8 - , - bbstored-certs - - 8 - , - raidfile-config - - 8 - - - - - Authors - - - Ben Summers - - - - Per Thomsen - - - - James O'Gorman - - - diff --git a/docs/bbstored.conf.xml b/docs/bbstored.conf.xml deleted file mode 100644 index 136b1bd0..00000000 --- a/docs/bbstored.conf.xml +++ /dev/null @@ -1,211 +0,0 @@ - - - - bbstored.conf - - 5 - - Box Backup - - Box Backup - - 0.11 - - - - bbstored.conf - - Box Backup store daemon configuration file - - - - - /etc/box/bbstored.conf - - - - - Description - - The following configuration options are valid: - - - - RaidFileConf - - - Specifies the path to the - raidfile.conf - - 5 - . This is normally - /etc/box/raidfile.conf. - - - - - AccountDatabase - - - Specifies the path to the account database created by - - bbstoreaccounts - - 8 - . This is usually - /etc/box/bbstored/accounts.txt. - - - - - ExtendedLogging - - - Specifies whether extended logging should be enabled to show - what commands are being received from clients. - - - - - TimeBetweenHousekeeping - - - How long between scanning for files which need - deleting. - - - - - Server - - - These options relate to the actual daemon. - - PidFile - - - The location of the pidfile, where the daemon's - process ID is kept. - - - - - User - - - The user to run as. - - - - - ListenAddresses - - - The interface addresses to listen on. Hostnames may be - used instead of IP addresses. The format is: - or - . - - - - - CertificateFile - - - The path to the server's public certificate. - - - - - PrivateKeyFile - - - The path to the server's private key. This should only - be readable by root and/or the . - - - - - TrustedCAsFile - - - The Certificate Authority created by - bbstored-certs - - 8 - . - - - - - - - - - - Examples - - The following is an example bbstored.conf: - - RaidFileConf = /etc/box/raidfile.conf -AccountDatabase = /etc/box/bbstored/accounts.txt - -TimeBetweenHousekeeping = 900 - -Server -{ - PidFile = /var/run/bbstored.pid - User = _bbstored - ListenAddresses = inet:server.example.com - CertificateFile = /etc/box/bbstored/server.example.com-cert.pem - PrivateKeyFile = /etc/box/bbstored/server.example.com-key.pem - TrustedCAsFile = /etc/box/bbstored/clientCA.pem -} - - - - Files - - /etc/box/bbstored.conf - - - - See Also - - - bbstored - - 8 - , - bbstored-config - - 8 - , - raidfile-config - - 8 - - - - - Authors - - - Ben Summers - - - - Per Thomsen - - - - James O'Gorman - - - diff --git a/docs/bbstored.xml b/docs/bbstored.xml deleted file mode 100644 index 81891a8d..00000000 --- a/docs/bbstored.xml +++ /dev/null @@ -1,90 +0,0 @@ - - - - bbstored - - 8 - - Box Backup - - Box Backup - - 0.11 - - - - bbstored - - Box Backup store daemon - - - - - bbstored - - config-file - - - - - Description - - bbstored runs on a central server. Clients - running bbackupd connect to the server and upload - files. - - The only argument is optional and specifies a non-default - configuration file. By default it will look for the configuration file as - /etc/box/bbstored.conf. - - - - Files - - /etc/box/bbstored.conf - - - - See Also - - - bbstored.conf - - 5 - , - bbstored-config - - 8 - , - raidfile-config - - 8 - , - raidfile.conf - - 5 - - - - - Authors - - - Ben Summers - - - - Per Thomsen - - - - James O'Gorman - - - diff --git a/docs/docbook/adminguide.xml b/docs/docbook/adminguide.xml new file mode 100644 index 00000000..edb0a58c --- /dev/null +++ b/docs/docbook/adminguide.xml @@ -0,0 +1,1981 @@ + + +]> + + Box Backup administrator's guide + + + License + + Copyright © 2003 - 2007, Ben Summers and contributors. All rights + reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + + + Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + + + Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + + + All use of this software and associated advertising materials + must display the following acknowledgement: This product includes + software developed by Ben Summers and contributors. + + + + The names of the Authors may not be used to endorse or promote + products derived from this software without specific prior written + permission. + + + + [Where legally impermissible the Authors do not disclaim liability + for direct physical injury or death caused solely by defects in the + software unless it is modified by a third party.] + + THIS SOFTWARE IS PROVIDED BY THE AUTHORS "AS IS" AND ANY EXPRESS OR + IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN + NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED + TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + + + Configuration + +
+ System configuration + +
+ Server + + After you've downloaded and compiled the programs you need to + install the programs on your server. As root do the following: + + make install-backup-server + + This assumes that you are installing on the same server that you + compiled the software on. If not, copy the + boxbackup-x.xx-backup-server-OSNAME.tgz file to the server you want to + run on, and install there. For example (on Mac OS X): + + tar zxvf boxbackup-0.10-server-darwin8.5.0.tgz +cd boxbackup-0.10-server-darwin8.5.0 +./install-backup-server + + Then create the user for the backup daemon on the server: + + useradd _bbstored + + Box Backup has a built-in software RAID facility (redundant + array of inexpensive disks) for the backup store. This allows you to + spread the store data over three disks, and recover from the loss of + any one disk without losing data. However, this is now deprecated, and + you are recommended to use the software or hardware RAID facilities of + your operating system instead. Use the following command if you want + to create a simple server without Box Backup RAID: + + mkdir /tmp/boxbackupRepository # Create the directory +chown _bbstored /tmp/boxbackupRepository/ # Change the owner to the new boxbackup daemon user + +/usr/local/sbin/raidfile-config /etc/box/ 1024 /tmp/boxbackupRepository + +#substitute 1024 with the desired blocksize +#substitute /tmp/boxbackupRepository with a directory that exists where you want the backup store located +#/usr/local/sbin/raidfile-config --help shows you the options + + Then create the configuration file /etc/box/bbstored.conf The + hostname is tricky as it is used for two things: The name of the + server in the certificate and the address the server is listening on. + Since you might be using NAT, might move the server around or the + domain name might change, choose a name that describes the server. + When the network address of the server changes, you need to update the + ListenAddresses directive in the + /etc/box/bbstored.conf file. + + /usr/local/sbin/bbstored-config /etc/box hostname _bbstored + + This last step outputs 5 instructions that you must execute to + the letter. A lot of questions are raised on the mailing list because + these steps have not been followed properly. + + TODO: Expand on this. Explain the 5 steps in detail. + + If you want to run the server as a non-root user, look here. +
+ +
+ Certificate Management + + There are two steps involved to create an account. You need to + create the account on the server, and sign a certificate to give the + client permission to connect to the server. + + Running a Certification Authority for TLS (SSL) connections is + not trivial. However, a script to does most of the work in a way which + should be good enough for most deployments. + + + The certificate authority directory is intended to be stored + on another server. It should not be kept on the backup server, in + order to limit the impact of a server compromise. The instructions + and the script assume that it will be kept elsewhere, so will ask + you to copy files to and from the CA. + + + + SSL certificates contain validity dates, including a "valid + from" time. If the clock on the machine which signs the certificates + is not syncronised to the clocks of the machines using these + certificates, you will probably get strange errors until the start + time is reached on all machines. If you get strange errors when + attempting to use new certificates, check the clocks on all machines + (client, store and CA). You will probably just need to wait a while + until the certificates become valid, rather than having to + regenerate them. + + +
+ Set up a Certificate Authority + + It is recommended that you keep your Certificate Authority on + a separate machine than either the client or the server, preferably + without direct network access. The contents of this directory + control who can access your backup store server. + + To setup the basic key structure, do the following: + + /usr/local/sbin/bbstored-certs ca init + + (See OpenSSL notes if you + get an OpenSSL error) + + This creates the directory called ca in + the current directory, and initialises it with basic keys. +
+ +
+ Sign a server certificate + + When you use the bbstored-config script to + set up a config file for a server, it will generate a certificate + request (CSR) for you. Transfer it to the machine with your CA, then + do: + + /usr/local/sbin/bbstored-certs ca sign-server hostname-csr.pem + + This signs the certificate for the server. Follow the + instructions in the output on which files to install on the server. + The CSR file is now no longer needed. Make sure you run this command + from the directory above the directory 'ca'. + + TODO: Explain instructions in output. +
+ +
+ Set up an account + + Choose an account number for the user. This must be unique on + the server, and is presented as a 31 bit number in hex greater than + 0, for example, 1 or 75AB23C. Then on the backup store server, + create the account with: + + /usr/local/sbin/bbstoreaccounts create 75AB23C 0 4096M 4505M + + This looks complicated. The numbers are, in order: + + + + The account number allocated (hex) + + + + The RAID disc set (0 if you use raidfile-config and don't + add a new set) + + + + Soft limit (size) + + + + Hard limit (size) + + + + The sizes are are specified in Mb, Gb, or blocks, depending on + the suffix. 1M specifies 1 Mb, 1G specifies 1 Gb, and 1B specifies 1 + block, the size of which depends on how you have configured the + raidfile system with raidfile-config. + + In this example, I have allocated 4Gb (assuming you use 2048 + byte blocks as per my example) as the soft limit, and 4Gb + 10% as + the hard limit. + + NOTE The sizes specified here are pre-RAID. So if you are + using userland RAID, you are actually allocating two-thirds of this + amount. This means that, when you take compression into account, + that if you allocate 2Gb on the server, it'll probably hold about + 2Gb of backed up files (depending on the compressability of those + files). + + The backup client will (voluntarily) try not to upload more + data than is allowed by the soft limit. The store server will refuse + to accept a file if it would take it over the hard limit, and when + doing housekeeping for this account, try and delete old versions and + deleted files to reduce the space taken to below the soft + limit. + + This command will create some files on disc in the raid file + directories (if you run as root, the utility will change to the user + specified in the bbstored.conf file to write them) and update the + accounts file. A server restart is not required. + + NOTE If you get a message saying 'Exception: RaidFile (2/8)', + the directories you specified in the raidfile.conf are not writable + by the _bbstored user -- fix it, and try again. + + Finally, tell the user their account number, and the hostname + of your server. They will use this to set up the backup client, and + send you a CSR. This has the account number embedded in it, and you + should be sure that it has the right account number in it. + + Sign this CSR with this command: + + /usr/local/sbin/bbstored-certs ca sign 75AB23C-csr.pem + + Don't forget to check that the embedded account number is + correct! Then send the two files back to the user, as instructed by + the script. + + Please read the Troubleshooting page if you have + problems. + + TODO: Link to troubleshooting... +
+
+ +
+ Log Files + + You may wish to see what's going on with the server. Edit + /etc/syslog.conf, and add: + + local6.info /var/log/box +local5.info /var/log/raidfile + + Note: Separators must be tabs, + otherwise these entries will be ignored. + + touch /var/log/box +touch /var/log/raidfile + + Set up log rotation for these new log files. For example, if you + have /etc/newsyslog.conf, add the following lines + to it: + + /var/log/box 644 7 2000 * Z +/var/log/raidfile 644 7 2000 * Z + + If you have /etc/logrotate.d, create a new + file in there (for example + /etc/logrotate.d/boxbackup) containing the + following: + + /var/log/box /var/log/raidfile { + weekly + create + compress + rotate 52 +} + + Then restart syslogd, for example: + + /etc/init.d/syslogd restart +
+ +
+ Configuring a client + + Before you can do any configuration, you need to know the + hostname of the server you will be using, and your account number on + that server. + + Later in the process, you will need to send a certificate + request to the administrator of that server for it to be + signed. + + Installation is covered in the compiling and installing section. + You only need the backup-client parcel. + + It is important that you read all the output of the config + scripts. See the end of this page for an example. + + The backup client has to be run as root, because it needs to + read all your files to back them up, although it is possible to back + up a single user's files by running it as that user. (Tip: specify a + directory other than /etc/box, and then give the + alternate config file as the first argument to + bbackupd). However, it will fall over if you don't + give yourself read access to one of your files. + +
+ Basic configuration + + Run the bbackupd-config script to generate + the configuration files and generate a private key and certificate + request. + + /usr/local/sbin/bbackupd-config /etc/box lazy 999 hostname /var/bbackupd /home + + (See OpenSSL notes if you + get an OpenSSL error) + + The items in bold need to be changed. In order, they are the + account number, the hostname of the server you're using, and + finally, the directories you want backed up. You can include as many + you want here. + + However, the directories you specify must not contain other + mounted file systems within them at any depth. Specify them + separately, one per mount point. No checks are currently made to + catch bad configuration of this nature! + + You may also want to consider changing the mode from lazy to + snapshot, depending on what your system is used for: + + + + Lazy Mode + + + This mode regularly scans the files, with only a rough + schedule. It uploads files as and when they are changed, if + the latest version is more than a set age. This is good for + backing up user's documents stored on a server, and spreads + the load out over the day. + + + + + Snapshot Mode + + + This mode emulates the traditional backup behaviour of + taking a snapshot of the filesystem. The backup daemon does + absolutely nothing until it is instructed to make a backup + using the bbackupctl utility (probably as a cron job), at + which point it uploads all files which have been changed since + the last time it uploaded. + + + + + When you run the config script, it will tell you what you need + to do next. Don't forget to read all the output. An example is shown + at the end of this page, but the instructions for your installation + may be different. +
+ +
+ Certificates + + After you have sent your certificate request off to the server + administrator and received your certificate and CA root back, + install them where instructed by the bbackupd-config script during + basic bbackupd configuration. + + You can then run the daemon (as root) by running + /usr/local/sbin/bbackupd, and of course, adding it + to your system's startup scripts. The first time it's run it will + upload everything. Interrupting it and restarting it will only + upload files which were not uploaded before - it's very + tolerant. + + If you run in snapshot mode, you will need to add a cron job + to schedule backups. The config script will tell you the exact + command to use for your system. + + Please read the Troubleshooting page if you have + problems. + + Remember to make a traditional backup of the keys file, as + instructed. You cannot restore files without it. + + It is recommended that you backup up all of /etc/box as it + will make things easier if you need to restore files. But only the + keys are absolutely essential. + + If you want to see what it's doing in more detail (probably a + good idea), follow the instructions in the server setup to create + new log files with syslog. +
+ +
+ Adding and removing backed up locations + + By editing the /etc/box/bbackupd.conf file, you can add and + remove directories to back up - see comments in this file for help. + Send bbackupd a HUP signal after you modify it. + + When you remove a location, it will not be marked as deleted + immediately. Instead, bbackupd waits about two days before doing so, + just in case you change your mind. After this, it will be eventually + removed from the store by the housekeeping process. Run as + root. + + The backup client is designed to be run as root. It is + possible to run without root, but this is not recommended. Clock + synchronisation for file servers. + + If you are using the backup client to backup a filesystem + served from a fileserver, you should ideally ensure that the + fileserver clocks are synchronised with the fileserver. + + bbackupd will cope perfectly well if the clocks are not + synchronised. Errors up to about half an hour cause no problems. + Larger discrepancies cause a loss of efficiency and the potential to + back up a file during a write process. + + There is a configuration parameter MaxFileTimeInFuture, which + specifies how far in the future a file must be for it to be uploaded + as soon as it is seen. You should not need to adjust this (default + is 2 days). Instead, get those clocks synchronised. Excluding files + and directories from the backup. + + Within the bbackupd.conf file, there is a section named + BackupLocations which specifies which locations on disc should be + backed up. It has subsections, each of which is in the + format: + + name + { + Path = /path/of/directory + (optional exclude directives) + } + + name is derived from the Path + by the config script, but should merely be unique. + + The exclude directives are of the form: + + [Exclude|AlwaysInclude][File|Dir][|sRegex] = regex or full pathname + + (The regex suffix is shown as 'sRegex' to make File or Dir + plural) + + For example: + + ExcludeDir = /home/guest-user + ExcludeFilesRegex = *.(mp3|MP3)\$ + AlwaysIncludeFile = /home/username/veryimportant.mp3 + + This excludes the directory /home/guest-user from the backup + along with all mp3 files, except one MP3 file in particular. + + In general, Exclude excludes a file or directory, unless the + directory is explicitly mentioned in a AlwaysInclude + directive. + + If a directive ends in Regex, then it is a regular expression + rather than a explicit full pathname. See + + man 7 re_format + + for the regex syntax on your platform. +
+ +
+ Example configuration output + + This is an example of output from the bbstored-config + script. + + + Follow the instructions output by your script, not the ones + here -- they may be different for your system. + + + /usr/local/sbin/bbackupd-config /etc/box lazy 51 server.example.com /var/bbackupd /home /etc/samba + +Setup bbackupd config utility. + +Configuration: + Writing configuration file: /etc/box/bbackupd.conf + Account: 51 + Server hostname: server.example.com + Directories to back up: + /home + /etc/samba + +Note: If other file systems are mounted inside these directories, then problems may occur +with files on the store server being renamed incorrectly. This will cause efficiency +problems, but not affect the integrity of the backups. + +WARNING: Directories not checked against mountpoints. Check mounted filesystems manually. + +Creating /etc/box... +Creating /etc/box/bbackupd +Generating private key... + [OpenSSL output omitted] + +Generating keys for file backup +Writing notify script /etc/box/bbackupd/NotifyStoreFull.sh +Writing configuration file /etc/box/bbackupd.conf + +=================================================================== + +bbackupd basic configuration complete. + +What you need to do now... + +1) Make a backup of /etc/box/bbackupd/51-FileEncKeys.raw + This should be a secure offsite backup. + Without it, you cannot restore backups. Everything else can + be replaced. But this cannot. + KEEP IT IN A SAFE PLACE, OTHERWISE YOUR BACKUPS ARE USELESS. + +2) Send /etc/box/bbackupd/51-csr.pem + to the administrator of the backup server, and ask for it to + be signed. + +3) The administrator will send you two files. Install them as + /etc/box/bbackupd/51-cert.pem + /etc/box/bbackupd/serverCA.pem + after checking their authenticity. + +4) You may wish to read the configuration file + /etc/box/bbackupd.conf + and adjust as appropraite. + + There are some notes in it on excluding files you do not + wish to be backed up. + +5) Review the script + /etc/box/bbackupd/NotifyStoreFull.sh + and check that it will email the right person when the store + becomes full. This is important -- when the store is full, no + more files will be backed up. You want to know about this. + +6) Start the backup daemon with the command + /usr/local/sbin/bbackupd + in /etc/rc.local, or your local equivalent. + Note that bbackupd must run as root. + +=================================================================== + + Remember to make a secure, offsite backup of your backup keys, + as described in Basic + configuration above. If you do not, and that key is lost, you + have no backups. +
+
+ +
+ Configuration Options + + Box Backup has many options in its configuration file. We will + try to list them all here. + + First of all, here is an example configuration file, for + reference: + + + Example Configuration File + + StoreHostname = localhost +AccountNumber = 0x2 + +KeysFile = /etc/box/2-FileEncKeys.raw +CertificateFile = /etc/box/2-cert.pem +PrivateKeyFile = /etc/box/2-key.pem +TrustedCAsFile = /etc/box/serverCA.pem +DataDirectory = /var/run/boxbackup +NotifyScript = /etc/box/NotifySysadmin.sh +CommandSocket = /var/run/box/bbackupd.sock + +UpdateStoreInterval = 86400 +MinimumFileAge = 3600 +MaxUploadWait = 7200 +FileTrackingSizeThreshold = 65536 +DiffingUploadSizeThreshold = 65536 +MaximumDiffingTime = 20 +ExtendedLogging = no +LogAllFileAccess = yes + +Server +{ + PidFile = /var/run/bbackupd.pid +} +BackupLocations +{ + etc + { + Path = /etc + } + home + { + Path = /home + ExcludeDir = /home/shared + ExcludeDir = /home/chris/.ccache + ExcludeDir = /home/chris/.mozilla/firefox/vvvkq3vp.default/Cache + } +} + + + As you can see from the example above, the configuration file + has a number of subsections, enclosed in curly braces {}. Some options + appear outside of any subsection, and we will refer to these as root options. The available options in + each section are described below. + + Every option has the form name = value. Names are + not case-sensitive, but values are. Depending on the option, the value + may be: + + + + a path (to a file or directory); + + + + a number (usually in seconds or bytes); + + + + a boolean (the word Yes or No); + + + + a hostname (or IP address). + + + + Paths are specified in native format, i.e. a full Windows path + with drive letter on Windows clients, or a full Unix path on Unix + clients. + + + Example: + + StoreObjectInfoFile = + /var/state/boxbackup/bbackupd.dat + + StoreObjectInfoFile = C:\Program Files\Box + Backup\data\bbackupd.dat + The use of relative paths (which do not start with a + forward slash on Unix, or a drive specification on Windows) is + possible but not recommended, since they are interpreted relative to + the current working directory when bbackupd was started, which is + liable to change unexpectedly over time. + + Numbers which start with "0x" are interpreted as hexadecimal. + Numbers which do not start with "0x" are interpreted as + decimal. + +
+ Root Options + + These options appear outside of any subsection. By convention + they are at the beginning of the configuration file. + + Some options are required, and some are optional. + + + + StoreHostname (required) + + + The Internet host name (DNS name) or IP address of the + server. This is only used to connect to the server. + + + + + AccountNumber (required) + + + The number of the client's account on the server. This + must be provided by the server operator, and must match the + account number in the client's certificate, otherwise the + client will not be able to log into the server. + + The account number may be specified in hexadecimal + (starting with 0x, as in the example above) or in decimal, but + since the server operator works in hexadecimal, that format is + highly recommended and is the default. + + + + + KeysFile (required) + + + The path to the file containing the encryption key used + for data encryption of client file data and filenames. This is + the most important file to keep safe, since without it your + backups cannot be decrypted and are useless. Likewise, if an + attacker gets access to this key and to your encrypted + backups, he can decrypt them and read all your data. + + Do not change the encryption key without deleting all + files from the account on the server first. None of your old + files on the store will be readable if you do so, and if you + change it back, none of the files uploaded with the new key + will be readable. + + + + + CertificateFile (required) + + + The path to the OpenSSL client certificate in PEM + format. This is supplied by the server operator in response to + the certificate request which you send to them. Together with + the PrivateKeyFile, this provides access to the store server + and the encrypted data stored there. + + It is not critical to protect this file or to back it up + safely, since it can be regenerated by creating a new + certificate request, and asking the server operator to sign + it. You may wish to back it up, together with the + PrivateKeyFile, to avoid this inconvenience if you lose all + your data and need quick access to your backups. + + If you do back them up, you should keep them in a + separate location to the KeysFile, since any person holding + the KeysFile and the PrivateKeyFile can gain access to your + encrypted data and decrypt it. + + + + + PrivateKeyFile (required) + + + The path to the OpenSSL private key in PEM format. This + is generated at the same time as the certificate request, but + there is no need to send it to the server operator, and you + should not do so, in case the communication is intercepted by + an attacker. Together with the CertificateFile, this provides + access to the store server and the encrypted data stored + there. + + See the notes under CertificateFile for information + about backing up this file. + + + + + TrustedCAsFile (required) + + + The path to the OpenSSL certificate of the Client + Certificate Authority (CCA), in PEM format. This is supplied + by the server operator along with your account details, or + along with your signed client certificate. This is used to + verify that the server which you are connecting to is + authorised by the person who signed your certificate. It + protects you against DNS and ARP poisoning and IP spoofing + attacks. + + + + + DataDirectory (required) + + + The path to a directory where bbackupd will keep local + state information. This consists of timestamp files which + identify the last backup start and end times, used by + bbackupquery to determine whether files + have changed, and optionally a database of inode numbers, + which are used to check for files being renamed. The database + is only saved if Box Backup is built with Berkeley Database + (BDB) support. + + + + + NotifyScript (optional) + + + The path to the script or command to run when the Box + Backup client detects an error during the backup process. This + is normally used to notify the client system administrator by + e-mail when a backup fails for any reason. + + The script or command is called with one of the + following additional arguments to identify the cause of the + problem: + + + + store-full + + + The backup store is full. No new files are being + uploaded. If some files are marked as deleted, they + should be removed in due course by the server's + housekeeping process. Otherwise, you need to remove some + files from your backup set, or ask the store operator + for more space. + + + + + read-error + + + One or more files which were supposed to be backed + up could not be read. This could be due to: + + running the server as a non-root user; + + + + backing up a mounted filesystem such as + NFS; + + + + access control lists being applied to some + files; + + + + SELinux being enabled; + + + + trying to back up open files under + Windows; + + + + strange directory permissions such as 0000 or + 0400. + + Check the client logs, e.g. + /var/log/bbackupd on Unix, or the Windows Event Viewer + in Control Panel > Administrative Tools, for more + information about which files are not being backed up + and why. + + + + + backup-error + + + There was a communications error with the server, + or an unexpected exception was encountered during a + backup run. Check the client logs, e.g. + /var/log/box on Unix, or the + Windows Event Viewer in Control Panel > + Administrative Tools, for more information about the + problem. + + You may wish to check your Internet access to the + server, check that the server is running, and ask your + server operator to check your account on the + server. + + + + + + + + CommandSocket (optional) + + + The path to the Unix socket which + bbackupd creates when running, and which + bbackupctl uses to communicate with it, for + example to force a sync or a configuration reload. If this + option is omitted, no socket will be created, and + bbackupctl will not function. + + Unix sockets appear within the filesystem on Unix, as a + special type of file, and must be created in a directory which + exists and to which bbackupd has write access, and bbackupctl + has read access. + + On Windows, the path is ignored, and a named + pipe is created instead. This does not currently + have any security attached, so it can be accessed by any user. + Unlike a Unix socket it can also be accessed remotely. Please + use this option with extreme caution on Windows, and only on + fully trusted networks. + + + + + AutomaticBackup (optional) + + + Enable or disable the client from connecting + automatically to the store every + UpdateStoreInterval seconds. When + enabled (set to Yes), the client is in + Lazy Mode. When disabled (set to + No), it is in Snapshot + Mode. This setting is optional, and the default + value is Yes. + + + + + UpdateStoreInterval (required) + + + The approximate time between successive connections to + the server, in seconds, when the client is in Lazy + Mode. The actual time is randomised slightly to + prevent "rush hour" traffic jams on the server, where many + clients try to connect at the same time. + + This value is ignored if the client is in + Snapshot Mode. However, it is still + required. It can be set to zero in this case. + + You will probably need to experiment with the value of + this option. A good value to start with is probably 86400 + seconds, which is one day. + + + + + MinimumFileAge (required) + + + The number of seconds since a file was last modified + before it will be backed up. The reason for this is to avoid + repeatedly backing up files which are repeatedly changing. A + good value is about 3600 seconds (one hour). If set to zero, + files which have changed will always be backed up on the next + backup run. + + The MaxUploadWait option + overrides this option in some circumstances. + + + + + MaxUploadWait (required) + + + The number of seconds since a file was last uploaded + before it will be uploaded again, even if it keeps changing. + The reason for this is to ensure that files which are + continuously modified are eventually uploaded anyway. This + should be no less than the value of + MinimumFileAge. A good value is about + 14400 seconds (4 hours). + + + + + MaxFileTimeInFuture (optional) + + + The maximum time that a file's timestamp can be in the + future, before it will be backed up anyway. Due to clock + synchronisation problems, it is inevitable that you will + occasionally see files timestamped in the future. Normally, + for files which are dated only slightly in the future, you + will want to wait until after the file's date before backing + it up. However, for files whose dates are very wrong (more + than a few hours) you will normally prefer to back them up + immediately. + + A good value is about 7200 seconds (2 hours) to cope + with potential problems when moving in and out of daylight + saving time, if applicable in your timezone. The default + value, if this setting is not provided, is 172800 seconds (2 + days). + + + + + FileTrackingSizeThreshold (required) + + + The minimum size of files which will be tracked by inode + number to detect renames. It is not worth detecting renames of + small files, since they are quick to upload again in full, and + keeping their inode numbers in memory increases the client's + memory usage and slows down searches. Larger files should be + tracked to avoid wasting space on the store and long + uploads. + + A good value is about 65536 bytes (64 kilobytes). + + + + + DiffingUploadSizeThreshold (required) + + + The minimum size of files which will be compared to the + old file on the server, and for which only changes will be + uploaded. It is not worth comparing small files, since they + are quick to upload again in full, and sending the entire file + reduces the risk of data loss if the store is accidentally + corrupted. Larger files should have only their differences + uploaded to avoid wasting space on the store and long + uploads. + + A good value is about 65536 bytes (64 kilobytes). + + + + + MaximumDiffingTime (optional) + + + The maximum time for which the client will attempt to + find differences between the current version and the old + version in the store, before giving up and uploading the + entire file again. Very large files (several gigabytes) may + take a very long time to scan for changes, but would also take + a very long time to upload again and use a lot of space on the + store, so it is normally worth omitting this value. + + Use this option only if, for some bizarre reason, you + prefer to upload really large files in full rather than spend + a long time scanning them for changes. + + + + + KeepAliveTime (optional) + + + The interval (in seconds) between sending Keep-Alive + messages to the server while performing long operations such + as finding differences in large files, or scanning large + directories. + + These messages ensure that the SSL connection is not + closed by the server, or an intervening firewall, due to lack + of activity. + + The server will normally wait up to 15 minutes (900 + seconds) before disconnecting the client, so the value should + be given and should be less than 900. Some firewalls may time + out inactive connections after 10 or 5 minutes. + + A good value is 300 seconds (5 minutes). You may need to + reduce this if you frequently see TLSReadFailed or + TLSWriteFailed errors on the client. + + + + + StoreObjectInfoFile (optional) + + + Enables the use of a state file, which stores the + client's internal state when the client is not running. This + is useful on clients machines which are frequently shut down, + for example desktop and laptop computers, because it removes + the need for the client to recontact the store and rescan all + directories on the first backup run, which may take some time. + This feature is somewhat experimental and not well tested. + + + This is option is disabled by default, in which case the + state is stored in memory only. The value is the path to the + state file. + + + + + ExtendedLogging (optional) + + + Enables the connection debugging mode of the client, + which writes all commands sent to or received from the server + to the system logs. This generates a lot + of output, so it should only be used when instructed, or when + you suspect a connection problem or client-server protocol + error (and you know how to interpret the output). + + This is a boolean value, which may be set to + Yes or No. The default is of + course No. + + + + + ExtendedLogFile (optional, new in 0.11) + + + Enables the same debugging output as + ExtendedLogging, but written to a file + instead of the system logs. This is useful if you need + extended logging, but do not have access to the system logs, + for example if you are not the administrator of the + computer. + + The value is the path to the file where these logs will + be written. If omitted, extended logs will not be written to a + file. This is entirely independent of the + ExtendedLogging option. It does not + make much sense to use both at the same time. + + + + + LogAllFileAccess (optional, new in 0.11) + + + Enables logging of all local file and directory access, + file uploads (full and differential), and excluded files. This + may be useful if the client is failing to upload a particular + file, or crashing while trying to upload it. The logs will be + sent to the system log or Windows Event Viewer. + + This generates a lot + of output, so it should only be used when instructed, or when + you suspect that bbackupd is skipping some files and want to + know why. Because it is verbose, the messages are hidden by + default even if the option is enabled. To see them, you must + run bbackupd with at least one -v option. + + This is a boolean value, which may be set to + Yes or No. The default is of + course No. + + + + + SyncAllowScript (optional) + + + The path to the script or command to run when the client + is about to start an automatic backup run, and wishes to know + whether it is safe to do so. This is useful for clients which + do not always have access to the server, for example laptops + and computers on dial-up Internet connections. + + The script should either output the word + now if the backup should proceed, or else a + number, in seconds, which indicates how long the client should + wait before trying to connect again. Any other output will + result in an error on the client, and the backup will not + run. + + This value is optional, and by default no such script is + used. + + + +
+ +
+ Server Section + + These options appear within the Server subsection, which is at + the root level. + + + + PidFile + + + This option enables the client to write its processs + identifier (PID) to the specified file after starting. The + file will be deleted when the client daemon exits for any + reason. This is disabled by default, but is recommended + whenever you run the client daemon as a daemon (in the + background), which is usually the case. This file can be used + by scripts to determine whether the daemon is still running, + and to send it messages to reload its configuration or to + terminate. + + + Example Server Section + + Server +{ + PidFile = /var/state/boxbackup/bbackupd.pid +} + + + + +
+ +
+ Backup Locations Section + + This section serves only as a container for all defined backup + locations. + + + Example Backup Locations Section + + BackupLocations +{ + etc + { + Path = /etc + } + home + { + Path = /home + ExcludeDir = /home/shared + ExcludeDir = /home/chris/.ccache + ExcludeDir = /home/chris/.mozilla/firefox/vvvkq3vp.default/Cache + } +} + + + Each subsection is a backup location. The name of the + subsection is the name that will be used on the server. The root + directory of the account on the server contains one subdirectory per + location. The name should be simple, not containing any spaces or + special characters. + + If you do not define any locations, the client will not back + up any files! + + It is currently not recommended to back up the root directory + of the filesystem on Unix. Box Backup is designed to back up + important data and configuration files, not full systems. + Nevertheless, nothing prevents you from doing so if you + desire. + + On Windows, it is currently not possible to back up files + which are open (currently in use), such as open documents in + Microsoft Office, and system files such as the registry and the + paging file. You will get an error for each open file which the + client attempts to back up. Once the file has been closed, it will + be backed up normally. System files will always be open, and should + be excluded from your backups. +
+
+
+ + + + Administration + + This chapter deals with the dauily running and management of the Box + Backup system. It explains most day-to-day tasks. + +
+ Regular Maintenance + + The steps involved in maintaining and keeping the backup sets + healthy are outlined in this section. + +
+ Controlling a backup client + + The bbackupctl program sends control commands to the bbackupd + daemon. It must be run as the same user as the daemon, and there is no + exception for root. + + The command line syntax is: + + /usr/local/sbin/bbackupctl [-q] [-c config-file] command + + The -q option reduces the amount of output the program emits, + and -c allows an alternative configuration file to be + specified. + + Valid commands are: + + + + terminate + + Stop the bbackupd daemon now (equivalent to kill) + + + + reload + + Reload the configuration file (equivalent to kill + -HUP) + + + + sync + + Connect to the server and synchronise files now + + + + bbackupctl communicates with + the server via a UNIX domain socket, specified in bbackupd.conf with + the CommandSocket directive. This does not need to be specified, and + bbackupd will run without the command + socket, but in this case bbackupctl will not be able to communicate + with the daemon. + + Some platforms cannot check the user id of the connecting + process, so this command socket becomes a denial of service security + risk. bbackupd will warn you when it + starts up if this is the case on your platform, and you should + consider removing the CommandSocket directive on these + platforms. +
+ +
+ Using bbackupctl to perform snapshots + + bbackupctl's main purpose is to + implement snapshot based backups, emulating the behaviour of + traditional backup software. + + Use bbackupd-config to write a configuration file in snapshot + mode, and then run the following command as a cron job. + + /usr/local/sbin/bbackupctl -q sync + + This will cause the backup daemon to upload all changed files + immediately. bbackupctl will exit + almost immediately, and will not output anything unless there is an + error. +
+ +
+ Checking storage space used on the server + +
+ From the client machine + + bbackupquery can tell you how much space is used on the server + for this account. Either use the usage command in interactive mode, + or type: + + /usr/local/sbin/bbackupquery -q usage quit + + to show the space used as a single command. +
+ +
+ On the server + + bbstoreaccounts allows you to query the space used, and change + the limits. To display the space used on the server for an account, + use: + + /usr/local/sbin/bbstoreaccounts info 75AB23C + + To adjust the soft and hard limits on an account, use: + + /usr/local/sbin/bbstoreaccounts setlimit 75AB23C new-soft-limit new-hard-limit + + You do not need to restart the server. +
+
+ +
+ Verify and restore files + + Backups are no use unless you can restore them. The bbackupquery + utility does this and more. + + You don't provide any login information to it, as it just picks + up the data it needs from /etc/box/bbackupd.conf. You should run it as + root so it can find everything it needs. + + Full documentation can be found in the bbackupquery manual page. It follows + the model of a command line sftp client quite closely. + + TODO: Link to bbackupquery man-page here. + + On systems where GNU readline is available (by default) it uses + that for command line history and editing. Otherwise it falls back to + very basic UNIX text entry. + + TODO: Did the readline dependency change to editline? + +
+ Using bbackupquery + + bbackupquery is the tool you use to verify, restore and + investigate your backup files with. When invoked, it simply logs + into the server using the certificates you have listed in + bbackupd.conf. + + After you run bbackupquery, you will see a prompt, allowing + you to execute commands. The list (or ls) command lets you view + files in the store. It works much like unix ls, but with different + options. An example: + + [pthomsen@host bbackupquery]$ bbackupquery +Box Backup Query Tool v0.10, (c) Ben Summers and contributors 2003-2006 +Using configuration file /etc/box/bbackupd.conf +Connecting to store... +Handshake with store... +Login to store... +Login complete. + +Type "help" for a list of commands. + +query > ls +00000002 -d---- mp3 +00000003 -d---- video +00000004 -d---- home-pthomsen +00000005 -d---- root +query > + + The ls commands shows the directories that are backed up. Now + we'll take a closer look at the home-pthomsen directory: + + query > cd home-pthomsen +query > ls +00002809 f----- sample.tiff +0000280a f----- s3.tiff +0000280b f----- s4.tiff +0000280d f----- s2.tiff +0000280e f----- foo.pdf +0000286c f----- core.28720 +0000339a -d---- .emacs.d +0000339d -d---- bbackup-contrib +00003437 f----- calnut.compare.txt +0000345d f----- DSCN1783.jpg +0000345e f----- DSCN1782.jpg +query > + + The ls command takes the following options; + + + + -r -- recursively list + all files + + + + -d -- list deleted + files/directories + + + + -o -- list old versions + of files/directories + + + + -I -- don't display + object ID + + + + -F -- don't display + flags + + + + -t -- show file + modification time (and attr mod time if has the object has + attributes, ~ separated) + + + + -s -- show file size in + blocks used on server (only very approximate indication of size + locally) + + + + The flags displayed from the ls command are as follows: + + + f = file + + d = directory + + X = deleted + + o = old version + + R = remove from server as soon as marked deleted or + old + + a = has attributes stored in directory record which + override attributes in backup file + +
+ +
+ Verify backups + + As with any backup system, you should frequently check that + your backups are working properly by comparing them. Box Backup + makes this very easy and completely automatic. All you have to do is + schedule the bbackupquery compare command to run + regularly, and check its output. You can run the command manually as + follows: + + /usr/local/sbin/bbackupquery "compare -a" quit + + This command will report all the differences found between the + store and the files on disc. It will download everything, so may + take a while. You should expect to see some differences on a typical + compare, because files which have recently changed are unlikely to + have been uploaded yet. It will also tell you how many files have + been modified since the last backup run, since these will normally + have changed, and such failures are expected. + + You are strongly recommended to add this command as a + cron job, at least once a month, and to check the + output for anything suspicious, particularly a large number of + compare failures, failures on files that have not been modified, or + any error (anything except a compare mismatch) that occurs during + the compare operation. + + Consider keeping a record of these messages and comparing them + with a future verification. + + If you would like to do a "quick" check which just downloads + file checksums and compares against that, then run: + + /usr/local/sbin/bbackupquery "compare -aq" quit + + However, this does not check that the file attributes are + correct, and since the checksums are generated on the client they + may not reflect the data on the server if there is a problem -- the + server cannot check the encrypted contents. View this as a quick + indication, rather than a definite check that your backup verifies + correctly. +
+ +
+ Restore backups + + You will need the keys file created when you configured the + server. Without it, you cannot restore the files; this is the + downside of encrypted backups. However, by keeping the small keys + file safe, you indirectly keep your entire backup safe. + + The first step is to recreate the configuration of the backup + client. It's probably best to have stored the /etc/box directory + with your keys. But if you're recreating it, all you really need is + to have got the login infomation correct (ie the certs and + keys). + + Don't run bbackupd yet! It will mark all your files as deleted + if you do, which is not hugely bad in terms of losing data, just a + major inconvenience. (This assumes that you are working from a blank + slate. If you want to restore some files to a different location, + it's fine to restore while bbackupd is running, just do it outside a + backed up directory to make sure it doesn't start uploading the + restored files.) + + Type: + + /usr/local/sbin/bbackupquery + + to run it in interactive mode. + + Type: + + list + + to see a list of the locations stored on the server. + + For each location you want to restore, type: + + restore name-on-server local-dir-name + + The directory specified by local-dir-name must not exist yet. + If the restore is interrupted for any reason, repeat the above + steps, but add the -r flag to the + restore command to tell it to resume. +
+ +
+ Retrieving deleted and old files + + Box Backup makes old versions of files and files you have + deleted available, subject to there being enough disc space on the + server to hold them. + + This is how to retrieve them using bbackupquery. Future + versions will make this far more user-friendly. + + Firstly, run bbackupquery in interactive mode. It behaves in a + similar manner to a command line sftp client. + + /usr/local/sbin/bbackupquery + + Then navigate to the directory containing the file you want, + using list, cd and pwd. + + query > cd home/profiles/USERNAME + + List the directory, using the "o" option to list the files + available without filtering out everything apart from the current + version. (if you want to see deleted files as well, use list + -odt) + + query > list -ot +00000078 f--o- 2004-01-21T20:17:48 NTUSER.DAT +00000079 f--o- 2004-01-21T20:17:48 ntuser.dat.LOG +0000007a f--o- 2004-01-21T17:55:12 ntuser.ini +0000007b f---- 2004-01-12T15:32:00 ntuser.pol +0000007c -d--- 1970-01-01T00:00:00 Templates +00000089 -d--- 1970-01-01T00:00:00 Start Menu +000000a0 -d--- 1970-01-01T00:00:00 SendTo +000000a6 -d--- 1970-01-01T00:00:00 Recent +00000151 -d--- 1970-01-01T00:00:00 PrintHood +00000152 -d--- 1970-01-01T00:00:00 NetHood +00000156 -d--- 1970-01-01T00:00:00 My Documents +0000018d -d--- 1970-01-01T00:00:00 Favorites +00000215 -d--- 1970-01-01T00:00:00 Desktop +00000219 -d--- 1970-01-01T00:00:00 Cookies +0000048b -d--- 1970-01-01T00:00:00 Application Data +000005da -d--- 1970-01-01T00:00:00 UserData +0000437e f--o- 2004-01-24T02:45:43 NTUSER.DAT +0000437f f--o- 2004-01-24T02:45:43 ntuser.dat.LOG +00004380 f--o- 2004-01-23T17:01:29 ntuser.ini +00004446 f--o- 2004-01-24T02:45:43 NTUSER.DAT +00004447 f--o- 2004-01-24T02:45:43 ntuser.dat.LOG +000045f4 f---- 2004-01-26T15:54:16 NTUSER.DAT +000045f5 f---- 2004-01-26T15:54:16 ntuser.dat.LOG +000045f6 f---- 2004-01-26T16:54:31 ntuser.ini + + (this is a listing from a server which is used as a Samba + server for a network of Windows clients.) You now need to fetch the + file using it's ID, rather than it's name. The ID is the hex number + in the first column. Fetch it like this: + + query > get -i 0000437e NTUSER.DAT +Object ID 0000437e fetched successfully. + + The object is now available on your local machine. You can use + lcd to move around, and sh ls to list directories on your local + machine. +
+
+
+ +
+ Fixing corruptions of store data + + This section gives help on what to do if your server has suffered + corruption, for example, after an unclean shutdown or other operating + system or hardware problem. + + In general, as updates to the store are made in an atomic manner, + the most likely result is wasted disc space. However, if really bad + things happen, or you believe that there is a lot of wasted space, then + these instructions will help to restore your data. + + You know you will need to do something if you get strange errors, + and bbackupd attempts to contact the server every 100 seconds or so. Or + if one of the discs in your RAID disc set has failed. + + After following these instructions, the end result will be that + bbackupquery will be able to see all the files which were stored on your + server, and retrieve them. Some of them may be in lost+found directories + in the root of the store (or in their original position if they have + been moved) but they will all be able to be retrieved. + + After you have retrieved the files you want, bbackupd will upload + new versions where necessary, and after about two days, mark any + lost+found directories as deleted. Finally, those directories will be + removed by the housekeeping process on the server. + + These instructions assume you're working on account 1234. Replace + this with the account number that you actually want to check (the one + that is experiencing errors). These steps will need to be repeated for + all affected accounts. + +
+ Stop bbackupd + + First, make sure that bbackupd is not running on the client + machine for the account you are going to recover. Use + bbackupctl terminate to stop it. This step is not + strictly necessary, but is recommended. During any checks on the + account, bbackupd will be unable to log in, and after they are + complete, the account is marked as changed on the server so bbackupd + will perform a complete scan. +
+ +
+ Are you using RAID on the server? + + The raidfile recovery tools have not been written, and probably + will not be, since Box Backup RAID is deprecated. However, when two + out of three files are available, the server will successfully allow + access to your data, even if it complains a lot in the logs. The best + thing to do is to fix the accounts, if necessary, and retrieve any + files you need. Then move the old store directories aside (in case you + need them) and start afresh with new accounts, and let the clients + upload all their data again. +
+ +
+ Check and fix the account + + First, run the check utility, and see what errors it + reports. + + /usr/local/sbin/bbstoreaccounts check 1234 + + This will take some time, and use a fair bit of memory (about 16 + bytes per file and directory). If the output looks plausible and + reports errors which need fixing, run it again but with the fix + flag: + + /usr/local/sbin/bbstoreaccounts check 1234 fix + + This will fix any errors, and remove unrecoverable files. + Directories will be recreated if necessary. + + NOTE: The utility may adjust + the soft and hard limits on the account to make sure that housekeeping + will not remove anything -- check these afterwards. +
+ +
+ Grab any files you need with bbackupquery + + At this point, you will have a working store. Every file which + was on the server, and wasn't corrupt, will be available. + + On the client, use bbackupquery to log in and examine the store. + (type help at the prompt for instructions). Retrieve any files you + need, paying attention to any lost+found directories in the root + directory of the store. + + You can skip this step if you are sure that the client machine + is fine -- in this case, bbackupd will bring the store up to + date. +
+ +
+ Restart bbackupd + + Restart bbackupd on the client machine. The store account will + be brought up to date, and files in the wrong place will be marked for + eventual deletion. +
+
+ +
+ Troubleshooting + + If you are trying to fix a store after your disc has been + corrupted, see Fixing corruptions of + store data. + + Unfortunately, the error messages are not particularly helpful at + the moment. This page lists some of the common errors, and the most + likely causes of them. + + When an error occurs, you will see a message like 'Exception: + RaidFile/OSFileError (2/8)' either on the screen or in your log files. + (it is recommended you set up another log file as recommended in the + server setup instructions.) + + This error may not be particularly helpful, although some do have + extra information about probable causes. To get further information, + check the ExceptionCodes.txt file in the root of the distribution. This + file is generated by the ./configure script, so you will need to have + run that first. + + Some common causes of exceptions are listed below. + + Please email me with any other codes you get, and I will let you + know what they mean, and add notes here. + +
+ RaidFile (2/8) + + This is found either when running bbstoreaccounts or in the + bbstored logs. + + Problem: The directories you + specified in the raidfile.conf are not writable by the _bbstored + user. + + Resolution: Change permissions + appropriately. +
+ +
+ Common (1/2) + + This usually occurs when the configuration files can't be + opened. + + Problem: You created your + configurations in non-standard locations, and the programs cannot find + them. + + Resolution: Explicitly specify + configuration file locations to daemons and programs. For + example + + /usr/local/sbin/bbstored /some/other/dir/bbstored.config /usr/local/sbin/bbackupquery -c /some/other/dir/bbackupd.config + + (daemons specify the name as the first argument, utility + programs with the -c option). + + Problem: bbstored can't find + the raidfile.conf file specified in bbstored.conf. + + Resolution: Edit bbstored.conf + to point to the correct location of this additional configuration + file. +
+ +
+ Server (3/16) + + The server can't listen for connections on the IP address + specified when you configured it. + + Problem: This probably means + you've specified the wrong hostname to bbstored-config -- maybe your + server is behind a NAT firewall? + + Resolution: Edit bbstored.conf + and correct the ListenAddresses line. You should replace the server + address with the IP address of your machine. +
+ +
+ Connection (7/x) + + These errors all relate to connections failing -- you may see + them during operation if there are network failures or other problems + between the client and server. The backup system will recover from + them automatically. + +
+ Connection (7/30) - SSL problems + + Log snippet from client side: + + bbackupd[1904]: Opening connection to server xxxx.xxx... +bbackupd[1904]: SSL err during Connect: error:xxxxxxxx:rsa routines:RSA_padding_check_PKCS1_type_1:block type is not 01 +bbackupd[1904]: SSL err during Connect: error:xxxxxxxx:rsa routines:RSA_EAY_PUBLIC_DECRYPT:padding check failed +bbackupd[1904]: SSL err during Connect: error:xxxxxxxx:asn1 encoding routines:ASN1_verify:EVP lib +bbackupd[1904]: SSL err during Connect: error:xxxxxxxx:SSL routines:SSL3_GET_SERVER_CERTIFICATE:certificate verify failed +bbackupd[1904]: TRACE: Exception thrown: ConnectionException(Conn_TLSHandshakeFailed) at SocketStreamTLS.cpp(237) +bbackupd[1904]: Exception caught (7/30), reset state and waiting to retry... + + And from the server: + + bbstored[19291]: Incoming connection from xx.xxx.xx.xxx port xxxxx (handling in child xxxxx) +bbstored[21588]: SSL err during Accept: error:xxxxxxxx:SSL routines:SSL3_READ_BYTES:tlsv1 alert decrypt error +bbstored[21588]: in server child, exception Connection TLSHandshakeFailed (7/30) -- terminating child + + Solution: Create a new CA on + the server side and re-generate the client certificate. Re-creating + the client certificate request is not necessary. +
+
+ +
+ Advanced troubleshooting + + If this really doesn't help, then using the DEBUG builds of the + system will give you much more information -- a more descriptive + exception message and the file and line number where the error + occurred. + + For example, if you are having problems with bbstoreaccounts, + build the debug version with: + + cd boxbackup-0.0 +cd bin/bbstoreaccounts +make + + Within the module directories, make defaults to building the + debug version. At the top level, it defaults to release. + + This will build an executable in debug/bin/bbstoreaccounts which + you can then use instead of the release version. It will give far more + useful error messages. + + When you get an error message, use the file and line number to + locate where the error occurs in the code. There will be comments + around that line to explain why the exception happened. + + If you are using a debug version of a daemon, these extended + messages are found in the log files. +
+
+ + + &__ExceptionCodes__elfjz3fu; + + + Running without root + + It is possible to run both the server and client without root + privileges. + +
+ Server + + The server, by default, runs as a non-root user. However, it + expects to be run as root and changes user to a specified user as soon + as it can, simply for administrative convenience. The server uses a port + greater than 1024, so it doesn't need root to start. + + To run it entirely as a non-root user, edit the bbstored.conf + file, and remove the User directive from the Server section. Then simply + run the server as your desired user. +
+ +
+ Client + + The client requires root for normal operation, since it must be + able to access all files to back them up. However, it is possible to run + the client as a non-root user, with certain limitations. + + Follow the installation instructions, but install the executable + files manually to somewhere in your home directory. Then use + bbackupd-config to configure the daemon, but use a directory other than + /etc/box, probably somewhere in your home directory. + + All directories you specify to be backed up must be readable, and + all files must be owned by the user and readable to that user. + + Important: If any file or directory is not readable by this user, + the backup process will skip that file or directory. Keep an eye on the + logs for reports of this failure. + + Non-root operation of the backup client is recommended only for + testing, and should not be relied on in a production environment. +
+ + diff --git a/docs/docbook/bb-book.xsl b/docs/docbook/bb-book.xsl new file mode 100644 index 00000000..a4f05fdb --- /dev/null +++ b/docs/docbook/bb-book.xsl @@ -0,0 +1,17 @@ + + + + + + + + + + + + + diff --git a/docs/docbook/bb-man.xsl.tmpl b/docs/docbook/bb-man.xsl.tmpl new file mode 100644 index 00000000..e20eedd5 --- /dev/null +++ b/docs/docbook/bb-man.xsl.tmpl @@ -0,0 +1,9 @@ + + + + + + + + diff --git a/docs/docbook/bb-nochunk-book.xsl b/docs/docbook/bb-nochunk-book.xsl new file mode 100644 index 00000000..86574122 --- /dev/null +++ b/docs/docbook/bb-nochunk-book.xsl @@ -0,0 +1,17 @@ + + + + + + + + + + + + + diff --git a/docs/docbook/bbackupctl.xml b/docs/docbook/bbackupctl.xml new file mode 100644 index 00000000..b4880bfd --- /dev/null +++ b/docs/docbook/bbackupctl.xml @@ -0,0 +1,205 @@ + + + + bbackupctl + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + bbackupctl + + Control the Box Backup client daemon + + + + + bbackupctl + + -q + + -c config-file + + command + + + + + Description + + bbackupctl sends commands to a running + bbackupd daemon on a client machine. It can be used to + force an immediate backup, tell the daemon to reload its configuration + files or stop the daemon. If bbackupd is configured in + snapshot mode, it will not back up automatically, and the + bbackupctl must be used to tell it when to start a + backup. + + Communication with the bbackupd daemon takes place over a local + socket (not over the network). Some platforms (notably Windows) can't + determine if the user connecting on this socket has the correct + credentials to execute the commands. On these platforms, ANY local user + can interfere with bbackupd. To avoid this, remove the CommandSocket + option from bbackupd.conf, which will also disable bbackupctl. See the + Client Configuration page for more information. + + bbackupctl needs to read the + bbackupd configuration file to find out the name of the + CommandSocket. If you have to tell bbackupd where to + find the configuration file, you will have to tell + bbackupctl as well. The default on Unix systems is + usually /etc/box/bbackupd.conf. On Windows systems, + it is bbackupd.conf in the same directory where + bbackupd.exe is located. If + bbackupctl cannot find or read the configuration file, + it will log an error message and exit. + + bbackupctl usually writes error messages to the + console and the system logs. If it is not doing what you expect, please + check these outputs first of all. + + + + + + + Run in quiet mode. + + + + + config-file + + + Specify configuration file. + + + + + + Commands + + The following commands are available in bbackupctl: + + + + terminate + + + This command cleanly shuts down bbackupd. + This is better than killing or terminating it any other + way. + + + + + reload + + + Causes the bbackupd daemon to re-read all + its configuration files. Equivalent to kill + -HUP. + + + + + sync + + + Initiates a backup. If no files need to be backed up, no + connection will be made to the server. + + + + + force-sync + + + Initiates a backup, even if the + SyncAllowScript says that no backup should run + now. + + + + + wait-for-sync + + + Passively waits until the next backup starts of its own + accord, and then terminates. + + + + + wait-for-end + + + Passively waits until the next backup starts of its own + accord and finishes, and then terminates. + + + + + sync-and-wait + + + Initiates a backup, waits for it to finish, and then + terminates. + + + + + + + + Files + + /etc/box/bbackupd.conf + + + + See Also + + + bbackupd.conf + + 5 + , + bbackupd-config + + 8 + , + bbackupctl + + 8 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbackupd-config.xml b/docs/docbook/bbackupd-config.xml new file mode 100644 index 00000000..41bb6587 --- /dev/null +++ b/docs/docbook/bbackupd-config.xml @@ -0,0 +1,153 @@ + + + + bbackupd-config + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + bbackupd-config + + Box Backup client daemon configuration file + generator + + + + + bbackupd-config + + config-dir + + backup-mode + + account-num + + server-hostname + + working-dir + + backup-dir + + backup-dir ... + + + + + Description + + The bbackupd-config script creates configuration files and client + certificates. It takes at least six parameters: + + + + config-dir + + + Configuration directory. Usually + /etc/box. + + + + + backup-mode + + + Either lazy or snapshot. + + + + + account-num + + + The client account number. This is set by the bbstored + administrator. + + + + + server-hostname + + + The hostname or IP address of the bbstored server. + + + + + working-dir + + + A directory to keep temporary state files. This is usually + something like /var/bbackupd. This can be + changed in bbackupd.conf later on if + required. + + + + + backup-dir + + + A space-separated list of directories to be backed up. Note + that this does not traverse mount + points. + + + + + + + Files + + /etc/box/bbackupd.conf + + /etc/box/bbackupd/NotifySysAdmin.sh + + + + See Also + + + bbackupd.conf + + 5 + , + bbackupd + + 8 + , + bbackupctl + + 8 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbackupd.conf.xml b/docs/docbook/bbackupd.conf.xml new file mode 100644 index 00000000..43ae2bed --- /dev/null +++ b/docs/docbook/bbackupd.conf.xml @@ -0,0 +1,479 @@ + + + + bbackupd.conf + + 5 + + Box Backup + + Box Backup + + 0.11 + + + + bbackupd.conf + + Box Backup client daemon configuration file + + + + + /etc/box/bbackupd.conf + + + + + Description + + + + AccountNumber + + + The account number of this client. This is set by the admin of + the store server. + + + + + UpdateStoreInterval + + + Specifies the interval between scanning of the local discs. To + avoid cycles of load on the server, this time is randomly adjusted + by a small percentage as the daemon runs. Defaults to 1 hour. + + + + + MinimumFileAge + + + Specifies how long since a file was last modified before it + will be uploaded. Defaults to 6 hours. + + + + + MaxUploadWait + + + If a file is repeatedly modified it won't be uploaded + immediately in case it's modified again. However it should be + uploaded eventually. This is how long we should wait after first + noticing a change. Defaults to 1 day. + + + + + MaxFileTimeInFuture + + + + + + + + AutomaticBackup + + + + + + + + SyncAllowScript + + + Use this to temporarily stop bbackupd from syncronising or + connecting to the store. This specifies a program or script script + which is run just before each sync, and ideally the full path to the + interpreter. It will be run as the same user bbackupd is running as, + usually root. + + The script prints either "now" or a number to STDOUT (and a + terminating newline, no quotes). If the result was "now", then the + sync will happen. If it's a number, then the script will be asked + again in that number of seconds. + + For example, you could use this on a laptop to only backup + when on a specific network. + + + + + MaximumDiffingTime + + + How much time should be spent on diffing files. + + + + + DeleteRedundantLocationsAfter + + + + + + + + FileTrackingSizeThreshold + + + + + + + + DiffingUploadSizeThreshold + + + + + + + + StoreHostname + + + The hostname or IP address of the + bbstored + + 8 + server. + + + + + StorePort + + + The port used by the server. Defaults to 2201. + + + + + ExtendedLogging + + + Logs everything that happens between the client and server. + The + bbackupd + + 8 + client must also be started with + . + + + + + ExtendedLogFile + + + + + + + + LogAllFileAccess + + + + + + + + LogFile + + + + + + + + LogFileLevel + + + + + + + + CommandSocket + + + Where the command socket is created in the filesystem. + + + + + KeepAliveTime + + + + + + + + StoreObjectInfoFile + + + + + + + + NotifyScript + + + The location of the script which runs at certain events. This + script is generated by + bbackupd-config + + 8 + . Defaults to + /etc/box/bbackupd/NotifySysAdmin.sh. + + + + + NotifyAlways + + + + + + + + CertificateFile + + + The path to the client's public certificate. + + + + + PrivateKeyFile + + + The path to the client's private key. This should only be + readable by root. + + + + + TrustedCAsFile + + + The Certificate Authority created by + bbstored-certs + + 8 + . + + + + + KeysFile + + + The data encryption key. This must be kept safe at all costs, your data is + useless without it! + + + + + DataDirectory + + + A directory to keep temporary state files. This is usually + something like /var/bbackupd. + + + + + Server + + + This section relates to the running daemon. + + + + PidFile + + + The location of the process ID file. Defaults to + /var/run/bbackupd.pid. + + + + + + + + BackupLocations + + + This section defines each directory to be backed up. Each + entry must have at least a Path entry and, optionally, include and + exclude directives. + + Multiple include and exclude directives may appear. + + + + Path + + + The path to back up. + + + + + ExcludeFile + + + Exclude a single file. + + + + + ExcludeFilesRegex + + + Exclude multiple files based on a regular expression. + See + re_format + + 7 + . + + + + + ExcludeDir + + + Exclude a single directory. + + + + + ExcludeDirsRegex + + + Exclude multiple directories based on a regular + expression. See + re_format + + 7 + . + + + + + AlwaysIncludeFile + + + Include a single file from a directory which has been + excluded. + + + + + AlwaysIncludeFilesRegex + + + Include multiple files from an excluded directory, + based on a regular expression. + + + + + AlwaysIncludeDir + + + Include a single directory from a directory which has + been excluded. + + + + + AlwaysIncludeDirsRegex + + + Include multiple directories from an excluded + directory, based on a regular expression. + + + + + + + + + + Examples + + The following is an example of a backup location: + + home +{ + Path = /home + ExcludeDir = /home/guest + ExcludeDir = /home/[^/]+/tmp + ExcludeFilesRegex = .*\.(mp3|MP3)$ + AlwaysIncludeFile = /home/someuser/importantspeech.mp3 +} + + + + Files + + /etc/box/bbackupd.conf + + + + See Also + + + bbackupd + + 8 + , + bbackupd-config + + 8 + , + bbackupctl + + 8 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbackupd.xml b/docs/docbook/bbackupd.xml new file mode 100644 index 00000000..11de0f3f --- /dev/null +++ b/docs/docbook/bbackupd.xml @@ -0,0 +1,209 @@ + + + + bbackupd + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + bbackupd + + Box Backup client daemon + + + + + bbackupd + + -DFkqvVT + + -c config-file + + -t tag + + + + + Description + + bbackupd runs on client computers in the + background, finding new files to back up. When it is time for a backup, + bbackupd will connect to the server + (bbstored) to upload the files. + + A running bbackupd daemon can be controlled with + the bbackupctl command, to make it shut down, reload + its configuration, or start an immediate backup. + + bbackupd needs to be configured to tell it which + files to back up, how often, and to which server (running + bbstored). See the Client Configuration page for more + information. For this, you must write a configuration file. You must + either place it in the default location, or tell + bbackupd where to find it. + + You can check the default location with the + option. The default on Unix systems is usually + /etc/box/bbackupd.conf. On Windows systems, it is + bbackupd.conf in the same directory where + bbackupd.exe is located. If bbackupd cannot find or + read the configuration file, it will log an error message and exit. + + bbackupd usually writes log messages to the + system logs, using the facility local5, which you can + use to filter them to send them to a separate file. It can also write them + to the console, see options below. If bbackupd is not + doing what you expect, please check the logs first of all. + + + Options + + + + config-file + + + Use the specified configuration file. If + is omitted, the last argument is the configuration file. If none + is specified, the default is used (see above). + + + + + + + + Debugging mode. Do not fork into the background (do not run + as a daemon). Not available on Windows. + + + + + + + + No-fork mode. Same as for bbackupd. Not + available on Windows. + + + + + + + + Keep console open after fork, keep writing log messages to + it. Not available on Windows. + + + + + + + + Run more quietly. Reduce verbosity level by one. Available + levels are NOTHING, FATAL, + ERROR, WARNING, + NOTICE, INFO, + TRACE, EVERYTHING. Default + level is NOTICE in non-debugging builds. Use + once to drop to WARNING level, twice for + ERROR level, four times for no logging at + all. + + + + + -v + + + Run more verbosely. Increase verbosity level by one. Use + once to raise to INFO level, twice for + TRACE level, three times for + EVERYTHING (currently the same as + TRACE). + + + + + + + + Run at maximum verbosity (EVERYTHING + level). + + + + + tag + + + Tag each console message with specified marker. Mainly + useful in testing when running multiple daemons on the same + console. + + + + + + + + Timestamp each line of console output. + + + + + + + + Files + + /etc/box/bbackupd.conf + + + + See Also + + + bbackupd.conf + + 5 + , + bbackupd-config + + 8 + , + bbackupctl + + 8 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbackupquery.xml b/docs/docbook/bbackupquery.xml new file mode 100644 index 00000000..016d5c7d --- /dev/null +++ b/docs/docbook/bbackupquery.xml @@ -0,0 +1,506 @@ + + + + bbackupquery + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + bbackupquery + + Box Backup store query and file retrieval + + + + + bbackupquery + + -q + + -c configfile + + command ... + + + + + Description + + bbackupquery is the main way of interacting with + the backup store from a Box Backup client machine. It supports both + interactive and batch modes of operation. + + It can be used to reviewing the status of a client machine's backup + store, getting status from the store server. The main use is to retrieve + files and directories when needed. + + bbackupquery supports interactive and batch modes + of operation. Interactive mode allows for interaction with the server much + like an interactive FTP client. + + Batch mode is invoked by putting commands into the invocation of + bbackupquery. Example: + + bbackupquery "list home-dirs" quit + + Note that commands that contain spaces are enclosed in double + quotes. If the quit command is omitted, after the + preceding commands are completed, bbackupquery will + enter interactive mode. + + + + Options + + + + + + + Quiet. Suppresses status output while running. + + + + + + + + Use configfile instead of the default bbackupd.conf file. + Can be a relative or full path. + + + + + + + Commands + + The commands that can be used in bbackupquery are listed + below. + + + + help + + + Displays the basic help message, which gives information about + the commands available in bbackupquery. Use the + form help command to get help on a specific + command. + + + + + quit + + + End the session with the store server, and quit + bbackupquery. + + + + + cd options + directory-name + + + Change directory. Options: + + + + + consider deleted directories for traversal + + + + + + + + consider old versions of directories for traversal. + This option should never be useful in a correctly formed + store. + + + + + + + + lcd + local-directory-name + + + Change directory on the client machine. To list the contents + of the local directory, type sh ls (on Unix-like + machines). + + + + + list options + directory-name + + + The list (or its synonym ls) command lists + the content of the current, or specified, directory. The options are + as follows: + + + + + + + recursively list all files + + + + + + + + list deleted files and directories + + + + + + + + list old versions of files and directories + + + + + + + + don't display object IDs + + + + + + + + don't display flags + + + + + + + + show file modification time (and attr mod time, if the + object has attributes). + + + + + + + + show file size in blocks used on server. Note that + this is only a very approximate indication of local file + size. + + + + + + + + ls options + directory-name + + + Synonym for list. + + + + + pwd + + + Print current directory, always relative to the backup store + root. + + + + + sh shell-command + + + Everything after the sh is passed to a shell and run. All + output from the command is displayed in the client. + + Example: to list the contents of the current directory on the + client machine type sh ls. + + + + + compare -a + + + + + + + + compare -l + location-name + + + + + + + + compare store-dir-name + local-dir-name + + + Compare the current data in the store with the data on the + disc. Please note that all the data will be downloaded from the + store, so this can be a very lengthy process depending on the size + of the store, and the size of the part you are comparing. + + Options: + + + + + + + compare all locations. + + + + + + + + compare one backup location as specified in the + configuration file. This compares one of the top level store + directories. + + + + + + + + set return code. The return code is set to the + following values, if quit is the next command. So, if + another command is run after the compare, the return code + will not refer to the compare. This option is very useful + for automating compares. Return code values: + + -- no differences were + found + + + + -- differences were + found + + + + -- an error occured + + + + + + + + + + get object-filename + local-filename + + + + + + + + get -i object-id + local-filename + + + Gets a file from the store. Object is specified as the + filename within the current directory. Local filename is optional. + Ignores old and deleted files when searching the directory for the + file to retrieve. + + To get an old or deleted file, use the + option and select the object as a hex object ID (first column in + listing). The local filename must be specified. + + + + + getobject object-id + local-filename + + + Gets the object specified by the object id (in hex) and stores + the raw contents in the local file specified. Note: This is only + useful for debugging as it does not decode files from the stored + format, which is encrypted and compressed. + + + + + restore -d + directory-name + local-directory-name + + + + + + + + restore -r + + + Restores a directory to the local disc. The local directory + specified must not exist (unless a previous restore is being + restarted). The root cannot be restored -- restore locations + individually. + + Options: + + + + + + + restore a deleted directory + + + + + + + + resume an interrupted restore + + + If a restore operation is interrupted for any + reason, it can be restarted using the switch. + Restore progress information is saved in a file at regular intervals + during the restore operation to allow restarts. + + + + + usage -m + + + Show space used on the server for this account. Display + fields: + + Used: Total amount of space used on + the server + + + + Old files: Space used by old + files + + + + Deleted files: Space used by + deleted files + + + + Directories: Space used by the + directory structure + + + + When Used exceeds the soft limit, the + server will start to remove old and deleted files until the usage + drops below the soft limit. After a while, you should expect to see + the usage stay at just below the soft limit. You only need more + space if the space used by old and deleted files is near + zero. + + The option displays output in + machine-readable form. + + + + + + + Bugs + + If you find a bug in Box Backup and you want to let us know about + it, join the mailing + list and send us a description of the problem there. + + To report a bug, give us at least the following information: + + + + The version of Box Backup you are running + + + + The platform you are running on (hardware and OS), for both + client and server. + + + + If possible attach your config files (bbstored.conf, + bbackupd.conf) to the bug report. + + + + Also attach any log file output that helps shed light on the + problem you are seeing. + + + + And last but certainly not least, a description of what you are + seeing, in as much detail as possible. + + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbstoreaccounts.xml b/docs/docbook/bbstoreaccounts.xml new file mode 100644 index 00000000..08092acf --- /dev/null +++ b/docs/docbook/bbstoreaccounts.xml @@ -0,0 +1,386 @@ + + + + bbstoreaccounts + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + bbstoreaccounts + + Box Backup store accounts manager + + + + + bbstoreaccounts + + -c config-file + + command + + account-id + + command-specific arguments + + + + + Description + + bbstoreaccounts is the tool for managing accounts + on the store server. It can be used to view information related to + accounts, as well as create, change and delete accounts on the store + server. + + bbstoreaccounts always takes at least 2 + parameters: the command name and the account ID. Some commands require + additional parameters, and some commands have optional parameters. + + + Options + + + + + + + The configfile to use for connecting to the store. Default + is /etc/box/bbstored.conf. + + + + + + + Commands + + The commands tells bbstoreaccounts what action to perform. + + + + check account-id + fix + + + The check command verifies the + integrity of the store account given, and optionally fixes any + corruptions. Note: It is + recommended to run the 'simple' check command (without + fix) before using the fix + option. This gives an overview of the extent of any problems, + before attempting to fix them. + + + + + create account-id + disc-set soft-limit + hard-limit + + + Creates a new store account with the parameters given. The + parameters are as follows: + + + + + + + The ID of the new account to be created. A 32-bit + hexadecimal number. Cannot already exist on the + server. + + + + + + + + The disc set from + raidfile.conf + + 5 + where the backups for this client will + be stored. A number. Each RAID-file set has a number in + raidfile.conf. This number is what's used. + + + + + + + + The soft limit is the amount of storage that the + server will guarantee to be available for + storage. + + + + + + + + The amount of storage that the the server will + allow, before rejecting uploads, and starting to + eliminate old and deleted files to get back down to + soft-limit. + + + + + + + + delete account-id + yes + + + Deletes the account from the store server completely. + Removes all backups and deletes all references to the account in + the config files. + + delete will ask for confirmation from + the user, when called. Using the flag, + eliminates that need. This is useful when deleting accounts from + within a script or some other automated means. 0 + + + + + info account-id + + + Display information about the given account. + Example:[root]# bbstoreaccounts info 1 + Account ID: 00000001 + Last object ID: 58757 + Blocks used: 9864063 (38531.50Mb) + Blocks used by old files: 62058 (242.41Mb) +Blocks used by deleted files: 34025 (132.91Mb) + Blocks used by directories: 6679 (26.09Mb) + Block soft limit: 11796480 (46080.00Mb) + Block hard limit: 13107200 (51200.00Mb) + Client store marker: 1139559852000000 + + Explanation: + + + + Account ID + + + The account ID being displayed. + + + + + Last Object ID + + + A counter that keeps track of the objects that + have been backed up. This number refers to the last file + that was written to the store. The ID is displayed as a + decimal number, and the object ID can be converted to a + path name to a file as follows: convert the number to + hex (e.g.: 58757 => 0xE585); The last backed up file + will be (relative from the client's store root): + e5/o85.rfw. Longer numbers infer + more directories in the structure, so as an example + 3952697264 as the last object ID gives 0xEB995FB0, which + translates to a backup pathname of + eb/99/5f/ob0.rfw. + + + + + Blocks used + + + The number of blocks used by the store. The size + in Mb depends on the number of blocks, as well as the + block size for the disc set given in + raidfile.conf + + 5 + . In this case the block size is + 4096. + + + + + Blocks used by old files + + + The number of blocks occupied by files that have + newer versions in the store. This data is at risk for + being removed during housekeeping. + + + + + Blocks used by deleted files + + + The number of blocks used by files that have been + deleted on the client. This data is at risk for being + removed during housekeeping. + + + + + Blocks used by directories + + + The number of blocks used by directories in the + store. + + + + + Block soft limit + + + The soft limit in blocks. The soft limit is the + maximum guaranteed storage space available to the + account. When housekeeping starts, and the old and + deleted files are removed, they are removed in + chronological order (oldest first), until the data used + is less than the soft limit. + + + + + Block hard limit + + + The hard limit in blocks. The hard limit is the + most amount of storage the server will allow in an + account. Any data above this amount will be rejected. + Housekeeping will reduce the storage use, so more data + can be uploaded. + + + + + Client store marker + + + + bbstored + + 8 + uses this number to determine if it + needs to rescan the entire store. If this number is + different from the last time it checked, a rescan will + take place. + + + + + + + + setlimit account-id + soft-limit hard-limit + + + Changes the storage space allocation for the given + account. No server restart is needed. + + Parameters: + + + + + + + The ID of the account to be modified. + + + + + + + + The soft limit is the amount of storage that the + server will guarantee to be available for + storage. + + + + + + + + The amount of storage that the the server will + allow before rejecting uploads and starting to eliminate + old and deleted files to get back down to + . + + + + + + + + + + + Examples + + Create an account with ID 3af on disc set 0, with a 20GB soft-limit + and a 22GB hard-limit:bbstoreaccounts create 3af 0 20G 22GAlter + existing account ID 20 to have a 50GB soft-limit and a 55GB + hard-limit:bbstoreaccounts setlimit 20 50G 55G + + + + Files + + /etc/box/bbstored/accounts.txt + + + + See Also + + + bbstored + + 8 + , + bbstored-config + + 8 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbstored-certs.xml b/docs/docbook/bbstored-certs.xml new file mode 100644 index 00000000..79308089 --- /dev/null +++ b/docs/docbook/bbstored-certs.xml @@ -0,0 +1,180 @@ + + + + bbstored-certs + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + bbstored-certs + + Manage certificates for the Box Backup system + + + + + bbstored-certs + + certs-dir + + command + + arguments + + + + + Description + + bbstored-certs creates and signs certificates for + use in Box Backup. It allows the user to create and sign the server keys, + as well as signing client keys. + + All commands must be followed by the certs-dir, + which is the directory in which the certificates are stored. + + + Commands + + There are 3 commands: + + + + init + + + Create the certs-dir, and generate the + server keys for bbstored. certs-dir cannot + exist before running the command. + + + + + sign-server + servercsrfile + + + Sign the server certificate. The + servercsrfile is the file generated by the + init command. + + + + + sign + clientcsrfile + + + Sign a client certificate. The + clientcsrfile is generated during client setup. + See + bbackupd-config + + 8 + . Send the signed certificate back to the client, + and install according to the instructions given by + bbackupd-config. + + + + + + + + Files + + + raidfile-config + + 8 + generates the + raidfile.conf + + 5 + file. + + + + Bugs + + If you find a bug in Box Backup, and you want to let us know about + it, join the mailing + list, and send a description of the problem there. + + To report a bug, give us at least the following information: + + + + The version of Box Backup you are running + + + + The platform you are running on (hardware and OS), for both + client and server. + + + + If possible attach your config files (bbstored.conf, + bbackupd.conf) to the bug report. + + + + Also attach any log file output that helps shed light on the + problem you are seeing. + + + + And last but certainly not least, a description of what you are + seeing, in as much detail as possible. + + + + + + See Also + + + bbstored-config + + 8 + , + bbstored.conf + + 5 + , + bbstoreaccounts + + 8 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbstored-config.xml b/docs/docbook/bbstored-config.xml new file mode 100644 index 00000000..6d0113c1 --- /dev/null +++ b/docs/docbook/bbstored-config.xml @@ -0,0 +1,148 @@ + + + + bbstored-config + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + bbstored-config + + Box Backup store daemon configuration file + generator + + + + + bbstored-config + + configdir + + servername + + username + + + + + Description + + The bbstored-config script creates configuration files and server + certificates for a bbstored instance. It takes three parameters: + + + + configdir + + + The directory where config files will reside. A + bbstored subdirectory will be created where + several config files will reside. The + bbstored.conf file will be created in + configdir. + + + + + servername + + + The name of the server that is being configured. Usually the + fully qualified domain name of the machine in question. + + + + + username + + + The name of the user that should be running the + bbstored process. Recommended name: + _bbstored. + + + + + A valid + raidfile.conf + + 5 + must be found in configdir. Several steps are taken + during the run of bbstored-config: + + + + Server certificates are created. This requires interaction from + the operator. + + + + The RAID volumes are checked to ensure that the configuration is + consistent and will work. + + + + Instructions for next steps to take are shown. These steps may + be different for different OS platforms, so pay close attention to + these instructions. + + + + + + Files + + /etc/box/bbstored.conf + + + + See Also + + + bbstored.conf + + 5 + , + bbstored + + 8 + , + bbstored-certs + + 8 + , + raidfile-config + + 8 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbstored.conf.xml b/docs/docbook/bbstored.conf.xml new file mode 100644 index 00000000..136b1bd0 --- /dev/null +++ b/docs/docbook/bbstored.conf.xml @@ -0,0 +1,211 @@ + + + + bbstored.conf + + 5 + + Box Backup + + Box Backup + + 0.11 + + + + bbstored.conf + + Box Backup store daemon configuration file + + + + + /etc/box/bbstored.conf + + + + + Description + + The following configuration options are valid: + + + + RaidFileConf + + + Specifies the path to the + raidfile.conf + + 5 + . This is normally + /etc/box/raidfile.conf. + + + + + AccountDatabase + + + Specifies the path to the account database created by + + bbstoreaccounts + + 8 + . This is usually + /etc/box/bbstored/accounts.txt. + + + + + ExtendedLogging + + + Specifies whether extended logging should be enabled to show + what commands are being received from clients. + + + + + TimeBetweenHousekeeping + + + How long between scanning for files which need + deleting. + + + + + Server + + + These options relate to the actual daemon. + + PidFile + + + The location of the pidfile, where the daemon's + process ID is kept. + + + + + User + + + The user to run as. + + + + + ListenAddresses + + + The interface addresses to listen on. Hostnames may be + used instead of IP addresses. The format is: + or + . + + + + + CertificateFile + + + The path to the server's public certificate. + + + + + PrivateKeyFile + + + The path to the server's private key. This should only + be readable by root and/or the . + + + + + TrustedCAsFile + + + The Certificate Authority created by + bbstored-certs + + 8 + . + + + + + + + + + + Examples + + The following is an example bbstored.conf: + + RaidFileConf = /etc/box/raidfile.conf +AccountDatabase = /etc/box/bbstored/accounts.txt + +TimeBetweenHousekeeping = 900 + +Server +{ + PidFile = /var/run/bbstored.pid + User = _bbstored + ListenAddresses = inet:server.example.com + CertificateFile = /etc/box/bbstored/server.example.com-cert.pem + PrivateKeyFile = /etc/box/bbstored/server.example.com-key.pem + TrustedCAsFile = /etc/box/bbstored/clientCA.pem +} + + + + Files + + /etc/box/bbstored.conf + + + + See Also + + + bbstored + + 8 + , + bbstored-config + + 8 + , + raidfile-config + + 8 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/bbstored.xml b/docs/docbook/bbstored.xml new file mode 100644 index 00000000..81891a8d --- /dev/null +++ b/docs/docbook/bbstored.xml @@ -0,0 +1,90 @@ + + + + bbstored + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + bbstored + + Box Backup store daemon + + + + + bbstored + + config-file + + + + + Description + + bbstored runs on a central server. Clients + running bbackupd connect to the server and upload + files. + + The only argument is optional and specifies a non-default + configuration file. By default it will look for the configuration file as + /etc/box/bbstored.conf. + + + + Files + + /etc/box/bbstored.conf + + + + See Also + + + bbstored.conf + + 5 + , + bbstored-config + + 8 + , + raidfile-config + + 8 + , + raidfile.conf + + 5 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/html/bbdoc-man.css b/docs/docbook/html/bbdoc-man.css new file mode 100644 index 00000000..0345560e --- /dev/null +++ b/docs/docbook/html/bbdoc-man.css @@ -0,0 +1,104 @@ +body { + font-family: Verdana, Geneva, Arial, sans-serif; + background-color: #edeef3; + font-size: .75em; + line-height: 180%; + text-align: left; + margin-top: 20px; + margin-right: 100px; + margin-left: 250px; + position: relative; + width: auto; } + +table { + font-family: Verdana, Geneva, Arial, sans-serif; + background-color: #edeef3; + font-size: 10pt; + line-height: 100%; } + + +code { + font-size: 11pt; } + + + +div.navheader { + font-family: Verdana, Geneva, Arial, sans-serif; + background-color: #edeef3; + line-height: 100%; } + +#header { + background-color: #e4e6ed; + text-align: left; + padding-top: 10px; + margin-right: -100px; + margin-left: -250px; + top: 20px; + border-top: 1px solid #c4c4d5; + border-bottom: 1px solid white } + +#logo { + position: relative; + margin-left: 200px } + + +#page { + font-size: .75em; + line-height: 180%; + text-align: left; + margin-top: 50px; + margin-right: 100px; + margin-left: 250px; + position: relative; + width: auto } + +#disc { } + +.informaltable td,tr {font-size: 1em; + line-height: 140%; + text-align: left; + background-color: #e4e6ed; + padding: 4px } + +tr,td {font-size: 1em; + line-height: 100%; + background-color: #edeef3; } + +pre, tt { font-size: 1.3em; + color: #088; + letter-spacing: 1px; + word-spacing: 2px} + +h1 { + color: #c00; + font-size: 16pt; + margin-bottom: 2em; + margin-left: -50px } + +h2 { + color: #324e95; + font-size: 12pt; + margin-top: 2em; + margin-left: -50px } + +h3 { + color: #324e95; + font-size: 10pt; + margin-top: 2em; + margin-left: -50px } + +dt { font-weight: bold } + +a:link { + color: #324e95; + text-decoration: none; + background-color: transparent } + +a:visited { + color: #90c; + text-decoration: none } + +a:hover { + color: #c00; + text-decoration: underline; + background-color: transparent } diff --git a/docs/docbook/html/bbdoc.css b/docs/docbook/html/bbdoc.css new file mode 100644 index 00000000..d3b4a1c2 --- /dev/null +++ b/docs/docbook/html/bbdoc.css @@ -0,0 +1,112 @@ +body { + font-family: Verdana, Geneva, Arial, sans-serif; + background-color: #edeef3; + font-size: .75em; + line-height: 180%; + text-align: left; + margin-top: 20px; + margin-right: 100px; + margin-left: 250px; + position: relative; + width: auto; } + +table { + font-family: Verdana, Geneva, Arial, sans-serif; + background-color: #edeef3; + font-size: 10pt; + line-height: 100%; } + + + + +div.navheader { + font-family: Verdana, Geneva, Arial, sans-serif; + background-color: #edeef3; + line-height: 100%; } + +#header { + background-color: #e4e6ed; + text-align: left; + padding-top: 10px; + margin-right: -100px; + margin-left: -250px; + top: 20px; + border-top: 1px solid #c4c4d5; + border-bottom: 1px solid white } + +#logo { + position: relative; + margin-left: 200px } + + +#page { + font-size: .75em; + line-height: 180%; + text-align: left; + margin-top: 50px; + margin-right: 100px; + margin-left: 250px; + position: relative; + width: auto } + +#disc { } + +.informaltable td,tr {font-size: 1em; + line-height: 140%; + text-align: left; + background-color: #e4e6ed; + padding: 4px } + +tr,td {font-size: 1em; + line-height: 100%; + background-color: #edeef3; } + +pre, tt { font-size: 1.3em; + color: #088; + letter-spacing: 1px; + word-spacing: 2px} + +h1 { + color: #c00; + font-size: 16pt; + margin-bottom: 2em; + margin-left: -50px } + +h2 { + color: #324e95; + font-size: 12pt; + margin-top: 2em; + margin-left: -50px } + +h3 { + color: #324e95; + font-size: 10pt; + margin-top: 2em; + margin-left: -50px } + +dt { font-weight: bold } + +ul { + list-style-image: url(images/arrow.png) } + +ul li { + background-color: #e4e6ed; + margin: 1em 6em 1em -2em; + padding: 0.2em 0.5em 0.2em 1em; + border-style: solid; + border-width: 1px; + border-color: #c4c4d5 #fff #fff #c4c4d5 } + +a:link { + color: #324e95; + text-decoration: none; + background-color: transparent } + +a:visited { + color: #90c; + text-decoration: none } + +a:hover { + color: #c00; + text-decoration: underline; + background-color: transparent } diff --git a/docs/docbook/html/favicon.ico b/docs/docbook/html/favicon.ico new file mode 100644 index 00000000..f8cbb3b3 Binary files /dev/null and b/docs/docbook/html/favicon.ico differ diff --git a/docs/docbook/html/images/arrow.png b/docs/docbook/html/images/arrow.png new file mode 100644 index 00000000..4c60805d Binary files /dev/null and b/docs/docbook/html/images/arrow.png differ diff --git a/docs/docbook/html/images/bblogo.png b/docs/docbook/html/images/bblogo.png new file mode 100644 index 00000000..b33230b4 Binary files /dev/null and b/docs/docbook/html/images/bblogo.png differ diff --git a/docs/docbook/html/images/stepahead.png b/docs/docbook/html/images/stepahead.png new file mode 100644 index 00000000..9ac05b8c Binary files /dev/null and b/docs/docbook/html/images/stepahead.png differ diff --git a/docs/docbook/instguide.xml b/docs/docbook/instguide.xml new file mode 100644 index 00000000..addef297 --- /dev/null +++ b/docs/docbook/instguide.xml @@ -0,0 +1,766 @@ + + + + Box Backup Build and Installation Guide + + + License + + Copyright © 2003 - 2007, Ben Summers and contributors. + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + + + Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + + + Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + + + + All use of this software and associated advertising materials + must display the following acknowledgement: This product includes + software developed by Ben Summers. + + + + The names of the Authors may not be used to endorse or promote + products derived from this software without specific prior written + permission. + + + + + [Where legally impermissible the Authors do not disclaim liability + for direct physical injury or death caused solely by defects in the + software unless it is modified by a third party.] + + THIS SOFTWARE IS PROVIDED BY THE AUTHORS "AS IS" AND ANY EXPRESS + OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR + ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING + IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + POSSIBILITY OF SUCH DAMAGE. + + + + Introduction + + The backup daemon, bbackupd, runs on all machines to be backed up. + The store server daemon, bbstored runs on a central server. Data is sent + to the store server, which stores all data on local filesystems, that is, + only on local hard drives. Tape or other archive media is not used. + + The system is designed to be easy to set up and run, and cheap to + use. Once set up, there should be no need for user or administrative + intervention, apart from usual system maintenance. + +
+ Client daemon + + bbackupd is configured with a list of directories to back up. It + has a lazy approach to backing up data. Every so often, the directories + are scanned, and new data is uploaded to the server. This new data must + be over a set age before it is uploaded. This prevents rapid revisions + of a file resulting in many uploads of the same file in a short period + of time. + + It can also operate in a snapshot mode, which behaves like + traditional backup software. When instructed by an external bbackupctl + program, it will upload all changed files to the server. + + The daemon is always running, although sleeping most of the time. + In lazy mode, it is completely self contained -- scripts running under + cron jobs are not used. The objective is to keep files backed up, not to + make snapshots of the filesystem at particular points in time + available. + + If an old version of the file is present on the server, a modified + version of the rsync algorithm is used to upload only the changed + portions of the file. + + After a new version is uploaded, the old version is still + available (subject to disc space on the server). Similarly, a deleted + file is still available. The only limit to their availability is space + allocated to this account on the server + + Future versions will add the ability to mark the current state of + files on the server, and restore from this mark. This will emulate the + changing of tapes in a tape backup system. + +
+ Restoration + + Restoring files is performed using a query tool, bbackupquery. + This can be used to restore entire directories, or as an 'FTP-like' + tool to list and retrieve individual files. Old versions and deleted + files can be retrieved using this tool for as long as they are kept on + the server. +
+ +
+ Client Resource Usage + + bbackupd uses only a minimal amount of disc space to store + records on uploaded files -- less than 32 bytes per directory and file + over a set size threshold. However, it minimises the amount of queries + it must make to the server by storing, in memory, a data structure + which allows it to determine what data is new. It does not need to + store a record of all files, essentially just the directory names and + last modification times. This is not a huge amount of memory. + + If there are no changes to the directories, then the client will + not even connect to the server. +
+
+ +
+ Security + + Box Backup is designed to be secure in several ways. The data + stored on the backup store server is encrypted using secret-key + cryptography. Additionally, the transport layer is encrypted using TLS, + to ensure that the communications can't be snooped. + +
+ Encryption + + The files, directories, filenames and file attributes are all + encrypted. By examining the stored files on the server, it is only + possible to determine the approximate sizes of a files and the tree + structure of the disc (not names, just number of files and + subdirectories in a directory). By monitoring the actions performed by + a client, it is possible to determine the frequency and approximate + scope of changes to files and directories. + + The connections between the server and client are encrypted + using TLS (latest version of SSL). Traffic analysis is possible to + some degree, but limited in usefulness. + + An attacker will not be able to recover the backed up data + without the encryption keys. Of course, you won't be able to recover + your files without the keys either, so you must make a conventional, + secure, backup of these keys. +
+ +
+ Authentication + + SSL certificates are used to authenticate clients. UNIX user + accounts are not used to minimise the dependence on the configuration + of the operating system hosting the server. + + A script is provided to run the necessary certification + authority with minimal effort. +
+
+ +
+ Server daemon + + The server daemon is designed to be simple to deploy, and run on + the cheapest hardware possible. To avoid the necessity to use expensive + hardware RAID or software RAID with complex setup, it (optionally) + stores files using RAID techniques. + + It does not need to run as a privileged user. + + Each account has a set amount of disc space allocated, with a soft + and a hard limit. If the account exceeds the soft limit, a housekeeping + process will start deleting old versions and deleted files to reduce the + space used to below the soft limit. If the backup client attempts to + upload a file which causes the store to exceed the hard limit, the + upload will be refused. +
+ + + + Building and installing + +
+ Before you start + + Firstly, check that all the clocks on your clients, servers and + signing machines are accurate and in sync. A disagreement in time + between a client and a server is the biggest cause of installation + difficulties, as the times in the generated certificates will cause + login failures if the start date is in the future. +
+ +
+ Box Backup compile + + In the following instructions, replace 0.00 with the actual + version number of the archive you have downloaded. + + For help building on Windows, see the Windows + Compile Appendix. And if you want to build a Linux RPM, look here. + + You need the latest version of OpenSSL, as some of the extra APIs + it provides are required. You should have this anyway, as earlier + versions have security flaws. (If you have an earlier version installed, + the configuration script will give you instructions on enabling + experimental support for older versions.) + + See OpenSSL notes for more information + on OpenSSL issues. + + There are some notes in the archive about compiling on various + platforms within the boxbackup-0.00 directory -- read them first. For + example, if you are compiling under Linux, look for LINUX.txt as + boxbackup-0.00/LINUX.txt after untaring the archive. + + Download the archive, then in that directory type + + tar xvzf boxbackup-0.00.tgz +cd boxbackup-0.00 +./configure +make + + The server and client will be built and packaged up for + installation on this machine, or ready to be transferred as tar files to + another machine for installation. + + This builds two parcels of binaries and scripts, 'backup-client' + and 'backup-server'. The generated installation scripts assumes you want + everything installed in /usr/local/bin + + Optionally, type make test to run + all the tests. +
+ +
+ Local installation + + Type make install-backup-client + to install the backup client. + + Type make install-backup-server + to install the backup server. +
+ +
+ Remote installation + + In the parcels directory, there are tar files for each parcel. The + name reflects the version and platform you have built it for. + + Transfer this tar file to the remote server, and unpack it, then + run the install script. For example: + + tar xvzf boxbackup-0.00-backup-server-OpenBSD.tgz +cd boxbackup-0.00-backup-server-OpenBSD +./install-backup-server +
+ +
+ Configure options + + You can use arguments to the configure script to adjust the + compile and link lines in the generated Makefiles, should this be + necessary for your platform. The configure script takes the usual GNU + autoconf arguments, a full list of which can be obtained with --help. Additional options for Box Backup + include: + + + + + + --enable-gnu-readline + + Use GNU readline if present. Linking Box Backup against + GNU readline may create licence implications if you then + distribute the binaries. libeditline is also supported as a safe + alternative, and is used by default if available. + + + + --disable-largefile + + Omit support for large files + + + + --with-bdb-lib=DIR + + Specify Berkeley DB library location + + + + --with-bdb-headers=DIR + + Specify Berkeley DB headers location + + + + --with-random=FILE + + Use FILE as random number seed (normally + auto-detected) + + + + --with-tmp-dir=DIR + + Directory for temporary files (normally /tmp) + + + + + + See OpenSSL notes for the OpenSSL + specific options. +
+ +
+ Tests + + There are a number of unit tests provided. To compile and run one + type: + + ./runtest.pl bbackupd release +./runtest.pl common debug +./runtest.pl ALL + + The runtest.pl script will compile and run the test. The first + argument is the test name, and the second the type of build. Use ALL as + a test name to run all the tests. + + The output from the tests is slightly muddled using this script. + If you're developing, porting or trying out new things, it might be + better to use the following scheme: + + cd test/bbackupd +make +cd ../../debug/test/bbackupd +./t + + or in release mode... + + cd test/bbackupd +make -D RELEASE +cd ../../release/test/bbackupd +./t + + (use RELEASE=1 with GNU make) + + I tend to use two windows, one for compilation, and one for + running tests. +
+ + + + Box Backup and SSL + +
+ General notes + + Ideally, you need to use version 0.9.7 or later of OpenSSL. If + this is installed on your system by default (and it is on most recent + releases of UNIX like OSes) then everything should just work. + + However, if it isn't, you have a few options. + +
+ Upgrade your installation + + The best option is to upgrade your installation to use 0.9.7. + Hopefully your package manager will make this easy for you. This may + require reinstallation of lots of software which depends on OpenSSL, + so may not be ideal. + + (But as there have been a few security flaws in OpenSSL + recently, you probably want to upgrade it anyway.) +
+ +
+ Install another OpenSSL + + The second best option is to install another copy. If you + download and install from source, it will probably install into + /usr/local/ssl. You can then configure Box Backup to use it + using: + + ./configure --with-ssl-headers=/usr/local/ssl/include --with-ssl-lib=/usr/local/ssl/lib + + which will set up the various includes and libraries for + you. + + The configuration scripts may be a problem, depending on your + installation. See below for more information. +
+ +
+ Use the old version of OpenSSL + + If you have an old version installed, the configuration script + will give you instructions on how to enable support for older + versions. Read the warnings, and please, whatever you do, don't + release binary packages or ports which enable this option. + + You may have issues with the configuration scripts, see + below. +
+
+ +
+ If you have problems with the config scripts + + If you get OpenSSL related errors with the configuration scripts, + there are two things to check: + + + + The bin directory within your OpenSSL directory is in the path + (if you have installed another version) + + + + You have an openssl.cnf file which works and can be + found. + + + +
+ OpenSSL config file + + You need to have an openssl.cnf file. The default will generally + work well (see example at end). Make sure the openssl utility can find + it, either set the OPENSSL_CONF environment variable, or install it + into the location that is mentioned in the error messages. + + Example OpenSSL config file: + + # +# OpenSSL example configuration file. +# This is mostly being used for generation of certificate requests. +# + +RANDFILE = /dev/arandom + +#################################################################### +[ req ] +default_bits = 1024 +default_keyfile = privkey.pem +distinguished_name = req_distinguished_name +attributes = req_attributes + +[ req_distinguished_name ] +countryName = Country Name (2 letter code) +#countryName_default = AU +countryName_min = 2 +countryName_max = 2 + +stateOrProvinceName = State or Province Name (full name) +#stateOrProvinceName_default = Some-State + +localityName = Locality Name (eg, city) + +0.organizationName = Organization Name (eg, company) +#0.organizationName_default = Internet Widgits Pty Ltd + +# we can do this but it is not needed normally :-) +#1.organizationName = Second Organization Name (eg, company) +#1.organizationName_default = CryptSoft Pty Ltd + +organizationalUnitName = Organizational Unit Name (eg, section) +#organizationalUnitName_default = + +commonName = Common Name (eg, fully qualified host name) +commonName_max = 64 + +emailAddress = Email Address +emailAddress_max = 64 + +[ req_attributes ] +challengePassword = A challenge password +challengePassword_min = 4 +challengePassword_max = 20 + +unstructuredName = An optional company name + +[ x509v3_extensions ] + +nsCaRevocationUrl = http://www.cryptsoft.com/ca-crl.pem +nsComment = "This is a comment" + +# under ASN.1, the 0 bit would be encoded as 80 +nsCertType = 0x40 +
+
+ + + + Compiling bbackupd on Windows using Visual C++ + + This Appendix explains how to build the bbackupd daemon for Windows + using the Visual C++ compiler. + + If you have any problems following these instructions, please sign + up to the mailing + list and report them to us, or send an email to Chris Wilson. Thanks! + + Note: bbstored will not be built + with this process. bbstored is not currently supported on Windows. There + are no plans for bbstored support on Windows. + +
+ Tools + + You will need quite a bit of software to make this work. All of it + is available for free on the Internet, although Visual C++ Express has + license restrictions and a time limit. + +
+ Visual C++ + + Microsoft's Visual C++ compiler and development environment are + part of their commercial product Visual Studio. Visual + Studio 2005 is supported, and 2003 should work as well. + + You can also download + a free copy of Visual C++ 2005 Express. This copy must be registered + (activated) within 30 days, and is free for one year. + + You will need the Platform + SDK to allow you to compile Windows applications. +
+ +
+ Perl + + Download and install ActivePerl for + Windows, which you can probably find here. +
+ +
+ Libraries + + You will need to download and install several libraries. They + must all be built in the same directory, to be able to link + properly. + + Choose a directory where you will unpack and compile OpenSSL, + Zlib and Box Backup. We will call this the base directory. An example + might be: + + C:\Documents and Settings\Your Username\Desktop\Box + + Make sure you know the full path to this directory. + +
+ OpenSSL + + You will need to compile OpenSSL using Visual C++. The latest + release at this time, OpenSSL 0.9.8a, does not compile with Visual + C++ 2005 out of the box, so you need a + patched version. The original + source and patch + are also available. + + To compile OpenSSL: + + + + Use a Windows unzipper such as WinZip (free trial) to + extract the openssl-0.9.8a-vc2005.tar.gz archive, + which you just downloaded, into the base directory. + + + + Rename the folder from openssl-0.9.8a-vc2005 to openssl + + + + Open a command shell (run cmd.exe) and cd to the openssl directory + + + + Run the following commands: + + perl Configure VC-WIN32 +ms\do_ms +"c:\program files\Microsoft Visual Studio 8\Common7\Tools\vsvars32.bat" +nmake -f ms\ntdll.mak + + +
+ +
+ Zlib + + You will need to download the Zlib compiled DLL. + Create a directory called zlib in + the base directory, and unzip the file you just downloaded into that + directory. You don't need to compile anything. +
+
+ +
+ Download Box Backup + + The first version of Box Backup that's known to compile and with + Visual C++ 2005 is available on the Subversion + server. However, this version has not been extensively tested + and may be out of date. + + The changes are expected to be merged into the Subversion trunk + at some point, and this page should then be updated. If in doubt, + please sign up to the mailing + list and ask us. + + To get the source code out of Subversion you will need a Subversion + client for Windows. After installing it, open a new command + prompt, go to the base directory, and type: + + svn co http://www.boxbackup.org/svn/box/chris/win32/vc2005-compile-fixes/ boxbackup + + This should create a directory called boxbackup inside the base directory. +
+ +
+ Configure Box Backup + + Open a command prompt, change to the base directory then + boxbackup, and run win32.bat to configure the sources. Otherwise, + Visual C++ will complain about missing files whose names start with + autogen, and missing config.h. +
+ +
+ Compile Box Backup + + Open Visual C++. Choose "File/Open/Project", navigate to the + base directory, then to boxbackup\infrastructure\msvc\2005 (or + 2003 if using Visual Studio 2003), + and open any project or solution file in that directory. + + Press F7 to compile Box Backup. If the compilation is + successful, boxbackup\Debug\bbackupd.exe will be + created. +
+ +
+ Install Box Backup + + Create the destination directory, C:\Program Files\Box Backup\bbackupd. + + Write a configuration file, keys and certificate on a Unix + machine, and copy them into the Box + Backup directory, together with the following files from + the base directory: + + + + boxbackup\Debug\bbackupd.exe + + + + openssl\out32dll\libeay32.dll + + + + openssl\out32dll\ssleay32.dll + + + + zlib\zlib1.dll + + + + Ensure that the user running Box Backup can read from the + Box Backup directory, and write to + the bbackupd directory inside + it. + + Run Box Backup by double-clicking on it, and check that it + connects to the server. If the window opens and closes immediately, + it's probably due to a problem with the configuration file - check the + Windows Event Viewer for details. +
+ +
+ Windows Service + + Box Backup can also run as a Windows service, in which case it + will be automatically started at boot time in the background. To + install this, open a command prompt, and run: + + cd "C:\Program Files\Box Backup" +bbackupd.exe -i + + This should output Box Backup service installed. +
+
+ + + + Compilation and installation by building an RPM on + Linux + + It is very easy to build an RPM of Box Backup on Linux platforms. + This should work on all Red Hat distributions (including Fedora), SuSE, + and probably others too. + + Given that you have the correct development packages installed + simply execute this command (replacing the version number): + + rpmbuild -ta boxbackup-0.00.tgz + + rpmbuild will report where the packages have been written to, and + these can be installed in the normal manner. + + If you have never built an RPM before you should set up a convenient + build area as described in the RPM + book. + + If you wish to customise the package you can find the spec file in + the contrib/rpm directory. + + diff --git a/docs/docbook/raidfile-config.xml b/docs/docbook/raidfile-config.xml new file mode 100644 index 00000000..78a3e94c --- /dev/null +++ b/docs/docbook/raidfile-config.xml @@ -0,0 +1,198 @@ + + + + raidfile-config + + 8 + + Box Backup + + Box Backup + + 0.11 + + + + raidfile-config + + Configure Box Backup's RAID files + + + + + raidfile-config + + config-dir + + blocksize + + dir1 dir2 dir3 + + + + + Description + + raidfile-config creates a raidfile.conf file for Box Backup. This + file holds information about the directories used to store backups in. Box + Backup supports userland RAID, in a restricted RAID5 configuration, where + 3 and only 3 'drives' are supported. You can read more about RAID5 (and + other RAID-levels) here. + + + Parameters + + The parameters are as follows: + + + + config-dir + + + The directory path where configuration files are located. + Usually this is /etc/box. + raidfile.conf will be written in this + directory. + + + + + blocksize + + + The block size used for file storage in the system, in + bytes. Using a multiple of the file system block size is a good + strategy. Depending on the size of the files you will be backing + up, this multiple varies. Of course it also depends on the native + block size of your file system. + + + + + dir1 + + + The first directory in the built-in RAID array. + + + + + dir2 + + + The second directory in the built-in RAID array. If you are + not using the built-in RAID functionality, this field should be + ignored. You should not use the built-in RAID if you have a + hardware RAID solution or if you're using another type of software + RAID (like md on Linux). + + + + + dir3 + + + The third directory in the built-in RAID array. The same + notes that apply to dir2 also apply to + dir3. + + + + + Note that there are currently no way to add multiple disk sets to + the raidfile.conf file using command line tools, etc. See + raidfile.conf + + 5 + for details on adding more disks. + + + + + Bugs + + If you find a bug in Box Backup, and you want to let us know about + it, join the mailing + list, and send a description of the problem there. + + To report a bug, give us at least the following information: + + + + The version of Box Backup you are running + + + + The platform you are running on (hardware and OS), for both + client and server. + + + + If possible attach your config files (bbstored.conf, + bbackupd.conf) to the bug report. + + + + Also attach any log file output that helps shed light on the + problem you are seeing. + + + + And last but certainly not least, a description of what you are + seeing, in as much detail as possible. + + + + + + Files + + raidfile-config generates the + raidfile.conf + + 5 + file. + + + + See Also + + + bbstored-config + + 8 + , + bbstored.conf + + 5 + , + raidfile.conf + + 5 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/docbook/raidfile.conf.xml b/docs/docbook/raidfile.conf.xml new file mode 100644 index 00000000..7a8b6410 --- /dev/null +++ b/docs/docbook/raidfile.conf.xml @@ -0,0 +1,143 @@ + + + + raidfile.conf + + 5 + + Box Backup + + Box Backup + + 0.11 + + + + raidfile.conf + + Userland RAID for Box Backup + + + + + /etc/box/raidfile.conf + + + + + Description + + The raidfile.conf is usually generated by + raidfile-config + + 8 + but may be manually edited if the store locations move + or if more than one disc set is required. + + + + discX + + + Specifies a set of discs. + + + + SetNumber + + + The set number of the RAID disc, referenced by each + account. + + + + + BlockSize + + + The block size of the file system (usually 2048). + Under BSD with FFS, set this to your file system's + fragment size (most likely an 8th of the block + size). + + + + + Dir0 + + + The first directory in the RAID array. + + + + + Dir1 + + + The second directory in the RAID array. If you do + not wish to use the built-in RAID functionality, this + field should be set to the same as + Dir0. You should not use the built-in + RAID if you have a hardware RAID solution or if you're + using another type of software RAID (like md on + Linux). + + + + + Dir2 + + + The third directory in the RAID array. The same + notes that apply to Dir2 also apply to + Dir3. + + + + + + + + + + Files + + /etc/box/raidfile.conf + + + + See Also + + + raidfile-config + + 8 + , + bbstored.conf + + 5 + + + + + Authors + + + Ben Summers + + + + Per Thomsen + + + + James O'Gorman + + + diff --git a/docs/favicon.ico b/docs/favicon.ico deleted file mode 100644 index f8cbb3b3..00000000 Binary files a/docs/favicon.ico and /dev/null differ diff --git a/docs/generate_except_xml.pl b/docs/generate_except_xml.pl deleted file mode 100644 index 914ef456..00000000 --- a/docs/generate_except_xml.pl +++ /dev/null @@ -1,74 +0,0 @@ -#!/usr/bin/perl -w -use strict; - -open (EXCEPT, "<../ExceptionCodes.txt") or die "Can't open ../ExceptionCodes.txt: $!\n"; -open (DOCBOOK, ">ExceptionCodes.xml") or die "Can't open Exceptioncodes.xml for writing: $!\n"; - -print DOCBOOK < - - - Exception codes - -EOD -my $sectionName; -my $sectionNum; -my $sectionDesc; -my $exceptionCode; -my $exceptionShortDesc; -my $exceptionLongDesc; -while() -{ - next if(m/^#/); - chomp; - if(m/^EXCEPTION TYPE (\w+) (\d+)/) - { - $sectionName = ucfirst(lc($1)); - $sectionNum = $2; - if($sectionName ne "Common") - { - $sectionDesc = "the " . $sectionName; - } - else - { - $sectionDesc = "any"; - } - print DOCBOOK < - $sectionName Exceptions ($sectionNum) - - These are exceptions that can occur in $sectionDesc module - of the system. - - -EOD - } - - # The END TYPE line - if(m/^END TYPE$/) - { - print DOCBOOK " \n \n"; - } - - # The actual exceptions - if(m/(\(\d+\/\d+\)) - (\w+ \w+)(?: - )?(.*)$/) - { - $exceptionCode = $1; - $exceptionShortDesc = $2; - $exceptionLongDesc = $3; - - print DOCBOOK " \n "; - print DOCBOOK $exceptionCode . ": " . $exceptionShortDesc . ""; - if($exceptionLongDesc ne "") - { - print DOCBOOK " -- " . $exceptionLongDesc; - } - print DOCBOOK "\n \n"; - } -} - -print DOCBOOK "\n"; - -close EXCEPT; -close DOCBOOK; - diff --git a/docs/html/bbdoc-man.css b/docs/html/bbdoc-man.css deleted file mode 100644 index 0345560e..00000000 --- a/docs/html/bbdoc-man.css +++ /dev/null @@ -1,104 +0,0 @@ -body { - font-family: Verdana, Geneva, Arial, sans-serif; - background-color: #edeef3; - font-size: .75em; - line-height: 180%; - text-align: left; - margin-top: 20px; - margin-right: 100px; - margin-left: 250px; - position: relative; - width: auto; } - -table { - font-family: Verdana, Geneva, Arial, sans-serif; - background-color: #edeef3; - font-size: 10pt; - line-height: 100%; } - - -code { - font-size: 11pt; } - - - -div.navheader { - font-family: Verdana, Geneva, Arial, sans-serif; - background-color: #edeef3; - line-height: 100%; } - -#header { - background-color: #e4e6ed; - text-align: left; - padding-top: 10px; - margin-right: -100px; - margin-left: -250px; - top: 20px; - border-top: 1px solid #c4c4d5; - border-bottom: 1px solid white } - -#logo { - position: relative; - margin-left: 200px } - - -#page { - font-size: .75em; - line-height: 180%; - text-align: left; - margin-top: 50px; - margin-right: 100px; - margin-left: 250px; - position: relative; - width: auto } - -#disc { } - -.informaltable td,tr {font-size: 1em; - line-height: 140%; - text-align: left; - background-color: #e4e6ed; - padding: 4px } - -tr,td {font-size: 1em; - line-height: 100%; - background-color: #edeef3; } - -pre, tt { font-size: 1.3em; - color: #088; - letter-spacing: 1px; - word-spacing: 2px} - -h1 { - color: #c00; - font-size: 16pt; - margin-bottom: 2em; - margin-left: -50px } - -h2 { - color: #324e95; - font-size: 12pt; - margin-top: 2em; - margin-left: -50px } - -h3 { - color: #324e95; - font-size: 10pt; - margin-top: 2em; - margin-left: -50px } - -dt { font-weight: bold } - -a:link { - color: #324e95; - text-decoration: none; - background-color: transparent } - -a:visited { - color: #90c; - text-decoration: none } - -a:hover { - color: #c00; - text-decoration: underline; - background-color: transparent } diff --git a/docs/html/bbdoc.css b/docs/html/bbdoc.css deleted file mode 100644 index d3b4a1c2..00000000 --- a/docs/html/bbdoc.css +++ /dev/null @@ -1,112 +0,0 @@ -body { - font-family: Verdana, Geneva, Arial, sans-serif; - background-color: #edeef3; - font-size: .75em; - line-height: 180%; - text-align: left; - margin-top: 20px; - margin-right: 100px; - margin-left: 250px; - position: relative; - width: auto; } - -table { - font-family: Verdana, Geneva, Arial, sans-serif; - background-color: #edeef3; - font-size: 10pt; - line-height: 100%; } - - - - -div.navheader { - font-family: Verdana, Geneva, Arial, sans-serif; - background-color: #edeef3; - line-height: 100%; } - -#header { - background-color: #e4e6ed; - text-align: left; - padding-top: 10px; - margin-right: -100px; - margin-left: -250px; - top: 20px; - border-top: 1px solid #c4c4d5; - border-bottom: 1px solid white } - -#logo { - position: relative; - margin-left: 200px } - - -#page { - font-size: .75em; - line-height: 180%; - text-align: left; - margin-top: 50px; - margin-right: 100px; - margin-left: 250px; - position: relative; - width: auto } - -#disc { } - -.informaltable td,tr {font-size: 1em; - line-height: 140%; - text-align: left; - background-color: #e4e6ed; - padding: 4px } - -tr,td {font-size: 1em; - line-height: 100%; - background-color: #edeef3; } - -pre, tt { font-size: 1.3em; - color: #088; - letter-spacing: 1px; - word-spacing: 2px} - -h1 { - color: #c00; - font-size: 16pt; - margin-bottom: 2em; - margin-left: -50px } - -h2 { - color: #324e95; - font-size: 12pt; - margin-top: 2em; - margin-left: -50px } - -h3 { - color: #324e95; - font-size: 10pt; - margin-top: 2em; - margin-left: -50px } - -dt { font-weight: bold } - -ul { - list-style-image: url(images/arrow.png) } - -ul li { - background-color: #e4e6ed; - margin: 1em 6em 1em -2em; - padding: 0.2em 0.5em 0.2em 1em; - border-style: solid; - border-width: 1px; - border-color: #c4c4d5 #fff #fff #c4c4d5 } - -a:link { - color: #324e95; - text-decoration: none; - background-color: transparent } - -a:visited { - color: #90c; - text-decoration: none } - -a:hover { - color: #c00; - text-decoration: underline; - background-color: transparent } diff --git a/docs/html/images/arrow.png b/docs/html/images/arrow.png deleted file mode 100644 index 4c60805d..00000000 Binary files a/docs/html/images/arrow.png and /dev/null differ diff --git a/docs/html/images/bblogo.png b/docs/html/images/bblogo.png deleted file mode 100644 index b33230b4..00000000 Binary files a/docs/html/images/bblogo.png and /dev/null differ diff --git a/docs/html/images/stepahead.png b/docs/html/images/stepahead.png deleted file mode 100644 index 9ac05b8c..00000000 Binary files a/docs/html/images/stepahead.png and /dev/null differ diff --git a/docs/images/bblogo-alpha.xcf b/docs/images/bblogo-alpha.xcf new file mode 100644 index 00000000..18f494f9 Binary files /dev/null and b/docs/images/bblogo-alpha.xcf differ diff --git a/docs/instguide.xml b/docs/instguide.xml deleted file mode 100644 index addef297..00000000 --- a/docs/instguide.xml +++ /dev/null @@ -1,766 +0,0 @@ - - - - Box Backup Build and Installation Guide - - - License - - Copyright © 2003 - 2007, Ben Summers and contributors. - All rights reserved. - - Redistribution and use in source and binary forms, with or without - modification, are permitted provided that the following conditions are - met: - - - - Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - - - Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following disclaimer - in the documentation and/or other materials provided with the - distribution. - - - - All use of this software and associated advertising materials - must display the following acknowledgement: This product includes - software developed by Ben Summers. - - - - The names of the Authors may not be used to endorse or promote - products derived from this software without specific prior written - permission. - - - - - [Where legally impermissible the Authors do not disclaim liability - for direct physical injury or death caused solely by defects in the - software unless it is modified by a third party.] - - THIS SOFTWARE IS PROVIDED BY THE AUTHORS "AS IS" AND ANY EXPRESS - OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED - WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR - PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR - ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, - STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING - IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - POSSIBILITY OF SUCH DAMAGE. - - - - Introduction - - The backup daemon, bbackupd, runs on all machines to be backed up. - The store server daemon, bbstored runs on a central server. Data is sent - to the store server, which stores all data on local filesystems, that is, - only on local hard drives. Tape or other archive media is not used. - - The system is designed to be easy to set up and run, and cheap to - use. Once set up, there should be no need for user or administrative - intervention, apart from usual system maintenance. - -
- Client daemon - - bbackupd is configured with a list of directories to back up. It - has a lazy approach to backing up data. Every so often, the directories - are scanned, and new data is uploaded to the server. This new data must - be over a set age before it is uploaded. This prevents rapid revisions - of a file resulting in many uploads of the same file in a short period - of time. - - It can also operate in a snapshot mode, which behaves like - traditional backup software. When instructed by an external bbackupctl - program, it will upload all changed files to the server. - - The daemon is always running, although sleeping most of the time. - In lazy mode, it is completely self contained -- scripts running under - cron jobs are not used. The objective is to keep files backed up, not to - make snapshots of the filesystem at particular points in time - available. - - If an old version of the file is present on the server, a modified - version of the rsync algorithm is used to upload only the changed - portions of the file. - - After a new version is uploaded, the old version is still - available (subject to disc space on the server). Similarly, a deleted - file is still available. The only limit to their availability is space - allocated to this account on the server - - Future versions will add the ability to mark the current state of - files on the server, and restore from this mark. This will emulate the - changing of tapes in a tape backup system. - -
- Restoration - - Restoring files is performed using a query tool, bbackupquery. - This can be used to restore entire directories, or as an 'FTP-like' - tool to list and retrieve individual files. Old versions and deleted - files can be retrieved using this tool for as long as they are kept on - the server. -
- -
- Client Resource Usage - - bbackupd uses only a minimal amount of disc space to store - records on uploaded files -- less than 32 bytes per directory and file - over a set size threshold. However, it minimises the amount of queries - it must make to the server by storing, in memory, a data structure - which allows it to determine what data is new. It does not need to - store a record of all files, essentially just the directory names and - last modification times. This is not a huge amount of memory. - - If there are no changes to the directories, then the client will - not even connect to the server. -
-
- -
- Security - - Box Backup is designed to be secure in several ways. The data - stored on the backup store server is encrypted using secret-key - cryptography. Additionally, the transport layer is encrypted using TLS, - to ensure that the communications can't be snooped. - -
- Encryption - - The files, directories, filenames and file attributes are all - encrypted. By examining the stored files on the server, it is only - possible to determine the approximate sizes of a files and the tree - structure of the disc (not names, just number of files and - subdirectories in a directory). By monitoring the actions performed by - a client, it is possible to determine the frequency and approximate - scope of changes to files and directories. - - The connections between the server and client are encrypted - using TLS (latest version of SSL). Traffic analysis is possible to - some degree, but limited in usefulness. - - An attacker will not be able to recover the backed up data - without the encryption keys. Of course, you won't be able to recover - your files without the keys either, so you must make a conventional, - secure, backup of these keys. -
- -
- Authentication - - SSL certificates are used to authenticate clients. UNIX user - accounts are not used to minimise the dependence on the configuration - of the operating system hosting the server. - - A script is provided to run the necessary certification - authority with minimal effort. -
-
- -
- Server daemon - - The server daemon is designed to be simple to deploy, and run on - the cheapest hardware possible. To avoid the necessity to use expensive - hardware RAID or software RAID with complex setup, it (optionally) - stores files using RAID techniques. - - It does not need to run as a privileged user. - - Each account has a set amount of disc space allocated, with a soft - and a hard limit. If the account exceeds the soft limit, a housekeeping - process will start deleting old versions and deleted files to reduce the - space used to below the soft limit. If the backup client attempts to - upload a file which causes the store to exceed the hard limit, the - upload will be refused. -
- - - - Building and installing - -
- Before you start - - Firstly, check that all the clocks on your clients, servers and - signing machines are accurate and in sync. A disagreement in time - between a client and a server is the biggest cause of installation - difficulties, as the times in the generated certificates will cause - login failures if the start date is in the future. -
- -
- Box Backup compile - - In the following instructions, replace 0.00 with the actual - version number of the archive you have downloaded. - - For help building on Windows, see the Windows - Compile Appendix. And if you want to build a Linux RPM, look here. - - You need the latest version of OpenSSL, as some of the extra APIs - it provides are required. You should have this anyway, as earlier - versions have security flaws. (If you have an earlier version installed, - the configuration script will give you instructions on enabling - experimental support for older versions.) - - See OpenSSL notes for more information - on OpenSSL issues. - - There are some notes in the archive about compiling on various - platforms within the boxbackup-0.00 directory -- read them first. For - example, if you are compiling under Linux, look for LINUX.txt as - boxbackup-0.00/LINUX.txt after untaring the archive. - - Download the archive, then in that directory type - - tar xvzf boxbackup-0.00.tgz -cd boxbackup-0.00 -./configure -make - - The server and client will be built and packaged up for - installation on this machine, or ready to be transferred as tar files to - another machine for installation. - - This builds two parcels of binaries and scripts, 'backup-client' - and 'backup-server'. The generated installation scripts assumes you want - everything installed in /usr/local/bin - - Optionally, type make test to run - all the tests. -
- -
- Local installation - - Type make install-backup-client - to install the backup client. - - Type make install-backup-server - to install the backup server. -
- -
- Remote installation - - In the parcels directory, there are tar files for each parcel. The - name reflects the version and platform you have built it for. - - Transfer this tar file to the remote server, and unpack it, then - run the install script. For example: - - tar xvzf boxbackup-0.00-backup-server-OpenBSD.tgz -cd boxbackup-0.00-backup-server-OpenBSD -./install-backup-server -
- -
- Configure options - - You can use arguments to the configure script to adjust the - compile and link lines in the generated Makefiles, should this be - necessary for your platform. The configure script takes the usual GNU - autoconf arguments, a full list of which can be obtained with --help. Additional options for Box Backup - include: - - - - - - --enable-gnu-readline - - Use GNU readline if present. Linking Box Backup against - GNU readline may create licence implications if you then - distribute the binaries. libeditline is also supported as a safe - alternative, and is used by default if available. - - - - --disable-largefile - - Omit support for large files - - - - --with-bdb-lib=DIR - - Specify Berkeley DB library location - - - - --with-bdb-headers=DIR - - Specify Berkeley DB headers location - - - - --with-random=FILE - - Use FILE as random number seed (normally - auto-detected) - - - - --with-tmp-dir=DIR - - Directory for temporary files (normally /tmp) - - - - - - See OpenSSL notes for the OpenSSL - specific options. -
- -
- Tests - - There are a number of unit tests provided. To compile and run one - type: - - ./runtest.pl bbackupd release -./runtest.pl common debug -./runtest.pl ALL - - The runtest.pl script will compile and run the test. The first - argument is the test name, and the second the type of build. Use ALL as - a test name to run all the tests. - - The output from the tests is slightly muddled using this script. - If you're developing, porting or trying out new things, it might be - better to use the following scheme: - - cd test/bbackupd -make -cd ../../debug/test/bbackupd -./t - - or in release mode... - - cd test/bbackupd -make -D RELEASE -cd ../../release/test/bbackupd -./t - - (use RELEASE=1 with GNU make) - - I tend to use two windows, one for compilation, and one for - running tests. -
- - - - Box Backup and SSL - -
- General notes - - Ideally, you need to use version 0.9.7 or later of OpenSSL. If - this is installed on your system by default (and it is on most recent - releases of UNIX like OSes) then everything should just work. - - However, if it isn't, you have a few options. - -
- Upgrade your installation - - The best option is to upgrade your installation to use 0.9.7. - Hopefully your package manager will make this easy for you. This may - require reinstallation of lots of software which depends on OpenSSL, - so may not be ideal. - - (But as there have been a few security flaws in OpenSSL - recently, you probably want to upgrade it anyway.) -
- -
- Install another OpenSSL - - The second best option is to install another copy. If you - download and install from source, it will probably install into - /usr/local/ssl. You can then configure Box Backup to use it - using: - - ./configure --with-ssl-headers=/usr/local/ssl/include --with-ssl-lib=/usr/local/ssl/lib - - which will set up the various includes and libraries for - you. - - The configuration scripts may be a problem, depending on your - installation. See below for more information. -
- -
- Use the old version of OpenSSL - - If you have an old version installed, the configuration script - will give you instructions on how to enable support for older - versions. Read the warnings, and please, whatever you do, don't - release binary packages or ports which enable this option. - - You may have issues with the configuration scripts, see - below. -
-
- -
- If you have problems with the config scripts - - If you get OpenSSL related errors with the configuration scripts, - there are two things to check: - - - - The bin directory within your OpenSSL directory is in the path - (if you have installed another version) - - - - You have an openssl.cnf file which works and can be - found. - - - -
- OpenSSL config file - - You need to have an openssl.cnf file. The default will generally - work well (see example at end). Make sure the openssl utility can find - it, either set the OPENSSL_CONF environment variable, or install it - into the location that is mentioned in the error messages. - - Example OpenSSL config file: - - # -# OpenSSL example configuration file. -# This is mostly being used for generation of certificate requests. -# - -RANDFILE = /dev/arandom - -#################################################################### -[ req ] -default_bits = 1024 -default_keyfile = privkey.pem -distinguished_name = req_distinguished_name -attributes = req_attributes - -[ req_distinguished_name ] -countryName = Country Name (2 letter code) -#countryName_default = AU -countryName_min = 2 -countryName_max = 2 - -stateOrProvinceName = State or Province Name (full name) -#stateOrProvinceName_default = Some-State - -localityName = Locality Name (eg, city) - -0.organizationName = Organization Name (eg, company) -#0.organizationName_default = Internet Widgits Pty Ltd - -# we can do this but it is not needed normally :-) -#1.organizationName = Second Organization Name (eg, company) -#1.organizationName_default = CryptSoft Pty Ltd - -organizationalUnitName = Organizational Unit Name (eg, section) -#organizationalUnitName_default = - -commonName = Common Name (eg, fully qualified host name) -commonName_max = 64 - -emailAddress = Email Address -emailAddress_max = 64 - -[ req_attributes ] -challengePassword = A challenge password -challengePassword_min = 4 -challengePassword_max = 20 - -unstructuredName = An optional company name - -[ x509v3_extensions ] - -nsCaRevocationUrl = http://www.cryptsoft.com/ca-crl.pem -nsComment = "This is a comment" - -# under ASN.1, the 0 bit would be encoded as 80 -nsCertType = 0x40 -
-
- - - - Compiling bbackupd on Windows using Visual C++ - - This Appendix explains how to build the bbackupd daemon for Windows - using the Visual C++ compiler. - - If you have any problems following these instructions, please sign - up to the mailing - list and report them to us, or send an email to Chris Wilson. Thanks! - - Note: bbstored will not be built - with this process. bbstored is not currently supported on Windows. There - are no plans for bbstored support on Windows. - -
- Tools - - You will need quite a bit of software to make this work. All of it - is available for free on the Internet, although Visual C++ Express has - license restrictions and a time limit. - -
- Visual C++ - - Microsoft's Visual C++ compiler and development environment are - part of their commercial product Visual Studio. Visual - Studio 2005 is supported, and 2003 should work as well. - - You can also download - a free copy of Visual C++ 2005 Express. This copy must be registered - (activated) within 30 days, and is free for one year. - - You will need the Platform - SDK to allow you to compile Windows applications. -
- -
- Perl - - Download and install ActivePerl for - Windows, which you can probably find here. -
- -
- Libraries - - You will need to download and install several libraries. They - must all be built in the same directory, to be able to link - properly. - - Choose a directory where you will unpack and compile OpenSSL, - Zlib and Box Backup. We will call this the base directory. An example - might be: - - C:\Documents and Settings\Your Username\Desktop\Box - - Make sure you know the full path to this directory. - -
- OpenSSL - - You will need to compile OpenSSL using Visual C++. The latest - release at this time, OpenSSL 0.9.8a, does not compile with Visual - C++ 2005 out of the box, so you need a - patched version. The original - source and patch - are also available. - - To compile OpenSSL: - - - - Use a Windows unzipper such as WinZip (free trial) to - extract the openssl-0.9.8a-vc2005.tar.gz archive, - which you just downloaded, into the base directory. - - - - Rename the folder from openssl-0.9.8a-vc2005 to openssl - - - - Open a command shell (run cmd.exe) and cd to the openssl directory - - - - Run the following commands: - - perl Configure VC-WIN32 -ms\do_ms -"c:\program files\Microsoft Visual Studio 8\Common7\Tools\vsvars32.bat" -nmake -f ms\ntdll.mak - - -
- -
- Zlib - - You will need to download the Zlib compiled DLL. - Create a directory called zlib in - the base directory, and unzip the file you just downloaded into that - directory. You don't need to compile anything. -
-
- -
- Download Box Backup - - The first version of Box Backup that's known to compile and with - Visual C++ 2005 is available on the Subversion - server. However, this version has not been extensively tested - and may be out of date. - - The changes are expected to be merged into the Subversion trunk - at some point, and this page should then be updated. If in doubt, - please sign up to the mailing - list and ask us. - - To get the source code out of Subversion you will need a Subversion - client for Windows. After installing it, open a new command - prompt, go to the base directory, and type: - - svn co http://www.boxbackup.org/svn/box/chris/win32/vc2005-compile-fixes/ boxbackup - - This should create a directory called boxbackup inside the base directory. -
- -
- Configure Box Backup - - Open a command prompt, change to the base directory then - boxbackup, and run win32.bat to configure the sources. Otherwise, - Visual C++ will complain about missing files whose names start with - autogen, and missing config.h. -
- -
- Compile Box Backup - - Open Visual C++. Choose "File/Open/Project", navigate to the - base directory, then to boxbackup\infrastructure\msvc\2005 (or - 2003 if using Visual Studio 2003), - and open any project or solution file in that directory. - - Press F7 to compile Box Backup. If the compilation is - successful, boxbackup\Debug\bbackupd.exe will be - created. -
- -
- Install Box Backup - - Create the destination directory, C:\Program Files\Box Backup\bbackupd. - - Write a configuration file, keys and certificate on a Unix - machine, and copy them into the Box - Backup directory, together with the following files from - the base directory: - - - - boxbackup\Debug\bbackupd.exe - - - - openssl\out32dll\libeay32.dll - - - - openssl\out32dll\ssleay32.dll - - - - zlib\zlib1.dll - - - - Ensure that the user running Box Backup can read from the - Box Backup directory, and write to - the bbackupd directory inside - it. - - Run Box Backup by double-clicking on it, and check that it - connects to the server. If the window opens and closes immediately, - it's probably due to a problem with the configuration file - check the - Windows Event Viewer for details. -
- -
- Windows Service - - Box Backup can also run as a Windows service, in which case it - will be automatically started at boot time in the background. To - install this, open a command prompt, and run: - - cd "C:\Program Files\Box Backup" -bbackupd.exe -i - - This should output Box Backup service installed. -
-
- - - - Compilation and installation by building an RPM on - Linux - - It is very easy to build an RPM of Box Backup on Linux platforms. - This should work on all Red Hat distributions (including Fedora), SuSE, - and probably others too. - - Given that you have the correct development packages installed - simply execute this command (replacing the version number): - - rpmbuild -ta boxbackup-0.00.tgz - - rpmbuild will report where the packages have been written to, and - these can be installed in the normal manner. - - If you have never built an RPM before you should set up a convenient - build area as described in the RPM - book. - - If you wish to customise the package you can find the spec file in - the contrib/rpm directory. - - diff --git a/docs/raidfile-config.xml b/docs/raidfile-config.xml deleted file mode 100644 index 78a3e94c..00000000 --- a/docs/raidfile-config.xml +++ /dev/null @@ -1,198 +0,0 @@ - - - - raidfile-config - - 8 - - Box Backup - - Box Backup - - 0.11 - - - - raidfile-config - - Configure Box Backup's RAID files - - - - - raidfile-config - - config-dir - - blocksize - - dir1 dir2 dir3 - - - - - Description - - raidfile-config creates a raidfile.conf file for Box Backup. This - file holds information about the directories used to store backups in. Box - Backup supports userland RAID, in a restricted RAID5 configuration, where - 3 and only 3 'drives' are supported. You can read more about RAID5 (and - other RAID-levels) here. - - - Parameters - - The parameters are as follows: - - - - config-dir - - - The directory path where configuration files are located. - Usually this is /etc/box. - raidfile.conf will be written in this - directory. - - - - - blocksize - - - The block size used for file storage in the system, in - bytes. Using a multiple of the file system block size is a good - strategy. Depending on the size of the files you will be backing - up, this multiple varies. Of course it also depends on the native - block size of your file system. - - - - - dir1 - - - The first directory in the built-in RAID array. - - - - - dir2 - - - The second directory in the built-in RAID array. If you are - not using the built-in RAID functionality, this field should be - ignored. You should not use the built-in RAID if you have a - hardware RAID solution or if you're using another type of software - RAID (like md on Linux). - - - - - dir3 - - - The third directory in the built-in RAID array. The same - notes that apply to dir2 also apply to - dir3. - - - - - Note that there are currently no way to add multiple disk sets to - the raidfile.conf file using command line tools, etc. See - raidfile.conf - - 5 - for details on adding more disks. - - - - - Bugs - - If you find a bug in Box Backup, and you want to let us know about - it, join the mailing - list, and send a description of the problem there. - - To report a bug, give us at least the following information: - - - - The version of Box Backup you are running - - - - The platform you are running on (hardware and OS), for both - client and server. - - - - If possible attach your config files (bbstored.conf, - bbackupd.conf) to the bug report. - - - - Also attach any log file output that helps shed light on the - problem you are seeing. - - - - And last but certainly not least, a description of what you are - seeing, in as much detail as possible. - - - - - - Files - - raidfile-config generates the - raidfile.conf - - 5 - file. - - - - See Also - - - bbstored-config - - 8 - , - bbstored.conf - - 5 - , - raidfile.conf - - 5 - - - - - Authors - - - Ben Summers - - - - Per Thomsen - - - - James O'Gorman - - - diff --git a/docs/raidfile.conf.xml b/docs/raidfile.conf.xml deleted file mode 100644 index 7a8b6410..00000000 --- a/docs/raidfile.conf.xml +++ /dev/null @@ -1,143 +0,0 @@ - - - - raidfile.conf - - 5 - - Box Backup - - Box Backup - - 0.11 - - - - raidfile.conf - - Userland RAID for Box Backup - - - - - /etc/box/raidfile.conf - - - - - Description - - The raidfile.conf is usually generated by - raidfile-config - - 8 - but may be manually edited if the store locations move - or if more than one disc set is required. - - - - discX - - - Specifies a set of discs. - - - - SetNumber - - - The set number of the RAID disc, referenced by each - account. - - - - - BlockSize - - - The block size of the file system (usually 2048). - Under BSD with FFS, set this to your file system's - fragment size (most likely an 8th of the block - size). - - - - - Dir0 - - - The first directory in the RAID array. - - - - - Dir1 - - - The second directory in the RAID array. If you do - not wish to use the built-in RAID functionality, this - field should be set to the same as - Dir0. You should not use the built-in - RAID if you have a hardware RAID solution or if you're - using another type of software RAID (like md on - Linux). - - - - - Dir2 - - - The third directory in the RAID array. The same - notes that apply to Dir2 also apply to - Dir3. - - - - - - - - - - Files - - /etc/box/raidfile.conf - - - - See Also - - - raidfile-config - - 8 - , - bbstored.conf - - 5 - - - - - Authors - - - Ben Summers - - - - Per Thomsen - - - - James O'Gorman - - - diff --git a/docs/tools/generate_except_xml.pl b/docs/tools/generate_except_xml.pl new file mode 100644 index 00000000..38ee9074 --- /dev/null +++ b/docs/tools/generate_except_xml.pl @@ -0,0 +1,74 @@ +#!/usr/bin/perl -w +use strict; + +open (EXCEPT, "< $ARGV[0]") or die "Can't open $ARGV[0]: $!\n"; +open (DOCBOOK, "> $ARGV[1]") or die "Can't open $ARGV[1] for writing: $!\n"; + +print DOCBOOK < + + + Exception codes + +EOD +my $sectionName; +my $sectionNum; +my $sectionDesc; +my $exceptionCode; +my $exceptionShortDesc; +my $exceptionLongDesc; +while() +{ + next if(m/^#/); + chomp; + if(m/^EXCEPTION TYPE (\w+) (\d+)/) + { + $sectionName = ucfirst(lc($1)); + $sectionNum = $2; + if($sectionName ne "Common") + { + $sectionDesc = "the " . $sectionName; + } + else + { + $sectionDesc = "any"; + } + print DOCBOOK < + $sectionName Exceptions ($sectionNum) + + These are exceptions that can occur in $sectionDesc module + of the system. + + +EOD + } + + # The END TYPE line + if(m/^END TYPE$/) + { + print DOCBOOK " \n \n"; + } + + # The actual exceptions + if(m/(\(\d+\/\d+\)) - (\w+ \w+)(?: - )?(.*)$/) + { + $exceptionCode = $1; + $exceptionShortDesc = $2; + $exceptionLongDesc = $3; + + print DOCBOOK " \n "; + print DOCBOOK $exceptionCode . ": " . $exceptionShortDesc . ""; + if($exceptionLongDesc ne "") + { + print DOCBOOK " -- " . $exceptionLongDesc; + } + print DOCBOOK "\n \n"; + } +} + +print DOCBOOK "\n"; + +close EXCEPT; +close DOCBOOK; + -- cgit v1.2.3