diff options
40 files changed, 8694 insertions, 4708 deletions
diff --git a/.github/workflows/random-tests.yml b/.github/workflows/random-tests.yml new file mode 100644 index 0000000..2d15a9e --- /dev/null +++ b/.github/workflows/random-tests.yml @@ -0,0 +1,67 @@ +name: Random Tests +on: + push: + branches: + - 'bug_fixes' + - 'master' + paths: + - '**.h' + - '**.cpp' + - 'bowtie2' + - 'bowtie2-build' + - 'bowtie2-inspect' + - 'Makefile' + - 'scripts/sim/**' + - 'scripts/test/**' + pull_request: + branches: + - 'master' + paths: + - '**.h' + - '**.cpp' + - 'bowtie2' + - 'bowtie2-build' + - 'bowtie2-inspect' + - 'Makefile' + - 'scripts/sim/**' + - 'scripts/test/**' +jobs: + setup: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v2 + - name: Install zlib development files + run: | + sudo apt-get update + sudo apt-get install zlib1g-dev + linux: + runs-on: ubuntu-latest + needs: setup + steps: + - uses: actions/checkout@v2 + - name: Run random tests + run: | + make clean; make -j4 allall + export NUM_CORES=1 + make random-test + linux-avx2: + runs-on: ubuntu-latest + needs: linux + steps: + - uses: actions/checkout@v2 + - name: Run random tests using avx2 + run: | + export SSE_AVX2=1 + make clean; make -j4 allall + export NUM_CORES=1 + make random-test + macos: + runs-on: macos-latest + steps: + - uses: actions/checkout@v2 + - name: Run random tests + run: | + make clean; make -j4 allall + export NUM_CORES=1 + make random-test diff --git a/.github/workflows/bowtie2.yml b/.github/workflows/simple-tests.yml index a64a8a3..b65054c 100644 --- a/.github/workflows/bowtie2.yml +++ b/.github/workflows/simple-tests.yml @@ -1,4 +1,4 @@ -name: Basic compile and test on Ubuntu and MacOS +name: Simple Tests on: push: branches: @@ -42,7 +42,17 @@ jobs: - uses: actions/checkout@v2 - name: Run simple tests run: | - make allall + make clean; make -j4 allall + make simple-test + linux-avx2: + runs-on: ubuntu-latest + needs: linux + steps: + - uses: actions/checkout@v2 + - name: Run simple tests using AVX2 + run: | + export SSE_AVX2=1 + make clean; make -j4 allall make simple-test macos: runs-on: macos-latest @@ -50,6 +60,5 @@ jobs: - uses: actions/checkout@v2 - name: Run simple tests run: | - make allall + make clean; make -j4 allall make simple-test - make random-test diff --git a/.gitmodules b/.gitmodules index 018b984..34085b9 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,6 @@ path = third_party/simde url = https://github.com/simd-everywhere/simde-no-tests.git branch = master +[submodule "third_party/libsais"] + path = third_party/libsais + url = https://github.com/ch4rr0/libsais diff --git a/BOWTIE2_VERSION b/BOWTIE2_VERSION index 26f8b8b..fad066f 100644 --- a/BOWTIE2_VERSION +++ b/BOWTIE2_VERSION @@ -1 +1 @@ -2.4.5
\ No newline at end of file +2.5.0
\ No newline at end of file @@ -99,9 +99,6 @@ Bowtie 2 is available from various package managers, notably Bioconda. With Bioconda installed, you should be able to install Bowtie 2 with conda install bowtie2. -Containerized versions of Bowtie 2 are also available via the -Biocontainers project (e.g. via Docker Hub). - You can also download Bowtie 2 sources and binaries from the Download section of the Sourceforge site. Binaries are available for the x86_64 architecture running Linux, Mac OS X, and Windows. FreeBSD users can @@ -1596,8 +1593,8 @@ by tabs; from left to right, the fields are: 6. CIGAR string representation of alignment 7. Name of reference sequence where mate's alignment occurs. Set to = - if the mate's reference sequence is the same as this alignment's, or - * if there is no mate. + if the mate's reference sequence is the same as this alignment's, + or * if there is no mate. 8. 1-based offset into the forward reference strand where leftmost character of the mate's alignment occurs. Offset is 0 if there is no diff --git a/MANUAL.markdown b/MANUAL.markdown index fcb5cf6..828d9e2 100644 --- a/MANUAL.markdown +++ b/MANUAL.markdown @@ -104,9 +104,6 @@ Bowtie 2 is available from various package managers, notably [Bioconda](https:// With Bioconda installed, you should be able to install Bowtie 2 with `conda install bowtie2`. -Containerized versions of Bowtie 2 are also available via the [Biocontainers](https://BioContainers.pro) -project (e.g. [via Docker Hub](https://hub.docker.com/r/biocontainers/bowtie2/)). - You can also download Bowtie 2 sources and binaries from the [Download] section of the Sourceforge site. Binaries are available for the `x86_64` architecture running Linux, Mac OS X, and Windows. @@ -64,7 +64,12 @@ endif BITS := 32 ifneq (,$(findstring $(shell uname -m), x86_64 amd64)) BITS := 64 - SSE_FLAG := -msse2 + ifeq (1, $(SSE_AVX2)) + SSE_FLAG := -mavx2 -faligned-new -DSSE_AVX2 + else + SSE_FLAG := -msse2 + endif + POPCNT_CAPABILITY ?= 1 else ifneq (,$(findstring $(shell uname -m), aarch64 arm64 s390x ppc64 ppc64le)) BITS := 64 @@ -74,6 +79,13 @@ else ifneq (,$(findstring $(shell uname -m), aarch64 arm64 s390x ppc64 ppc64le)) POPCNT_CAPABILITY ?= 0 endif +ifdef USE_SAIS + CPPFLAGS += -Ithird_party/libsais/include + CXXFLAGS += -fopenmp -DUSE_SAIS + LDFLAGS += -Lthird_party/libsais/lib + LDLIBS += -lsais +endif + # msys will always be 32 bit so look at the cpu arch instead. ifneq (,$(findstring AMD64,$(PROCESSOR_ARCHITEW6432))) ifeq (1,$(MINGW)) @@ -176,6 +188,7 @@ SEARCH_CPPS := qual.cpp pat.cpp sam.cpp \ ifeq (1, $(WITH_ZSTD)) LDLIBS += -lzstd + CXXFLAGS += -DWITH_ZSTD SHARED_CPPS += zstd_decompress.cpp endif @@ -305,7 +318,6 @@ endif # # bowtie2-build targets # - bowtie2-build-s-sanitized bowtie2-build-s: bt2_build.cpp $(SHARED_CPPS) $(HEADERS) $(CXX) $(RELEASE_FLAGS) $(RELEASE_DEFS) $(CXXFLAGS) \ $(DEFS) -DBOWTIE2 $(NOASSERT_FLAGS) -Wall \ @@ -544,16 +556,16 @@ static-libs: cd $(CURDIR)/.tmp ; \ DL=$$( ( which wget >/dev/null 2>&1 && echo "wget --no-check-certificate" ) || echo "curl -LOk") ; \ if [ ! -f "$(CURDIR)/.tmp/include/zlib.h" ] ; then \ - $$DL https://zlib.net/zlib-1.2.11.tar.gz && tar xzf zlib-1.2.11.tar.gz && cd zlib-1.2.11 ; \ + $$DL https://zlib.net/zlib-1.2.13.tar.gz && tar xzf zlib-1.2.13.tar.gz && cd zlib-1.2.13 ; \ $(if $(MINGW), mingw32-make -f win32/Makefile.gcc, ./configure --static && make) ; \ cp zlib.h zconf.h $(CURDIR)/.tmp/include && cp libz.a $(CURDIR)/.tmp/lib ; \ - rm -f zlib-1.2.11 ; \ + rm -f zlib-1.2.13 ; \ fi ; \ if [ ! -f "$(CURDIR)/.tmp/include/zstd.h" ]; then \ cd $(CURDIR)/.tmp ; \ - $$DL https://github.com/facebook/zstd/releases/download/v1.4.9/zstd-1.4.9.tar.gz && tar xzf zstd-1.4.9.tar.gz ; \ - cd zstd-1.4.9 && $(MAKE) ; \ - cd $(CURDIR)/.tmp/zstd-1.4.9/lib && cp zstd.h $(CURDIR)/.tmp/include && cp libzstd.a $(CURDIR)/.tmp/lib ; \ + $$DL https://github.com/facebook/zstd/releases/download/v1.5.1/zstd-1.5.1.tar.gz && tar xzf zstd-1.5.1.tar.gz ; \ + cd zstd-1.5.1 && $(MAKE) lib ; \ + cd $(CURDIR)/.tmp/zstd-1.5.1/lib && cp zstd.h $(CURDIR)/.tmp/include && cp libzstd.a $(CURDIR)/.tmp/lib ; \ fi .PHONY: sra-deps @@ -4,7 +4,7 @@ Bowtie 2 NEWS Bowtie 2 is available for download from the project website, http://bowtie-bio.sf.net/bowtie2 and on Github, https://github.com/BenLangmead/bowtie2/releases. 2.0.0-beta1 is -the first version released to the public and 2.4.5 is the latest +the first version released to the public and 2.5.0 is the latest version. Bowtie 2 is licensed under the GPLv3 license. See `LICENSE' file for details. @@ -19,6 +19,18 @@ Please report any issues to the Bowtie 2 Github page or using the Sourceforge bu Version Release History ======================= +## Version 2.5.0 - Oct 31, 2022 ## + +### bowtie2 ### + * Overall improvements in the use of prefetch instructions. (contribution by Igor Sfiligoi) + * Made input/output fully asynchronous by using a dedicated thread. (contribution by Igor Sfiligoi) + * Added support for AVX2 256-bit instructions with can be enabled by + setting the `SSE_AXV2` environment variable at compile time. (contribution by Igor Sfiligoi) + * Fixed an issue causing `bowtie2` to crash when processing ZSTD + files with high compression ratios. + * Changed the way that unique alignments are counted in summary + message to better match up with filters on SAM output + ## Version 2.4.5 - Jan 15, 2022 ## ### bowtie2 ### @@ -1,7 +1,8 @@ <!-- badges: start --> - -[](https://shields.io/) + + +[](https://shields.io/) <!-- [](https://travis-ci.org/BenLangmead/bowtie2) --> [](https://www.gnu.org/licenses/gpl-3.0) <!--badges: end --> diff --git a/aligner_bt.cpp b/aligner_bt.cpp index 64ae15f..26cd0c2 100644 --- a/aligner_bt.cpp +++ b/aligner_bt.cpp @@ -19,6 +19,7 @@ #include "aligner_bt.h" #include "mask.h" +#include "aligner_swsse.h" using namespace std; @@ -626,7 +627,7 @@ void BtBranchTracer::squareFill( const TAlScore sc_rfe = prob_.sc_->refGapExtend(); const bool local = !prob_.sc_->monotone; const CpQuad *qup = NULL; - const __m128i *qlf = NULL; + const SSERegI *qlf = NULL; size_t per = prob_.cper_->per_; ASSERT_ONLY(size_t nrow = prob_.cper_->nrow()); size_t ncol = prob_.cper_->ncol(); @@ -637,7 +638,7 @@ void BtBranchTracer::squareFill( qup = prob_.cper_->qrows_.ptr() + (ncol * (ydiv-1)) + xdiv * per; } if(!left) { - // Set up the column pointers to point to the first __m128i word in the + // Set up the column pointers to point to the first SSERegI word in the // relevant column size_t off = (niter << 2) * (xdiv-1); qlf = prob_.cper_->qcols_.ptr() + off; @@ -647,8 +648,8 @@ void BtBranchTracer::squareFill( size_t xi = xedge, yi = yedge; // iterators for columns, rows size_t ii = 0; // iterator into packed square // Iterate over rows, then over columns - size_t m128mod = yi % prob_.cper_->niter_; - size_t m128div = yi / prob_.cper_->niter_; + size_t regmod = yi % prob_.cper_->niter_; + size_t regdiv = yi / prob_.cper_->niter_; int16_t sc_h_dg_lastrow = MIN_I16; for(size_t i = 0; i <= ymod; i++, yi++) { assert_lt(yi, nrow); @@ -700,7 +701,7 @@ void BtBranchTracer::squareFill( if(j == 0 && xi > 0) { // Get values for left neighbors from the checkpoint if(is8_) { - size_t vecoff = (m128mod << 6) + m128div; + size_t vecoff = (regmod << (ROWSTRIDE_LOG2+BYTES_LOG2_PER_REG)) + regdiv; sc_e_lf = ((uint8_t*)(qlf + 0))[vecoff]; sc_h_lf = ((uint8_t*)(qlf + 2))[vecoff]; if(local) { @@ -712,7 +713,7 @@ void BtBranchTracer::squareFill( else sc_e_lf -= 0xff; } } else { - size_t vecoff = (m128mod << 5) + m128div; + size_t vecoff = (regmod << (ROWSTRIDE_LOG2+BYTES_LOG2_PER_REG-1)) + regdiv; sc_e_lf = ((int16_t*)(qlf + 0))[vecoff]; sc_h_lf = ((int16_t*)(qlf + 2))[vecoff]; if(local) { @@ -856,11 +857,11 @@ void BtBranchTracer::squareFill( sc_h_lf_last = sc_best; sc_e_lf_last = sc_e_best; } - // Update m128mod, m128div - m128mod++; - if(m128mod == prob_.cper_->niter_) { - m128mod = 0; - m128div++; + // Update regmod, regdiv + regmod++; + if(regmod == prob_.cper_->niter_) { + regmod = 0; + regdiv++; } // update qup ii += sq_ncol; diff --git a/aligner_cache.h b/aligner_cache.h index 7b61c1c..23dd28c 100644 --- a/aligner_cache.h +++ b/aligner_cache.h @@ -65,6 +65,39 @@ #define CACHE_PAGE_SZ (16 * 1024) +class BwtTopBot { +public: + BwtTopBot() : topf(0), botf(0), topb(0), botb(0) {} + + BwtTopBot( + TIndexOffU _topf, // top in BWT + TIndexOffU _botf, // bot in BWT + TIndexOffU _topb, // top in BWT' + TIndexOffU _botb) // bot in BWT' + : topf(_topf) + , botf(_botf) + , topb(_topb) + , botb(_botb) + {} + + void set( + TIndexOffU _topf, // top in BWT + TIndexOffU _botf, // bot in BWT + TIndexOffU _topb, // top in BWT' + TIndexOffU _botb) // bot in BWT' + { + topf = _topf; + botf = _botf; + topb = _topb; + botb = _botb; + } + + TIndexOffU topf; // top in BWT + TIndexOffU botf; // bot in BWT + TIndexOffU topb; // top in BWT' + TIndexOffU botb; // bot in BWT' +}; + typedef PListSlice<TIndexOffU, CACHE_PAGE_SZ> TSlice; /** @@ -753,8 +786,6 @@ public: * Returns: * -1 if out of memory * 0 if key was found in cache - * 1 if key was not found in cache (and there's enough memory to - * add a new key) */ int beginAlign( const BTDnaString& seq, @@ -764,13 +795,6 @@ public: { assert(repOk()); qk_.init(seq ASSERT_ONLY(, tmpdnastr_)); - //if(qk_.cacheable() && (qv_ = current_->query(qk_, getLock)) != NULL) { - // // qv_ holds the answer - // assert(qv_->valid()); - // qv = *qv_; - // resetRead(); - // return 1; // found in cache - //} else if(qk_.cacheable()) { // Make a QNode for this key and possibly add the QNode to the // Red-Black map; but if 'seq' isn't cacheable, just create the @@ -865,7 +889,7 @@ public: * compiled for the current read in the local cache. */ bool addOnTheFly( - const BTDnaString& rfseq, // reference sequence close to read seq + const SAKey& sak, // the key holding the reference substring TIndexOffU topf, // top in BWT index TIndexOffU botf, // bot in BWT index TIndexOffU topb, // top in BWT' index @@ -875,8 +899,6 @@ public: assert(aligning()); assert(repOk()); - ASSERT_ONLY(BTDnaString tmp); - SAKey sak(rfseq ASSERT_ONLY(, tmp)); //assert(sak.cacheable()); if(current_->addOnTheFly((*qv_), sak, topf, botf, topb, botb, getLock)) { rangen_++; @@ -886,6 +908,20 @@ public: return false; } + bool addOnTheFly( + const BTDnaString& rfseq, // reference sequence close to read seq + TIndexOffU topf, // top in BWT index + TIndexOffU botf, // bot in BWT index + TIndexOffU topb, // top in BWT' index + TIndexOffU botb, // bot in BWT' index + bool getLock = true) // true -> lock is not held by caller + { + + ASSERT_ONLY(BTDnaString tmp); + SAKey sak(rfseq ASSERT_ONLY(, tmp)); + return addOnTheFly(sak, topf, botf, topb, botb, getLock); + } + /** * Given a QVal, populate the given EList of SATuples with records * describing all of the cached information about the QVal's diff --git a/aligner_seed.cpp b/aligner_seed.cpp index 1f18a82..fa1cc90 100644 --- a/aligner_seed.cpp +++ b/aligner_seed.cpp @@ -17,6 +17,7 @@ * along with Bowtie 2. If not, see <http://www.gnu.org/licenses/>. */ +#include <array> #include "aligner_cache.h" #include "aligner_seed.h" #include "search_globals.h" @@ -75,6 +76,131 @@ Constraint Constraint::editBased(int edits) { return c; } +// Input to seachSeedBi +class SeedAligner::SeedAlignerSearchParams { +public: + class CacheAndSeed { + public: + CacheAndSeed( + SeedSearchCache &_cache, // local seed alignment cache + const InstantiatedSeed& _seed, // current instantiated seed + const Ebwt* ebwtFw, // forward index (BWT) + const Ebwt* ebwtBw // backward/mirror index (BWT') + + ) : cache(_cache) + , seed(_seed) + , hasi0(false), fwi0(0), bwi0(0) // just set a default + { + int off = seed.steps[0]; + bool ltr = off > 0; + off = abs(off)-1; + int ftabLen = ebwtFw->eh().ftabChars(); + hasi0 = (ftabLen > 1 && ftabLen <= seed.maxjump); + if(hasi0) { + if(!ltr) { + assert_geq(off+1, ftabLen-1); + off = off - ftabLen + 1; + } + // startSearchSeedBi will need them, start prefetching now + fwi0 = ebwtFw->ftabSeqToInt( cache.getSeq(), off, false); + ebwtFw->ftabLoHiPrefetch(fwi0); + if(ebwtBw!=NULL) { + bwi0 = ebwtBw->ftabSeqToInt( cache.getSeq(), off, false); + ebwtBw->ftabLoHiPrefetch(bwi0); + } + } + } + + CacheAndSeed(CacheAndSeed &other) = default; + CacheAndSeed(CacheAndSeed &&other) = default; + + SeedSearchCache &cache; // local seed alignment cache + const InstantiatedSeed& seed; // current instantiated seed + bool hasi0; + TIndexOffU fwi0; // Idx of fw ftab + TIndexOffU bwi0; // Idx of bw ftab + }; + + SeedAlignerSearchParams( + CacheAndSeed &_cs, + const int _step, // depth into steps[] array + const int _depth, // recursion depth + const BwtTopBot &_bwt, // The 4 BWT idxs + const SideLocus &_tloc, // locus for top (perhaps unititialized) + const SideLocus &_bloc, // locus for bot (perhaps unititialized) + const std::array<Constraint,3> _cv, // constraints to enforce in seed zones + const Constraint &_overall, // overall constraints to enforce + DoublyLinkedList<Edit> *_prevEdit) // previous edit + : cs(_cs) + , step(_step) + , depth(_depth) + , bwt(_bwt) + , tloc(_tloc) + , bloc(_bloc) + , cv(_cv) + , overall(_overall) + , prevEdit(_prevEdit) + {} + + SeedAlignerSearchParams( + CacheAndSeed &_cs, + const int _step, // depth into steps[] array + const int _depth, // recursion depth + const BwtTopBot &_bwt, // The 4 BWT idxs + const SideLocus &_tloc, // locus for top (perhaps unititialized) + const SideLocus &_bloc, // locus for bot (perhaps unititialized) + const Constraint &_c0, // constraints to enforce in seed zone 0 + const Constraint &_c1, // constraints to enforce in seed zone 1 + const Constraint &_c2, // constraints to enforce in seed zone 2 + const Constraint &_overall, // overall constraints to enforce + DoublyLinkedList<Edit> *_prevEdit) // previous edit + : cs(_cs) + , step(_step) + , depth(_depth) + , bwt(_bwt) + , tloc(_tloc) + , bloc(_bloc) + , cv{ _c0, _c1, _c2 } + , overall(_overall) + , prevEdit(_prevEdit) + {} + + // create an empty bwt, tloc and bloc, with step=0 + // and constratins from seed, for initial searchSeedBi invocation + SeedAlignerSearchParams( + SeedSearchCache &cache, // local seed alignment cache + const InstantiatedSeed& seed, // current instantiated seed + const Ebwt* ebwtFw, // forward index (BWT) + const Ebwt* ebwtBw) // backward/mirror index (BWT') + : cs(cache, seed, ebwtFw, ebwtBw) + , step(0) + , depth(0) + , bwt() + , tloc() + , bloc() + , cv{ seed.cons[0], seed.cons[1], seed.cons[2] } + , overall(seed.overall) + , prevEdit(NULL) + {} + + void checkCV() const { + assert(cv[0].acceptable()); + assert(cv[1].acceptable()); + assert(cv[2].acceptable()); + } + + CacheAndSeed cs; // local seed alignment cache and associated instatiated seed + int step; // depth into steps[] array + int depth; // recursion depth + BwtTopBot bwt; // The 4 BWT idxs + SideLocus tloc; // locus for top (perhaps unititialized) + SideLocus bloc; // locus for bot (perhaps unititialized) + std::array<Constraint,3> cv; // constraints to enforce in seed zones + Constraint overall; // overall constraints to enforce + DoublyLinkedList<Edit> *prevEdit; // previous edit +}; + + // // Some static methods for constructing some standard SeedPolicies // @@ -487,74 +613,89 @@ void SeedAligner::searchAllSeeds( ebwtBw_ = ebwtBw; sc_ = &pens; read_ = &read; - ca_ = &cache; bwops_ = bwedits_ = 0; uint64_t possearches = 0, seedsearches = 0, intrahits = 0, interhits = 0, ooms = 0; - // For each instantiated seed - for(int i = 0; i < (int)sr.numOffs(); i++) { - size_t off = sr.idx2off(i); - for(int fwi = 0; fwi < 2; fwi++) { - bool fw = (fwi == 0); + + /** + * TODO: Define is somewhere else + * Note: The ideal may be dependent on the CPU model, but 8 seems to work fine. + * 2 is too small for prefetch to be fully effective, 4 seems already OK, + * and 32 is too big (cache trashing). + **/ + const int ibatch_size = 8; + + SeedSearchMultiCache mcache; + std::vector<SeedAlignerSearchParams> paramVec; + + mcache.reserve(ibatch_size); + paramVec.reserve(ibatch_size*16); // assume no more than 16 iss per cache, on average + + for(int fwi = 0; fwi < 2; fwi++) { + const bool fw = (fwi == 0); + int i =0; + // For each instantiated seed, but batched + while (i < (int)sr.numOffs()) { + const int ibatch_max = std::min(i+ibatch_size,(int)sr.numOffs()); + mcache.clear(); + paramVec.clear(); + // start aligning and find list of seeds to search + for(; i < ibatch_max; i++) { assert(sr.repOk(&cache.current())); EList<InstantiatedSeed>& iss = sr.instantiatedSeeds(fw, i); if(iss.empty()) { // Cache hit in an across-read cache continue; } - QVal qv; - seq_ = &sr.seqs(fw)[i]; // seed sequence - qual_ = &sr.quals(fw)[i]; // seed qualities - off_ = off; // seed offset (from 5') - fw_ = fw; // seed orientation - // Tell the cache that we've started aligning, so the cache can - // expect a series of on-the-fly updates - int ret = cache.beginAlign(*seq_, *qual_, qv); - ASSERT_ONLY(hits_.clear()); - if(ret == -1) { - // Out of memory when we tried to add key to map - ooms++; - continue; - } - bool abort = false; - if(ret == 0) { - // Not already in cache - assert(cache.aligning()); + const BTDnaString& seq = sr.seqs(fw)[i]; // seed sequence + const BTString& qual = sr.quals(fw)[i]; // seed qualities + mcache.emplace_back(seq, qual, i, fw); + const size_t mnr = mcache.size()-1; + SeedSearchCache &srcache = mcache[mnr]; + { possearches++; for(size_t j = 0; j < iss.size(); j++) { - // Set seq_ and qual_ appropriately, using the seed sequences + // Set seq and qual appropriately, using the seed sequences // and qualities already installed in SeedResults assert_eq(fw, iss[j].fw); assert_eq(i, (int)iss[j].seedoffidx); - s_ = &iss[j]; - // Do the search with respect to seq_, qual_ and s_. - if(!searchSeedBi()) { - // Memory exhausted during search - ooms++; - abort = true; - break; - } + paramVec.emplace_back(srcache, iss[j], ebwtFw_, ebwtBw_); seedsearches++; - assert(cache.aligning()); } - if(!abort) { - qv = cache.finishAlign(); - } - } else { - // Already in cache - assert_eq(1, ret); - assert(qv.valid()); - intrahits++; } - assert(abort || !cache.aligning()); - if(qv.valid()) { + } // internal i (batch) loop + + // do the searches + if (!paramVec.empty()) searchSeedBi(paramVec.size(), &(paramVec[0])); + + // finish aligning and add to SeedResult + for (size_t mnr=0; mnr<mcache.size(); mnr++) { + SeedSearchCache &srcache = mcache[mnr]; + // Tell the cache that we've started aligning, so the cache can + // expect a series of on-the-fly updates + int ret = srcache.beginAlign(cache); + if(ret == -1) { + // Out of memory when we tried to add key to map + ooms++; + continue; + } + assert(srcache.aligning()); + if(!srcache.addAllCached()){ + // Memory exhausted during copy + ooms++; + continue; + } + srcache.finishAlign(); + assert(!srcache.aligning()); + if(srcache.qvValid()) { sr.add( - qv, // range of ranges in cache + srcache.getQv(), // range of ranges in cache cache.current(), // cache - i, // seed index (from 5' end) - fw); // whether seed is from forward read + mcache.getSeedOffIdx(mnr), // seed index (from 5' end) + mcache.getFw(mnr)); // whether seed is from forward read } - } - } + } // mnr loop + } // external i while + } // for fwi prm.nSeedRanges = sr.numRanges(); prm.nSeedElts = sr.numElts(); prm.nSeedRangesFw = sr.numRangesFw(); @@ -606,6 +747,104 @@ bool SeedAligner::sanityPartial( return true; } +inline void exactSweepInit( + const Ebwt& ebwt, + const BTDnaString& seq, + const int ftabLen, + const size_t len, + size_t &dep, + TIndexOffU &top, + TIndexOffU &bot + ) +{ + top = bot = 0; + + const size_t left = len - dep; + assert_gt(left, 0); + bool doFtab = ftabLen > 1 && left >= (size_t)ftabLen; + if(doFtab) { + const size_t endi = len-dep-1; + // Does N interfere with use of Ftab? + for(size_t i = 0; i < (size_t)ftabLen; i++) { + int c = seq[endi-i]; + if(c > 3) { + doFtab = false; + break; + } + } + } + if(doFtab) { + // Use ftab + ebwt.ftabLoHi(seq, left - ftabLen, false, top, bot); + dep += (size_t)ftabLen; + } else { + // Use fchr + int c = seq[len-dep-1]; + if(c < 4) { + top = ebwt.fchr()[c]; + bot = ebwt.fchr()[c+1]; + } + dep++; + } +} + +inline void exactSweepMapLF( + const Ebwt& ebwt, + const BTDnaString& seq, + const size_t len, + const size_t dep, + const SideLocus &tloc, + const SideLocus &bloc, + TIndexOffU &top, + TIndexOffU &bot, + uint64_t &bwops // Burrows-Wheeler operations +) +{ + int c = seq[len-dep-1]; + if(c > 3) { + top = bot = 0; + } else { + if(bloc.valid()) { + bwops += 2; + top = ebwt.mapLF(tloc, c); + bot = ebwt.mapLF(bloc, c); + } else { + bwops++; + top = ebwt.mapLF1(top, tloc, c); + if(top == OFF_MASK) { + top = bot = 0; + } else { + bot = top+1; + } + } + } +} + + +inline bool exactSweepStep( + const Ebwt& ebwt, // BWT index + const TIndexOffU top, + const TIndexOffU bot, + const size_t mineMax, // don't care about edit bounds > this + SideLocus &tloc, + SideLocus &bloc, + size_t &mineCnt, // minimum # edits + size_t &nedit, + bool &done + ) +{ + if(bot <= top) { + nedit++; + if(nedit >= mineMax) { + mineCnt = nedit; + done = true; + } + return true; + } + INIT_LOCS(top, bot, tloc, bloc, ebwt); + return false; +} + /** * Sweep right-to-left and left-to-right using exact matching. Remember all * the SA ranges encountered along the way. Report exact matches if there are @@ -626,112 +865,103 @@ size_t SeedAligner::exactSweep( SeedSearchMetrics& met) // metrics { assert_gt(mineMax, 0); - TIndexOffU top = 0, bot = 0; - SideLocus tloc, bloc; const size_t len = read.length(); + const int ftabLen = ebwt.eh().ftabChars(); + size_t nelt = 0; + + std::array<SideLocus,2> tloc; + std::array<SideLocus,2> bloc; + TIndexOffU top[2] = {0, 0}; + TIndexOffU bot[2] = {0, 0}; + + size_t dep[2] = {0, 0}; + size_t nedit[2] = {0, 0}; + bool doInit[2] = {true, true}; + + size_t prefetch_count = 0; + bool done[2] = {nofw, norc}; + for(int fwi = 0; fwi < 2; fwi++) { - bool fw = (fwi == 0); - if( fw && nofw) continue; - if(!fw && norc) continue; - const BTDnaString& seq = fw ? read.patFw : read.patRc; - assert(!seq.empty()); - int ftabLen = ebwt.eh().ftabChars(); - size_t dep = 0; - size_t nedit = 0; - bool done = false; - while(dep < len && !done) { - top = bot = 0; - size_t left = len - dep; - assert_gt(left, 0); - bool doFtab = ftabLen > 1 && left >= (size_t)ftabLen; - if(doFtab) { - // Does N interfere with use of Ftab? - for(size_t i = 0; i < (size_t)ftabLen; i++) { - int c = seq[len-dep-1-i]; - if(c > 3) { - doFtab = false; - break; + if (!done[fwi]) { + bool fw = (fwi == 0); + const BTDnaString& seq = fw ? read.patFw : read.patRc; + assert(!seq.empty()); + __builtin_prefetch(&(seq[len-1])); + if (len>48) __builtin_prefetch(&(seq[len-49])); // HW prefetch prediction assumes forward, help it + } + } + + while( (dep[0] < len && !done[0]) || (dep[1] < len && !done[1]) ) { + prefetch_count++; + if (prefetch_count>=48) { // cache line is 64 bytes, but we may skip some deps + for(int fwi = 0; fwi < 2; fwi++) { + if (dep[fwi] < len && !done[fwi]) { + bool fw = (fwi == 0); + const BTDnaString& seq = fw ? read.patFw : read.patRc; + const size_t left = len-dep[fwi]; + if (left>48) { + __builtin_prefetch(&(seq[left-49])); // HW prefetch prediction assumes forward, help it } } } - if(doFtab) { - // Use ftab - ebwt.ftabLoHi(seq, len - dep - ftabLen, false, top, bot); - dep += (size_t)ftabLen; - } else { - // Use fchr - int c = seq[len-dep-1]; - if(c < 4) { - top = ebwt.fchr()[c]; - bot = ebwt.fchr()[c+1]; - } - dep++; - } - if(bot <= top) { - nedit++; - if(nedit >= mineMax) { - if(fw) { mineFw = nedit; } else { mineRc = nedit; } - break; - } - continue; - } - INIT_LOCS(top, bot, tloc, bloc, ebwt); - // Keep going - while(dep < len) { - int c = seq[len-dep-1]; - if(c > 3) { - top = bot = 0; - } else { - if(bloc.valid()) { - bwops_ += 2; - top = ebwt.mapLF(tloc, c); - bot = ebwt.mapLF(bloc, c); - } else { - bwops_++; - top = ebwt.mapLF1(top, tloc, c); - if(top == OFF_MASK) { - top = bot = 0; - } else { - bot = top+1; - } + prefetch_count=0; + } + // by doing both fw in the internal loop, I give the prefetch in exactSweepStep to be effective + for(int fwi = 0; fwi < 2; fwi++) { + if (dep[fwi] < len && !done[fwi]) { + bool fw = (fwi == 0); + const BTDnaString& seq = fw ? read.patFw : read.patRc; + + if (doInit[fwi]) { + exactSweepInit(ebwt, seq, ftabLen, len, // in + dep[fwi], top[fwi], bot[fwi]); // out + if ( exactSweepStep(ebwt, top[fwi], bot[fwi], mineMax, + tloc[fwi], bloc[fwi], + fw ? mineFw : mineRc, + nedit[fwi], done[fwi]) ) { + continue; } + doInit[fwi]=false; } - if(bot <= top) { - nedit++; - if(nedit >= mineMax) { - if(fw) { mineFw = nedit; } else { mineRc = nedit; } - done = true; + + if (dep[fwi]< len) { + exactSweepMapLF(ebwt, seq, len, dep[fwi], tloc[fwi], bloc[fwi], + top[fwi], bot[fwi], bwops_); + + if ( exactSweepStep(ebwt, top[fwi], bot[fwi], mineMax, + tloc[fwi], bloc[fwi], + fw ? mineFw : mineRc, + nedit[fwi], done[fwi]) ) { + doInit[fwi]=true; } - break; + dep[fwi]++; } - INIT_LOCS(top, bot, tloc, bloc, ebwt); - dep++; } - if(done) { - break; - } - if(dep == len) { - // Set the minimum # edits - if(fw) { mineFw = nedit; } else { mineRc = nedit; } - // Done - if(nedit == 0 && bot > top) { - if(repex) { - // This is an exact hit - int64_t score = len * sc.match(); - if(fw) { - hits.addExactEeFw(top, bot, NULL, NULL, fw, score); - assert(ebwt.contains(seq, NULL, NULL)); - } else { - hits.addExactEeRc(top, bot, NULL, NULL, fw, score); - assert(ebwt.contains(seq, NULL, NULL)); - } + } + } + + for(int fwi = 0; fwi < 2; fwi++) { + if( (!done[fwi]) && (dep[fwi] >= len) ) { + const bool fw = (fwi == 0); + + // Set the minimum # edits + if(fw) { mineFw = nedit[fwi]; } else { mineRc = nedit[fwi]; } + // Done + if(nedit[fwi] == 0 && bot[fwi] > top[fwi]) { + if(repex) { + // This is an exact hit + int64_t score = len * sc.match(); + if(fw) { + hits.addExactEeFw(top[fwi], bot[fwi], NULL, NULL, fw, score); + assert(ebwt.contains(fw ? read.patFw : read.patRc, NULL, NULL)); + } else { + hits.addExactEeRc(top[fwi], bot[fwi], NULL, NULL, fw, score); + assert(ebwt.contains(fw ? read.patFw : read.patRc, NULL, NULL)); } - nelt += (bot - top); } - break; + nelt += (bot[fwi] - top[fwi]); } - dep++; } } return nelt; @@ -1092,21 +1322,9 @@ bool SeedAligner::oneMmSearch( return results; } -/** - * Wrapper for initial invcation of searchSeed. - */ -bool -SeedAligner::searchSeedBi() { - return searchSeedBi( - 0, 0, - 0, 0, 0, 0, - SideLocus(), SideLocus(), - s_->cons[0], s_->cons[1], s_->cons[2], s_->overall, - NULL); -} - inline void SeedAligner::prefetchNextLocsBi( + const InstantiatedSeed& seed, // current instantiated seed TIndexOffU topf, // top in BWT TIndexOffU botf, // bot in BWT TIndexOffU topb, // top in BWT' @@ -1114,9 +1332,9 @@ SeedAligner::prefetchNextLocsBi( int step // step to get ready for ) { - if(step == (int)s_->steps.size()) return; // no more steps! + if(step == (int)seed.steps.size()) return; // no more steps! // Which direction are we going in next? - if(s_->steps[step] > 0) { + if(seed.steps[step] > 0) { // Left to right; use BWT' if(botb - topb == 1) { // Already down to 1 row; just init top locus @@ -1145,8 +1363,9 @@ SeedAligner::prefetchNextLocsBi( */ inline void SeedAligner::nextLocsBi( - SideLocus& tloc, // top locus - SideLocus& bloc, // bot locus + const InstantiatedSeed& seed, // current instantiated seed + SideLocus& tloc, // top locus + SideLocus& bloc, // bot locus TIndexOffU topf, // top in BWT TIndexOffU botf, // bot in BWT TIndexOffU topb, // top in BWT' @@ -1162,9 +1381,9 @@ SeedAligner::nextLocsBi( assert(ebwtBw_ == NULL || botb > 0); assert_geq(step, 0); // next step can't be first one assert(ebwtBw_ == NULL || botf-topf == botb-topb); - if(step == (int)s_->steps.size()) return; // no more steps! + if(step == (int)seed.steps.size()) return; // no more steps! // Which direction are we going in next? - if(s_->steps[step] > 0) { + if(seed.steps[step] > 0) { // Left to right; use BWT' if(botb - topb == 1) { // Already down to 1 row; just init top locus @@ -1194,8 +1413,8 @@ SeedAligner::nextLocsBi( // Inherit state from the predecessor ot = *prevOt; } - bool ltr = s_->steps[step-1] > 0; - int adj = abs(s_->steps[step-1])-1; + bool ltr = seed.steps[step-1] > 0; + int adj = abs(seed.steps[step-1])-1; const Ebwt* ebwt = ltr ? ebwtBw_ : ebwtFw_; ot.update( ltr ? topb : topf, // top @@ -1219,8 +1438,11 @@ SeedAligner::nextLocsBi( * we're done, which actually adds the hit to the cache. Returns result from * calling reportHit(). */ -bool +void SeedAligner::extendAndReportHit( + SeedSearchCache &cache, // local seed alignment cache + size_t off, // offset of seed currently being searched + bool fw, // orientation of seed currently being searched TIndexOffU topf, // top in BWT TIndexOffU botf, // bot in BWT TIndexOffU topb, // top in BWT' @@ -1232,11 +1454,11 @@ SeedAligner::extendAndReportHit( TIndexOffU t[4], b[4]; TIndexOffU tp[4], bp[4]; SideLocus tloc, bloc; - if(off_ > 0) { + if(off > 0) { const Ebwt *ebwt = ebwtFw_; assert(ebwt != NULL); // Extend left using forward index - const BTDnaString& seq = fw_ ? read_->patFw : read_->patRc; + const BTDnaString& seq = fw ? read_->patFw : read_->patRc; // See what we get by extending TIndexOffU top = topf, bot = botf; t[0] = t[1] = t[2] = t[3] = 0; @@ -1245,7 +1467,7 @@ SeedAligner::extendAndReportHit( bp[0] = bp[1] = bp[2] = bp[3] = botb; SideLocus tloc, bloc; INIT_LOCS(top, bot, tloc, bloc, *ebwt); - for(size_t ii = off_; ii > 0; ii--) { + for(size_t ii = off; ii > 0; ii--) { size_t i = ii-1; // Get char from read int rdc = seq.get(i); @@ -1287,12 +1509,12 @@ SeedAligner::extendAndReportHit( } } size_t rdlen = read_->length(); - size_t nright = rdlen - off_ - len; + size_t nright = rdlen - off - len; if(nright > 0 && ebwtBw_ != NULL) { const Ebwt *ebwt = ebwtBw_; assert(ebwt != NULL); // Extend right using backward index - const BTDnaString& seq = fw_ ? read_->patFw : read_->patRc; + const BTDnaString& seq = fw ? read_->patFw : read_->patRc; // See what we get by extending TIndexOffU top = topb, bot = botb; t[0] = t[1] = t[2] = t[3] = 0; @@ -1300,7 +1522,7 @@ SeedAligner::extendAndReportHit( tp[0] = tp[1] = tp[2] = tp[3] = topb; bp[0] = bp[1] = bp[2] = bp[3] = botb; INIT_LOCS(top, bot, tloc, bloc, *ebwt); - for(size_t i = off_ + len; i < rdlen; i++) { + for(size_t i = off + len; i < rdlen; i++) { // Get char from read int rdc = seq.get(i); // See what we get by extending @@ -1341,17 +1563,18 @@ SeedAligner::extendAndReportHit( } } assert_lt(nlex, rdlen); - assert_leq(nlex, off_); + assert_leq(nlex, off); assert_lt(nrex, rdlen); - return reportHit(topf, botf, topb, botb, len, prevEdit); + reportHit(cache, topf, botf, topb, botb, len, prevEdit); } /** * Report a seed hit found by searchSeedBi() by adding it to the cache. Return * false if the hit could not be reported because of, e.g., cache exhaustion. */ -bool +void SeedAligner::reportHit( + SeedSearchCache &cache, // local seed alignment cache TIndexOffU topf, // top in BWT TIndexOffU botf, // bot in BWT TIndexOffU topb, // top in BWT' @@ -1359,6 +1582,8 @@ SeedAligner::reportHit( uint16_t len, // length of hit DoublyLinkedList<Edit> *prevEdit) // previous edit { + const BTDnaString& seq = cache.getSeq(); + // Add information about the seed hit to AlignmentCache. This // information eventually makes its way back to the SeedResults // object when we call finishAlign(...). @@ -1368,21 +1593,20 @@ SeedAligner::reportHit( if(prevEdit != NULL) { prevEdit->toList(edits_); Edit::sort(edits_); - assert(Edit::repOk(edits_, *seq_)); - Edit::toRef(*seq_, edits_, rf); + assert(Edit::repOk(edits_, seq)); + Edit::toRef(seq, edits_, rf); } else { - rf = *seq_; + rf = seq; } + // Note: Disabled, as we now use memory cache // Sanity check: shouldn't add the same hit twice. If this // happens, it may be because our zone Constraints are not set up // properly and erroneously return true from acceptable() when they // should return false in some cases. - assert_eq(hits_.size(), ca_->curNumRanges()); - assert(hits_.insert(rf)); - if(!ca_->addOnTheFly(rf, topf, botf, topb, botb)) { - return false; - } - assert_eq(hits_.size(), ca_->curNumRanges()); + //assert_eq(hits_.size(), cache.curNumRanges()); + //assert(hits_.insert(rf)); + cache.addOnTheFly(rf, topf, botf, topb, botb); + //assert_eq(hits_.size(), cache.curNumRanges()); #ifndef NDEBUG // Sanity check that the topf/botf and topb/botb ranges really // correspond to the reference sequence aligned to @@ -1402,240 +1626,349 @@ SeedAligner::reportHit( } } #endif - return true; + return; } -/** - * Given a seed, search. Assumes zone 0 = no backtracking. - * - * Return a list of Seed hits. - * 1. Edits - * 2. Bidirectional BWT range(s) on either end - */ +// return true, if we are already done bool -SeedAligner::searchSeedBi( - int step, // depth into steps_[] array - int depth, // recursion depth - TIndexOffU topf, // top in BWT - TIndexOffU botf, // bot in BWT - TIndexOffU topb, // top in BWT' - TIndexOffU botb, // bot in BWT' - SideLocus tloc, // locus for top (perhaps unititialized) - SideLocus bloc, // locus for bot (perhaps unititialized) - Constraint c0, // constraints to enforce in seed zone 0 - Constraint c1, // constraints to enforce in seed zone 1 - Constraint c2, // constraints to enforce in seed zone 2 - Constraint overall, // overall constraints to enforce - DoublyLinkedList<Edit> *prevEdit // previous edit -#if 0 - , const SABWOffTrack* prevOt // prev off tracker (if tracking started) -#endif - ) +SeedAligner::startSearchSeedBi(SeedAligner::SeedAlignerSearchParams &p) { - assert(s_ != NULL); - const InstantiatedSeed& s = *s_; - assert_gt(s.steps.size(), 0); + SeedSearchCache &cache = p.cs.cache; + const InstantiatedSeed& seed = p.cs.seed; + const BTDnaString& seq = cache.getSeq(); + + assert_gt(seed.steps.size(), 0); assert(ebwtBw_ == NULL || ebwtBw_->eh().ftabChars() == ebwtFw_->eh().ftabChars()); #ifndef NDEBUG for(int i = 0; i < 4; i++) { assert(ebwtBw_ == NULL || ebwtBw_->fchr()[i] == ebwtFw_->fchr()[i]); } #endif - if(step == (int)s.steps.size()) { + if(p.step == (int)seed.steps.size()) { // Finished aligning seed - assert(c0.acceptable()); - assert(c1.acceptable()); - assert(c2.acceptable()); - if(!reportHit(topf, botf, topb, botb, seq_->length(), prevEdit)) { - return false; // Memory exhausted - } + p.checkCV(); + reportHit(cache, p.bwt, seq.length(), p.prevEdit); return true; } #ifndef NDEBUG - if(depth > 0) { - assert(botf - topf == 1 || bloc.valid()); - assert(botf - topf > 1 || !bloc.valid()); + if(p.depth > 0) { + assert(p.bwt.botf - p.bwt.topf == 1 || p.bloc.valid()); + assert(p.bwt.botf - p.bwt.topf > 1 || !p.bloc.valid()); } #endif - int off; - TIndexOffU tp[4], bp[4]; // dest BW ranges for "prime" index - if(step == 0) { + if(p.step == 0) { // Just starting - assert(prevEdit == NULL); - assert(!tloc.valid()); - assert(!bloc.valid()); - off = s.steps[0]; - bool ltr = off > 0; + assert(p.prevEdit == NULL); + assert(!p.tloc.valid()); + assert(!p.bloc.valid()); + int off = seed.steps[0]; off = abs(off)-1; // Check whether/how far we can jump using ftab or fchr int ftabLen = ebwtFw_->eh().ftabChars(); - if(ftabLen > 1 && ftabLen <= s.maxjump) { - if(!ltr) { - assert_geq(off+1, ftabLen-1); - off = off - ftabLen + 1; - } - ebwtFw_->ftabLoHi(*seq_, off, false, topf, botf); + if(p.cs.hasi0) { //if(ftabLen > 1 && ftabLen <= seed.maxjump) + ebwtFw_->ftabLoHi(p.cs.fwi0, p.bwt.topf, p.bwt.botf); #ifdef NDEBUG - if(botf - topf == 0) return true; + if(p.bwt.botf - p.bwt.topf == 0) return true; #endif #ifdef NDEBUG if(ebwtBw_ != NULL) { - topb = ebwtBw_->ftabHi(*seq_, off); - botb = topb + (botf-topf); + p.bwt.topb = ebwtBw_->ftabHi(p.cs.bwi0); + p.bwt.botb = p.bwt.topb + (p.bwt.botf-p.bwt.topf); } #else if(ebwtBw_ != NULL) { - ebwtBw_->ftabLoHi(*seq_, off, false, topb, botb); - assert_eq(botf-topf, botb-topb); + ebwtBw_->ftabLoHi(p.cs.bwi0, p.bwt.topb, p.bwt.botb); + assert_eq(p.bwt.botf-p.bwt.topf, p.bwt.botb-p.bwt.topb); } - if(botf - topf == 0) return true; + if(p.bwt.botf - p.bwt.topf == 0) return true; #endif - step += ftabLen; - } else if(s.maxjump > 0) { + p.step += ftabLen; + } else if(seed.maxjump > 0) { // Use fchr - int c = (*seq_)[off]; + int c = seq[off]; assert_range(0, 3, c); - topf = topb = ebwtFw_->fchr()[c]; - botf = botb = ebwtFw_->fchr()[c+1]; - if(botf - topf == 0) return true; - step++; + p.bwt.topf = p.bwt.topb = ebwtFw_->fchr()[c]; + p.bwt.botf = p.bwt.botb = ebwtFw_->fchr()[c+1]; + if(p.bwt.botf - p.bwt.topf == 0) return true; + p.step++; } else { - assert_eq(0, s.maxjump); - topf = topb = 0; - botf = botb = ebwtFw_->fchr()[4]; + assert_eq(0, seed.maxjump); + p.bwt.topf = p.bwt.topb = 0; + p.bwt.botf = p.bwt.botb = ebwtFw_->fchr()[4]; } - if(step == (int)s.steps.size()) { + if(p.step == (int)seed.steps.size()) { // Finished aligning seed - assert(c0.acceptable()); - assert(c1.acceptable()); - assert(c2.acceptable()); - if(!reportHit(topf, botf, topb, botb, seq_->length(), prevEdit)) { - return false; // Memory exhausted - } + p.checkCV(); + reportHit(cache, p.bwt, seq.length(), p.prevEdit); return true; } - nextLocsBi(tloc, bloc, topf, botf, topb, botb, step); - assert(tloc.valid()); - } else assert(prevEdit != NULL); - assert(tloc.valid()); - assert(botf - topf == 1 || bloc.valid()); - assert(botf - topf > 1 || !bloc.valid()); - assert_geq(step, 0); - TIndexOffU t[4], b[4]; // dest BW ranges - Constraint* zones[3] = { &c0, &c1, &c2 }; - ASSERT_ONLY(TIndexOffU lasttot = botf - topf); - for(int i = step; i < (int)s.steps.size(); i++) { - assert_gt(botf, topf); - assert(botf - topf == 1 || bloc.valid()); - assert(botf - topf > 1 || !bloc.valid()); - assert(ebwtBw_ == NULL || botf-topf == botb-topb); - assert(tloc.valid()); - off = s.steps[i]; - bool ltr = off > 0; - const Ebwt* ebwt = ltr ? ebwtBw_ : ebwtFw_; - assert(ebwt != NULL); + nextLocsBi(seed, p.tloc, p.bloc, p.bwt, p.step); + assert(p.tloc.valid()); + } else assert(p.prevEdit != NULL); + assert(p.tloc.valid()); + assert(p.bwt.botf - p.bwt.topf == 1 || p.bloc.valid()); + assert(p.bwt.botf - p.bwt.topf > 1 || !p.bloc.valid()); + assert_geq(p.step, 0); + + return false; +} + +class SeedAlignerSearchState { +public: + TIndexOffU tp[4], bp[4]; // dest BW ranges for "prime" index + TIndexOffU t[4], b[4]; // dest BW ranges + TIndexOffU *tf, *tb, *bf, *bb; // depend on ltr + const Ebwt* ebwt; + + TIndexOffU ntop; + int off; + bool ltr; + bool done; +public: + SeedAlignerSearchState() + : tp{0,0,0,0}, bp{0,0,0,0} + , t{0,0,0,0}, b{0,0,0,0} + , tf(NULL), tb(NULL), bf(NULL), bb(NULL) + , ebwt(NULL) + , ntop(0) + , off(0) + , ltr(false) + , done(false) + {} + + void setOff( + size_t _off, + const BwtTopBot &bwt, // The 4 BWT idxs + const Ebwt* ebwtFw_, // forward index (BWT) + const Ebwt* ebwtBw_) // backward/mirror index (BWT') + { + off = _off; + ltr = off > 0; + t[0] = t[1] = t[2] = t[3] = b[0] = b[1] = b[2] = b[3] = 0; off = abs(off)-1; - __builtin_prefetch(&((*seq_)[off])); if(ltr) { - tp[0] = tp[1] = tp[2] = tp[3] = topf; - bp[0] = bp[1] = bp[2] = bp[3] = botf; + ebwt = ebwtBw_; + tp[0] = tp[1] = tp[2] = tp[3] = bwt.topf; + bp[0] = bp[1] = bp[2] = bp[3] = bwt.botf; + tf = tp; tb = t; + bf = bp; bb = b; + ntop = bwt.topb; } else { - tp[0] = tp[1] = tp[2] = tp[3] = topb; - bp[0] = bp[1] = bp[2] = bp[3] = botb; + ebwt = ebwtFw_; + tp[0] = tp[1] = tp[2] = tp[3] = bwt.topb; + bp[0] = bp[1] = bp[2] = bp[3] = bwt.botb; + tf = t; tb = tp; + bf = b; bb = bp; + ntop = bwt.topf; } - t[0] = t[1] = t[2] = t[3] = b[0] = b[1] = b[2] = b[3] = 0; - if(bloc.valid()) { + assert(ebwt != NULL); + } + +public: +#ifndef NDEBUG + TIndexOffU lasttot; + + void initLastTot(TIndexOffU tot) { lasttot = tot;} + void assertLeqAndSetLastTot(TIndexOffU tot) { + assert_leq(tot, lasttot); + lasttot = tot; + } +#else + // noop in production code + void initLastTot(TIndexOffU tot) {} + void assertLeqAndSetLastTot(TIndexOffU tot) {}; +#endif + +}; + +class SeedAlignerSearchSave { +public: + SeedAlignerSearchSave( + Constraint &cons, Constraint &ovCons, + SideLocus &tloc, SideLocus &bloc) + : orgCons(cons), orgOvCons(ovCons) + , orgTloc(tloc), orgBloc(bloc) + , oldCons(cons), oldOvCons(ovCons) + , oldTloc(tloc), oldBloc(bloc) + {} + + ~SeedAlignerSearchSave() { + orgCons = oldCons; orgOvCons = oldOvCons; + orgTloc = oldTloc; orgBloc = oldBloc; + } + +private: + // reference to the original variables + Constraint &orgCons; + Constraint &orgOvCons; + SideLocus &orgTloc; + SideLocus &orgBloc; + + // copy of the originals + const Constraint oldCons; + const Constraint oldOvCons; + const SideLocus oldTloc; + const SideLocus oldBloc; +}; + +// State used for recursion +class SeedAlignerSearchRecState { +public: + SeedAlignerSearchRecState( + int j, + int c, + const SeedAlignerSearchState &sstate, + DoublyLinkedList<Edit> *prevEdit) + : bwt(sstate.tf[j], sstate.bf[j], sstate.tb[j], sstate.bb[j]) + , edit(sstate.off, j, c, EDIT_TYPE_MM, false) + , editl() + , _prevEdit(prevEdit) + { + assert(_prevEdit == NULL || _prevEdit->next == NULL); + editl.payload = edit; + if(_prevEdit != NULL) { + _prevEdit->next = &editl; + editl.prev = _prevEdit; + } + assert(editl.next == NULL); + } + + ~SeedAlignerSearchRecState() { + if(_prevEdit != NULL) _prevEdit->next = NULL; + } + + BwtTopBot bwt; + Edit edit; + DoublyLinkedList<Edit> editl; +private: + DoublyLinkedList<Edit> *_prevEdit; +}; + +/** + * Given a seed, search. Assumes zone 0 = no backtracking. + * + * Return a list of Seed hits. + * 1. Edits + * 2. Bidirectional BWT range(s) on either end + */ +void +SeedAligner::searchSeedBi(const size_t nparams, SeedAligner::SeedAlignerSearchParams paramVec[]) +{ + size_t nleft = nparams; // will keep track of how many are not done yet + std::vector<SeedAlignerSearchState> sstateVec(nparams); + + for (size_t n=0; n<nparams; n++) { + SeedAlignerSearchParams& p= paramVec[n]; + const bool done = startSearchSeedBi(p); + + SeedAlignerSearchState& sstate = sstateVec[n]; + sstate.done = done; + if(done) { + nleft--; + } else { + sstate.initLastTot(p.bwt.botf - p.bwt.topf); + } + } + + while (nleft>0) { + // Note: We can do the params in any order we want + // but we must do the steps inside the same param in order + // Will loop over all of them, and just check which ones are invalid + for (size_t n=0; n<nparams; n++) { + SeedAlignerSearchState& sstate = sstateVec[n]; + if (sstate.done) continue; + + SeedAlignerSearchParams& p= paramVec[n]; + const InstantiatedSeed& seed = p.cs.seed; + if (p.step >= (int) seed.steps.size()) { + sstate.done = true; + nleft--; + continue; + } + size_t i = p.step; // call the stepIdx i for historical reasons + p.step++; // get ready for the next iteration + + SeedSearchCache &cache = p.cs.cache; + const BTDnaString& seq = cache.getSeq(); + const BTString& qual = cache.getQual(); + + assert_gt(p.bwt.botf, p.bwt.topf); + assert(p.bwt.botf - p.bwt.topf == 1 || p.bloc.valid()); + assert(p.bwt.botf - p.bwt.topf > 1 || !p.bloc.valid()); + assert(ebwtBw_ == NULL || p.bwt.botf-p.bwt.topf == p.bwt.botb-p.bwt.topb); + assert(p.tloc.valid()); + sstate.setOff(seed.steps[i], p.bwt, ebwtFw_, ebwtBw_); + __builtin_prefetch(&(seq[sstate.off])); + __builtin_prefetch(&(qual[sstate.off])); + if(p.bloc.valid()) { // Range delimited by tloc/bloc has size >1. If size == 1, // we use a simpler query (see if(!bloc.valid()) blocks below) bwops_++; - ebwt->mapBiLFEx(tloc, bloc, t, b, tp, bp); - ASSERT_ONLY(TIndexOffU tot = (b[0]-t[0])+(b[1]-t[1])+(b[2]-t[2])+(b[3]-t[3])); - ASSERT_ONLY(TIndexOffU totp = (bp[0]-tp[0])+(bp[1]-tp[1])+(bp[2]-tp[2])+(bp[3]-tp[3])); + sstate.ebwt->mapBiLFEx(p.tloc, p.bloc, sstate.t, sstate.b, sstate.tp, sstate.bp); + ASSERT_ONLY(TIndexOffU tot = (sstate.b[0]-sstate.t[0])+(sstate.b[1]-sstate.t[1])+(sstate.b[2]-sstate.t[2])+(sstate.b[3]-sstate.t[3])); + ASSERT_ONLY(TIndexOffU totp = (sstate.bp[0]-sstate.tp[0])+(sstate.bp[1]-sstate.tp[1])+(sstate.bp[2]-sstate.tp[2])+(sstate.bp[3]-sstate.tp[3])); assert_eq(tot, totp); - assert_leq(tot, lasttot); - ASSERT_ONLY(lasttot = tot); +#ifndef NDEBUG + sstate.assertLeqAndSetLastTot(tot); +#endif } - TIndexOffU *tf = ltr ? tp : t, *tb = ltr ? t : tp; - TIndexOffU *bf = ltr ? bp : b, *bb = ltr ? b : bp; - int c = (*seq_)[off]; assert_range(0, 4, c); - // not 100% sure we need it, but redundant prefetches are not dangerous - // and helps in the average case - prefetchNextLocsBi(tf[c], bf[c], tb[c], bb[c], i+1); - + int c = seq[sstate.off]; assert_range(0, 4, c); // - bool leaveZone = s.zones[i].first < 0; + bool leaveZone = seed.zones[i].first < 0; //bool leaveZoneIns = zones_[i].second < 0; - Constraint& cons = *zones[abs(s.zones[i].first)]; - //Constraint& insCons = *zones[abs(s.zones[i].second)]; - int q = (*qual_)[off]; + Constraint& cons = p.cv[abs(seed.zones[i].first)]; + //Constraint& insCons = p.cv[abs(seed.zones[i].second)]; // Is it legal for us to advance on characters other than 'c'? - if(!(cons.mustMatch() && !overall.mustMatch()) || c == 4) { + if(!(cons.mustMatch() && !p.overall.mustMatch()) || c == 4) { // There may be legal edits bool bail = false; - if(!bloc.valid()) { + if(!p.bloc.valid()) { // Range delimited by tloc/bloc has size 1 - TIndexOffU ntop = ltr ? topb : topf; bwops_++; - int cc = ebwt->mapLF1(ntop, tloc); + int cc = sstate.ebwt->mapLF1(sstate.ntop, p.tloc); assert_range(-1, 3, cc); if(cc < 0) bail = true; - else { t[cc] = ntop; b[cc] = ntop+1; } + else { sstate.t[cc] = sstate.ntop; sstate.b[cc] = sstate.ntop+1; } } if(!bail) { - if((cons.canMismatch(q, *sc_) && overall.canMismatch(q, *sc_)) || c == 4) { - Constraint oldCons = cons, oldOvCons = overall; - SideLocus oldTloc = tloc, oldBloc = bloc; + int q = qual[sstate.off]; + if((cons.canMismatch(q, *sc_) && p.overall.canMismatch(q, *sc_)) || c == 4) { + SeedAlignerSearchSave save(cons, p.overall, p.tloc, p.bloc); if(c != 4) { cons.chargeMismatch(q, *sc_); - overall.chargeMismatch(q, *sc_); + p.overall.chargeMismatch(q, *sc_); } // Can leave the zone as-is - if(!leaveZone || (cons.acceptable() && overall.acceptable())) { + if(!leaveZone || (cons.acceptable() && p.overall.acceptable())) { for(int j = 0; j < 4; j++) { - if(j == c || b[j] == t[j]) continue; + if(j == c || sstate.b[j] == sstate.t[j]) continue; // Potential mismatch - nextLocsBi(tloc, bloc, tf[j], bf[j], tb[j], bb[j], i+1); - int loff = off; - if(!ltr) loff = (int)(s.steps.size() - loff - 1); - assert(prevEdit == NULL || prevEdit->next == NULL); - Edit edit(off, j, c, EDIT_TYPE_MM, false); - DoublyLinkedList<Edit> editl; - editl.payload = edit; - if(prevEdit != NULL) { - prevEdit->next = &editl; - editl.prev = prevEdit; - } - assert(editl.next == NULL); + SeedAlignerSearchRecState rstate(j, c, sstate, p.prevEdit); + nextLocsBi(seed, p.tloc, p.bloc, rstate.bwt, i+1); bwedits_++; - if(!searchSeedBi( - i+1, // depth into steps_[] array - depth+1, // recursion depth - tf[j], // top in BWT - bf[j], // bot in BWT - tb[j], // top in BWT' - bb[j], // bot in BWT' - tloc, // locus for top (perhaps unititialized) - bloc, // locus for bot (perhaps unititialized) - c0, // constraints to enforce in seed zone 0 - c1, // constraints to enforce in seed zone 1 - c2, // constraints to enforce in seed zone 2 - overall, // overall constraints to enforce - &editl)) // latest edit - { - return false; - } - if(prevEdit != NULL) prevEdit->next = NULL; + SeedAlignerSearchParams p2( + p.cs, + i+1, // depth into steps[] array + p.depth+1, // recursion depth + rstate.bwt, // The 4 BWT idxs + p.tloc, // locus for top (perhaps unititialized) + p.bloc, // locus for bot (perhaps unititialized) + p.cv, // constraints to enforce in seed zones + p.overall, // overall constraints to enforce + &rstate.editl); // latest edit + // recursion is rare, so just do one at a time + searchSeedBi(1, &p2); + // as rstate gets out of scope, p.prevEdit->next is updated } } else { // Not enough edits to make this path // non-redundant with other seeds } - cons = oldCons; - overall = oldOvCons; - tloc = oldTloc; - bloc = oldBloc; + + // as olds gets out of scope, + // restores cons, p.overall, p.tloc, p.bloc } - if(cons.canGap() && overall.canGap()) { + if(cons.canGap() && p.overall.canGap()) { throw 1; // TODO // int delEx = 0; // if(cons.canDelete(delEx, *sc_) && overall.canDelete(delEx, *sc_)) { @@ -1648,49 +1981,55 @@ SeedAligner::searchSeedBi( } } // if(!bail) } - if(c == 4) { - return true; // couldn't handle the N + if(c == 4) { // couldn't handle the N + sstate.done = true; + nleft--; + continue; } - if(leaveZone && (!cons.acceptable() || !overall.acceptable())) { + if(leaveZone && (!cons.acceptable() || !p.overall.acceptable())) { // Not enough edits to make this path non-redundant with // other seeds - return true; + sstate.done = true; + nleft--; + continue; } - if(!bloc.valid()) { - assert(ebwtBw_ == NULL || bp[c] == tp[c]+1); + if(!p.bloc.valid()) { + assert(ebwtBw_ == NULL || sstate.bp[c] == sstate.tp[c]+1); // Range delimited by tloc/bloc has size 1 - TIndexOffU top = ltr ? topb : topf; bwops_++; - t[c] = ebwt->mapLF1(top, tloc, c); - if(t[c] == OFF_MASK) { - return true; + sstate.t[c] = sstate.ebwt->mapLF1(sstate.ntop, p.tloc, c); + if(sstate.t[c] == OFF_MASK) { + sstate.done = true; + nleft--; + continue; } - assert_geq(t[c], ebwt->fchr()[c]); - assert_lt(t[c], ebwt->fchr()[c+1]); - b[c] = t[c]+1; - assert_gt(b[c], 0); + assert_geq(sstate.t[c], sstate.ebwt->fchr()[c]); + assert_lt(sstate.t[c], sstate.ebwt->fchr()[c+1]); + sstate.b[c] = sstate.t[c]+1; + assert_gt(sstate.b[c], 0); } - assert(ebwtBw_ == NULL || bf[c]-tf[c] == bb[c]-tb[c]); - assert_leq(bf[c]-tf[c], lasttot); - ASSERT_ONLY(lasttot = bf[c]-tf[c]); - if(b[c] == t[c]) { - return true; + assert(ebwtBw_ == NULL || sstate.bf[c]-sstate.tf[c] == sstate.bb[c]-sstate.tb[c]); +#ifndef NDEBUG + sstate.assertLeqAndSetLastTot(sstate.bf[c]-sstate.tf[c]); +#endif + if(sstate.b[c] == sstate.t[c]) { + sstate.done = true; + nleft--; + continue; } - topf = tf[c]; botf = bf[c]; - topb = tb[c]; botb = bb[c]; - if(i+1 == (int)s.steps.size()) { + p.bwt.set(sstate.tf[c], sstate.bf[c], sstate.tb[c], sstate.bb[c]); + if(i+1 == seed.steps.size()) { // Finished aligning seed - assert(c0.acceptable()); - assert(c1.acceptable()); - assert(c2.acceptable()); - if(!reportHit(topf, botf, topb, botb, seq_->length(), prevEdit)) { - return false; // Memory exhausted - } - return true; + p.checkCV(); + reportHit(cache, p.bwt, seq.length(), p.prevEdit); + sstate.done = true; + nleft--; + continue; } - nextLocsBi(tloc, bloc, tf[c], bf[c], tb[c], bb[c], i+1); - } - return true; + nextLocsBi(seed, p.tloc, p.bloc, p.bwt, i+1); + } // for n + } // while + return; } #ifdef ALIGNER_SEED_MAIN diff --git a/aligner_seed.h b/aligner_seed.h index 56ba057..26cce6e 100644 --- a/aligner_seed.h +++ b/aligner_seed.h @@ -23,6 +23,7 @@ #include <iostream> #include <utility> #include <limits> +#include <vector> #include "qual.h" #include "ds.h" #include "sstring.h" @@ -65,7 +66,7 @@ struct Constraint { * Return true iff penalities and constraints prevent us from * adding any edits. */ - bool mustMatch() { + bool mustMatch() const { assert(instantiated); return (mms == 0 && edits == 0) || penalty == 0 || @@ -75,7 +76,7 @@ struct Constraint { /** * Return true iff a mismatch of the given quality is permitted. */ - bool canMismatch(int q, const Scoring& cm) { + bool canMismatch(int q, const Scoring& cm) const { assert(instantiated); return (mms > 0 || edits > 0) && penalty >= cm.mm(q); @@ -84,7 +85,7 @@ struct Constraint { /** * Return true iff a mismatch of the given quality is permitted. */ - bool canN(int q, const Scoring& cm) { + bool canN(int q, const Scoring& cm) const { assert(instantiated); return (mms > 0 || edits > 0) && penalty >= cm.n(q); @@ -94,7 +95,7 @@ struct Constraint { * Return true iff a mismatch of *any* quality (even qual=1) is * permitted. */ - bool canMismatch() { + bool canMismatch() const { assert(instantiated); return (mms > 0 || edits > 0) && penalty > 0; } @@ -103,7 +104,7 @@ struct Constraint { * Return true iff a mismatch of *any* quality (even qual=1) is * permitted. */ - bool canN() { + bool canN() const { assert(instantiated); return (mms > 0 || edits > 0); } @@ -112,7 +113,7 @@ struct Constraint { * Return true iff a deletion of the given extension (0=open, 1=1st * extension, etc) is permitted. */ - bool canDelete(int ex, const Scoring& cm) { + bool canDelete(int ex, const Scoring& cm) const { assert(instantiated); return (dels > 0 && edits > 0) && penalty >= cm.del(ex); @@ -121,7 +122,7 @@ struct Constraint { /** * Return true iff a deletion of any extension is permitted. */ - bool canDelete() { + bool canDelete() const { assert(instantiated); return (dels > 0 || edits > 0) && penalty > 0; @@ -131,7 +132,7 @@ struct Constraint { * Return true iff an insertion of the given extension (0=open, * 1=1st extension, etc) is permitted. */ - bool canInsert(int ex, const Scoring& cm) { + bool canInsert(int ex, const Scoring& cm) const { assert(instantiated); return (ins > 0 || edits > 0) && penalty >= cm.ins(ex); @@ -140,7 +141,7 @@ struct Constraint { /** * Return true iff an insertion of any extension is permitted. */ - bool canInsert() { + bool canInsert() const { assert(instantiated); return (ins > 0 || edits > 0) && penalty > 0; @@ -149,7 +150,7 @@ struct Constraint { /** * Return true iff a gap of any extension is permitted */ - bool canGap() { + bool canGap() const { assert(instantiated); return ((ins > 0 || dels > 0) || edits > 0) && penalty > 0; } @@ -213,7 +214,7 @@ struct Constraint { * are helpful to resolve instances where two search roots would * otherwise overlap in what alignments they can find. */ - bool acceptable() { + bool acceptable() const { assert(instantiated); return edits <= editsCeil && mms <= mmsCeil && @@ -1455,6 +1456,186 @@ struct SeedSearchMetrics { }; /** + * Wrap the search cache with all the relevant objects + */ +class SeedSearchCache { + +public: + SeedSearchCache( + const BTDnaString& _seq, // sequence of current seed + const BTString& _qual // quality string for current seed + ) + : qv() + , seq(_seq) + , qual(_qual) + , cachedEls() + , cachep(NULL) + { + cachedEls.reserve(16); // do not expect I will need more + } + + /** + * This function is called whenever we start to align a new read or + * read substring. + * + * See AlignmentCacheIface::beginAlign for details + */ + int beginAlign(AlignmentCacheIface& cache) + { + int ret = cache.beginAlign(seq, qual, qv); + if (ret>=0) { + cachep = &cache; + } + return ret; + } + + /** + * Add an alignment to the running list of alignments being + * compiled for the current read in the local cache. + */ + bool addAllCached(bool getLock = true) + { + if (!aligning()) return false; + const size_t nEls = cachedEls.size(); + bool success = true; + for(size_t i=0; i<nEls; i++) { + AddEl &el = cachedEls[i]; + success &= cachep->addOnTheFly(el.sak, el.topf, el.botf, el.topb, el.botb, getLock); + } + cachedEls.clear(); + return success; + } + + + /** + * Called when is finished aligning a read (and so is finished + * adding associated reference strings). Returns a copy of the + * final QVal object and resets the alignment state of the + * current-read cache. + * + * Also, if the alignment is cacheable, it commits it to the next + * cache up in the cache hierarchy. + */ + void finishAlign(bool getLock = true) + { + assert(cachep!=NULL); + qv = cachep->finishAlign(getLock); + cachep = NULL; + } + + /** + * Add an alignment to the running list of alignments being + * compiled for the current read in the local memory buffer. + */ + void addOnTheFly( + const BTDnaString& rfseq, // reference sequence close to read seq + TIndexOffU topf, // top in BWT index + TIndexOffU botf, // bot in BWT index + TIndexOffU topb, // top in BWT' index + TIndexOffU botb) // bot in BWT' index + { + cachedEls.emplace_back(rfseq, topf, botf, topb, botb); + } + + /** + * Return true iff we're in the middle of aligning a sequence. + */ + bool aligning() const { return ((cachep!=NULL) && (cachep->aligning())); } + + bool qvValid() const { return qv.valid();} + + const QVal& getQv() const {return qv;} + const BTDnaString& getSeq() const {return seq;} + const BTString& getQual() const {return qual;} + +protected: + class AddEl { + public: + AddEl( + const BTDnaString& rfseq, // reference sequence close to read seq + TIndexOffU _topf, // top in BWT index + TIndexOffU _botf, // bot in BWT index + TIndexOffU _topb, // top in BWT' index + TIndexOffU _botb // bot in BWT' index + ) : + ASSERT_ONLY(tmp(), ) + sak(rfseq ASSERT_ONLY(, tmp)), + topf(_topf), botf(_botf), topb(_topb), botb(_botb) + {} + + ASSERT_ONLY(BTDnaString tmp;) + SAKey sak; + TIndexOffU topf; // top in BWT index + TIndexOffU botf; // bot in BWT index + TIndexOffU topb; // top in BWT' index + TIndexOffU botb; // bot in BWT' index + }; + + QVal qv; + const BTDnaString& seq; // sequence of current seed + const BTString& qual; // quality string for current seed + + std::vector<AddEl> cachedEls; // tmp storage of values that will go in the cache + AlignmentCacheIface* cachep; // local alignment cache for seed alignment, set at beginAliginings +}; + +/** + * Wrap the search cache with all the relevant objects + */ +class SeedSearchMultiCache { + +public: + SeedSearchMultiCache( + ) + : cacheVec() + {} + + void emplace_back( + const BTDnaString& seq, // sequence of current seed + const BTString& qual, // quality string for current seed + int seedoffidx, // seed index + bool fw // is it fw? + ) + { + cacheVec.emplace_back(seq, qual, seedoffidx, fw); + } + + // Same semantics as std::vector + void reserve(size_t new_cap) { cacheVec.reserve(new_cap); } + size_t size() const {return cacheVec.size(); } + void clear() { cacheVec.clear(); } + void pop_back() { cacheVec.pop_back(); } + + // Access one of the search caches + const SeedSearchCache& operator[](size_t idx) const { return cacheVec[idx].srcache; } + SeedSearchCache& operator[](size_t idx) { return cacheVec[idx].srcache; } + + int getSeedOffIdx(size_t idx) const { return cacheVec[idx].seedoffidx; } + bool getFw(size_t idx) const { return cacheVec[idx].fw; } + +protected: + class CacheEl { + public: + CacheEl( + const BTDnaString& _seq, // sequence of current seed + const BTString& _qual, // quality string for current seed + int _seedoffidx, // seed index + bool _fw // is it fw? + ) + : srcache(_seq, _qual) + , seedoffidx(_seedoffidx) + , fw(_fw) {} + + + SeedSearchCache srcache; // search wrapper + int seedoffidx; // seed index + bool fw; // is it fw? + }; + + std::vector<CacheEl> cacheVec; +}; + +/** * Given an index and a seeding scheme, searches for seed hits. */ class SeedAligner { @@ -1562,6 +1743,7 @@ public: SeedSearchMetrics& met); // metrics protected: + class SeedAlignerSearchParams; /** * Report a seed hit found by searchSeedBi(), but first try to extend it out in @@ -1570,7 +1752,10 @@ protected: * we're done, which actually adds the hit to the cache. Returns result from * calling reportHit(). */ - bool extendAndReportHit( + void extendAndReportHit( + SeedSearchCache &cache, // local seed alignment cache + size_t off, // offset of seed currently being searched + bool fw, // orientation of seed currently being searched TIndexOffU topf, // top in BWT TIndexOffU botf, // bot in BWT TIndexOffU topb, // top in BWT' @@ -1579,44 +1764,38 @@ protected: DoublyLinkedList<Edit> *prevEdit); // previous edit /** - * Report a seed hit found by searchSeedBi() by adding it to the cache. Return - * false if the hit could not be reported because of, e.g., cache exhaustion. + * Report a seed hit found by searchSeedBi() by adding it to the cache. */ - bool reportHit( + void reportHit( + SeedSearchCache &cache, // local seed alignment cache TIndexOffU topf, // top in BWT TIndexOffU botf, // bot in BWT TIndexOffU topb, // top in BWT' TIndexOffU botb, // bot in BWT' uint16_t len, // length of hit DoublyLinkedList<Edit> *prevEdit); // previous edit - - /** - * Given an instantiated seed (in s_ and other fields), search - */ - bool searchSeedBi(); - + + void reportHit( + SeedSearchCache &cache, // local seed alignment cache + const BwtTopBot &bwt, // The 4 BWT idxs + uint16_t len, // length of hit + DoublyLinkedList<Edit> *prevEdit) // previous edit + { reportHit(cache, bwt.topf, bwt.botf, bwt.topb, bwt.botb, len, prevEdit); } + /** * Main, recursive implementation of the seed search. + * Given a vector of instantiated seeds, search */ - bool searchSeedBi( - int step, // depth into steps_[] array - int depth, // recursion depth - TIndexOffU topf, // top in BWT - TIndexOffU botf, // bot in BWT - TIndexOffU topb, // top in BWT' - TIndexOffU botb, // bot in BWT' - SideLocus tloc, // locus for top (perhaps unititialized) - SideLocus bloc, // locus for bot (perhaps unititialized) - Constraint c0, // constraints to enforce in seed zone 0 - Constraint c1, // constraints to enforce in seed zone 1 - Constraint c2, // constraints to enforce in seed zone 2 - Constraint overall, // overall constraints - DoublyLinkedList<Edit> *prevEdit); // previous edit - + void searchSeedBi(const size_t nparams, SeedAlignerSearchParams paramVec[]); + + // helper function + bool startSearchSeedBi(SeedAlignerSearchParams &p); + /** * Get tloc and bloc ready for the next step. */ - inline void nextLocsBi( + void nextLocsBi( + const InstantiatedSeed& seed, // current instantiated seed SideLocus& tloc, // top locus SideLocus& bloc, // bot locus TIndexOffU topf, // top in BWT @@ -1625,30 +1804,37 @@ protected: TIndexOffU botb, // bot in BWT' int step); // step to get ready for - inline void prefetchNextLocsBi( + void nextLocsBi( + const InstantiatedSeed& seed, // current instantiated seed + SideLocus& tloc, // top locus + SideLocus& bloc, // bot locus + const BwtTopBot &bwt, // The 4 BWT idxs + int step) // step to get ready for + { nextLocsBi(seed, tloc, bloc, bwt.topf, bwt.botf, bwt.topb, bwt.botb, step); } + + void prefetchNextLocsBi( + const InstantiatedSeed& seed, // current instantiated seed TIndexOffU topf, // top in BWT TIndexOffU botf, // bot in BWT TIndexOffU topb, // top in BWT' TIndexOffU botb, // bot in BWT' int step); // step to get ready for + void prefetchNextLocsBi( + const InstantiatedSeed& seed, // current instantiated seed + const BwtTopBot &bwt, // The 4 BWT idxs + int step) // step to get ready for + { prefetchNextLocsBi(seed, bwt.topf, bwt.botf, bwt.topb, bwt.botb, step); } + // Following are set in searchAllSeeds then used by searchSeed() // and other protected members. const Ebwt* ebwtFw_; // forward index (BWT) const Ebwt* ebwtBw_; // backward/mirror index (BWT') const Scoring* sc_; // scoring scheme - const InstantiatedSeed* s_;// current instantiated seed const Read* read_; // read whose seeds are currently being aligned - // The following are set just before a call to searchSeedBi() - const BTDnaString* seq_; // sequence of current seed - const BTString* qual_; // quality string for current seed - size_t off_; // offset of seed currently being searched - bool fw_; // orientation of seed currently being searched - EList<Edit> edits_; // temporary place to sort edits - AlignmentCacheIface *ca_; // local alignment cache for seed alignments EList<uint32_t> offIdx2off_;// offset idx to read offset map, set up instantiateSeeds() uint64_t bwops_; // Burrows-Wheeler operations uint64_t bwedits_; // Burrows-Wheeler edits diff --git a/aligner_sw_driver.cpp b/aligner_sw_driver.cpp index f0c0e0a..f202c37 100644 --- a/aligner_sw_driver.cpp +++ b/aligner_sw_driver.cpp @@ -1071,7 +1071,7 @@ int SwDriver::extendSeeds( swmSeed.ungapsucc++; } } - int64_t pastedRefoff = (int64_t)wr.toff - rdoff; + // int64_t pastedRefoff = (int64_t)wr.toff - rdoff; DPRect rect; if(state == FOUND_NONE) { found = dpframe.frameSeedExtensionRect( @@ -1090,7 +1090,7 @@ int SwDriver::extendSeeds( continue; } } - int64_t leftShift = refoff - rect.refl; + // int64_t leftShift = refoff - rect.refl; size_t nwindow = 0; if((int64_t)toff >= rect.refl) { nwindow = (size_t)(toff - rect.refl); @@ -1098,7 +1098,7 @@ int SwDriver::extendSeeds( // NOTE: We might be taking off more than we should because the // pasted string omits non-A/C/G/T characters, but we included them // when calculating leftShift. We'll account for this later. - pastedRefoff -= leftShift; + // pastedRefoff -= leftShift; size_t nsInLeftShift = 0; if(state == FOUND_NONE) { if(!swa.initedRead()) { @@ -1766,7 +1766,7 @@ int SwDriver::extendSeedsPaired( swmSeed.ungapsucc++; } } - int64_t pastedRefoff = (int64_t)wr.toff - rdoff; + // int64_t pastedRefoff = (int64_t)wr.toff - rdoff; DPRect rect; if(state == FOUND_NONE) { found = dpframe.frameSeedExtensionRect( @@ -1785,7 +1785,7 @@ int SwDriver::extendSeedsPaired( continue; } } - int64_t leftShift = refoff - rect.refl; + // int64_t leftShift = refoff - rect.refl; size_t nwindow = 0; if((int64_t)toff >= rect.refl) { nwindow = (size_t)(toff - rect.refl); @@ -1793,7 +1793,7 @@ int SwDriver::extendSeedsPaired( // NOTE: We might be taking off more than we should because the // pasted string omits non-A/C/G/T characters, but we included them // when calculating leftShift. We'll account for this later. - pastedRefoff -= leftShift; + // pastedRefoff -= leftShift; size_t nsInLeftShift = 0; if(state == FOUND_NONE) { if(!swa.initedRead()) { diff --git a/aligner_swsse.cpp b/aligner_swsse.cpp index d4f7d78..5d9c727 100644 --- a/aligner_swsse.cpp +++ b/aligner_swsse.cpp @@ -23,7 +23,7 @@ /** * Given a number of rows (nrow), a number of columns (ncol), and the - * number of words to fit inside a single __m128i vector, initialize the + * number of words to fit inside a single SSERegI vector, initialize the * matrix buffer to accomodate the needed configuration of vectors. */ void SSEMatrix::init( @@ -47,8 +47,13 @@ void SSEMatrix::init( << " vectors per cell" << endl; throw e; } - assert(wperv_ == 8 || wperv_ == 16); - vecshift_ = (wperv_ == 8) ? 3 : 4; + { + // compute vecshift_=log2(wperv) + size_t wp = wperv; + assert(wp == (NBYTES_PER_REG/2) || wp == NBYTES_PER_REG); + vecshift_ = 0; + while( wp>>=1 ) vecshift_++; + } nvecrow_ = (nrow + (wperv_-1)) >> vecshift_; nveccol_ = ncol; colstride_ = nvecPerCol_ * nvecPerCell_; @@ -79,10 +84,10 @@ int SSEMatrix::eltSlow(size_t row, size_t col, size_t mat) const { size_t rowelt = row / nvecrow_; size_t rowvec = row % nvecrow_; size_t eltvec = (col * colstride_) + (rowvec * rowstride_) + mat; - if(wperv_ == 16) { + if(wperv_ == NBYTES_PER_REG) { return (int)((uint8_t*)(matbuf_.ptr() + eltvec))[rowelt]; } else { - assert_eq(8, wperv_); + assert_eq((NBYTES_PER_REG/2), wperv_); return (int)((int16_t*)(matbuf_.ptr() + eltvec))[rowelt]; } } diff --git a/aligner_swsse.h b/aligner_swsse.h index 7d8759c..aa8d1e3 100644 --- a/aligner_swsse.h +++ b/aligner_swsse.h @@ -82,7 +82,7 @@ struct SSEMetrics { * * Matrix memory is laid out as follows: * - * - Elements (individual cell scores) are packed into __m128i vectors + * - Elements (individual cell scores) are packed into SSERegI vectors * - Vectors are packed into quartets, quartet elements correspond to: a vector * from E, one from F, one from H, and one that's "reserved" * - Quartets are packed into columns, where the number of quartets is @@ -109,7 +109,7 @@ struct SSEMatrix { /** * Return a pointer to the matrix buffer. */ - inline __m128i *ptr() { + inline SSERegI *ptr() { assert(inited_); return matbuf_.ptr(); } @@ -118,7 +118,7 @@ struct SSEMatrix { * Return a pointer to the E vector at the given row and column. Note: * here row refers to rows of vectors, not rows of elements. */ - inline __m128i* evec(size_t row, size_t col) { + inline SSERegI* evec(size_t row, size_t col) { assert_lt(row, nvecrow_); assert_lt(col, nveccol_); size_t elt = row * rowstride() + col * colstride() + E; @@ -130,7 +130,7 @@ struct SSEMatrix { * Like evec, but it's allowed to ask for a pointer to one column after the * final one. */ - inline __m128i* evecUnsafe(size_t row, size_t col) { + inline SSERegI* evecUnsafe(size_t row, size_t col) { assert_lt(row, nvecrow_); assert_leq(col, nveccol_); size_t elt = row * rowstride() + col * colstride() + E; @@ -142,7 +142,7 @@ struct SSEMatrix { * Return a pointer to the F vector at the given row and column. Note: * here row refers to rows of vectors, not rows of elements. */ - inline __m128i* fvec(size_t row, size_t col) { + inline SSERegI* fvec(size_t row, size_t col) { assert_lt(row, nvecrow_); assert_lt(col, nveccol_); size_t elt = row * rowstride() + col * colstride() + F; @@ -154,7 +154,7 @@ struct SSEMatrix { * Return a pointer to the H vector at the given row and column. Note: * here row refers to rows of vectors, not rows of elements. */ - inline __m128i* hvec(size_t row, size_t col) { + inline SSERegI* hvec(size_t row, size_t col) { assert_lt(row, nvecrow_); assert_lt(col, nveccol_); size_t elt = row * rowstride() + col * colstride() + H; @@ -166,7 +166,7 @@ struct SSEMatrix { * Return a pointer to the TMP vector at the given row and column. Note: * here row refers to rows of vectors, not rows of elements. */ - inline __m128i* tmpvec(size_t row, size_t col) { + inline SSERegI* tmpvec(size_t row, size_t col) { assert_lt(row, nvecrow_); assert_lt(col, nveccol_); size_t elt = row * rowstride() + col * colstride() + TMP; @@ -178,7 +178,7 @@ struct SSEMatrix { * Like tmpvec, but it's allowed to ask for a pointer to one column after * the final one. */ - inline __m128i* tmpvecUnsafe(size_t row, size_t col) { + inline SSERegI* tmpvecUnsafe(size_t row, size_t col) { assert_lt(row, nvecrow_); assert_leq(col, nveccol_); size_t elt = row * rowstride() + col * colstride() + TMP; @@ -188,7 +188,7 @@ struct SSEMatrix { /** * Given a number of rows (nrow), a number of columns (ncol), and the - * number of words to fit inside a single __m128i vector, initialize the + * number of words to fit inside a single SSERegI vector, initialize the * matrix buffer to accomodate the needed configuration of vectors. */ void init( @@ -197,13 +197,13 @@ struct SSEMatrix { size_t wperv); /** - * Return the number of __m128i's you need to skip over to get from one + * Return the number of SSERegI's you need to skip over to get from one * cell to the cell one column over from it. */ inline size_t colstride() const { return colstride_; } /** - * Return the number of __m128i's you need to skip over to get from one + * Return the number of SSERegI's you need to skip over to get from one * cell to the cell one row down from it. */ inline size_t rowstride() const { return rowstride_; } @@ -228,10 +228,10 @@ struct SSEMatrix { size_t rowvec = row % nvecrow_; size_t eltvec = (col * colstride_) + (rowvec * rowstride_) + mat; assert_lt(eltvec, matbuf_.size()); - if(wperv_ == 16) { + if(wperv_ == NBYTES_PER_REG) { return (int)((uint8_t*)(matbuf_.ptr() + eltvec))[rowelt]; } else { - assert_eq(8, wperv_); + assert_eq((NBYTES_PER_REG/2), wperv_); return (int)((int16_t*)(matbuf_.ptr() + eltvec))[rowelt]; } } @@ -395,7 +395,7 @@ struct SSEMatrix { size_t nvecPerCell_; // # vectors per matrix cell (4) size_t colstride_; // # vectors b/t adjacent cells in same row size_t rowstride_; // # vectors b/t adjacent cells in same col - EList_m128i matbuf_; // buffer for holding vectors + EList_sse matbuf_; // buffer for holding vectors ELList<uint16_t> masks_; // buffer for masks/backtracking flags EList<bool> reset_; // true iff row in masks_ has been reset }; @@ -406,15 +406,15 @@ struct SSEMatrix { */ struct SSEData { SSEData(int cat = 0) : profbuf_(cat), mat_(cat) { } - EList_m128i profbuf_; // buffer for query profile & temp vecs - EList_m128i vecbuf_; // buffer for 2 column vectors (not using mat_) + EList_sse profbuf_; // buffer for query profile & temp vecs + EList_sse vecbuf_; // buffer for 2 column vectors (not using mat_) size_t qprofStride_; // stride for query profile size_t gbarStride_; // gap barrier for query profile SSEMatrix mat_; // SSE matrix for holding all E, F, H vectors size_t maxPen_; // biggest penalty of all size_t maxBonus_; // biggest bonus of all - size_t lastIter_; // which 128-bit striped word has final row? - size_t lastWord_; // which word within 128-word has final row? + size_t lastIter_; // which N-bit striped word has final row? + size_t lastWord_; // which word within N-word has final row? int bias_; // all scores shifted up by this for unsigned }; @@ -495,5 +495,6 @@ inline void SSEMatrix::fMaskSet( #define ROWSTRIDE_2COL 4 #define ROWSTRIDE 4 +#define ROWSTRIDE_LOG2 2 #endif /*ndef ALIGNER_SWSSE_H_*/ diff --git a/aligner_swsse_ee_i16.cpp b/aligner_swsse_ee_i16.cpp index 2e761be..fb16c2d 100644 --- a/aligner_swsse_ee_i16.cpp +++ b/aligner_swsse_ee_i16.cpp @@ -55,8 +55,7 @@ #include <limits> #include "aligner_sw.h" -static const size_t NBYTES_PER_REG = 16; -static const size_t NWORDS_PER_REG = 8; +static const size_t NWORDS_PER_REG = NBYTES_PER_REG/2; static const size_t NBITS_PER_WORD = 16; static const size_t NBYTES_PER_WORD = 2; @@ -83,14 +82,14 @@ void SwAligner::buildQueryProfileEnd2EndSseI16(bool fw) { const BTString* qu = fw ? qufw_ : qurc_; const size_t len = rd->length(); const size_t seglen = (len + (NWORDS_PER_REG-1)) / NWORDS_PER_REG; - // How many __m128i's are needed - size_t n128s = + // How many SSERegI's are needed + size_t nsses = 64 + // slack bytes, for alignment? (seglen * ALPHA_SIZE) // query profile data * 2; // & gap barrier data - assert_gt(n128s, 0); + assert_gt(nsses, 0); SSEData& d = fw ? sseI16fw_ : sseI16rc_; - d.profbuf_.resizeNoCopy(n128s); + d.profbuf_.resizeNoCopy(nsses); assert(!d.profbuf_.empty()); d.maxPen_ = d.maxBonus_ = 0; d.lastIter_ = d.lastWord_ = 0; @@ -234,43 +233,43 @@ static bool cellOkEnd2EndI16( #else #define assert_all_eq0(x) { \ - __m128i z = _mm_setzero_si128(); \ - __m128i tmp = _mm_setzero_si128(); \ - z = _mm_xor_si128(z, z); \ - tmp = _mm_cmpeq_epi16(x, z); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI z = sse_setzero_siall(); \ + SSERegI tmp = sse_setzero_siall(); \ + z = sse_xor_siall(z, z); \ + tmp = sse_cmpeq_epi16(x, z); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #define assert_all_gt(x, y) { \ - __m128i tmp = _mm_cmpgt_epi16(x, y); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI tmp = sse_cmpgt_epi16(x, y); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #define assert_all_gt_lo(x) { \ - __m128i z = _mm_setzero_si128(); \ - __m128i tmp = _mm_setzero_si128(); \ - z = _mm_xor_si128(z, z); \ - tmp = _mm_cmpgt_epi16(x, z); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI z = sse_setzero_siall(); \ + SSERegI tmp = sse_setzero_siall(); \ + z = sse_xor_siall(z, z); \ + tmp = sse_cmpgt_epi16(x, z); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #define assert_all_lt(x, y) { \ - __m128i tmp = _mm_cmplt_epi16(x, y); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI tmp = sse_cmplt_epi16(x, y); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #define assert_all_leq(x, y) { \ - __m128i tmp = _mm_cmpgt_epi16(x, y); \ - assert_eq(0x0000, _mm_movemask_epi8(tmp)); \ + SSERegI tmp = sse_cmpgt_epi16(x, y); \ + assert_eq(0x0000, sse_movemask_epi8(tmp)); \ } #define assert_all_lt_hi(x) { \ - __m128i z = _mm_setzero_si128(); \ - __m128i tmp = _mm_setzero_si128(); \ - z = _mm_cmpeq_epi16(z, z); \ - z = _mm_srli_epi16(z, 1); \ - tmp = _mm_cmplt_epi16(x, z); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI z = sse_setzero_siall(); \ + SSERegI tmp = sse_setzero_siall(); \ + z = sse_cmpeq_epi16(z, z); \ + z = sse_srli_epi16(z, 1); \ + tmp = sse_cmplt_epi16(x, z); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #endif @@ -320,8 +319,8 @@ TAlScore SwAligner::alignGatherEE16(int& flag, bool debug) { // we'll call "left" and "right". d.vecbuf_.resize(4 * 2 * iter); d.vecbuf_.zero(); - __m128i *vbuf_l = d.vecbuf_.ptr(); - __m128i *vbuf_r = d.vecbuf_.ptr() + (4 * iter); + SSERegI *vbuf_l = d.vecbuf_.ptr(); + SSERegI *vbuf_r = d.vecbuf_.ptr() + (4 * iter); // This is the data structure that holds candidate cells per diagonal. const size_t ndiags = rff_ - rfi_ + dpRows() - 1; @@ -354,91 +353,81 @@ TAlScore SwAligner::alignGatherEE16(int& flag, bool debug) { // Much of the implmentation below is adapted from Michael's code. // Set all elts to reference gap open penalty - __m128i rfgapo = _mm_setzero_si128(); - __m128i rfgape = _mm_setzero_si128(); - __m128i rdgapo = _mm_setzero_si128(); - __m128i rdgape = _mm_setzero_si128(); - __m128i vlo = _mm_setzero_si128(); - __m128i vhi = _mm_setzero_si128(); - __m128i vhilsw = _mm_setzero_si128(); - __m128i vlolsw = _mm_setzero_si128(); - __m128i ve = _mm_setzero_si128(); - __m128i vf = _mm_setzero_si128(); - __m128i vh = _mm_setzero_si128(); - __m128i vhd = _mm_setzero_si128(); - __m128i vhdtmp = _mm_setzero_si128(); - __m128i vtmp = _mm_setzero_si128(); + SSERegI rfgapo = sse_setzero_siall(); + SSERegI rfgape = sse_setzero_siall(); + SSERegI rdgapo = sse_setzero_siall(); + SSERegI rdgape = sse_setzero_siall(); + SSERegI vlo = sse_setzero_siall(); + SSERegI vhilsw = sse_setzero_siall(); + SSERegI vlolsw = sse_setzero_siall(); + SSERegI ve = sse_setzero_siall(); + SSERegI vf = sse_setzero_siall(); + SSERegI vh = sse_setzero_siall(); + SSERegI vhd = sse_setzero_siall(); + SSERegI vhdtmp = sse_setzero_siall(); + SSERegI vtmp = sse_setzero_siall(); assert_gt(sc_->refGapOpen(), 0); assert_leq(sc_->refGapOpen(), MAX_I16); - rfgapo = _mm_insert_epi16(rfgapo, sc_->refGapOpen(), 0); - rfgapo = _mm_shufflelo_epi16(rfgapo, 0); - rfgapo = _mm_shuffle_epi32(rfgapo, 0); + sse_fill_i16(sc_->refGapOpen(), rfgapo); // Set all elts to reference gap extension penalty assert_gt(sc_->refGapExtend(), 0); assert_leq(sc_->refGapExtend(), MAX_I16); assert_leq(sc_->refGapExtend(), sc_->refGapOpen()); - rfgape = _mm_insert_epi16(rfgape, sc_->refGapExtend(), 0); - rfgape = _mm_shufflelo_epi16(rfgape, 0); - rfgape = _mm_shuffle_epi32(rfgape, 0); + sse_fill_i16(sc_->refGapExtend(), rfgape); // Set all elts to read gap open penalty assert_gt(sc_->readGapOpen(), 0); assert_leq(sc_->readGapOpen(), MAX_I16); - rdgapo = _mm_insert_epi16(rdgapo, sc_->readGapOpen(), 0); - rdgapo = _mm_shufflelo_epi16(rdgapo, 0); - rdgapo = _mm_shuffle_epi32(rdgapo, 0); + sse_fill_i16(sc_->readGapOpen(), rdgapo); // Set all elts to read gap extension penalty assert_gt(sc_->readGapExtend(), 0); assert_leq(sc_->readGapExtend(), MAX_I16); assert_leq(sc_->readGapExtend(), sc_->readGapOpen()); - rdgape = _mm_insert_epi16(rdgape, sc_->readGapExtend(), 0); - rdgape = _mm_shufflelo_epi16(rdgape, 0); - rdgape = _mm_shuffle_epi32(rdgape, 0); + sse_fill_i16(sc_->readGapExtend(), rdgape); // Set all elts to 0x8000 (min value for signed 16-bit) - vlo = _mm_cmpeq_epi16(vlo, vlo); // all elts = 0xffff - vlo = _mm_slli_epi16(vlo, NBITS_PER_WORD-1); // all elts = 0x8000 - + sse_fill_i16(0x8000, vlo); + +#ifndef NDEBUG // Set all elts to 0x7fff (max value for signed 16-bit) - vhi = _mm_cmpeq_epi16(vhi, vhi); // all elts = 0xffff - vhi = _mm_srli_epi16(vhi, 1); // all elts = 0x7fff - + SSERegI vhi = sse_setzero_siall(); + sse_fill_i16(0x7fff, vhi); +#endif + // vlolsw: topmost (least sig) word set to 0x8000, all other words=0 - vlolsw = _mm_shuffle_epi32(vlo, 0); - vlolsw = _mm_srli_si128(vlolsw, NBYTES_PER_REG - NBYTES_PER_WORD); + sse_set_low_i16(0x8000, vlolsw); // vhilsw: topmost (least sig) word set to 0x7fff, all other words=0 - vhilsw = _mm_shuffle_epi32(vhi, 0); - vhilsw = _mm_srli_si128(vhilsw, NBYTES_PER_REG - NBYTES_PER_WORD); + sse_set_low_i16(0x7fff, vhilsw); - // Points to a long vector of __m128i where each element is a block of + // Points to a long vector of SSERegI where each element is a block of // contiguous cells in the E, F or H matrix. If the index % 3 == 0, then // the block of cells is from the E matrix. If index % 3 == 1, they're // from the F matrix. If index % 3 == 2, then they're from the H matrix. // Blocks of cells are organized in the same interleaved manner as they are // calculated by the Farrar algorithm. - const __m128i *pvScore; // points into the query profile + const SSERegI *pvScore; // points into the query profile const size_t colstride = ROWSTRIDE_2COL * iter; // Initialize the H and E vectors in the first matrix column - __m128i *pvELeft = vbuf_l + 0; __m128i *pvERight = vbuf_r + 0; - /* __m128i *pvFLeft = vbuf_l + 1; */ __m128i *pvFRight = vbuf_r + 1; - __m128i *pvHLeft = vbuf_l + 2; __m128i *pvHRight = vbuf_r + 2; + SSERegI *pvELeft = vbuf_l + 0; SSERegI *pvERight = vbuf_r + 0; + /* SSERegI *pvFLeft = vbuf_l + 1; */ SSERegI *pvFRight = vbuf_r + 1; + SSERegI *pvHLeft = vbuf_l + 2; SSERegI *pvHRight = vbuf_r + 2; // Maximum score in final row bool found = false; TCScore lrmax = MIN_I16; for(size_t i = 0; i < iter; i++) { - _mm_store_si128(pvERight, vlo); pvERight += ROWSTRIDE_2COL; + sse_store_siall(pvERight, vlo); pvERight += ROWSTRIDE_2COL; // Could initialize Hs to high or low. If high, cells in the lower // triangle will have somewhat more legitiate scores, but still won't // be exhaustively scored. - _mm_store_si128(pvHRight, vlo); pvHRight += ROWSTRIDE_2COL; + sse_store_siall(pvHRight, vlo); pvHRight += ROWSTRIDE_2COL; } assert_gt(sc_->gapbar, 0); @@ -474,49 +463,49 @@ TAlScore SwAligner::alignGatherEE16(int& flag, bool debug) { pvScore = d.profbuf_.ptr() + off; // even elts = query profile, odd = gap barrier // Set all cells to low value - vf = _mm_cmpeq_epi16(vf, vf); - vf = _mm_slli_epi16(vf, NBITS_PER_WORD-1); - vf = _mm_or_si128(vf, vlolsw); + vf = sse_cmpeq_epi16(vf, vf); + vf = sse_slli_epi16(vf, NBITS_PER_WORD-1); + vf = sse_or_siall(vf, vlolsw); // Load H vector from the final row of the previous column - vh = _mm_load_si128(pvHLeft + colstride - ROWSTRIDE_2COL); + vh = sse_load_siall(pvHLeft + colstride - ROWSTRIDE_2COL); // Shift 2 bytes down so that topmost (least sig) cell gets 0 - vh = _mm_slli_si128(vh, NBYTES_PER_WORD); + vh = sse_slli_siall(vh, NBYTES_PER_WORD); // Fill topmost (least sig) cell with high value - vh = _mm_or_si128(vh, vhilsw); + vh = sse_or_siall(vh, vhilsw); // For each character in the reference text: size_t j; for(j = 0; j < iter; j++) { // Load cells from E, calculated previously - ve = _mm_load_si128(pvELeft); - vhd = _mm_load_si128(pvHLeft); + ve = sse_load_siall(pvELeft); + vhd = sse_load_siall(pvHLeft); assert_all_lt(ve, vhi); pvELeft += ROWSTRIDE_2COL; // Store cells in F, calculated previously - vf = _mm_adds_epi16(vf, pvScore[1]); // veto some ref gap extensions - vf = _mm_adds_epi16(vf, pvScore[1]); // veto some ref gap extensions - _mm_store_si128(pvFRight, vf); + vf = sse_adds_epi16(vf, pvScore[1]); // veto some ref gap extensions + vf = sse_adds_epi16(vf, pvScore[1]); // veto some ref gap extensions + sse_store_siall(pvFRight, vf); pvFRight += ROWSTRIDE_2COL; // Factor in query profile (matches and mismatches) - vh = _mm_adds_epi16(vh, pvScore[0]); + vh = sse_adds_epi16(vh, pvScore[0]); // Update H, factoring in E and F - vh = _mm_max_epi16(vh, vf); + vh = sse_max_epi16(vh, vf); // Update vE value vhdtmp = vhd; - vhd = _mm_subs_epi16(vhd, rdgapo); - vhd = _mm_adds_epi16(vhd, pvScore[1]); // veto some read gap opens - vhd = _mm_adds_epi16(vhd, pvScore[1]); // veto some read gap opens - ve = _mm_subs_epi16(ve, rdgape); - ve = _mm_max_epi16(ve, vhd); - vh = _mm_max_epi16(vh, ve); + vhd = sse_subs_epi16(vhd, rdgapo); + vhd = sse_adds_epi16(vhd, pvScore[1]); // veto some read gap opens + vhd = sse_adds_epi16(vhd, pvScore[1]); // veto some read gap opens + ve = sse_subs_epi16(ve, rdgape); + ve = sse_max_epi16(ve, vhd); + vh = sse_max_epi16(vh, ve); // Save the new vH values - _mm_store_si128(pvHRight, vh); + sse_store_siall(pvHRight, vh); pvHRight += ROWSTRIDE_2COL; vtmp = vh; assert_all_lt(ve, vhi); @@ -526,49 +515,49 @@ TAlScore SwAligner::alignGatherEE16(int& flag, bool debug) { pvHLeft += ROWSTRIDE_2COL; // Save E values - _mm_store_si128(pvERight, ve); + sse_store_siall(pvERight, ve); pvERight += ROWSTRIDE_2COL; // Update vf value - vtmp = _mm_subs_epi16(vtmp, rfgapo); - vf = _mm_subs_epi16(vf, rfgape); + vtmp = sse_subs_epi16(vtmp, rfgapo); + vf = sse_subs_epi16(vf, rfgape); assert_all_lt(vf, vhi); - vf = _mm_max_epi16(vf, vtmp); + vf = sse_max_epi16(vf, vtmp); pvScore += 2; // move on to next query profile / gap veto } // pvHStore, pvELoad, pvEStore have all rolled over to the next column pvFRight -= colstride; // reset to start of column - vtmp = _mm_load_si128(pvFRight); + vtmp = sse_load_siall(pvFRight); pvHRight -= colstride; // reset to start of column - vh = _mm_load_si128(pvHRight); + vh = sse_load_siall(pvHRight); pvScore = d.profbuf_.ptr() + off + 1; // reset veto vector // vf from last row gets shifted down by one to overlay the first row // rfgape has already been subtracted from it. - vf = _mm_slli_si128(vf, NBYTES_PER_WORD); - vf = _mm_or_si128(vf, vlolsw); + vf = sse_slli_siall(vf, NBYTES_PER_WORD); + vf = sse_or_siall(vf, vlolsw); - vf = _mm_adds_epi16(vf, *pvScore); // veto some ref gap extensions - vf = _mm_adds_epi16(vf, *pvScore); // veto some ref gap extensions - vf = _mm_max_epi16(vtmp, vf); - vtmp = _mm_cmpgt_epi16(vf, vtmp); - int cmp = _mm_movemask_epi8(vtmp); + vf = sse_adds_epi16(vf, *pvScore); // veto some ref gap extensions + vf = sse_adds_epi16(vf, *pvScore); // veto some ref gap extensions + vf = sse_max_epi16(vtmp, vf); + vtmp = sse_cmpgt_epi16(vf, vtmp); + int cmp = sse_movemask_epi8(vtmp); // If any element of vtmp is greater than H - gap-open... j = 0; while(cmp != 0x0000) { // Store this vf - _mm_store_si128(pvFRight, vf); + sse_store_siall(pvFRight, vf); pvFRight += ROWSTRIDE_2COL; // Update vh w/r/t new vf - vh = _mm_max_epi16(vh, vf); + vh = sse_max_epi16(vh, vf); // Save vH values - _mm_store_si128(pvHRight, vh); + sse_store_siall(pvHRight, vh); pvHRight += ROWSTRIDE_2COL; pvScore += 2; @@ -576,31 +565,31 @@ TAlScore SwAligner::alignGatherEE16(int& flag, bool debug) { assert_lt(j, iter); if(++j == iter) { pvFRight -= colstride; - vtmp = _mm_load_si128(pvFRight); // load next vf ASAP + vtmp = sse_load_siall(pvFRight); // load next vf ASAP pvHRight -= colstride; - vh = _mm_load_si128(pvHRight); // load next vh ASAP + vh = sse_load_siall(pvHRight); // load next vh ASAP pvScore = d.profbuf_.ptr() + off + 1; j = 0; - vf = _mm_slli_si128(vf, NBYTES_PER_WORD); - vf = _mm_or_si128(vf, vlolsw); + vf = sse_slli_siall(vf, NBYTES_PER_WORD); + vf = sse_or_siall(vf, vlolsw); } else { - vtmp = _mm_load_si128(pvFRight); // load next vf ASAP - vh = _mm_load_si128(pvHRight); // load next vh ASAP + vtmp = sse_load_siall(pvFRight); // load next vf ASAP + vh = sse_load_siall(pvHRight); // load next vh ASAP } // Update F with another gap extension - vf = _mm_subs_epi16(vf, rfgape); - vf = _mm_adds_epi16(vf, *pvScore); // veto some ref gap extensions - vf = _mm_adds_epi16(vf, *pvScore); // veto some ref gap extensions - vf = _mm_max_epi16(vtmp, vf); - vtmp = _mm_cmpgt_epi16(vf, vtmp); - cmp = _mm_movemask_epi8(vtmp); + vf = sse_subs_epi16(vf, rfgape); + vf = sse_adds_epi16(vf, *pvScore); // veto some ref gap extensions + vf = sse_adds_epi16(vf, *pvScore); // veto some ref gap extensions + vf = sse_max_epi16(vtmp, vf); + vtmp = sse_cmpgt_epi16(vf, vtmp); + cmp = sse_movemask_epi8(vtmp); nfixup++; } // Check in the last row for the maximum so far - __m128i *vtmp = vbuf_r + 2 /* H */ + (d.lastIter_ * ROWSTRIDE_2COL); + SSERegI *vtmp = vbuf_r + 2 /* H */ + (d.lastIter_ * ROWSTRIDE_2COL); // Note: we may not want to extract from the final row TCScore lr = ((TCScore*)(vtmp))[d.lastWord_]; found = true; @@ -623,9 +612,9 @@ TAlScore SwAligner::alignGatherEE16(int& flag, bool debug) { // Save some elements to checkpoints if(checkpoint) { - __m128i *pvE = vbuf_r + 0; - __m128i *pvF = vbuf_r + 1; - __m128i *pvH = vbuf_r + 2; + SSERegI *pvE = vbuf_r + 0; + SSERegI *pvF = vbuf_r + 1; + SSERegI *pvH = vbuf_r + 2; size_t coli = i - rfi_; if(coli < cper_.locol_) cper_.locol_ = coli; if(coli > cper_.hicol_) cper_.hicol_ = coli; @@ -653,8 +642,8 @@ TAlScore SwAligner::alignGatherEE16(int& flag, bool debug) { row_div++; } size_t delt = idxrow + row; - size_t vecoff = (row_mod << 5) + row_div; - assert_lt(row_div, 8); + size_t vecoff = (row_mod << (ROWSTRIDE_LOG2+BYTES_LOG2_PER_REG-1)) + row_div; + assert_lt(row_div, (NBYTES_PER_REG/2)); int16_t h_sc = ((int16_t*)pvH)[vecoff]; int16_t e_sc = ((int16_t*)pvE)[vecoff]; int16_t f_sc = ((int16_t*)pvF)[vecoff]; @@ -694,7 +683,7 @@ TAlScore SwAligner::alignGatherEE16(int& flag, bool debug) { cpimod -= iter; cpidiv++; } - size_t vecoff = (cpimod << 5) + cpidiv; + size_t vecoff = (cpimod << (ROWSTRIDE_LOG2+BYTES_LOG2_PER_REG-1)) + cpidiv; cper_.commitMap_.push_back(vecoff); cpi += cper_.per_; cpimod += cper_.per_; @@ -725,16 +714,16 @@ TAlScore SwAligner::alignGatherEE16(int& flag, bool debug) { assert_gt(coli, 0); size_t wordspercol = cper_.niter_ * ROWSTRIDE_2COL; size_t coloff = (coli >> cper_.perpow2_) * wordspercol; - __m128i *dst = cper_.qcols_.ptr() + coloff; - memcpy(dst, vbuf_r, sizeof(__m128i) * wordspercol); + SSERegI *dst = cper_.qcols_.ptr() + coloff; + memcpy(dst, vbuf_r, sizeof(SSERegI) * wordspercol); } } if(cper_.debug_) { // Save the column using memcpys size_t wordspercol = cper_.niter_ * ROWSTRIDE_2COL; size_t coloff = coli * wordspercol; - __m128i *dst = cper_.qcolsD_.ptr() + coloff; - memcpy(dst, vbuf_r, sizeof(__m128i) * wordspercol); + SSERegI *dst = cper_.qcolsD_.ptr() + coloff; + memcpy(dst, vbuf_r, sizeof(SSERegI) * wordspercol); } } } @@ -809,9 +798,11 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseI16(int& flag, bool debug) { assert(!d.profbuf_.empty()); assert_eq(0, d.maxBonus_); - size_t iter = + const size_t iter = (dpRows() + (NWORDS_PER_REG-1)) / NWORDS_PER_REG; // iter = segLen + const size_t lastWordIdx = NWORDS_PER_REG*(d.lastIter_*ROWSTRIDE)+d.lastWord_; + // Many thanks to Michael Farrar for releasing his striped Smith-Waterman // implementation: // @@ -820,104 +811,97 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseI16(int& flag, bool debug) { // Much of the implmentation below is adapted from Michael's code. // Set all elts to reference gap open penalty - __m128i rfgapo = _mm_setzero_si128(); - __m128i rfgape = _mm_setzero_si128(); - __m128i rdgapo = _mm_setzero_si128(); - __m128i rdgape = _mm_setzero_si128(); - __m128i vlo = _mm_setzero_si128(); - __m128i vhi = _mm_setzero_si128(); - __m128i vhilsw = _mm_setzero_si128(); - __m128i vlolsw = _mm_setzero_si128(); - __m128i ve = _mm_setzero_si128(); - __m128i vf = _mm_setzero_si128(); - __m128i vh = _mm_setzero_si128(); + SSERegI rfgapo = sse_setzero_siall(); + SSERegI rfgape = sse_setzero_siall(); + SSERegI rdgapo = sse_setzero_siall(); + SSERegI rdgape = sse_setzero_siall(); + SSERegI vlo = sse_setzero_siall(); + SSERegI vhilsw = sse_setzero_siall(); + SSERegI vlolsw = sse_setzero_siall(); + SSERegI ve = sse_setzero_siall(); + SSERegI vf = sse_setzero_siall(); + SSERegI vh = sse_setzero_siall(); #if 0 - __m128i vhd = _mm_setzero_si128(); - __m128i vhdtmp = _mm_setzero_si128(); + SSERegI vhd = sse_setzero_siall(); + SSERegI vhdtmp = sse_setzero_siall(); #endif - __m128i vtmp = _mm_setzero_si128(); + SSERegI vtmp = sse_setzero_siall(); + + SSERegI vs0 = sse_setzero_siall(); + SSERegI vs1 = sse_setzero_siall(); assert_gt(sc_->refGapOpen(), 0); assert_leq(sc_->refGapOpen(), MAX_I16); - rfgapo = _mm_insert_epi16(rfgapo, sc_->refGapOpen(), 0); - rfgapo = _mm_shufflelo_epi16(rfgapo, 0); - rfgapo = _mm_shuffle_epi32(rfgapo, 0); + sse_fill_i16(sc_->refGapOpen(), rfgapo); // Set all elts to reference gap extension penalty assert_gt(sc_->refGapExtend(), 0); assert_leq(sc_->refGapExtend(), MAX_I16); assert_leq(sc_->refGapExtend(), sc_->refGapOpen()); - rfgape = _mm_insert_epi16(rfgape, sc_->refGapExtend(), 0); - rfgape = _mm_shufflelo_epi16(rfgape, 0); - rfgape = _mm_shuffle_epi32(rfgape, 0); + sse_fill_i16(sc_->refGapExtend(), rfgape); // Set all elts to read gap open penalty assert_gt(sc_->readGapOpen(), 0); assert_leq(sc_->readGapOpen(), MAX_I16); - rdgapo = _mm_insert_epi16(rdgapo, sc_->readGapOpen(), 0); - rdgapo = _mm_shufflelo_epi16(rdgapo, 0); - rdgapo = _mm_shuffle_epi32(rdgapo, 0); + sse_fill_i16(sc_->readGapOpen(), rdgapo); // Set all elts to read gap extension penalty assert_gt(sc_->readGapExtend(), 0); assert_leq(sc_->readGapExtend(), MAX_I16); assert_leq(sc_->readGapExtend(), sc_->readGapOpen()); - rdgape = _mm_insert_epi16(rdgape, sc_->readGapExtend(), 0); - rdgape = _mm_shufflelo_epi16(rdgape, 0); - rdgape = _mm_shuffle_epi32(rdgape, 0); + sse_fill_i16(sc_->readGapExtend(), rdgape); // Set all elts to 0x8000 (min value for signed 16-bit) - vlo = _mm_cmpeq_epi16(vlo, vlo); // all elts = 0xffff - vlo = _mm_slli_epi16(vlo, NBITS_PER_WORD-1); // all elts = 0x8000 - + sse_fill_i16(0x8000, vlo); + +#ifndef NDEBUG // Set all elts to 0x7fff (max value for signed 16-bit) - vhi = _mm_cmpeq_epi16(vhi, vhi); // all elts = 0xffff - vhi = _mm_srli_epi16(vhi, 1); // all elts = 0x7fff - + SSERegI vhi = sse_setzero_siall(); + sse_fill_i16(0x7fff, vhi); +#endif + // vlolsw: topmost (least sig) word set to 0x8000, all other words=0 - vlolsw = _mm_shuffle_epi32(vlo, 0); - vlolsw = _mm_srli_si128(vlolsw, NBYTES_PER_REG - NBYTES_PER_WORD); + sse_set_low_i16(0x8000, vlolsw); // vhilsw: topmost (least sig) word set to 0x7fff, all other words=0 - vhilsw = _mm_shuffle_epi32(vhi, 0); - vhilsw = _mm_srli_si128(vhilsw, NBYTES_PER_REG - NBYTES_PER_WORD); + sse_set_low_i16(0x7fff, vhilsw); - // Points to a long vector of __m128i where each element is a block of + // Points to a long vector of SSERegI where each element is a block of // contiguous cells in the E, F or H matrix. If the index % 3 == 0, then // the block of cells is from the E matrix. If index % 3 == 1, they're // from the F matrix. If index % 3 == 2, then they're from the H matrix. // Blocks of cells are organized in the same interleaved manner as they are // calculated by the Farrar algorithm. - const __m128i *pvScore; // points into the query profile + const SSERegI *pvScore; // points into the query profile d.mat_.init(dpRows(), rff_ - rfi_, NWORDS_PER_REG); const size_t colstride = d.mat_.colstride(); assert_eq(ROWSTRIDE, colstride / iter); // Initialize the H and E vectors in the first matrix column - __m128i *pvHTmp = d.mat_.tmpvec(0, 0); - __m128i *pvETmp = d.mat_.evec(0, 0); + SSERegI *pvHTmp = d.mat_.tmpvec(0, 0); + SSERegI *pvETmp = d.mat_.evec(0, 0); // Maximum score in final row bool found = false; TCScore lrmax = MIN_I16; for(size_t i = 0; i < iter; i++) { - _mm_store_si128(pvETmp, vlo); + sse_store_siall(pvETmp, vlo); // Could initialize Hs to high or low. If high, cells in the lower // triangle will have somewhat more legitiate scores, but still won't // be exhaustively scored. - _mm_store_si128(pvHTmp, vlo); + sse_store_siall(pvHTmp, vlo); pvETmp += ROWSTRIDE; pvHTmp += ROWSTRIDE; } // These are swapped just before the innermost loop - __m128i *pvHStore = d.mat_.hvec(0, 0); - __m128i *pvHLoad = d.mat_.tmpvec(0, 0); - __m128i *pvELoad = d.mat_.evec(0, 0); - __m128i *pvEStore = d.mat_.evecUnsafe(0, 1); - __m128i *pvFStore = d.mat_.fvec(0, 0); - __m128i *pvFTmp = NULL; + SSERegI *pvHStore = d.mat_.hvec(0, 0); + SSERegI *pvHLoad = d.mat_.tmpvec(0, 0); + SSERegI *pvELoad = d.mat_.evec(0, 0); + SSERegI *pvEStore = d.mat_.evecUnsafe(0, 1); + SSERegI *pvFStore = d.mat_.fvec(0, 0); + SSERegI *pvFTmp = NULL; assert_gt(sc_->gapbar, 0); size_t nfixup = 0; @@ -948,60 +932,64 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseI16(int& flag, bool debug) { pvScore = d.profbuf_.ptr() + off; // even elts = query profile, odd = gap barrier // Set all cells to low value - vf = _mm_cmpeq_epi16(vf, vf); - vf = _mm_slli_epi16(vf, NBITS_PER_WORD-1); - vf = _mm_or_si128(vf, vlolsw); + vf = sse_cmpeq_epi16(vf, vf); + vf = sse_slli_epi16(vf, NBITS_PER_WORD-1); + vf = sse_or_siall(vf, vlolsw); // Load H vector from the final row of the previous column - vh = _mm_load_si128(pvHLoad + colstride - ROWSTRIDE); + vh = sse_load_siall(pvHLoad + colstride - ROWSTRIDE); // Shift 2 bytes down so that topmost (least sig) cell gets 0 - vh = _mm_slli_si128(vh, NBYTES_PER_WORD); + vh = sse_slli_siall(vh, NBYTES_PER_WORD); // Fill topmost (least sig) cell with high value - vh = _mm_or_si128(vh, vhilsw); + vh = sse_or_siall(vh, vhilsw); // For each character in the reference text: size_t j; for(j = 0; j < iter; j++) { + vs0 = sse_load_siall(pvScore); + pvScore++; + vs1 = sse_load_siall(pvScore); + pvScore++; // Load cells from E, calculated previously - ve = _mm_load_si128(pvELoad); + ve = sse_load_siall(pvELoad); #if 0 - vhd = _mm_load_si128(pvHLoad); + vhd = sse_load_siall(pvHLoad); #endif assert_all_lt(ve, vhi); pvELoad += ROWSTRIDE; // Store cells in F, calculated previously - vf = _mm_adds_epi16(vf, pvScore[1]); // veto some ref gap extensions - vf = _mm_adds_epi16(vf, pvScore[1]); // veto some ref gap extensions - _mm_store_si128(pvFStore, vf); + vf = sse_adds_epi16(vf, vs1); // veto some ref gap extensions + vf = sse_adds_epi16(vf, vs1); // veto some ref gap extensions + sse_store_siall(pvFStore, vf); pvFStore += ROWSTRIDE; // Factor in query profile (matches and mismatches) - vh = _mm_adds_epi16(vh, pvScore[0]); + vh = sse_adds_epi16(vh, vs0); // Update H, factoring in E and F - vh = _mm_max_epi16(vh, ve); - vh = _mm_max_epi16(vh, vf); + vh = sse_max_epi16(vh, ve); + vh = sse_max_epi16(vh, vf); // Save the new vH values - _mm_store_si128(pvHStore, vh); + sse_store_siall(pvHStore, vh); pvHStore += ROWSTRIDE; // Update vE value vtmp = vh; #if 0 vhdtmp = vhd; - vhd = _mm_subs_epi16(vhd, rdgapo); - vhd = _mm_adds_epi16(vhd, pvScore[1]); // veto some read gap opens - vhd = _mm_adds_epi16(vhd, pvScore[1]); // veto some read gap opens - ve = _mm_subs_epi16(ve, rdgape); - ve = _mm_max_epi16(ve, vhd); + vhd = sse_subs_epi16(vhd, rdgapo); + vhd = sse_adds_epi16(vhd, vs1); // veto some read gap opens + vhd = sse_adds_epi16(vhd, vs1); // veto some read gap opens + ve = sse_subs_epi16(ve, rdgape); + ve = sse_max_epi16(ve, vhd); #else - vh = _mm_subs_epi16(vh, rdgapo); - vh = _mm_adds_epi16(vh, pvScore[1]); // veto some read gap opens - vh = _mm_adds_epi16(vh, pvScore[1]); // veto some read gap opens - ve = _mm_subs_epi16(ve, rdgape); - ve = _mm_max_epi16(ve, vh); + vh = sse_subs_epi16(vh, rdgapo); + vh = sse_adds_epi16(vh, vs1); // veto some read gap opens + vh = sse_adds_epi16(vh, vs1); // veto some read gap opens + ve = sse_subs_epi16(ve, rdgape); + ve = sse_max_epi16(ve, vh); #endif assert_all_lt(ve, vhi); @@ -1009,107 +997,106 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseI16(int& flag, bool debug) { #if 0 vh = vhdtmp; #else - vh = _mm_load_si128(pvHLoad); + vh = sse_load_siall(pvHLoad); #endif pvHLoad += ROWSTRIDE; // Save E values - _mm_store_si128(pvEStore, ve); + sse_store_siall(pvEStore, ve); pvEStore += ROWSTRIDE; // Update vf value - vtmp = _mm_subs_epi16(vtmp, rfgapo); - vf = _mm_subs_epi16(vf, rfgape); + vtmp = sse_subs_epi16(vtmp, rfgapo); + vf = sse_subs_epi16(vf, rfgape); assert_all_lt(vf, vhi); - vf = _mm_max_epi16(vf, vtmp); - - pvScore += 2; // move on to next query profile / gap veto + vf = sse_max_epi16(vf, vtmp); } + pvScore -= (iter*2) - 1; // reset veto vector + vs1 = sse_load_siall(pvScore); + // pvHStore, pvELoad, pvEStore have all rolled over to the next column pvFTmp = pvFStore; pvFStore -= colstride; // reset to start of column - vtmp = _mm_load_si128(pvFStore); + vtmp = sse_load_siall(pvFStore); + + // vf from last row gets shifted down by one to overlay the first row + // rfgape has already been subtracted from it. + vf = sse_slli_siall(vf, NBYTES_PER_WORD); + vf = sse_or_siall(vf, vlolsw); + + vf = sse_adds_epi16(vf, vs1); // veto some ref gap extensions + vf = sse_adds_epi16(vf, vs1); // veto some ref gap extensions + vf = sse_max_epi16(vtmp, vf); + vtmp = sse_cmpgt_epi16(vf, vtmp); + int cmp = sse_movemask_epi8(vtmp); + // Load after computing cmp, so the result is ready by the time it is tested in while pvHStore -= colstride; // reset to start of column - vh = _mm_load_si128(pvHStore); + vh = sse_load_siall(pvHStore); + pvHLoad = pvHStore; // new pvHLoad = pvHStore #if 0 #else pvEStore -= colstride; // reset to start of column - ve = _mm_load_si128(pvEStore); + ve = sse_load_siall(pvEStore); #endif - pvHLoad = pvHStore; // new pvHLoad = pvHStore - pvScore = d.profbuf_.ptr() + off + 1; // reset veto vector - - // vf from last row gets shifted down by one to overlay the first row - // rfgape has already been subtracted from it. - vf = _mm_slli_si128(vf, NBYTES_PER_WORD); - vf = _mm_or_si128(vf, vlolsw); - - vf = _mm_adds_epi16(vf, *pvScore); // veto some ref gap extensions - vf = _mm_adds_epi16(vf, *pvScore); // veto some ref gap extensions - vf = _mm_max_epi16(vtmp, vf); - vtmp = _mm_cmpgt_epi16(vf, vtmp); - int cmp = _mm_movemask_epi8(vtmp); - // If any element of vtmp is greater than H - gap-open... j = 0; while(cmp != 0x0000) { // Store this vf - _mm_store_si128(pvFStore, vf); + sse_store_siall(pvFStore, vf); pvFStore += ROWSTRIDE; // Update vh w/r/t new vf - vh = _mm_max_epi16(vh, vf); + vh = sse_max_epi16(vh, vf); // Save vH values - _mm_store_si128(pvHStore, vh); + sse_store_siall(pvHStore, vh); pvHStore += ROWSTRIDE; // Update E in case it can be improved using our new vh #if 0 #else - vh = _mm_subs_epi16(vh, rdgapo); - vh = _mm_adds_epi16(vh, *pvScore); // veto some read gap opens - vh = _mm_adds_epi16(vh, *pvScore); // veto some read gap opens - ve = _mm_max_epi16(ve, vh); - _mm_store_si128(pvEStore, ve); + vh = sse_subs_epi16(vh, rdgapo); + vh = sse_adds_epi16(vh, vs1); // veto some read gap opens + vh = sse_adds_epi16(vh, vs1); // veto some read gap opens + ve = sse_max_epi16(ve, vh); + sse_store_siall(pvEStore, ve); pvEStore += ROWSTRIDE; #endif pvScore += 2; assert_lt(j, iter); if(++j == iter) { + pvScore -= iter*2; + j = 0; pvFStore -= colstride; - vtmp = _mm_load_si128(pvFStore); // load next vf ASAP pvHStore -= colstride; - vh = _mm_load_si128(pvHStore); // load next vh ASAP #if 0 #else pvEStore -= colstride; - ve = _mm_load_si128(pvEStore); // load next ve ASAP -#endif - pvScore = d.profbuf_.ptr() + off + 1; - j = 0; - vf = _mm_slli_si128(vf, NBYTES_PER_WORD); - vf = _mm_or_si128(vf, vlolsw); - } else { - vtmp = _mm_load_si128(pvFStore); // load next vf ASAP - vh = _mm_load_si128(pvHStore); // load next vh ASAP -#if 0 -#else - ve = _mm_load_si128(pvEStore); // load next vh ASAP #endif + vf = sse_slli_siall(vf, NBYTES_PER_WORD); + vf = sse_or_siall(vf, vlolsw); } + vs1 = sse_load_siall(pvScore); + vtmp = sse_load_siall(pvFStore); // load next vf ASAP // Update F with another gap extension - vf = _mm_subs_epi16(vf, rfgape); - vf = _mm_adds_epi16(vf, *pvScore); // veto some ref gap extensions - vf = _mm_adds_epi16(vf, *pvScore); // veto some ref gap extensions - vf = _mm_max_epi16(vtmp, vf); - vtmp = _mm_cmpgt_epi16(vf, vtmp); - cmp = _mm_movemask_epi8(vtmp); + vf = sse_subs_epi16(vf, rfgape); + vf = sse_adds_epi16(vf, vs1); // veto some ref gap extensions + vf = sse_adds_epi16(vf, vs1); // veto some ref gap extensions + vf = sse_max_epi16(vtmp, vf); + vtmp = sse_cmpgt_epi16(vf, vtmp); + cmp = sse_movemask_epi8(vtmp); + + // Load after computing cmp, so the result is afailable by the time it is tested + vh = sse_load_siall(pvHStore); // load next vh ASAP +#if 0 +#else + ve = sse_load_siall(pvEStore); // load next vh ASAP +#endif nfixup++; } @@ -1130,9 +1117,8 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseI16(int& flag, bool debug) { } #endif - __m128i *vtmp = d.mat_.hvec(d.lastIter_, i-rfi_); // Note: we may not want to extract from the final row - TCScore lr = ((TCScore*)(vtmp))[d.lastWord_]; + TCScore lr = ((TCScore*)(pvHLoad))[lastWordIdx]; found = true; if(lr > lrmax) { lrmax = lr; @@ -1226,7 +1212,7 @@ bool SwAligner::gatherCellsNucleotidesEnd2EndSseI16(TAlScore best) { assert(!d.profbuf_.empty()); const size_t colstride = d.mat_.colstride(); ASSERT_ONLY(bool sawbest = false); - __m128i *pvH = d.mat_.hvec(d.lastIter_, 0); + SSERegI *pvH = d.mat_.hvec(d.lastIter_, 0); for(size_t j = 0; j < ncol; j++) { TAlScore sc = (TAlScore)(((TCScore*)pvH)[d.lastWord_] - 0x7fff); assert_leq(sc, best); @@ -1352,7 +1338,7 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseI16( size_t rowelt, rowvec, eltvec; size_t left_rowelt, up_rowelt, upleft_rowelt; size_t left_rowvec, up_rowvec, upleft_rowvec; - __m128i *cur_vec, *left_vec, *up_vec, *upleft_vec; + SSERegI *cur_vec, *left_vec, *up_vec, *upleft_vec; NEW_ROW_COL(row, col); while((int)row >= 0) { met.btcell++; diff --git a/aligner_swsse_ee_u8.cpp b/aligner_swsse_ee_u8.cpp index ee65dbf..86c0c38 100644 --- a/aligner_swsse_ee_u8.cpp +++ b/aligner_swsse_ee_u8.cpp @@ -55,8 +55,7 @@ #include <limits> #include "aligner_sw.h" -static const size_t NBYTES_PER_REG = 16; -static const size_t NWORDS_PER_REG = 16; +static const size_t NWORDS_PER_REG = NBYTES_PER_REG; //static const size_t NBITS_PER_WORD = 8; static const size_t NBYTES_PER_WORD = 1; @@ -83,14 +82,14 @@ void SwAligner::buildQueryProfileEnd2EndSseU8(bool fw) { const BTString* qu = fw ? qufw_ : qurc_; const size_t len = rd->length(); const size_t seglen = (len + (NWORDS_PER_REG-1)) / NWORDS_PER_REG; - // How many __m128i's are needed - size_t n128s = + // How many SSERegI's are needed + size_t nsses = 64 + // slack bytes, for alignment? (seglen * ALPHA_SIZE) // query profile data * 2; // & gap barrier data - assert_gt(n128s, 0); + assert_gt(nsses, 0); SSEData& d = fw ? sseU8fw_ : sseU8rc_; - d.profbuf_.resizeNoCopy(n128s); + d.profbuf_.resizeNoCopy(nsses); assert(!d.profbuf_.empty()); d.maxPen_ = d.maxBonus_ = 0; d.lastIter_ = d.lastWord_ = 0; @@ -237,41 +236,41 @@ static bool cellOkEnd2EndU8( #else #define assert_all_eq0(x) { \ - __m128i z = _mm_setzero_si128(); \ - __m128i tmp = _mm_setzero_si128(); \ - z = _mm_xor_si128(z, z); \ - tmp = _mm_cmpeq_epi16(x, z); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI z = sse_setzero_siall(); \ + SSERegI tmp = sse_setzero_siall(); \ + z = sse_xor_siall(z, z); \ + tmp = sse_cmpeq_epi16(x, z); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #define assert_all_gt(x, y) { \ - __m128i tmp = _mm_cmpgt_epu8(x, y); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI tmp = sse_cmpgt_epu8(x, y); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #define assert_all_gt_lo(x) { \ - __m128i z = _mm_setzero_si128(); \ - __m128i tmp = _mm_setzero_si128(); \ - z = _mm_xor_si128(z, z); \ - tmp = _mm_cmpgt_epu8(x, z); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI z = sse_setzero_siall(); \ + SSERegI tmp = sse_setzero_siall(); \ + z = sse_xor_siall(z, z); \ + tmp = sse_cmpgt_epu8(x, z); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #define assert_all_lt(x, y) { \ - __m128i z = _mm_setzero_si128(); \ - z = _mm_xor_si128(z, z); \ - __m128i tmp = _mm_subs_epu8(y, x); \ - tmp = _mm_cmpeq_epi16(tmp, z); \ - assert_eq(0x0000, _mm_movemask_epi8(tmp)); \ + SSERegI z = sse_setzero_siall(); \ + z = sse_xor_siall(z, z); \ + SSERegI tmp = sse_subs_epu8(y, x); \ + tmp = sse_cmpeq_epi16(tmp, z); \ + assert_eq(0x0000, sse_movemask_epi8(tmp)); \ } #define assert_all_lt_hi(x) { \ - __m128i z = _mm_setzero_si128(); \ - __m128i tmp = _mm_setzero_si128(); \ - z = _mm_cmpeq_epu8(z, z); \ - z = _mm_srli_epu8(z, 1); \ - tmp = _mm_cmplt_epu8(x, z); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI z = sse_setzero_siall(); \ + SSERegI tmp = sse_setzero_siall(); \ + z = sse_cmpeq_epu8(z, z); \ + z = sse_srli_epu8(z, 1); \ + tmp = sse_cmplt_epu8(x, z); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #endif @@ -317,14 +316,12 @@ TAlScore SwAligner::alignGatherEE8(int& flag, bool debug) { size_t iter = (dpRows() + (NWORDS_PER_REG-1)) / NWORDS_PER_REG; // iter = segLen - int dup; - // Now set up the score vectors. We just need two columns worth, which // we'll call "left" and "right". d.vecbuf_.resize(4 * 2 * iter); d.vecbuf_.zero(); - __m128i *vbuf_l = d.vecbuf_.ptr(); - __m128i *vbuf_r = d.vecbuf_.ptr() + (4 * iter); + SSERegI *vbuf_l = d.vecbuf_.ptr(); + SSERegI *vbuf_r = d.vecbuf_.ptr() + (4 * iter); // This is the data structure that holds candidate cells per diagonal. const size_t ndiags = rff_ - rfi_ + dpRows() - 1; @@ -357,86 +354,75 @@ TAlScore SwAligner::alignGatherEE8(int& flag, bool debug) { // Much of the implmentation below is adapted from Michael's code. // Set all elts to reference gap open penalty - __m128i rfgapo = _mm_setzero_si128(); - __m128i rfgape = _mm_setzero_si128(); - __m128i rdgapo = _mm_setzero_si128(); - __m128i rdgape = _mm_setzero_si128(); - __m128i vlo = _mm_setzero_si128(); - __m128i vhi = _mm_setzero_si128(); - __m128i ve = _mm_setzero_si128(); - __m128i vf = _mm_setzero_si128(); - __m128i vh = _mm_setzero_si128(); - __m128i vhd = _mm_setzero_si128(); - __m128i vhdtmp = _mm_setzero_si128(); - __m128i vtmp = _mm_setzero_si128(); - __m128i vzero = _mm_setzero_si128(); - __m128i vhilsw = _mm_setzero_si128(); + SSERegI rfgapo = sse_setzero_siall(); + SSERegI rfgape = sse_setzero_siall(); + SSERegI rdgapo = sse_setzero_siall(); + SSERegI rdgape = sse_setzero_siall(); + SSERegI vlo = sse_setzero_siall(); + SSERegI ve = sse_setzero_siall(); + SSERegI vf = sse_setzero_siall(); + SSERegI vh = sse_setzero_siall(); + SSERegI vhd = sse_setzero_siall(); + SSERegI vhdtmp = sse_setzero_siall(); + SSERegI vtmp = sse_setzero_siall(); + SSERegI vzero = sse_setzero_siall(); + SSERegI vhilsw = sse_setzero_siall(); assert_gt(sc_->refGapOpen(), 0); assert_leq(sc_->refGapOpen(), MAX_U8); - dup = (sc_->refGapOpen() << 8) | (sc_->refGapOpen() & 0x00ff); - rfgapo = _mm_insert_epi16(rfgapo, dup, 0); - rfgapo = _mm_shufflelo_epi16(rfgapo, 0); - rfgapo = _mm_shuffle_epi32(rfgapo, 0); + sse_fill_u8(sc_->refGapOpen(), rfgapo); // Set all elts to reference gap extension penalty assert_gt(sc_->refGapExtend(), 0); assert_leq(sc_->refGapExtend(), MAX_U8); assert_leq(sc_->refGapExtend(), sc_->refGapOpen()); - dup = (sc_->refGapExtend() << 8) | (sc_->refGapExtend() & 0x00ff); - rfgape = _mm_insert_epi16(rfgape, dup, 0); - rfgape = _mm_shufflelo_epi16(rfgape, 0); - rfgape = _mm_shuffle_epi32(rfgape, 0); + sse_fill_u8(sc_->refGapExtend(), rfgape); // Set all elts to read gap open penalty assert_gt(sc_->readGapOpen(), 0); assert_leq(sc_->readGapOpen(), MAX_U8); - dup = (sc_->readGapOpen() << 8) | (sc_->readGapOpen() & 0x00ff); - rdgapo = _mm_insert_epi16(rdgapo, dup, 0); - rdgapo = _mm_shufflelo_epi16(rdgapo, 0); - rdgapo = _mm_shuffle_epi32(rdgapo, 0); + sse_fill_u8(sc_->readGapOpen(), rdgapo); // Set all elts to read gap extension penalty assert_gt(sc_->readGapExtend(), 0); assert_leq(sc_->readGapExtend(), MAX_U8); assert_leq(sc_->readGapExtend(), sc_->readGapOpen()); - dup = (sc_->readGapExtend() << 8) | (sc_->readGapExtend() & 0x00ff); - rdgape = _mm_insert_epi16(rdgape, dup, 0); - rdgape = _mm_shufflelo_epi16(rdgape, 0); - rdgape = _mm_shuffle_epi32(rdgape, 0); - - vhi = _mm_cmpeq_epi16(vhi, vhi); // all elts = 0xffff - vlo = _mm_xor_si128(vlo, vlo); // all elts = 0 - - // vhilsw: topmost (least sig) word set to 0x7fff, all other words=0 - vhilsw = _mm_shuffle_epi32(vhi, 0); - vhilsw = _mm_srli_si128(vhilsw, NBYTES_PER_REG - NBYTES_PER_WORD); + sse_fill_u8(sc_->readGapExtend(), rdgape); + +#ifndef NDEBUG + SSERegI vhi = sse_setzero_siall(); + sse_fill_u8_opt(0xff, vhi) +#endif + sse_fill_u8_opt(0, vlo); + + // vhilsw: topmost (least sig) word set to 0xff, all other words=0 + sse_set_low_u8(0xff, vhilsw); - // Points to a long vector of __m128i where each element is a block of + // Points to a long vector of SSERegI where each element is a block of // contiguous cells in the E, F or H matrix. If the index % 3 == 0, then // the block of cells is from the E matrix. If index % 3 == 1, they're // from the F matrix. If index % 3 == 2, then they're from the H matrix. // Blocks of cells are organized in the same interleaved manner as they are // calculated by the Farrar algorithm. - const __m128i *pvScore; // points into the query profile + const SSERegI *pvScore; // points into the query profile const size_t colstride = ROWSTRIDE_2COL * iter; // Initialize the H and E vectors in the first matrix column - __m128i *pvELeft = vbuf_l + 0; __m128i *pvERight = vbuf_r + 0; - /* __m128i *pvFLeft = vbuf_l + 1; */ __m128i *pvFRight = vbuf_r + 1; - __m128i *pvHLeft = vbuf_l + 2; __m128i *pvHRight = vbuf_r + 2; + SSERegI *pvELeft = vbuf_l + 0; SSERegI *pvERight = vbuf_r + 0; + /* SSERegI *pvFLeft = vbuf_l + 1; */ SSERegI *pvFRight = vbuf_r + 1; + SSERegI *pvHLeft = vbuf_l + 2; SSERegI *pvHRight = vbuf_r + 2; // Maximum score in final row bool found = false; TCScore lrmax = MIN_U8; for(size_t i = 0; i < iter; i++) { - _mm_store_si128(pvERight, vlo); pvERight += ROWSTRIDE_2COL; + sse_store_siall(pvERight, vlo); pvERight += ROWSTRIDE_2COL; // Could initialize Hs to high or low. If high, cells in the lower // triangle will have somewhat more legitiate scores, but still won't // be exhaustively scored. - _mm_store_si128(pvHRight, vlo); pvHRight += ROWSTRIDE_2COL; + sse_store_siall(pvHRight, vlo); pvHRight += ROWSTRIDE_2COL; } assert_gt(sc_->gapbar, 0); @@ -472,45 +458,45 @@ TAlScore SwAligner::alignGatherEE8(int& flag, bool debug) { pvScore = d.profbuf_.ptr() + off; // even elts = query profile, odd = gap barrier // Set all cells to low value - vf = _mm_xor_si128(vf, vf); + vf = sse_xor_siall(vf, vf); // Load H vector from the final row of the previous column - vh = _mm_load_si128(pvHLeft + colstride - ROWSTRIDE_2COL); + vh = sse_load_siall(pvHLeft + colstride - ROWSTRIDE_2COL); // Shift 2 bytes down so that topmost (least sig) cell gets 0 - vh = _mm_slli_si128(vh, NBYTES_PER_WORD); + vh = sse_slli_siall(vh, NBYTES_PER_WORD); // Fill topmost (least sig) cell with high value - vh = _mm_or_si128(vh, vhilsw); + vh = sse_or_siall(vh, vhilsw); // For each character in the reference text: size_t j; for(j = 0; j < iter; j++) { // Load cells from E, calculated previously - ve = _mm_load_si128(pvELeft); - vhd = _mm_load_si128(pvHLeft); + ve = sse_load_siall(pvELeft); + vhd = sse_load_siall(pvHLeft); assert_all_lt(ve, vhi); pvELeft += ROWSTRIDE_2COL; // Store cells in F, calculated previously - vf = _mm_subs_epu8(vf, pvScore[1]); // veto some ref gap extensions - _mm_store_si128(pvFRight, vf); + vf = sse_subs_epu8(vf, pvScore[1]); // veto some ref gap extensions + sse_store_siall(pvFRight, vf); pvFRight += ROWSTRIDE_2COL; // Factor in query profile (matches and mismatches) - vh = _mm_subs_epu8(vh, pvScore[0]); + vh = sse_subs_epu8(vh, pvScore[0]); // Update H, factoring in E and F - vh = _mm_max_epu8(vh, vf); + vh = sse_max_epu8(vh, vf); // Update vE value vhdtmp = vhd; - vhd = _mm_subs_epu8(vhd, rdgapo); - vhd = _mm_subs_epu8(vhd, pvScore[1]); // veto some read gap opens - ve = _mm_subs_epu8(ve, rdgape); - ve = _mm_max_epu8(ve, vhd); - vh = _mm_max_epu8(vh, ve); + vhd = sse_subs_epu8(vhd, rdgapo); + vhd = sse_subs_epu8(vhd, pvScore[1]); // veto some read gap opens + ve = sse_subs_epu8(ve, rdgape); + ve = sse_max_epu8(ve, vhd); + vh = sse_max_epu8(vh, ve); // Save the new vH values - _mm_store_si128(pvHRight, vh); + sse_store_siall(pvHRight, vh); pvHRight += ROWSTRIDE_2COL; vtmp = vh; assert_all_lt(ve, vhi); @@ -520,49 +506,49 @@ TAlScore SwAligner::alignGatherEE8(int& flag, bool debug) { pvHLeft += ROWSTRIDE_2COL; // Save E values - _mm_store_si128(pvERight, ve); + sse_store_siall(pvERight, ve); pvERight += ROWSTRIDE_2COL; // Update vf value - vtmp = _mm_subs_epu8(vtmp, rfgapo); + vtmp = sse_subs_epu8(vtmp, rfgapo); - vf = _mm_subs_epu8(vf, rfgape); + vf = sse_subs_epu8(vf, rfgape); assert_all_lt(vf, vhi); - vf = _mm_max_epu8(vf, vtmp); + vf = sse_max_epu8(vf, vtmp); pvScore += 2; // move on to next query profile / gap veto } // pvHStore, pvELoad, pvEStore have all rolled over to the next column pvFRight -= colstride; // reset to start of column - vtmp = _mm_load_si128(pvFRight); + vtmp = sse_load_siall(pvFRight); pvHRight -= colstride; // reset to start of column - vh = _mm_load_si128(pvHRight); + vh = sse_load_siall(pvHRight); pvScore = d.profbuf_.ptr() + off + 1; // reset veto vector // vf from last row gets shifted down by one to overlay the first row // rfgape has already been subtracted from it. - vf = _mm_slli_si128(vf, NBYTES_PER_WORD); + vf = sse_slli_siall(vf, NBYTES_PER_WORD); - vf = _mm_subs_epu8(vf, *pvScore); // veto some ref gap extensions - vf = _mm_max_epu8(vtmp, vf); - vtmp = _mm_subs_epu8(vf, vtmp); - vtmp = _mm_cmpeq_epi8(vtmp, vzero); - int cmp = _mm_movemask_epi8(vtmp); + vf = sse_subs_epu8(vf, *pvScore); // veto some ref gap extensions + vf = sse_max_epu8(vtmp, vf); + vtmp = sse_subs_epu8(vf, vtmp); + vtmp = sse_cmpeq_epi8(vtmp, vzero); + int cmp = sse_movemask_epi8(vtmp); // If any element of vtmp is greater than H - gap-open... j = 0; - while(cmp != 0xffff) { + while(cmp != SSE_MASK_ALL) { // Store this vf - _mm_store_si128(pvFRight, vf); + sse_store_siall(pvFRight, vf); pvFRight += ROWSTRIDE_2COL; // Update vh w/r/t new vf - vh = _mm_max_epu8(vh, vf); + vh = sse_max_epu8(vh, vf); // Save vH values - _mm_store_si128(pvHRight, vh); + sse_store_siall(pvHRight, vh); pvHRight += ROWSTRIDE_2COL; pvScore += 2; @@ -570,29 +556,29 @@ TAlScore SwAligner::alignGatherEE8(int& flag, bool debug) { assert_lt(j, iter); if(++j == iter) { pvFRight -= colstride; - vtmp = _mm_load_si128(pvFRight); // load next vf ASAP + vtmp = sse_load_siall(pvFRight); // load next vf ASAP pvHRight -= colstride; - vh = _mm_load_si128(pvHRight); // load next vh ASAP + vh = sse_load_siall(pvHRight); // load next vh ASAP pvScore = d.profbuf_.ptr() + off + 1; j = 0; - vf = _mm_slli_si128(vf, NBYTES_PER_WORD); + vf = sse_slli_siall(vf, NBYTES_PER_WORD); } else { - vtmp = _mm_load_si128(pvFRight); // load next vf ASAP - vh = _mm_load_si128(pvHRight); // load next vh ASAP + vtmp = sse_load_siall(pvFRight); // load next vf ASAP + vh = sse_load_siall(pvHRight); // load next vh ASAP } // Update F with another gap extension - vf = _mm_subs_epu8(vf, rfgape); - vf = _mm_subs_epu8(vf, *pvScore); // veto some ref gap extensions - vf = _mm_max_epu8(vtmp, vf); - vtmp = _mm_subs_epu8(vf, vtmp); - vtmp = _mm_cmpeq_epi8(vtmp, vzero); - cmp = _mm_movemask_epi8(vtmp); + vf = sse_subs_epu8(vf, rfgape); + vf = sse_subs_epu8(vf, *pvScore); // veto some ref gap extensions + vf = sse_max_epu8(vtmp, vf); + vtmp = sse_subs_epu8(vf, vtmp); + vtmp = sse_cmpeq_epi8(vtmp, vzero); + cmp = sse_movemask_epi8(vtmp); nfixup++; } // Check in the last row for the maximum so far - __m128i *vtmp = vbuf_r + 2 /* H */ + (d.lastIter_ * ROWSTRIDE_2COL); + SSERegI *vtmp = vbuf_r + 2 /* H */ + (d.lastIter_ * ROWSTRIDE_2COL); // Note: we may not want to extract from the final row TCScore lr = ((TCScore*)(vtmp))[d.lastWord_]; found = true; @@ -615,9 +601,9 @@ TAlScore SwAligner::alignGatherEE8(int& flag, bool debug) { // Save some elements to checkpoints if(checkpoint) { - __m128i *pvE = vbuf_r + 0; - __m128i *pvF = vbuf_r + 1; - __m128i *pvH = vbuf_r + 2; + SSERegI *pvE = vbuf_r + 0; + SSERegI *pvF = vbuf_r + 1; + SSERegI *pvH = vbuf_r + 2; size_t coli = i - rfi_; if(coli < cper_.locol_) cper_.locol_ = coli; if(coli > cper_.hicol_) cper_.hicol_ = coli; @@ -645,8 +631,8 @@ TAlScore SwAligner::alignGatherEE8(int& flag, bool debug) { row_div++; } size_t delt = idxrow + row; - size_t vecoff = (row_mod << 6) + row_div; - assert_lt(row_div, 16); + size_t vecoff = (row_mod << (ROWSTRIDE_LOG2+BYTES_LOG2_PER_REG)) + row_div; + assert_lt(row_div, NBYTES_PER_REG); int16_t h_sc = ((uint8_t*)pvH)[vecoff]; int16_t e_sc = ((uint8_t*)pvE)[vecoff]; int16_t f_sc = ((uint8_t*)pvF)[vecoff]; @@ -689,7 +675,7 @@ TAlScore SwAligner::alignGatherEE8(int& flag, bool debug) { cpimod -= iter; cpidiv++; } - size_t vecoff = (cpimod << 6) + cpidiv; + size_t vecoff = (cpimod << (ROWSTRIDE_LOG2+BYTES_LOG2_PER_REG)) + cpidiv; cper_.commitMap_.push_back(vecoff); cpi += cper_.per_; cpimod += cper_.per_; @@ -723,16 +709,16 @@ TAlScore SwAligner::alignGatherEE8(int& flag, bool debug) { assert_gt(coli, 0); size_t wordspercol = cper_.niter_ * ROWSTRIDE_2COL; size_t coloff = (coli >> cper_.perpow2_) * wordspercol; - __m128i *dst = cper_.qcols_.ptr() + coloff; - memcpy(dst, vbuf_r, sizeof(__m128i) * wordspercol); + SSERegI *dst = cper_.qcols_.ptr() + coloff; + memcpy(dst, vbuf_r, sizeof(SSERegI) * wordspercol); } } if(cper_.debug_) { // Save the column using memcpys size_t wordspercol = cper_.niter_ * ROWSTRIDE_2COL; size_t coloff = coli * wordspercol; - __m128i *dst = cper_.qcolsD_.ptr() + coloff; - memcpy(dst, vbuf_r, sizeof(__m128i) * wordspercol); + SSERegI *dst = cper_.qcolsD_.ptr() + coloff; + memcpy(dst, vbuf_r, sizeof(SSERegI) * wordspercol); } } } @@ -807,11 +793,11 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseU8(int& flag, bool debug) { assert(!d.profbuf_.empty()); assert_eq(0, d.maxBonus_); - size_t iter = + const size_t iter = (dpRows() + (NWORDS_PER_REG-1)) / NWORDS_PER_REG; // iter = segLen - int dup; - + const size_t lastWordIdx = NWORDS_PER_REG*(d.lastIter_*ROWSTRIDE)+d.lastWord_; + // Many thanks to Michael Farrar for releasing his striped Smith-Waterman // implementation: // @@ -820,70 +806,60 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseU8(int& flag, bool debug) { // Much of the implmentation below is adapted from Michael's code. // Set all elts to reference gap open penalty - __m128i rfgapo = _mm_setzero_si128(); - __m128i rfgape = _mm_setzero_si128(); - __m128i rdgapo = _mm_setzero_si128(); - __m128i rdgape = _mm_setzero_si128(); - __m128i vlo = _mm_setzero_si128(); - __m128i vhi = _mm_setzero_si128(); - __m128i ve = _mm_setzero_si128(); - __m128i vf = _mm_setzero_si128(); - __m128i vh = _mm_setzero_si128(); + SSERegI rfgapo = sse_setzero_siall(); + SSERegI rfgape = sse_setzero_siall(); + SSERegI rdgapo = sse_setzero_siall(); + SSERegI rdgape = sse_setzero_siall(); + SSERegI vlo = sse_setzero_siall(); + SSERegI vhi = sse_setzero_siall(); + SSERegI ve = sse_setzero_siall(); + SSERegI vf = sse_setzero_siall(); + SSERegI vh = sse_setzero_siall(); #if 0 - __m128i vhd = _mm_setzero_si128(); - __m128i vhdtmp = _mm_setzero_si128(); + SSERegI vhd = sse_setzero_siall(); + SSERegI vhdtmp = sse_setzero_siall(); #endif - __m128i vtmp = _mm_setzero_si128(); - __m128i vzero = _mm_setzero_si128(); - __m128i vhilsw = _mm_setzero_si128(); + SSERegI vtmp = sse_setzero_siall(); + SSERegI vzero = sse_setzero_siall(); + SSERegI vhilsw = sse_setzero_siall(); + + SSERegI vs0 = sse_setzero_siall(); + SSERegI vs1 = sse_setzero_siall(); assert_gt(sc_->refGapOpen(), 0); assert_leq(sc_->refGapOpen(), MAX_U8); - dup = (sc_->refGapOpen() << 8) | (sc_->refGapOpen() & 0x00ff); - rfgapo = _mm_insert_epi16(rfgapo, dup, 0); - rfgapo = _mm_shufflelo_epi16(rfgapo, 0); - rfgapo = _mm_shuffle_epi32(rfgapo, 0); + sse_fill_u8(sc_->refGapOpen(), rfgapo); // Set all elts to reference gap extension penalty assert_gt(sc_->refGapExtend(), 0); assert_leq(sc_->refGapExtend(), MAX_U8); assert_leq(sc_->refGapExtend(), sc_->refGapOpen()); - dup = (sc_->refGapExtend() << 8) | (sc_->refGapExtend() & 0x00ff); - rfgape = _mm_insert_epi16(rfgape, dup, 0); - rfgape = _mm_shufflelo_epi16(rfgape, 0); - rfgape = _mm_shuffle_epi32(rfgape, 0); + sse_fill_u8(sc_->refGapExtend(), rfgape); // Set all elts to read gap open penalty assert_gt(sc_->readGapOpen(), 0); assert_leq(sc_->readGapOpen(), MAX_U8); - dup = (sc_->readGapOpen() << 8) | (sc_->readGapOpen() & 0x00ff); - rdgapo = _mm_insert_epi16(rdgapo, dup, 0); - rdgapo = _mm_shufflelo_epi16(rdgapo, 0); - rdgapo = _mm_shuffle_epi32(rdgapo, 0); + sse_fill_u8(sc_->readGapOpen(), rdgapo); // Set all elts to read gap extension penalty assert_gt(sc_->readGapExtend(), 0); assert_leq(sc_->readGapExtend(), MAX_U8); assert_leq(sc_->readGapExtend(), sc_->readGapOpen()); - dup = (sc_->readGapExtend() << 8) | (sc_->readGapExtend() & 0x00ff); - rdgape = _mm_insert_epi16(rdgape, dup, 0); - rdgape = _mm_shufflelo_epi16(rdgape, 0); - rdgape = _mm_shuffle_epi32(rdgape, 0); - - vhi = _mm_cmpeq_epi16(vhi, vhi); // all elts = 0xffff - vlo = _mm_xor_si128(vlo, vlo); // all elts = 0 + sse_fill_u8(sc_->readGapExtend(), rdgape); + + sse_fill_u8_opt(0xff, vhi); + sse_fill_u8_opt(0, vlo); - // vhilsw: topmost (least sig) word set to 0x7fff, all other words=0 - vhilsw = _mm_shuffle_epi32(vhi, 0); - vhilsw = _mm_srli_si128(vhilsw, NBYTES_PER_REG - NBYTES_PER_WORD); + // vhilsw: topmost (least sig) word set to 0xff, all other words=0 + sse_set_low_u8(0xff, vhilsw); - // Points to a long vector of __m128i where each element is a block of + // Points to a long vector of SSERegI where each element is a block of // contiguous cells in the E, F or H matrix. If the index % 3 == 0, then // the block of cells is from the E matrix. If index % 3 == 1, they're // from the F matrix. If index % 3 == 2, then they're from the H matrix. // Blocks of cells are organized in the same interleaved manner as they are // calculated by the Farrar algorithm. - const __m128i *pvScore; // points into the query profile + const SSERegI *pvScore; // points into the query profile d.mat_.init(dpRows(), rff_ - rfi_, NWORDS_PER_REG); const size_t colstride = d.mat_.colstride(); @@ -891,26 +867,26 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseU8(int& flag, bool debug) { assert_eq(ROWSTRIDE, colstride / iter); // Initialize the H and E vectors in the first matrix column - __m128i *pvHTmp = d.mat_.tmpvec(0, 0); - __m128i *pvETmp = d.mat_.evec(0, 0); + SSERegI *pvHTmp = d.mat_.tmpvec(0, 0); + SSERegI *pvETmp = d.mat_.evec(0, 0); // Maximum score in final row bool found = false; TCScore lrmax = MIN_U8; for(size_t i = 0; i < iter; i++) { - _mm_store_si128(pvETmp, vlo); - _mm_store_si128(pvHTmp, vlo); // start high in end-to-end mode + sse_store_siall(pvETmp, vlo); + sse_store_siall(pvHTmp, vlo); // start high in end-to-end mode pvETmp += ROWSTRIDE; pvHTmp += ROWSTRIDE; } // These are swapped just before the innermost loop - __m128i *pvHStore = d.mat_.hvec(0, 0); - __m128i *pvHLoad = d.mat_.tmpvec(0, 0); - __m128i *pvELoad = d.mat_.evec(0, 0); - __m128i *pvEStore = d.mat_.evecUnsafe(0, 1); - __m128i *pvFStore = d.mat_.fvec(0, 0); - __m128i *pvFTmp = NULL; + SSERegI *pvHStore = d.mat_.hvec(0, 0); + SSERegI *pvHLoad = d.mat_.tmpvec(0, 0); + SSERegI *pvELoad = d.mat_.evec(0, 0); + SSERegI *pvEStore = d.mat_.evecUnsafe(0, 1); + SSERegI *pvFStore = d.mat_.fvec(0, 0); + SSERegI *pvFTmp = NULL; assert_gt(sc_->gapbar, 0); size_t nfixup = 0; @@ -941,55 +917,59 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseU8(int& flag, bool debug) { pvScore = d.profbuf_.ptr() + off; // even elts = query profile, odd = gap barrier // Set all cells to low value - vf = _mm_xor_si128(vf, vf); + vf = sse_xor_siall(vf, vf); // Load H vector from the final row of the previous column - vh = _mm_load_si128(pvHLoad + colstride - ROWSTRIDE); + vh = sse_load_siall(pvHLoad + colstride - ROWSTRIDE); // Shift 2 bytes down so that topmost (least sig) cell gets 0 - vh = _mm_slli_si128(vh, NBYTES_PER_WORD); + vh = sse_slli_siall(vh, NBYTES_PER_WORD); // Fill topmost (least sig) cell with high value - vh = _mm_or_si128(vh, vhilsw); + vh = sse_or_siall(vh, vhilsw); // For each character in the reference text: size_t j; for(j = 0; j < iter; j++) { + vs0 = sse_load_siall(pvScore); + pvScore++; + vs1 = sse_load_siall(pvScore); + pvScore++; // Load cells from E, calculated previously - ve = _mm_load_si128(pvELoad); + ve = sse_load_siall(pvELoad); #if 0 - vhd = _mm_load_si128(pvHLoad); + vhd = sse_load_siall(pvHLoad); #endif assert_all_lt(ve, vhi); pvELoad += ROWSTRIDE; // Store cells in F, calculated previously - vf = _mm_subs_epu8(vf, pvScore[1]); // veto some ref gap extensions - _mm_store_si128(pvFStore, vf); + vf = sse_subs_epu8(vf, vs1); // veto some ref gap extensions + sse_store_siall(pvFStore, vf); pvFStore += ROWSTRIDE; // Factor in query profile (matches and mismatches) - vh = _mm_subs_epu8(vh, pvScore[0]); + vh = sse_subs_epu8(vh, vs0); // Update H, factoring in E and F - vh = _mm_max_epu8(vh, ve); - vh = _mm_max_epu8(vh, vf); + vh = sse_max_epu8(vh, ve); + vh = sse_max_epu8(vh, vf); // Save the new vH values - _mm_store_si128(pvHStore, vh); + sse_store_siall(pvHStore, vh); pvHStore += ROWSTRIDE; // Update vE value vtmp = vh; #if 0 vhdtmp = vhd; - vhd = _mm_subs_epu8(vhd, rdgapo); - vhd = _mm_subs_epu8(vhd, pvScore[1]); // veto some read gap opens - ve = _mm_subs_epu8(ve, rdgape); - ve = _mm_max_epu8(ve, vhd); + vhd = sse_subs_epu8(vhd, rdgapo); + vhd = sse_subs_epu8(vhd, vs1); // veto some read gap opens + ve = sse_subs_epu8(ve, rdgape); + ve = sse_max_epu8(ve, vhd); #else - vh = _mm_subs_epu8(vh, rdgapo); - vh = _mm_subs_epu8(vh, pvScore[1]); // veto some read gap opens - ve = _mm_subs_epu8(ve, rdgape); - ve = _mm_max_epu8(ve, vh); + vh = sse_subs_epu8(vh, rdgapo); + vh = sse_subs_epu8(vh, vs1); // veto some read gap opens + ve = sse_subs_epu8(ve, rdgape); + ve = sse_max_epu8(ve, vh); #endif assert_all_lt(ve, vhi); @@ -997,104 +977,101 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseU8(int& flag, bool debug) { #if 0 vh = vhdtmp; #else - vh = _mm_load_si128(pvHLoad); + vh = sse_load_siall(pvHLoad); #endif pvHLoad += ROWSTRIDE; // Save E values - _mm_store_si128(pvEStore, ve); + sse_store_siall(pvEStore, ve); pvEStore += ROWSTRIDE; // Update vf value - vtmp = _mm_subs_epu8(vtmp, rfgapo); - vf = _mm_subs_epu8(vf, rfgape); + vtmp = sse_subs_epu8(vtmp, rfgapo); + vf = sse_subs_epu8(vf, rfgape); assert_all_lt(vf, vhi); - vf = _mm_max_epu8(vf, vtmp); - - pvScore += 2; // move on to next query profile / gap veto + vf = sse_max_epu8(vf, vtmp); } + pvScore -= (iter*2) - 1; // reset veto vector + vs1 = sse_load_siall(pvScore); + // pvHStore, pvELoad, pvEStore have all rolled over to the next column pvFTmp = pvFStore; pvFStore -= colstride; // reset to start of column - vtmp = _mm_load_si128(pvFStore); + vtmp = sse_load_siall(pvFStore); + // vf from last row gets shifted down by one to overlay the first row + // rfgape has already been subtracted from it. + vf = sse_slli_siall(vf, NBYTES_PER_WORD); + + vf = sse_subs_epu8(vf, vs1); // veto some ref gap extensions + vf = sse_max_epu8(vtmp, vf); + vtmp = sse_subs_epu8(vf, vtmp); + vtmp = sse_cmpeq_epi8(vtmp, vzero); + int cmp = sse_movemask_epi8(vtmp); + + // Load after computing cmp, so the result is ready by the time it is tested in while pvHStore -= colstride; // reset to start of column - vh = _mm_load_si128(pvHStore); + vh = sse_load_siall(pvHStore); + pvHLoad = pvHStore; // new pvHLoad = pvHStore #if 0 #else pvEStore -= colstride; // reset to start of column - ve = _mm_load_si128(pvEStore); + ve = sse_load_siall(pvEStore); #endif - pvHLoad = pvHStore; // new pvHLoad = pvHStore - pvScore = d.profbuf_.ptr() + off + 1; // reset veto vector - - // vf from last row gets shifted down by one to overlay the first row - // rfgape has already been subtracted from it. - vf = _mm_slli_si128(vf, NBYTES_PER_WORD); - - vf = _mm_subs_epu8(vf, *pvScore); // veto some ref gap extensions - vf = _mm_max_epu8(vtmp, vf); - vtmp = _mm_subs_epu8(vf, vtmp); - vtmp = _mm_cmpeq_epi8(vtmp, vzero); - int cmp = _mm_movemask_epi8(vtmp); - // If any element of vtmp is greater than H - gap-open... j = 0; - while(cmp != 0xffff) { + while(cmp != SSE_MASK_ALL) { // Store this vf - _mm_store_si128(pvFStore, vf); + sse_store_siall(pvFStore, vf); pvFStore += ROWSTRIDE; // Update vh w/r/t new vf - vh = _mm_max_epu8(vh, vf); + vh = sse_max_epu8(vh, vf); // Save vH values - _mm_store_si128(pvHStore, vh); + sse_store_siall(pvHStore, vh); pvHStore += ROWSTRIDE; // Update E in case it can be improved using our new vh #if 0 #else - vh = _mm_subs_epu8(vh, rdgapo); - vh = _mm_subs_epu8(vh, *pvScore); // veto some read gap opens - ve = _mm_max_epu8(ve, vh); - _mm_store_si128(pvEStore, ve); + vh = sse_subs_epu8(vh, rdgapo); + vh = sse_subs_epu8(vh, vs1); // veto some read gap opens + ve = sse_max_epu8(ve, vh); + sse_store_siall(pvEStore, ve); pvEStore += ROWSTRIDE; #endif pvScore += 2; assert_lt(j, iter); if(++j == iter) { + pvScore -= iter*2; + j = 0; pvFStore -= colstride; - vtmp = _mm_load_si128(pvFStore); // load next vf ASAP pvHStore -= colstride; - vh = _mm_load_si128(pvHStore); // load next vh ASAP -#if 0 -#else pvEStore -= colstride; - ve = _mm_load_si128(pvEStore); // load next ve ASAP -#endif - pvScore = d.profbuf_.ptr() + off + 1; - j = 0; - vf = _mm_slli_si128(vf, NBYTES_PER_WORD); - } else { - vtmp = _mm_load_si128(pvFStore); // load next vf ASAP - vh = _mm_load_si128(pvHStore); // load next vh ASAP + vf = sse_slli_siall(vf, NBYTES_PER_WORD); + } + vs1 = sse_load_siall(pvScore); + vtmp = sse_load_siall(pvFStore); // load next vf ASAP + + // Update F with another gap extension + vf = sse_subs_epu8(vf, rfgape); + vf = sse_subs_epu8(vf, vs1); // veto some ref gap extensions + vf = sse_max_epu8(vtmp, vf); + vtmp = sse_subs_epu8(vf, vtmp); + vtmp = sse_cmpeq_epi8(vtmp, vzero); + cmp = sse_movemask_epi8(vtmp); + + // Load after computing cmp, so the result is afailable by the time it is tested + vh = sse_load_siall(pvHStore); // load next vh #if 0 #else - ve = _mm_load_si128(pvEStore); // load next vh ASAP + ve = sse_load_siall(pvEStore); // load next ve #endif - } - // Update F with another gap extension - vf = _mm_subs_epu8(vf, rfgape); - vf = _mm_subs_epu8(vf, *pvScore); // veto some ref gap extensions - vf = _mm_max_epu8(vtmp, vf); - vtmp = _mm_subs_epu8(vf, vtmp); - vtmp = _mm_cmpeq_epi8(vtmp, vzero); - cmp = _mm_movemask_epi8(vtmp); nfixup++; } @@ -1115,9 +1092,8 @@ TAlScore SwAligner::alignNucleotidesEnd2EndSseU8(int& flag, bool debug) { } #endif - __m128i *vtmp = d.mat_.hvec(d.lastIter_, i-rfi_); // Note: we may not want to extract from the final row - TCScore lr = ((TCScore*)(vtmp))[d.lastWord_]; + TCScore lr = ((TCScore*)(pvHLoad))[lastWordIdx]; found = true; if(lr > lrmax) { lrmax = lr; @@ -1211,7 +1187,7 @@ bool SwAligner::gatherCellsNucleotidesEnd2EndSseU8(TAlScore best) { assert(!d.profbuf_.empty()); const size_t colstride = d.mat_.colstride(); ASSERT_ONLY(bool sawbest = false); - __m128i *pvH = d.mat_.hvec(d.lastIter_, 0); + SSERegI *pvH = d.mat_.hvec(d.lastIter_, 0); for(size_t j = 0; j < ncol; j++) { TAlScore sc = (TAlScore)(((TCScore*)pvH)[d.lastWord_] - 0xff); assert_leq(sc, best); @@ -1339,7 +1315,7 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8( size_t rowelt, rowvec, eltvec; size_t left_rowelt, up_rowelt, upleft_rowelt; size_t left_rowvec, up_rowvec, upleft_rowvec; - __m128i *cur_vec, *left_vec, *up_vec, *upleft_vec; + SSERegI *cur_vec, *left_vec, *up_vec, *upleft_vec; NEW_ROW_COL(row, col); while((int)row >= 0) { met.btcell++; diff --git a/aligner_swsse_loc_i16.cpp b/aligner_swsse_loc_i16.cpp index b0fa0c8..4c49c50 100644 --- a/aligner_swsse_loc_i16.cpp +++ b/aligner_swsse_loc_i16.cpp @@ -55,8 +55,7 @@ #include <limits> #include "aligner_sw.h" -static const size_t NBYTES_PER_REG = 16; -static const size_t NWORDS_PER_REG = 8; +static const size_t NWORDS_PER_REG = NBYTES_PER_REG/2; static const size_t NBITS_PER_WORD = 16; static const size_t NBYTES_PER_WORD = 2; @@ -83,14 +82,14 @@ void SwAligner::buildQueryProfileLocalSseI16(bool fw) { const BTString* qu = fw ? qufw_ : qurc_; const size_t len = rd->length(); const size_t seglen = (len + (NWORDS_PER_REG-1)) / NWORDS_PER_REG; - // How many __m128i's are needed - size_t n128s = + // How many SSERegI's are needed + size_t nsses = 64 + // slack bytes, for alignment? (seglen * ALPHA_SIZE) // query profile data * 2; // & gap barrier data - assert_gt(n128s, 0); + assert_gt(nsses, 0); SSEData& d = fw ? sseI16fw_ : sseI16rc_; - d.profbuf_.resizeNoCopy(n128s); + d.profbuf_.resizeNoCopy(nsses); assert(!d.profbuf_.empty()); d.maxPen_ = d.maxBonus_ = 0; d.lastIter_ = d.lastWord_ = 0; @@ -234,43 +233,43 @@ static bool cellOkLocalI16( #else #define assert_all_eq0(x) { \ - __m128i z = _mm_setzero_si128(); \ - __m128i tmp = _mm_setzero_si128(); \ - z = _mm_xor_si128(z, z); \ - tmp = _mm_cmpeq_epi16(x, z); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI z = sse_setzero_siall(); \ + SSERegI tmp = sse_setzero_siall(); \ + z = sse_xor_siall(z, z); \ + tmp = sse_cmpeq_epi16(x, z); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #define assert_all_gt(x, y) { \ - __m128i tmp = _mm_cmpgt_epi16(x, y); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI tmp = sse_cmpgt_epi16(x, y); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #define assert_all_gt_lo(x) { \ - __m128i z = _mm_setzero_si128(); \ - __m128i tmp = _mm_setzero_si128(); \ - z = _mm_xor_si128(z, z); \ - tmp = _mm_cmpgt_epi16(x, z); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI z = sse_setzero_siall(); \ + SSERegI tmp = sse_setzero_siall(); \ + z = sse_xor_siall(z, z); \ + tmp = sse_cmpgt_epi16(x, z); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #define assert_all_lt(x, y) { \ - __m128i tmp = _mm_cmplt_epi16(x, y); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI tmp = sse_cmplt_epi16(x, y); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #define assert_all_leq(x, y) { \ - __m128i tmp = _mm_cmpgt_epi16(x, y); \ - assert_eq(0x0000, _mm_movemask_epi8(tmp)); \ + SSERegI tmp = sse_cmpgt_epi16(x, y); \ + assert_eq(0x0000, sse_movemask_epi8(tmp)); \ } #define assert_all_lt_hi(x) { \ - __m128i z = _mm_setzero_si128(); \ - __m128i tmp = _mm_setzero_si128(); \ - z = _mm_cmpeq_epi16(z, z); \ - z = _mm_srli_epi16(z, 1); \ - tmp = _mm_cmplt_epi16(x, z); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI z = sse_setzero_siall(); \ + SSERegI tmp = sse_setzero_siall(); \ + z = sse_cmpeq_epi16(z, z); \ + z = sse_srli_epi16(z, 1); \ + tmp = sse_cmplt_epi16(x, z); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #endif @@ -322,8 +321,8 @@ TAlScore SwAligner::alignGatherLoc16(int& flag, bool debug) { // we'll call "left" and "right". d.vecbuf_.resize(ROWSTRIDE_2COL * iter * 2); d.vecbuf_.zero(); - __m128i *vbuf_l = d.vecbuf_.ptr(); - __m128i *vbuf_r = d.vecbuf_.ptr() + (ROWSTRIDE_2COL * iter); + SSERegI *vbuf_l = d.vecbuf_.ptr(); + SSERegI *vbuf_r = d.vecbuf_.ptr() + (ROWSTRIDE_2COL * iter); // This is the data structure that holds candidate cells per diagonal. const size_t ndiags = rff_ - rfi_ + dpRows() - 1; @@ -356,95 +355,78 @@ TAlScore SwAligner::alignGatherLoc16(int& flag, bool debug) { // Much of the implmentation below is adapted from Michael's code. // Set all elts to reference gap open penalty - __m128i rfgapo = _mm_setzero_si128(); - __m128i rfgape = _mm_setzero_si128(); - __m128i rdgapo = _mm_setzero_si128(); - __m128i rdgape = _mm_setzero_si128(); - __m128i vlo = _mm_setzero_si128(); - __m128i vhi = _mm_setzero_si128(); - __m128i vlolsw = _mm_setzero_si128(); - __m128i vmax = _mm_setzero_si128(); - __m128i vcolmax = _mm_setzero_si128(); - __m128i vmaxtmp = _mm_setzero_si128(); - __m128i ve = _mm_setzero_si128(); - __m128i vf = _mm_setzero_si128(); - __m128i vh = _mm_setzero_si128(); - __m128i vhd = _mm_setzero_si128(); - __m128i vhdtmp = _mm_setzero_si128(); - __m128i vtmp = _mm_setzero_si128(); - __m128i vzero = _mm_setzero_si128(); - __m128i vminsc = _mm_setzero_si128(); + SSERegI rfgapo = sse_setzero_siall(); + SSERegI rfgape = sse_setzero_siall(); + SSERegI rdgapo = sse_setzero_siall(); + SSERegI rdgape = sse_setzero_siall(); + SSERegI vlo = sse_setzero_siall(); + SSERegI vlolsw = sse_setzero_siall(); + SSERegI vmax = sse_setzero_siall(); + SSERegI vcolmax = sse_setzero_siall(); + SSERegI ve = sse_setzero_siall(); + SSERegI vf = sse_setzero_siall(); + SSERegI vh = sse_setzero_siall(); + SSERegI vhd = sse_setzero_siall(); + SSERegI vhdtmp = sse_setzero_siall(); + SSERegI vtmp = sse_setzero_siall(); + SSERegI vzero = sse_setzero_siall(); assert_gt(sc_->refGapOpen(), 0); assert_leq(sc_->refGapOpen(), MAX_I16); - rfgapo = _mm_insert_epi16(rfgapo, sc_->refGapOpen(), 0); - rfgapo = _mm_shufflelo_epi16(rfgapo, 0); - rfgapo = _mm_shuffle_epi32(rfgapo, 0); + sse_fill_i16(sc_->refGapOpen(), rfgapo); // Set all elts to reference gap extension penalty assert_gt(sc_->refGapExtend(), 0); assert_leq(sc_->refGapExtend(), MAX_I16); assert_leq(sc_->refGapExtend(), sc_->refGapOpen()); - rfgape = _mm_insert_epi16(rfgape, sc_->refGapExtend(), 0); - rfgape = _mm_shufflelo_epi16(rfgape, 0); - rfgape = _mm_shuffle_epi32(rfgape, 0); + sse_fill_i16(sc_->refGapExtend(), rfgape); // Set all elts to read gap open penalty assert_gt(sc_->readGapOpen(), 0); assert_leq(sc_->readGapOpen(), MAX_I16); - rdgapo = _mm_insert_epi16(rdgapo, sc_->readGapOpen(), 0); - rdgapo = _mm_shufflelo_epi16(rdgapo, 0); - rdgapo = _mm_shuffle_epi32(rdgapo, 0); + sse_fill_i16(sc_->readGapOpen(), rdgapo); // Set all elts to read gap extension penalty assert_gt(sc_->readGapExtend(), 0); assert_leq(sc_->readGapExtend(), MAX_I16); assert_leq(sc_->readGapExtend(), sc_->readGapOpen()); - rdgape = _mm_insert_epi16(rdgape, sc_->readGapExtend(), 0); - rdgape = _mm_shufflelo_epi16(rdgape, 0); - rdgape = _mm_shuffle_epi32(rdgape, 0); + sse_fill_i16(sc_->readGapExtend(), rdgape); - // Set all elts to minimum score threshold. Actually, to 1 less than the - // threshold so we can use gt instead of geq. - vminsc = _mm_insert_epi16(vminsc, (int)minsc_-1, 0); - vminsc = _mm_shufflelo_epi16(vminsc, 0); - vminsc = _mm_shuffle_epi32(vminsc, 0); - // Set all elts to 0x8000 (min value for signed 16-bit) - vlo = _mm_cmpeq_epi16(vlo, vlo); // all elts = 0xffff - vlo = _mm_slli_epi16(vlo, NBITS_PER_WORD-1); // all elts = 0x8000 - + sse_fill_i16(0x8000, vlo); + +#ifndef NDEBUG // Set all elts to 0x7fff (max value for signed 16-bit) - vhi = _mm_cmpeq_epi16(vhi, vhi); // all elts = 0xffff - vhi = _mm_srli_epi16(vhi, 1); // all elts = 0x7fff + SSERegI vhi = sse_setzero_siall(); + sse_fill_i16(0x7fff, vhi); +#endif // Set all elts to 0x8000 (min value for signed 16-bit) vmax = vlo; // vlolsw: topmost (least sig) word set to 0x8000, all other words=0 - vlolsw = _mm_shuffle_epi32(vlo, 0); - vlolsw = _mm_srli_si128(vlolsw, NBYTES_PER_REG - NBYTES_PER_WORD); + sse_set_low_i16(0x8000, vlolsw); - // Points to a long vector of __m128i where each element is a block of + // Points to a long vector of SSERegI where each element is a block of // contiguous cells in the E, F or H matrix. If the index % 3 == 0, then // the block of cells is from the E matrix. If index % 3 == 1, they're // from the F matrix. If index % 3 == 2, then they're from the H matrix. // Blocks of cells are organized in the same interleaved manner as they are // calculated by the Farrar algorithm. - const __m128i *pvScore; // points into the query profile + const SSERegI *pvScore; // points into the query profile const size_t colstride = ROWSTRIDE_2COL * iter; // Initialize the H and E vectors in the first matrix column - __m128i *pvELeft = vbuf_l + 0; __m128i *pvERight = vbuf_r + 0; - //__m128i *pvFLeft = vbuf_l + 1; - __m128i *pvFRight = vbuf_r + 1; - __m128i *pvHLeft = vbuf_l + 2; __m128i *pvHRight = vbuf_r + 2; + SSERegI *pvELeft = vbuf_l + 0; SSERegI *pvERight = vbuf_r + 0; + //SSERegI *pvFLeft = vbuf_l + 1; + SSERegI *pvFRight = vbuf_r + 1; + SSERegI *pvHLeft = vbuf_l + 2; SSERegI *pvHRight = vbuf_r + 2; for(size_t i = 0; i < iter; i++) { // start low in local mode - _mm_store_si128(pvERight, vlo); pvERight += ROWSTRIDE_2COL; - _mm_store_si128(pvHRight, vlo); pvHRight += ROWSTRIDE_2COL; + sse_store_siall(pvERight, vlo); pvERight += ROWSTRIDE_2COL; + sse_store_siall(pvHRight, vlo); pvHRight += ROWSTRIDE_2COL; // Note: right and left are going to be swapped as soon as we enter // the outer loop below } @@ -490,47 +472,47 @@ TAlScore SwAligner::alignGatherLoc16(int& flag, bool debug) { // current iter's? The way we currently do it, seems like it will // almost always require at least one fixup loop iter (to recalculate // this topmost F). - vh = _mm_load_si128(pvHLeft + colstride - ROWSTRIDE_2COL); + vh = sse_load_siall(pvHLeft + colstride - ROWSTRIDE_2COL); // Set all F cells to low value - vf = _mm_cmpeq_epi16(vf, vf); - vf = _mm_slli_epi16(vf, NBITS_PER_WORD-1); - vf = _mm_or_si128(vf, vlolsw); + vf = sse_cmpeq_epi16(vf, vf); + vf = sse_slli_epi16(vf, NBITS_PER_WORD-1); + vf = sse_or_siall(vf, vlolsw); // vf now contains the vertical contribution // Store cells in F, calculated previously // No need to veto ref gap extensions, they're all 0x8000s - _mm_store_si128(pvFRight, vf); + sse_store_siall(pvFRight, vf); pvFRight += ROWSTRIDE_2COL; // Shift down so that topmost (least sig) cell gets 0 - vh = _mm_slli_si128(vh, NBYTES_PER_WORD); + vh = sse_slli_siall(vh, NBYTES_PER_WORD); // Fill topmost (least sig) cell with low value - vh = _mm_or_si128(vh, vlolsw); + vh = sse_or_siall(vh, vlolsw); // We pull out one loop iteration to make it easier to veto values in the top row // Load cells from E, calculated previously - ve = _mm_load_si128(pvELeft); - vhd = _mm_load_si128(pvHLeft); + ve = sse_load_siall(pvELeft); + vhd = sse_load_siall(pvHLeft); assert_all_lt(ve, vhi); pvELeft += ROWSTRIDE_2COL; // ve now contains the horizontal contribution // Factor in query profile (matches and mismatches) - vh = _mm_adds_epi16(vh, pvScore[0]); + vh = sse_adds_epi16(vh, pvScore[0]); // vh now contains the diagonal contribution // Update vE value vhdtmp = vhd; - vhd = _mm_subs_epi16(vhd, rdgapo); - vhd = _mm_adds_epi16(vhd, pvScore[1]); // veto some read gap opens - vhd = _mm_adds_epi16(vhd, pvScore[1]); // veto some read gap opens - ve = _mm_subs_epi16(ve, rdgape); - ve = _mm_max_epi16(ve, vhd); + vhd = sse_subs_epi16(vhd, rdgapo); + vhd = sse_adds_epi16(vhd, pvScore[1]); // veto some read gap opens + vhd = sse_adds_epi16(vhd, pvScore[1]); // veto some read gap opens + ve = sse_subs_epi16(ve, rdgape); + ve = sse_max_epi16(ve, vhd); // Update H, factoring in E and F - vh = _mm_max_epi16(vh, ve); + vh = sse_max_epi16(vh, ve); // F won't change anything! vf = vh; @@ -539,7 +521,7 @@ TAlScore SwAligner::alignGatherLoc16(int& flag, bool debug) { vcolmax = vh; // Save the new vH values - _mm_store_si128(pvHRight, vh); + sse_store_siall(pvHRight, vh); assert_all_lt(ve, vhi); @@ -550,11 +532,11 @@ TAlScore SwAligner::alignGatherLoc16(int& flag, bool debug) { pvHLeft += ROWSTRIDE_2COL; // Save E values - _mm_store_si128(pvERight, ve); + sse_store_siall(pvERight, ve); pvERight += ROWSTRIDE_2COL; // Update vf value - vf = _mm_subs_epi16(vf, rfgapo); + vf = sse_subs_epi16(vf, rfgapo); assert_all_lt(vf, vhi); pvScore += 2; // move on to next query profile @@ -563,37 +545,37 @@ TAlScore SwAligner::alignGatherLoc16(int& flag, bool debug) { size_t j; for(j = 1; j < iter; j++) { // Load cells from E, calculated previously - ve = _mm_load_si128(pvELeft); - vhd = _mm_load_si128(pvHLeft); + ve = sse_load_siall(pvELeft); + vhd = sse_load_siall(pvHLeft); assert_all_lt(ve, vhi); pvELeft += ROWSTRIDE_2COL; // Store cells in F, calculated previously - vf = _mm_adds_epi16(vf, pvScore[1]); // veto some ref gap extensions - vf = _mm_adds_epi16(vf, pvScore[1]); // veto some ref gap extensions - _mm_store_si128(pvFRight, vf); + vf = sse_adds_epi16(vf, pvScore[1]); // veto some ref gap extensions + vf = sse_adds_epi16(vf, pvScore[1]); // veto some ref gap extensions + sse_store_siall(pvFRight, vf); pvFRight += ROWSTRIDE_2COL; // Factor in query profile (matches and mismatches) - vh = _mm_adds_epi16(vh, pvScore[0]); - vh = _mm_max_epi16(vh, vf); + vh = sse_adds_epi16(vh, pvScore[0]); + vh = sse_max_epi16(vh, vf); // Update vE value vhdtmp = vhd; - vhd = _mm_subs_epi16(vhd, rdgapo); - vhd = _mm_adds_epi16(vhd, pvScore[1]); // veto some read gap opens - vhd = _mm_adds_epi16(vhd, pvScore[1]); // veto some read gap opens - ve = _mm_subs_epi16(ve, rdgape); - ve = _mm_max_epi16(ve, vhd); + vhd = sse_subs_epi16(vhd, rdgapo); + vhd = sse_adds_epi16(vhd, pvScore[1]); // veto some read gap opens + vhd = sse_adds_epi16(vhd, pvScore[1]); // veto some read gap opens + ve = sse_subs_epi16(ve, rdgape); + ve = sse_max_epi16(ve, vhd); - vh = _mm_max_epi16(vh, ve); + vh = sse_max_epi16(vh, ve); vtmp = vh; // Update highest score encountered this far - vcolmax = _mm_max_epi16(vcolmax, vh); + vcolmax = sse_max_epi16(vcolmax, vh); // Save the new vH values - _mm_store_si128(pvHRight, vh); + sse_store_siall(pvHRight, vh); vh = vhdtmp; @@ -602,78 +584,78 @@ TAlScore SwAligner::alignGatherLoc16(int& flag, bool debug) { pvHLeft += ROWSTRIDE_2COL; // Save E values - _mm_store_si128(pvERight, ve); + sse_store_siall(pvERight, ve); pvERight += ROWSTRIDE_2COL; // Update vf value - vtmp = _mm_subs_epi16(vtmp, rfgapo); - vf = _mm_subs_epi16(vf, rfgape); + vtmp = sse_subs_epi16(vtmp, rfgapo); + vf = sse_subs_epi16(vf, rfgape); assert_all_lt(vf, vhi); - vf = _mm_max_epi16(vf, vtmp); + vf = sse_max_epi16(vf, vtmp); pvScore += 2; // move on to next query profile / gap veto } // pvHStore, pvELoad, pvEStore have all rolled over to the next column pvFRight -= colstride; // reset to start of column - vtmp = _mm_load_si128(pvFRight); + vtmp = sse_load_siall(pvFRight); pvHRight -= colstride; // reset to start of column - vh = _mm_load_si128(pvHRight); + vh = sse_load_siall(pvHRight); pvScore = d.profbuf_.ptr() + off + 1; // reset veto vector // vf from last row gets shifted down by one to overlay the first row // rfgape has already been subtracted from it. - vf = _mm_slli_si128(vf, NBYTES_PER_WORD); - vf = _mm_or_si128(vf, vlolsw); + vf = sse_slli_siall(vf, NBYTES_PER_WORD); + vf = sse_or_siall(vf, vlolsw); - vf = _mm_adds_epi16(vf, *pvScore); // veto some ref gap extensions - vf = _mm_adds_epi16(vf, *pvScore); // veto some ref gap extensions - vf = _mm_max_epi16(vtmp, vf); - vtmp = _mm_cmpgt_epi16(vf, vtmp); - int cmp = _mm_movemask_epi8(vtmp); + vf = sse_adds_epi16(vf, *pvScore); // veto some ref gap extensions + vf = sse_adds_epi16(vf, *pvScore); // veto some ref gap extensions + vf = sse_max_epi16(vtmp, vf); + vtmp = sse_cmpgt_epi16(vf, vtmp); + int cmp = sse_movemask_epi8(vtmp); // If any element of vtmp is greater than H - gap-open... j = 0; while(cmp != 0x0000) { // Store this vf - _mm_store_si128(pvFRight, vf); + sse_store_siall(pvFRight, vf); pvFRight += ROWSTRIDE_2COL; // Update vh w/r/t new vf - vh = _mm_max_epi16(vh, vf); + vh = sse_max_epi16(vh, vf); // Save vH values - _mm_store_si128(pvHRight, vh); + sse_store_siall(pvHRight, vh); pvHRight += ROWSTRIDE_2COL; // Update highest score encountered so far. - vcolmax = _mm_max_epi16(vcolmax, vh); + vcolmax = sse_max_epi16(vcolmax, vh); pvScore += 2; assert_lt(j, iter); if(++j == iter) { pvFRight -= colstride; - vtmp = _mm_load_si128(pvFRight); // load next vf ASAP + vtmp = sse_load_siall(pvFRight); // load next vf ASAP pvHRight -= colstride; - vh = _mm_load_si128(pvHRight); // load next vh ASAP + vh = sse_load_siall(pvHRight); // load next vh ASAP pvScore = d.profbuf_.ptr() + off + 1; j = 0; - vf = _mm_slli_si128(vf, NBYTES_PER_WORD); - vf = _mm_or_si128(vf, vlolsw); + vf = sse_slli_siall(vf, NBYTES_PER_WORD); + vf = sse_or_siall(vf, vlolsw); } else { - vtmp = _mm_load_si128(pvFRight); // load next vf ASAP - vh = _mm_load_si128(pvHRight); // load next vh ASAP + vtmp = sse_load_siall(pvFRight); // load next vf ASAP + vh = sse_load_siall(pvHRight); // load next vh ASAP } // Update F with another gap extension - vf = _mm_subs_epi16(vf, rfgape); - vf = _mm_adds_epi16(vf, *pvScore); // veto some ref gap extensions - vf = _mm_adds_epi16(vf, *pvScore); // veto some ref gap extensions - vf = _mm_max_epi16(vtmp, vf); - vtmp = _mm_cmpgt_epi16(vf, vtmp); - cmp = _mm_movemask_epi8(vtmp); + vf = sse_subs_epi16(vf, rfgape); + vf = sse_adds_epi16(vf, *pvScore); // veto some ref gap extensions + vf = sse_adds_epi16(vf, *pvScore); // veto some ref gap extensions + vf = sse_max_epi16(vtmp, vf); + vtmp = sse_cmpgt_epi16(vf, vtmp); + cmp = sse_movemask_epi8(vtmp); nfixup++; } @@ -689,9 +671,9 @@ TAlScore SwAligner::alignGatherLoc16(int& flag, bool debug) { assert_lt(lastoff, MAX_SIZE_T); pvScore = d.profbuf_.ptr() + lastoff; // even elts = query profile, odd = gap barrier for(size_t k = 0; k < iter; k++) { - vh = _mm_load_si128(pvHLeft); - vtmp = _mm_cmpgt_epi16(pvScore[0], vzero); - int cmp = _mm_movemask_epi8(vtmp); + vh = sse_load_siall(pvHLeft); + vtmp = sse_cmpgt_epi16(pvScore[0], vzero); + int cmp = sse_movemask_epi8(vtmp); if(cmp != 0) { // At least one candidate in this mask. Now iterate // through vm/vh to evaluate individual cells. @@ -720,9 +702,9 @@ TAlScore SwAligner::alignGatherLoc16(int& flag, bool debug) { // Save some elements to checkpoints if(checkpoint) { - __m128i *pvE = vbuf_r + 0; - __m128i *pvF = vbuf_r + 1; - __m128i *pvH = vbuf_r + 2; + SSERegI *pvE = vbuf_r + 0; + SSERegI *pvF = vbuf_r + 1; + SSERegI *pvH = vbuf_r + 2; size_t coli = i - rfi_; if(coli < cper_.locol_) cper_.locol_ = coli; if(coli > cper_.hicol_) cper_.hicol_ = coli; @@ -750,8 +732,8 @@ TAlScore SwAligner::alignGatherLoc16(int& flag, bool debug) { row_div++; } size_t delt = idxrow + row; - size_t vecoff = (row_mod << 5) + row_div; - assert_lt(row_div, 8); + size_t vecoff = (row_mod << (ROWSTRIDE_LOG2+BYTES_LOG2_PER_REG-1)) + row_div; + assert_lt(row_div, (NBYTES_PER_REG/2)); int16_t h_sc = ((int16_t*)pvH)[vecoff]; int16_t e_sc = ((int16_t*)pvE)[vecoff]; int16_t f_sc = ((int16_t*)pvF)[vecoff]; @@ -791,7 +773,7 @@ TAlScore SwAligner::alignGatherLoc16(int& flag, bool debug) { cpimod -= iter; cpidiv++; } - size_t vecoff = (cpimod << 5) + cpidiv; + size_t vecoff = (cpimod << (ROWSTRIDE_LOG2+BYTES_LOG2_PER_REG-1)) + cpidiv; cper_.commitMap_.push_back(vecoff); cpi += cper_.per_; cpimod += cper_.per_; @@ -822,30 +804,24 @@ TAlScore SwAligner::alignGatherLoc16(int& flag, bool debug) { assert_gt(coli, 0); size_t wordspercol = cper_.niter_ * ROWSTRIDE_2COL; size_t coloff = (coli >> cper_.perpow2_) * wordspercol; - __m128i *dst = cper_.qcols_.ptr() + coloff; - memcpy(dst, vbuf_r, sizeof(__m128i) * wordspercol); + SSERegI *dst = cper_.qcols_.ptr() + coloff; + memcpy(dst, vbuf_r, sizeof(SSERegI) * wordspercol); } } if(cper_.debug_) { // Save the column using memcpys size_t wordspercol = cper_.niter_ * ROWSTRIDE_2COL; size_t coloff = coli * wordspercol; - __m128i *dst = cper_.qcolsD_.ptr() + coloff; - memcpy(dst, vbuf_r, sizeof(__m128i) * wordspercol); + SSERegI *dst = cper_.qcolsD_.ptr() + coloff; + memcpy(dst, vbuf_r, sizeof(SSERegI) * wordspercol); } } - vmax = _mm_max_epi16(vmax, vcolmax); + vmax = sse_max_epi16(vmax, vcolmax); { // Get single largest score in this column - vmaxtmp = vcolmax; - vtmp = _mm_srli_si128(vmaxtmp, 8); - vmaxtmp = _mm_max_epi16(vmaxtmp, vtmp); - vtmp = _mm_srli_si128(vmaxtmp, 4); - vmaxtmp = _mm_max_epi16(vmaxtmp, vtmp); - vtmp = _mm_srli_si128(vmaxtmp, 2); - vmaxtmp = _mm_max_epi16(vmaxtmp, vtmp); - int16_t ret = _mm_extract_epi16(vmaxtmp, 0); + int16_t ret = 0; + sse_max_score_i16(vcolmax, ret); TAlScore score = (TAlScore)(ret + 0x8000); if(ret == MIN_I16) { score = MIN_I64; @@ -878,9 +854,9 @@ TAlScore SwAligner::alignGatherLoc16(int& flag, bool debug) { assert_lt(lastoff, MAX_SIZE_T); pvScore = d.profbuf_.ptr() + lastoff; // even elts = query profile, odd = gap barrier for(size_t k = 0; k < iter; k++) { - vh = _mm_load_si128(pvHLeft); - vtmp = _mm_cmpgt_epi16(pvScore[0], vzero); - int cmp = _mm_movemask_epi8(vtmp); + vh = sse_load_siall(pvHLeft); + vtmp = sse_cmpgt_epi16(pvScore[0], vzero); + int cmp = sse_movemask_epi8(vtmp); if(cmp != 0) { // At least one candidate in this mask. Now iterate // through vm/vh to evaluate individual cells. @@ -907,13 +883,8 @@ TAlScore SwAligner::alignGatherLoc16(int& flag, bool debug) { } // Find largest score in vmax - vtmp = _mm_srli_si128(vmax, 8); - vmax = _mm_max_epi16(vmax, vtmp); - vtmp = _mm_srli_si128(vmax, 4); - vmax = _mm_max_epi16(vmax, vtmp); - vtmp = _mm_srli_si128(vmax, 2); - vmax = _mm_max_epi16(vmax, vtmp); - int16_t ret = _mm_extract_epi16(vmax, 0); + int16_t ret = 0; + sse_max_score_i16(vmax, ret); // Update metrics if(!debug) { @@ -996,72 +967,62 @@ TAlScore SwAligner::alignNucleotidesLocalSseI16(int& flag, bool debug) { // Much of the implmentation below is adapted from Michael's code. // Set all elts to reference gap open penalty - __m128i rfgapo = _mm_setzero_si128(); - __m128i rfgape = _mm_setzero_si128(); - __m128i rdgapo = _mm_setzero_si128(); - __m128i rdgape = _mm_setzero_si128(); - __m128i vlo = _mm_setzero_si128(); - __m128i vhi = _mm_setzero_si128(); - __m128i vlolsw = _mm_setzero_si128(); - __m128i vmax = _mm_setzero_si128(); - __m128i vcolmax = _mm_setzero_si128(); - __m128i vmaxtmp = _mm_setzero_si128(); - __m128i ve = _mm_setzero_si128(); - __m128i vf = _mm_setzero_si128(); - __m128i vh = _mm_setzero_si128(); - __m128i vtmp = _mm_setzero_si128(); + SSERegI rfgapo = sse_setzero_siall(); + SSERegI rfgape = sse_setzero_siall(); + SSERegI rdgapo = sse_setzero_siall(); + SSERegI rdgape = sse_setzero_siall(); + SSERegI vlo = sse_setzero_siall(); + SSERegI vlolsw = sse_setzero_siall(); + SSERegI vmax = sse_setzero_siall(); + SSERegI vcolmax = sse_setzero_siall(); + SSERegI ve = sse_setzero_siall(); + SSERegI vf = sse_setzero_siall(); + SSERegI vh = sse_setzero_siall(); + SSERegI vtmp = sse_setzero_siall(); assert_gt(sc_->refGapOpen(), 0); assert_leq(sc_->refGapOpen(), MAX_I16); - rfgapo = _mm_insert_epi16(rfgapo, sc_->refGapOpen(), 0); - rfgapo = _mm_shufflelo_epi16(rfgapo, 0); - rfgapo = _mm_shuffle_epi32(rfgapo, 0); + sse_fill_i16(sc_->refGapOpen(), rfgapo); // Set all elts to reference gap extension penalty assert_gt(sc_->refGapExtend(), 0); assert_leq(sc_->refGapExtend(), MAX_I16); assert_leq(sc_->refGapExtend(), sc_->refGapOpen()); - rfgape = _mm_insert_epi16(rfgape, sc_->refGapExtend(), 0); - rfgape = _mm_shufflelo_epi16(rfgape, 0); - rfgape = _mm_shuffle_epi32(rfgape, 0); + sse_fill_i16(sc_->refGapExtend(), rfgape); // Set all elts to read gap open penalty assert_gt(sc_->readGapOpen(), 0); assert_leq(sc_->readGapOpen(), MAX_I16); - rdgapo = _mm_insert_epi16(rdgapo, sc_->readGapOpen(), 0); - rdgapo = _mm_shufflelo_epi16(rdgapo, 0); - rdgapo = _mm_shuffle_epi32(rdgapo, 0); + sse_fill_i16(sc_->readGapOpen(), rdgapo); // Set all elts to read gap extension penalty assert_gt(sc_->readGapExtend(), 0); assert_leq(sc_->readGapExtend(), MAX_I16); assert_leq(sc_->readGapExtend(), sc_->readGapOpen()); - rdgape = _mm_insert_epi16(rdgape, sc_->readGapExtend(), 0); - rdgape = _mm_shufflelo_epi16(rdgape, 0); - rdgape = _mm_shuffle_epi32(rdgape, 0); + sse_fill_i16(sc_->readGapExtend(), rdgape); // Set all elts to 0x8000 (min value for signed 16-bit) - vlo = _mm_cmpeq_epi16(vlo, vlo); // all elts = 0xffff - vlo = _mm_slli_epi16(vlo, NBITS_PER_WORD-1); // all elts = 0x8000 - + sse_fill_i16(0x8000, vlo); + +#ifndef NDEBUG // Set all elts to 0x7fff (max value for signed 16-bit) - vhi = _mm_cmpeq_epi16(vhi, vhi); // all elts = 0xffff - vhi = _mm_srli_epi16(vhi, 1); // all elts = 0x7fff + SSERegI vhi = sse_setzero_siall(); + sse_fill_i16(0x7fff, vhi); +#endif // Set all elts to 0x8000 (min value for signed 16-bit) vmax = vlo; // vlolsw: topmost (least sig) word set to 0x8000, all other words=0 - vlolsw = _mm_shuffle_epi32(vlo, 0); - vlolsw = _mm_srli_si128(vlolsw, NBYTES_PER_REG - NBYTES_PER_WORD); + sse_set_low_i16(0x8000, vlolsw); - // Points to a long vector of __m128i where each element is a block of + // Points to a long vector of SSERegI where each element is a block of // contiguous cells in the E, F or H matrix. If the index % 3 == 0, then // the block of cells is from the E matrix. If index % 3 == 1, they're // from the F matrix. If index % 3 == 2, then they're from the H matrix. // Blocks of cells are organized in the same interleaved manner as they are // calculated by the Farrar algorithm. - const __m128i *pvScore; // points into the query profile + const SSERegI *pvScore; // points into the query profile d.mat_.init(dpRows(), rff_ - rfi_, NWORDS_PER_REG); const size_t colstride = d.mat_.colstride(); @@ -1069,22 +1030,22 @@ TAlScore SwAligner::alignNucleotidesLocalSseI16(int& flag, bool debug) { assert_eq(ROWSTRIDE, colstride / iter); // Initialize the H and E vectors in the first matrix column - __m128i *pvHTmp = d.mat_.tmpvec(0, 0); - __m128i *pvETmp = d.mat_.evec(0, 0); + SSERegI *pvHTmp = d.mat_.tmpvec(0, 0); + SSERegI *pvETmp = d.mat_.evec(0, 0); for(size_t i = 0; i < iter; i++) { - _mm_store_si128(pvETmp, vlo); - _mm_store_si128(pvHTmp, vlo); // start low in local mode + sse_store_siall(pvETmp, vlo); + sse_store_siall(pvHTmp, vlo); // start low in local mode pvETmp += ROWSTRIDE; pvHTmp += ROWSTRIDE; } // These are swapped just before the innermost loop - __m128i *pvHStore = d.mat_.hvec(0, 0); - __m128i *pvHLoad = d.mat_.tmpvec(0, 0); - __m128i *pvELoad = d.mat_.evec(0, 0); - __m128i *pvEStore = d.mat_.evecUnsafe(0, 1); - __m128i *pvFStore = d.mat_.fvec(0, 0); - __m128i *pvFTmp = NULL; + SSERegI *pvHStore = d.mat_.hvec(0, 0); + SSERegI *pvHLoad = d.mat_.tmpvec(0, 0); + SSERegI *pvELoad = d.mat_.evec(0, 0); + SSERegI *pvEStore = d.mat_.evecUnsafe(0, 1); + SSERegI *pvFStore = d.mat_.fvec(0, 0); + SSERegI *pvFTmp = NULL; assert_gt(sc_->gapbar, 0); size_t nfixup = 0; @@ -1116,69 +1077,69 @@ TAlScore SwAligner::alignNucleotidesLocalSseI16(int& flag, bool debug) { pvScore = d.profbuf_.ptr() + off; // even elts = query profile, odd = gap barrier // Load H vector from the final row of the previous column - vh = _mm_load_si128(pvHLoad + colstride - ROWSTRIDE); + vh = sse_load_siall(pvHLoad + colstride - ROWSTRIDE); // Set all F cells to low value - vf = _mm_cmpeq_epi16(vf, vf); - vf = _mm_slli_epi16(vf, NBITS_PER_WORD-1); - vf = _mm_or_si128(vf, vlolsw); + vf = sse_cmpeq_epi16(vf, vf); + vf = sse_slli_epi16(vf, NBITS_PER_WORD-1); + vf = sse_or_siall(vf, vlolsw); // vf now contains the vertical contribution // Store cells in F, calculated previously // No need to veto ref gap extensions, they're all 0x8000s - _mm_store_si128(pvFStore, vf); + sse_store_siall(pvFStore, vf); pvFStore += ROWSTRIDE; // Shift down so that topmost (least sig) cell gets 0 - vh = _mm_slli_si128(vh, NBYTES_PER_WORD); + vh = sse_slli_siall(vh, NBYTES_PER_WORD); // Fill topmost (least sig) cell with low value - vh = _mm_or_si128(vh, vlolsw); + vh = sse_or_siall(vh, vlolsw); // We pull out one loop iteration to make it easier to veto values in the top row // Load cells from E, calculated previously - ve = _mm_load_si128(pvELoad); + ve = sse_load_siall(pvELoad); assert_all_lt(ve, vhi); pvELoad += ROWSTRIDE; // ve now contains the horizontal contribution // Factor in query profile (matches and mismatches) - vh = _mm_adds_epi16(vh, pvScore[0]); + vh = sse_adds_epi16(vh, pvScore[0]); // vh now contains the diagonal contribution // Update H, factoring in E and F - vtmp = _mm_max_epi16(vh, ve); + vtmp = sse_max_epi16(vh, ve); // F won't change anything! vh = vtmp; // Update highest score so far vcolmax = vlo; - vcolmax = _mm_max_epi16(vcolmax, vh); + vcolmax = sse_max_epi16(vcolmax, vh); // Save the new vH values - _mm_store_si128(pvHStore, vh); + sse_store_siall(pvHStore, vh); pvHStore += ROWSTRIDE; // Update vE value vf = vh; - vh = _mm_subs_epi16(vh, rdgapo); - vh = _mm_adds_epi16(vh, pvScore[1]); // veto some read gap opens - vh = _mm_adds_epi16(vh, pvScore[1]); // veto some read gap opens - ve = _mm_subs_epi16(ve, rdgape); - ve = _mm_max_epi16(ve, vh); + vh = sse_subs_epi16(vh, rdgapo); + vh = sse_adds_epi16(vh, pvScore[1]); // veto some read gap opens + vh = sse_adds_epi16(vh, pvScore[1]); // veto some read gap opens + ve = sse_subs_epi16(ve, rdgape); + ve = sse_max_epi16(ve, vh); assert_all_lt(ve, vhi); // Load the next h value - vh = _mm_load_si128(pvHLoad); + vh = sse_load_siall(pvHLoad); pvHLoad += ROWSTRIDE; // Save E values - _mm_store_si128(pvEStore, ve); + sse_store_siall(pvEStore, ve); pvEStore += ROWSTRIDE; // Update vf value - vf = _mm_subs_epi16(vf, rfgapo); + vf = sse_subs_epi16(vf, rfgapo); assert_all_lt(vf, vhi); pvScore += 2; // move on to next query profile @@ -1187,131 +1148,131 @@ TAlScore SwAligner::alignNucleotidesLocalSseI16(int& flag, bool debug) { size_t j; for(j = 1; j < iter; j++) { // Load cells from E, calculated previously - ve = _mm_load_si128(pvELoad); + ve = sse_load_siall(pvELoad); assert_all_lt(ve, vhi); pvELoad += ROWSTRIDE; // Store cells in F, calculated previously - vf = _mm_adds_epi16(vf, pvScore[1]); // veto some ref gap extensions - vf = _mm_adds_epi16(vf, pvScore[1]); // veto some ref gap extensions - _mm_store_si128(pvFStore, vf); + vf = sse_adds_epi16(vf, pvScore[1]); // veto some ref gap extensions + vf = sse_adds_epi16(vf, pvScore[1]); // veto some ref gap extensions + sse_store_siall(pvFStore, vf); pvFStore += ROWSTRIDE; // Factor in query profile (matches and mismatches) - vh = _mm_adds_epi16(vh, pvScore[0]); + vh = sse_adds_epi16(vh, pvScore[0]); // Update H, factoring in E and F - vh = _mm_max_epi16(vh, ve); - vh = _mm_max_epi16(vh, vf); + vh = sse_max_epi16(vh, ve); + vh = sse_max_epi16(vh, vf); // Update highest score encountered this far - vcolmax = _mm_max_epi16(vcolmax, vh); + vcolmax = sse_max_epi16(vcolmax, vh); // Save the new vH values - _mm_store_si128(pvHStore, vh); + sse_store_siall(pvHStore, vh); pvHStore += ROWSTRIDE; // Update vE value vtmp = vh; - vh = _mm_subs_epi16(vh, rdgapo); - vh = _mm_adds_epi16(vh, pvScore[1]); // veto some read gap opens - vh = _mm_adds_epi16(vh, pvScore[1]); // veto some read gap opens - ve = _mm_subs_epi16(ve, rdgape); - ve = _mm_max_epi16(ve, vh); + vh = sse_subs_epi16(vh, rdgapo); + vh = sse_adds_epi16(vh, pvScore[1]); // veto some read gap opens + vh = sse_adds_epi16(vh, pvScore[1]); // veto some read gap opens + ve = sse_subs_epi16(ve, rdgape); + ve = sse_max_epi16(ve, vh); assert_all_lt(ve, vhi); // Load the next h value - vh = _mm_load_si128(pvHLoad); + vh = sse_load_siall(pvHLoad); pvHLoad += ROWSTRIDE; // Save E values - _mm_store_si128(pvEStore, ve); + sse_store_siall(pvEStore, ve); pvEStore += ROWSTRIDE; // Update vf value - vtmp = _mm_subs_epi16(vtmp, rfgapo); - vf = _mm_subs_epi16(vf, rfgape); + vtmp = sse_subs_epi16(vtmp, rfgapo); + vf = sse_subs_epi16(vf, rfgape); assert_all_lt(vf, vhi); - vf = _mm_max_epi16(vf, vtmp); + vf = sse_max_epi16(vf, vtmp); pvScore += 2; // move on to next query profile / gap veto } // pvHStore, pvELoad, pvEStore have all rolled over to the next column pvFTmp = pvFStore; pvFStore -= colstride; // reset to start of column - vtmp = _mm_load_si128(pvFStore); + vtmp = sse_load_siall(pvFStore); pvHStore -= colstride; // reset to start of column - vh = _mm_load_si128(pvHStore); + vh = sse_load_siall(pvHStore); pvEStore -= colstride; // reset to start of column - ve = _mm_load_si128(pvEStore); + ve = sse_load_siall(pvEStore); pvHLoad = pvHStore; // new pvHLoad = pvHStore pvScore = d.profbuf_.ptr() + off + 1; // reset veto vector // vf from last row gets shifted down by one to overlay the first row // rfgape has already been subtracted from it. - vf = _mm_slli_si128(vf, NBYTES_PER_WORD); - vf = _mm_or_si128(vf, vlolsw); + vf = sse_slli_siall(vf, NBYTES_PER_WORD); + vf = sse_or_siall(vf, vlolsw); - vf = _mm_adds_epi16(vf, *pvScore); // veto some ref gap extensions - vf = _mm_adds_epi16(vf, *pvScore); // veto some ref gap extensions - vf = _mm_max_epi16(vtmp, vf); - vtmp = _mm_cmpgt_epi16(vf, vtmp); - int cmp = _mm_movemask_epi8(vtmp); + vf = sse_adds_epi16(vf, *pvScore); // veto some ref gap extensions + vf = sse_adds_epi16(vf, *pvScore); // veto some ref gap extensions + vf = sse_max_epi16(vtmp, vf); + vtmp = sse_cmpgt_epi16(vf, vtmp); + int cmp = sse_movemask_epi8(vtmp); // If any element of vtmp is greater than H - gap-open... j = 0; while(cmp != 0x0000) { // Store this vf - _mm_store_si128(pvFStore, vf); + sse_store_siall(pvFStore, vf); pvFStore += ROWSTRIDE; // Update vh w/r/t new vf - vh = _mm_max_epi16(vh, vf); + vh = sse_max_epi16(vh, vf); // Save vH values - _mm_store_si128(pvHStore, vh); + sse_store_siall(pvHStore, vh); pvHStore += ROWSTRIDE; // Update highest score encountered this far - vcolmax = _mm_max_epi16(vcolmax, vh); + vcolmax = sse_max_epi16(vcolmax, vh); // Update E in case it can be improved using our new vh - vh = _mm_subs_epi16(vh, rdgapo); - vh = _mm_adds_epi16(vh, *pvScore); // veto some read gap opens - vh = _mm_adds_epi16(vh, *pvScore); // veto some read gap opens - ve = _mm_max_epi16(ve, vh); - _mm_store_si128(pvEStore, ve); + vh = sse_subs_epi16(vh, rdgapo); + vh = sse_adds_epi16(vh, *pvScore); // veto some read gap opens + vh = sse_adds_epi16(vh, *pvScore); // veto some read gap opens + ve = sse_max_epi16(ve, vh); + sse_store_siall(pvEStore, ve); pvEStore += ROWSTRIDE; pvScore += 2; assert_lt(j, iter); if(++j == iter) { pvFStore -= colstride; - vtmp = _mm_load_si128(pvFStore); // load next vf ASAP + vtmp = sse_load_siall(pvFStore); // load next vf ASAP pvHStore -= colstride; - vh = _mm_load_si128(pvHStore); // load next vh ASAP + vh = sse_load_siall(pvHStore); // load next vh ASAP pvEStore -= colstride; - ve = _mm_load_si128(pvEStore); // load next ve ASAP + ve = sse_load_siall(pvEStore); // load next ve ASAP pvScore = d.profbuf_.ptr() + off + 1; j = 0; - vf = _mm_slli_si128(vf, NBYTES_PER_WORD); - vf = _mm_or_si128(vf, vlolsw); + vf = sse_slli_siall(vf, NBYTES_PER_WORD); + vf = sse_or_siall(vf, vlolsw); } else { - vtmp = _mm_load_si128(pvFStore); // load next vf ASAP - vh = _mm_load_si128(pvHStore); // load next vh ASAP - ve = _mm_load_si128(pvEStore); // load next vh ASAP + vtmp = sse_load_siall(pvFStore); // load next vf ASAP + vh = sse_load_siall(pvHStore); // load next vh ASAP + ve = sse_load_siall(pvEStore); // load next vh ASAP } // Update F with another gap extension - vf = _mm_subs_epi16(vf, rfgape); - vf = _mm_adds_epi16(vf, *pvScore); // veto some ref gap extensions - vf = _mm_adds_epi16(vf, *pvScore); // veto some ref gap extensions - vf = _mm_max_epi16(vtmp, vf); - vtmp = _mm_cmpgt_epi16(vf, vtmp); - cmp = _mm_movemask_epi8(vtmp); + vf = sse_subs_epi16(vf, rfgape); + vf = sse_adds_epi16(vf, *pvScore); // veto some ref gap extensions + vf = sse_adds_epi16(vf, *pvScore); // veto some ref gap extensions + vf = sse_max_epi16(vtmp, vf); + vtmp = sse_cmpgt_epi16(vf, vtmp); + cmp = sse_movemask_epi8(vtmp); nfixup++; } @@ -1333,19 +1294,13 @@ TAlScore SwAligner::alignNucleotidesLocalSseI16(int& flag, bool debug) { #endif // Store column maximum vector in first element of tmp - vmax = _mm_max_epi16(vmax, vcolmax); - _mm_store_si128(d.mat_.tmpvec(0, i - rfi_), vcolmax); + vmax = sse_max_epi16(vmax, vcolmax); + sse_store_siall(d.mat_.tmpvec(0, i - rfi_), vcolmax); { // Get single largest score in this column - vmaxtmp = vcolmax; - vtmp = _mm_srli_si128(vmaxtmp, 8); - vmaxtmp = _mm_max_epi16(vmaxtmp, vtmp); - vtmp = _mm_srli_si128(vmaxtmp, 4); - vmaxtmp = _mm_max_epi16(vmaxtmp, vtmp); - vtmp = _mm_srli_si128(vmaxtmp, 2); - vmaxtmp = _mm_max_epi16(vmaxtmp, vtmp); - int16_t ret = _mm_extract_epi16(vmaxtmp, 0); + int16_t ret = 0; + sse_max_score_i16(vcolmax, ret); TAlScore score = (TAlScore)(ret + 0x8000); if(score < minsc_) { @@ -1370,13 +1325,8 @@ TAlScore SwAligner::alignNucleotidesLocalSseI16(int& flag, bool debug) { } // Find largest score in vmax - vtmp = _mm_srli_si128(vmax, 8); - vmax = _mm_max_epi16(vmax, vtmp); - vtmp = _mm_srli_si128(vmax, 4); - vmax = _mm_max_epi16(vmax, vtmp); - vtmp = _mm_srli_si128(vmax, 2); - vmax = _mm_max_epi16(vmax, vtmp); - int16_t ret = _mm_extract_epi16(vmax, 0); + int16_t ret = 0; + sse_max_score_i16(vmax, ret); // Update metrics if(!debug) { @@ -1495,21 +1445,16 @@ bool SwAligner::gatherCellsNucleotidesLocalSseI16(TAlScore best) { size_t nrow_hi = nrow; // First, check if there is a cell in this column with a score // above the score threshold - __m128i vmax = *d.mat_.tmpvec(0, j); - __m128i vtmp = _mm_srli_si128(vmax, 8); - vmax = _mm_max_epi16(vmax, vtmp); - vtmp = _mm_srli_si128(vmax, 4); - vmax = _mm_max_epi16(vmax, vtmp); - vtmp = _mm_srli_si128(vmax, 2); - vmax = _mm_max_epi16(vmax, vtmp); - TAlScore score = (TAlScore)((int16_t)_mm_extract_epi16(vmax, 0) + 0x8000); + int16_t ret = 0; + sse_max_score_i16(*d.mat_.tmpvec(0, j), ret); + TAlScore score = (TAlScore)(ret + 0x8000); assert_geq(score, 0); #ifndef NDEBUG { // Start in upper vector row and move down TAlScore max = 0; - vmax = *d.mat_.tmpvec(0, j); - __m128i *pvH = d.mat_.hvec(0, j); + SSERegI vmax = *d.mat_.tmpvec(0, j); + SSERegI *pvH = d.mat_.hvec(0, j); for(size_t i = 0; i < iter; i++) { for(size_t k = 0; k < NWORDS_PER_REG; k++) { TAlScore sc = (TAlScore)(((TCScore*)pvH)[k] + 0x8000); @@ -1529,11 +1474,11 @@ bool SwAligner::gatherCellsNucleotidesLocalSseI16(TAlScore best) { continue; } // Get pointer to first cell in column to examine: - __m128i *pvHorig = d.mat_.hvec(0, j); - __m128i *pvH = pvHorig; + SSERegI *pvHorig = d.mat_.hvec(0, j); + SSERegI *pvH = pvHorig; // Get pointer to the vector in the following column that corresponds // to the cells diagonally down and to the right from the cells in pvH - __m128i *pvHSucc = (j < ncol-1) ? d.mat_.hvec(0, j+1) : NULL; + SSERegI *pvHSucc = (j < ncol-1) ? d.mat_.hvec(0, j+1) : NULL; // Start in upper vector row and move down for(size_t i = 0; i < iter; i++) { if(pvHSucc != NULL) { @@ -1700,7 +1645,7 @@ bool SwAligner::backtraceNucleotidesLocalSseI16( size_t rowelt, rowvec, eltvec; size_t left_rowelt, up_rowelt, upleft_rowelt; size_t left_rowvec, up_rowvec, upleft_rowvec; - __m128i *cur_vec, *left_vec, *up_vec, *upleft_vec; + SSERegI *cur_vec, *left_vec, *up_vec, *upleft_vec; const size_t gbar = sc_->gapbar; NEW_ROW_COL(row, col); // If 'backEliminate' is true, then every time we visit a cell, we remove diff --git a/aligner_swsse_loc_u8.cpp b/aligner_swsse_loc_u8.cpp index 956b84a..0fb00fa 100644 --- a/aligner_swsse_loc_u8.cpp +++ b/aligner_swsse_loc_u8.cpp @@ -55,8 +55,7 @@ #include <limits> #include "aligner_sw.h" -//static const size_t NBYTES_PER_REG = 16; -static const size_t NWORDS_PER_REG = 16; +static const size_t NWORDS_PER_REG = NBYTES_PER_REG; //static const size_t NBITS_PER_WORD = 8; static const size_t NBYTES_PER_WORD = 1; @@ -83,14 +82,14 @@ void SwAligner::buildQueryProfileLocalSseU8(bool fw) { const BTString* qu = fw ? qufw_ : qurc_; const size_t len = rd->length(); const size_t seglen = (len + (NWORDS_PER_REG-1)) / NWORDS_PER_REG; - // How many __m128i's are needed - size_t n128s = + // How many SSERegI's are needed + size_t nsses = 64 + // slack bytes, for alignment? (seglen * ALPHA_SIZE) // query profile data * 2; // & gap barrier data - assert_gt(n128s, 0); + assert_gt(nsses, 0); SSEData& d = fw ? sseU8fw_ : sseU8rc_; - d.profbuf_.resizeNoCopy(n128s); + d.profbuf_.resizeNoCopy(nsses); assert(!d.profbuf_.empty()); d.maxPen_ = d.maxBonus_ = 0; d.lastIter_ = d.lastWord_ = 0; @@ -248,41 +247,41 @@ static bool cellOkLocalU8( #else #define assert_all_eq0(x) { \ - __m128i z = _mm_setzero_si128(); \ - __m128i tmp = _mm_setzero_si128(); \ - z = _mm_xor_si128(z, z); \ - tmp = _mm_cmpeq_epi16(x, z); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI z = sse_setzero_siall(); \ + SSERegI tmp = sse_setzero_siall(); \ + z = sse_xor_siall(z, z); \ + tmp = sse_cmpeq_epi16(x, z); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #define assert_all_gt(x, y) { \ - __m128i tmp = _mm_cmpgt_epu8(x, y); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI tmp = sse_cmpgt_epu8(x, y); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #define assert_all_gt_lo(x) { \ - __m128i z = _mm_setzero_si128(); \ - __m128i tmp = _mm_setzero_si128(); \ - z = _mm_xor_si128(z, z); \ - tmp = _mm_cmpgt_epu8(x, z); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI z = sse_setzero_siall(); \ + SSERegI tmp = sse_setzero_siall(); \ + z = sse_xor_siall(z, z); \ + tmp = sse_cmpgt_epu8(x, z); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #define assert_all_lt(x, y) { \ - __m128i z = _mm_setzero_si128(); \ - z = _mm_xor_si128(z, z); \ - __m128i tmp = _mm_subs_epu8(y, x); \ - tmp = _mm_cmpeq_epi16(tmp, z); \ - assert_eq(0x0000, _mm_movemask_epi8(tmp)); \ + SSERegI z = sse_setzero_siall(); \ + z = sse_xor_siall(z, z); \ + SSERegI tmp = sse_subs_epu8(y, x); \ + tmp = sse_cmpeq_epi16(tmp, z); \ + assert_eq(0x0000, sse_movemask_epi8(tmp)); \ } #define assert_all_lt_hi(x) { \ - __m128i z = _mm_setzero_si128(); \ - __m128i tmp = _mm_setzero_si128(); \ - z = _mm_cmpeq_epu8(z, z); \ - z = _mm_srli_epu8(z, 1); \ - tmp = _mm_cmplt_epu8(x, z); \ - assert_eq(0xffff, _mm_movemask_epi8(tmp)); \ + SSERegI z = sse_setzero_siall(); \ + SSERegI tmp = sse_setzero_siall(); \ + z = sse_cmpeq_epu8(z, z); \ + z = sse_srli_epu8(z, 1); \ + tmp = sse_cmplt_epu8(x, z); \ + assert_eq(SSE_MASK_ALL, sse_movemask_epi8(tmp)); \ } #endif @@ -335,8 +334,8 @@ TAlScore SwAligner::alignGatherLoc8(int& flag, bool debug) { // we'll call "left" and "right". d.vecbuf_.resize(ROWSTRIDE_2COL * iter * 2); d.vecbuf_.zero(); - __m128i *vbuf_l = d.vecbuf_.ptr(); - __m128i *vbuf_r = d.vecbuf_.ptr() + (ROWSTRIDE_2COL * iter); + SSERegI *vbuf_l = d.vecbuf_.ptr(); + SSERegI *vbuf_r = d.vecbuf_.ptr() + (ROWSTRIDE_2COL * iter); // This is the data structure that holds candidate cells per diagonal. const size_t ndiags = rff_ - rfi_ + dpRows() - 1; @@ -369,101 +368,75 @@ TAlScore SwAligner::alignGatherLoc8(int& flag, bool debug) { // Much of the implmentation below is adapted from Michael's code. // Set all elts to reference gap open penalty - __m128i rfgapo = _mm_setzero_si128(); - __m128i rfgape = _mm_setzero_si128(); - __m128i rdgapo = _mm_setzero_si128(); - __m128i rdgape = _mm_setzero_si128(); - __m128i vlo = _mm_setzero_si128(); - __m128i vhi = _mm_setzero_si128(); - __m128i vmax = _mm_setzero_si128(); - __m128i vcolmax = _mm_setzero_si128(); - __m128i vmaxtmp = _mm_setzero_si128(); - __m128i ve = _mm_setzero_si128(); - __m128i vf = _mm_setzero_si128(); - __m128i vh = _mm_setzero_si128(); - __m128i vhd = _mm_setzero_si128(); - __m128i vhdtmp = _mm_setzero_si128(); - __m128i vtmp = _mm_setzero_si128(); - __m128i vzero = _mm_setzero_si128(); - __m128i vbias = _mm_setzero_si128(); - __m128i vbiasm1 = _mm_setzero_si128(); - __m128i vminsc = _mm_setzero_si128(); - - int dup; + SSERegI rfgapo = sse_setzero_siall(); + SSERegI rfgape = sse_setzero_siall(); + SSERegI rdgapo = sse_setzero_siall(); + SSERegI rdgape = sse_setzero_siall(); + SSERegI vlo = sse_setzero_siall(); + SSERegI vhi = sse_setzero_siall(); + SSERegI vmax = sse_setzero_siall(); + SSERegI vcolmax = sse_setzero_siall(); + SSERegI ve = sse_setzero_siall(); + SSERegI vf = sse_setzero_siall(); + SSERegI vh = sse_setzero_siall(); + SSERegI vhd = sse_setzero_siall(); + SSERegI vhdtmp = sse_setzero_siall(); + SSERegI vtmp = sse_setzero_siall(); + SSERegI vzero = sse_setzero_siall(); + SSERegI vbias = sse_setzero_siall(); + SSERegI vbiasm1 = sse_setzero_siall(); assert_gt(sc_->refGapOpen(), 0); assert_leq(sc_->refGapOpen(), MAX_U8); - dup = (sc_->refGapOpen() << 8) | (sc_->refGapOpen() & 0x00ff); - rfgapo = _mm_insert_epi16(rfgapo, dup, 0); - rfgapo = _mm_shufflelo_epi16(rfgapo, 0); - rfgapo = _mm_shuffle_epi32(rfgapo, 0); + sse_fill_u8(sc_->refGapOpen(), rfgapo); // Set all elts to reference gap extension penalty assert_gt(sc_->refGapExtend(), 0); assert_leq(sc_->refGapExtend(), MAX_U8); assert_leq(sc_->refGapExtend(), sc_->refGapOpen()); - dup = (sc_->refGapExtend() << 8) | (sc_->refGapExtend() & 0x00ff); - rfgape = _mm_insert_epi16(rfgape, dup, 0); - rfgape = _mm_shufflelo_epi16(rfgape, 0); - rfgape = _mm_shuffle_epi32(rfgape, 0); + sse_fill_u8(sc_->refGapExtend(), rfgape); // Set all elts to read gap open penalty assert_gt(sc_->readGapOpen(), 0); assert_leq(sc_->readGapOpen(), MAX_U8); - dup = (sc_->readGapOpen() << 8) | (sc_->readGapOpen() & 0x00ff); - rdgapo = _mm_insert_epi16(rdgapo, dup, 0); - rdgapo = _mm_shufflelo_epi16(rdgapo, 0); - rdgapo = _mm_shuffle_epi32(rdgapo, 0); + sse_fill_u8(sc_->readGapOpen(), rdgapo); // Set all elts to read gap extension penalty assert_gt(sc_->readGapExtend(), 0); assert_leq(sc_->readGapExtend(), MAX_U8); assert_leq(sc_->readGapExtend(), sc_->readGapOpen()); - dup = (sc_->readGapExtend() << 8) | (sc_->readGapExtend() & 0x00ff); - rdgape = _mm_insert_epi16(rdgape, dup, 0); - rdgape = _mm_shufflelo_epi16(rdgape, 0); - rdgape = _mm_shuffle_epi32(rdgape, 0); + sse_fill_u8(sc_->readGapExtend(), rdgape); // Set all elts to minimum score threshold. Actually, to 1 less than the // threshold so we can use gt instead of geq. - dup = (((int)minsc_ - 1) << 8) | (((int)minsc_ - 1) & 0x00ff); - vminsc = _mm_insert_epi16(vminsc, dup, 0); - vminsc = _mm_shufflelo_epi16(vminsc, 0); - vminsc = _mm_shuffle_epi32(vminsc, 0); - - dup = ((d.bias_ - 1) << 8) | ((d.bias_ - 1) & 0x00ff); - vbiasm1 = _mm_insert_epi16(vbiasm1, dup, 0); - vbiasm1 = _mm_shufflelo_epi16(vbiasm1, 0); - vbiasm1 = _mm_shuffle_epi32(vbiasm1, 0); - vhi = _mm_cmpeq_epi16(vhi, vhi); // all elts = 0xffff - vlo = _mm_xor_si128(vlo, vlo); // all elts = 0 + sse_fill_u8((d.bias_ - 1), vbiasm1); + + sse_fill_u8_opt(0xff, vhi); + sse_fill_u8_opt(0, vlo); vmax = vlo; // Make a vector of bias offsets - dup = (d.bias_ << 8) | (d.bias_ & 0x00ff); - vbias = _mm_insert_epi16(vbias, dup, 0); - vbias = _mm_shufflelo_epi16(vbias, 0); - vbias = _mm_shuffle_epi32(vbias, 0); + sse_fill_u8(d.bias_, vbias); - // Points to a long vector of __m128i where each element is a block of + // Points to a long vector of SSERegI where each element is a block of // contiguous cells in the E, F or H matrix. If the index % 3 == 0, then // the block of cells is from the E matrix. If index % 3 == 1, they're // from the F matrix. If index % 3 == 2, then they're from the H matrix. // Blocks of cells are organized in the same interleaved manner as they are // calculated by the Farrar algorithm. - const __m128i *pvScore; // points into the query profile + const SSERegI *pvScore; // points into the query profile const size_t colstride = ROWSTRIDE_2COL * iter; // Initialize the H and E vectors in the first matrix column - __m128i *pvELeft = vbuf_l + 0; __m128i *pvERight = vbuf_r + 0; - /* __m128i *pvFLeft = vbuf_l + 1; */ __m128i *pvFRight = vbuf_r + 1; - __m128i *pvHLeft = vbuf_l + 2; __m128i *pvHRight = vbuf_r + 2; + SSERegI *pvELeft = vbuf_l + 0; SSERegI *pvERight = vbuf_r + 0; + /* SSERegI *pvFLeft = vbuf_l + 1; */ SSERegI *pvFRight = vbuf_r + 1; + SSERegI *pvHLeft = vbuf_l + 2; SSERegI *pvHRight = vbuf_r + 2; for(size_t i = 0; i < iter; i++) { // start low in local mode - _mm_store_si128(pvERight, vlo); pvERight += ROWSTRIDE_2COL; - _mm_store_si128(pvHRight, vlo); pvHRight += ROWSTRIDE_2COL; + sse_store_siall(pvERight, vlo); pvERight += ROWSTRIDE_2COL; + sse_store_siall(pvHRight, vlo); pvHRight += ROWSTRIDE_2COL; } assert_gt(sc_->gapbar, 0); @@ -507,48 +480,48 @@ TAlScore SwAligner::alignGatherLoc8(int& flag, bool debug) { // current iter's? The way we currently do it, seems like it will // almost always require at least one fixup loop iter (to recalculate // this topmost F). - vh = _mm_load_si128(pvHLeft + colstride - ROWSTRIDE_2COL); + vh = sse_load_siall(pvHLeft + colstride - ROWSTRIDE_2COL); // Set all cells to low value - vf = _mm_xor_si128(vf, vf); + vf = sse_xor_siall(vf, vf); // vf now contains the vertical contribution // Store cells in F, calculated previously // No need to veto ref gap extensions, they're all 0x00s - _mm_store_si128(pvFRight, vf); + sse_store_siall(pvFRight, vf); pvFRight += ROWSTRIDE_2COL; // Shift down so that topmost (least sig) cell gets 0 - vh = _mm_slli_si128(vh, NBYTES_PER_WORD); + vh = sse_slli_siall(vh, NBYTES_PER_WORD); // We pull out one loop iteration to make it easier to veto values in the top row // Load cells from E, calculated previously - ve = _mm_load_si128(pvELeft); - vhd = _mm_load_si128(pvHLeft); + ve = sse_load_siall(pvELeft); + vhd = sse_load_siall(pvHLeft); assert_all_lt(ve, vhi); pvELeft += ROWSTRIDE_2COL; // ve now contains the horizontal contribution // Factor in query profile (matches and mismatches) - vh = _mm_adds_epu8(vh, pvScore[0]); - vh = _mm_subs_epu8(vh, vbias); + vh = sse_adds_epu8(vh, pvScore[0]); + vh = sse_subs_epu8(vh, vbias); // vh now contains the diagonal contribution vhdtmp = vhd; - vhd = _mm_subs_epu8(vhd, rdgapo); - vhd = _mm_subs_epu8(vhd, pvScore[1]); // veto some read gap opens - ve = _mm_subs_epu8(ve, rdgape); - ve = _mm_max_epu8(ve, vhd); + vhd = sse_subs_epu8(vhd, rdgapo); + vhd = sse_subs_epu8(vhd, pvScore[1]); // veto some read gap opens + ve = sse_subs_epu8(ve, rdgape); + ve = sse_max_epu8(ve, vhd); - vh = _mm_max_epu8(vh, ve); + vh = sse_max_epu8(vh, ve); vf = vh; // Update highest score so far vcolmax = vh; // Save the new vH values - _mm_store_si128(pvHRight, vh); + sse_store_siall(pvHRight, vh); vh = vhdtmp; assert_all_lt(ve, vhi); @@ -556,11 +529,11 @@ TAlScore SwAligner::alignGatherLoc8(int& flag, bool debug) { pvHLeft += ROWSTRIDE_2COL; // Save E values - _mm_store_si128(pvERight, ve); + sse_store_siall(pvERight, ve); pvERight += ROWSTRIDE_2COL; // Update vf value - vf = _mm_subs_epu8(vf, rfgapo); + vf = sse_subs_epu8(vf, rfgapo); assert_all_lt(vf, vhi); pvScore += 2; // move on to next query profile @@ -569,37 +542,37 @@ TAlScore SwAligner::alignGatherLoc8(int& flag, bool debug) { size_t j; for(j = 1; j < iter; j++) { // Load cells from E, calculated previously - ve = _mm_load_si128(pvELeft); - vhd = _mm_load_si128(pvHLeft); + ve = sse_load_siall(pvELeft); + vhd = sse_load_siall(pvHLeft); assert_all_lt(ve, vhi); pvELeft += ROWSTRIDE_2COL; // Store cells in F, calculated previously - vf = _mm_subs_epu8(vf, pvScore[1]); // veto some ref gap extensions - _mm_store_si128(pvFRight, vf); + vf = sse_subs_epu8(vf, pvScore[1]); // veto some ref gap extensions + sse_store_siall(pvFRight, vf); pvFRight += ROWSTRIDE_2COL; // Factor in query profile (matches and mismatches) - vh = _mm_adds_epu8(vh, pvScore[0]); - vh = _mm_subs_epu8(vh, vbias); + vh = sse_adds_epu8(vh, pvScore[0]); + vh = sse_subs_epu8(vh, vbias); // Update H, factoring in E and F - vh = _mm_max_epu8(vh, vf); + vh = sse_max_epu8(vh, vf); vhdtmp = vhd; - vhd = _mm_subs_epu8(vhd, rdgapo); - vhd = _mm_subs_epu8(vhd, pvScore[1]); // veto some read gap opens - ve = _mm_subs_epu8(ve, rdgape); - ve = _mm_max_epu8(ve, vhd); + vhd = sse_subs_epu8(vhd, rdgapo); + vhd = sse_subs_epu8(vhd, pvScore[1]); // veto some read gap opens + ve = sse_subs_epu8(ve, rdgape); + ve = sse_max_epu8(ve, vhd); - vh = _mm_max_epu8(vh, ve); + vh = sse_max_epu8(vh, ve); vtmp = vh; // Update highest score encountered this far - vcolmax = _mm_max_epu8(vcolmax, vh); + vcolmax = sse_max_epu8(vcolmax, vh); // Save the new vH values - _mm_store_si128(pvHRight, vh); + sse_store_siall(pvHRight, vh); vh = vhdtmp; @@ -608,79 +581,79 @@ TAlScore SwAligner::alignGatherLoc8(int& flag, bool debug) { pvHLeft += ROWSTRIDE_2COL; // Save E values - _mm_store_si128(pvERight, ve); + sse_store_siall(pvERight, ve); pvERight += ROWSTRIDE_2COL; // Update vf value - vtmp = _mm_subs_epu8(vtmp, rfgapo); - vf = _mm_subs_epu8(vf, rfgape); + vtmp = sse_subs_epu8(vtmp, rfgapo); + vf = sse_subs_epu8(vf, rfgape); assert_all_lt(vf, vhi); - vf = _mm_max_epu8(vf, vtmp); + vf = sse_max_epu8(vf, vtmp); pvScore += 2; // move on to next query profile / gap veto } // pvHStore, pvELoad, pvEStore have all rolled over to the next column pvFRight -= colstride; // reset to start of column - vtmp = _mm_load_si128(pvFRight); + vtmp = sse_load_siall(pvFRight); pvHRight -= colstride; // reset to start of column - vh = _mm_load_si128(pvHRight); + vh = sse_load_siall(pvHRight); pvScore = d.profbuf_.ptr() + off + 1; // reset veto vector // vf from last row gets shifted down by one to overlay the first row // rfgape has already been subtracted from it. - vf = _mm_slli_si128(vf, NBYTES_PER_WORD); + vf = sse_slli_siall(vf, NBYTES_PER_WORD); - vf = _mm_subs_epu8(vf, *pvScore); // veto some ref gap extensions - vf = _mm_max_epu8(vtmp, vf); + vf = sse_subs_epu8(vf, *pvScore); // veto some ref gap extensions + vf = sse_max_epu8(vtmp, vf); // TODO: We're testing whether F changed. Can't we just assume that F // did change and instead check whether H changed? Might save us from // entering the fixup loop. - vtmp = _mm_subs_epu8(vf, vtmp); - vtmp = _mm_cmpeq_epi8(vtmp, vzero); - int cmp = _mm_movemask_epi8(vtmp); + vtmp = sse_subs_epu8(vf, vtmp); + vtmp = sse_cmpeq_epi8(vtmp, vzero); + int cmp = sse_movemask_epi8(vtmp); // If any element of vtmp is greater than H - gap-open... j = 0; - while(cmp != 0xffff) { + while(cmp != SSE_MASK_ALL) { // Store this vf - _mm_store_si128(pvFRight, vf); + sse_store_siall(pvFRight, vf); pvFRight += ROWSTRIDE_2COL; // Update vh w/r/t new vf - vh = _mm_max_epu8(vh, vf); + vh = sse_max_epu8(vh, vf); // Save vH values - _mm_store_si128(pvHRight, vh); + sse_store_siall(pvHRight, vh); pvHRight += ROWSTRIDE_2COL; // Update highest score encountered so far. - vcolmax = _mm_max_epu8(vcolmax, vh); + vcolmax = sse_max_epu8(vcolmax, vh); pvScore += 2; assert_lt(j, iter); if(++j == iter) { pvFRight -= colstride; - vtmp = _mm_load_si128(pvFRight); // load next vf ASAP + vtmp = sse_load_siall(pvFRight); // load next vf ASAP pvHRight -= colstride; - vh = _mm_load_si128(pvHRight); // load next vh ASAP + vh = sse_load_siall(pvHRight); // load next vh ASAP pvScore = d.profbuf_.ptr() + off + 1; j = 0; - vf = _mm_slli_si128(vf, NBYTES_PER_WORD); + vf = sse_slli_siall(vf, NBYTES_PER_WORD); } else { - vtmp = _mm_load_si128(pvFRight); // load next vf ASAP - vh = _mm_load_si128(pvHRight); // load next vh ASAP + vtmp = sse_load_siall(pvFRight); // load next vf ASAP + vh = sse_load_siall(pvHRight); // load next vh ASAP } // Update F with another gap extension - vf = _mm_subs_epu8(vf, rfgape); - vf = _mm_subs_epu8(vf, *pvScore); // veto some ref gap extensions - vf = _mm_max_epu8(vtmp, vf); - vtmp = _mm_subs_epu8(vf, vtmp); - vtmp = _mm_cmpeq_epi8(vtmp, vzero); - cmp = _mm_movemask_epi8(vtmp); + vf = sse_subs_epu8(vf, rfgape); + vf = sse_subs_epu8(vf, *pvScore); // veto some ref gap extensions + vf = sse_max_epu8(vtmp, vf); + vtmp = sse_subs_epu8(vf, vtmp); + vtmp = sse_cmpeq_epi8(vtmp, vzero); + cmp = sse_movemask_epi8(vtmp); nfixup++; } @@ -696,10 +669,10 @@ TAlScore SwAligner::alignGatherLoc8(int& flag, bool debug) { assert_lt(lastoff, MAX_SIZE_T); pvScore = d.profbuf_.ptr() + lastoff; // even elts = query profile, odd = gap barrier for(size_t k = 0; k < iter; k++) { - vh = _mm_load_si128(pvHLeft); - vtmp = _mm_cmpgt_epi8(pvScore[0], vbiasm1); - int cmp = _mm_movemask_epi8(vtmp); - if(cmp != 0xffff) { + vh = sse_load_siall(pvHLeft); + vtmp = sse_cmpgt_epi8(pvScore[0], vbiasm1); + int cmp = sse_movemask_epi8(vtmp); + if(cmp != SSE_MASK_ALL) { // At least one candidate in this mask. Now iterate // through vm/vh to evaluate individual cells. for(size_t m = 0; m < NWORDS_PER_REG; m++) { @@ -726,9 +699,9 @@ TAlScore SwAligner::alignGatherLoc8(int& flag, bool debug) { // Save some elements to checkpoints if(checkpoint) { - __m128i *pvE = vbuf_r + 0; - __m128i *pvF = vbuf_r + 1; - __m128i *pvH = vbuf_r + 2; + SSERegI *pvE = vbuf_r + 0; + SSERegI *pvF = vbuf_r + 1; + SSERegI *pvH = vbuf_r + 2; size_t coli = i - rfi_; if(coli < cper_.locol_) cper_.locol_ = coli; if(coli > cper_.hicol_) cper_.hicol_ = coli; @@ -756,8 +729,8 @@ TAlScore SwAligner::alignGatherLoc8(int& flag, bool debug) { row_div++; } size_t delt = idxrow + row; - size_t vecoff = (row_mod << 6) + row_div; - assert_lt(row_div, 16); + size_t vecoff = (row_mod << (ROWSTRIDE_LOG2+BYTES_LOG2_PER_REG)) + row_div; + assert_lt(row_div, NBYTES_PER_REG); int16_t h_sc = ((uint8_t*)pvH)[vecoff]; int16_t e_sc = ((uint8_t*)pvE)[vecoff]; int16_t f_sc = ((uint8_t*)pvF)[vecoff]; @@ -796,7 +769,7 @@ TAlScore SwAligner::alignGatherLoc8(int& flag, bool debug) { cpimod -= iter; cpidiv++; } - size_t vecoff = (cpimod << 6) + cpidiv; + size_t vecoff = (cpimod << (ROWSTRIDE_LOG2+BYTES_LOG2_PER_REG)) + cpidiv; cper_.commitMap_.push_back(vecoff); cpi += cper_.per_; cpimod += cper_.per_; @@ -824,35 +797,26 @@ TAlScore SwAligner::alignGatherLoc8(int& flag, bool debug) { assert_gt(coli, 0); size_t wordspercol = cper_.niter_ * ROWSTRIDE_2COL; size_t coloff = (coli >> cper_.perpow2_) * wordspercol; - __m128i *dst = cper_.qcols_.ptr() + coloff; - memcpy(dst, vbuf_r, sizeof(__m128i) * wordspercol); + SSERegI *dst = cper_.qcols_.ptr() + coloff; + memcpy(dst, vbuf_r, sizeof(SSERegI) * wordspercol); } } if(cper_.debug_) { // Save the column using memcpys size_t wordspercol = cper_.niter_ * ROWSTRIDE_2COL; size_t coloff = coli * wordspercol; - __m128i *dst = cper_.qcolsD_.ptr() + coloff; - memcpy(dst, vbuf_r, sizeof(__m128i) * wordspercol); + SSERegI *dst = cper_.qcolsD_.ptr() + coloff; + memcpy(dst, vbuf_r, sizeof(SSERegI) * wordspercol); } } // Store column maximum vector in first element of tmp - vmax = _mm_max_epu8(vmax, vcolmax); + vmax = sse_max_epu8(vmax, vcolmax); { // Get single largest score in this column - vmaxtmp = vcolmax; - vtmp = _mm_srli_si128(vmaxtmp, 8); - vmaxtmp = _mm_max_epu8(vmaxtmp, vtmp); - vtmp = _mm_srli_si128(vmaxtmp, 4); - vmaxtmp = _mm_max_epu8(vmaxtmp, vtmp); - vtmp = _mm_srli_si128(vmaxtmp, 2); - vmaxtmp = _mm_max_epu8(vmaxtmp, vtmp); - vtmp = _mm_srli_si128(vmaxtmp, 1); - vmaxtmp = _mm_max_epu8(vmaxtmp, vtmp); - int score = _mm_extract_epi16(vmaxtmp, 0); - score = score & 0x00ff; + int score = 0; + sse_max_score_u8(vcolmax, score); // Could we have saturated? if(score + d.bias_ >= 255) { @@ -888,10 +852,10 @@ TAlScore SwAligner::alignGatherLoc8(int& flag, bool debug) { assert_lt(lastoff, MAX_SIZE_T); pvScore = d.profbuf_.ptr() + lastoff; // even elts = query profile, odd = gap barrier for(size_t k = 0; k < iter; k++) { - vh = _mm_load_si128(pvHLeft); - vtmp = _mm_cmpgt_epi8(pvScore[0], vbiasm1); - int cmp = _mm_movemask_epi8(vtmp); - if(cmp != 0xffff) { + vh = sse_load_siall(pvHLeft); + vtmp = sse_cmpgt_epi8(pvScore[0], vbiasm1); + int cmp = sse_movemask_epi8(vtmp); + if(cmp != SSE_MASK_ALL) { // At least one candidate in this mask. Now iterate // through vm/vh to evaluate individual cells. for(size_t m = 0; m < NWORDS_PER_REG; m++) { @@ -915,16 +879,6 @@ TAlScore SwAligner::alignGatherLoc8(int& flag, bool debug) { } } - // Find largest score in vmax - vtmp = _mm_srli_si128(vmax, 8); - vmax = _mm_max_epu8(vmax, vtmp); - vtmp = _mm_srli_si128(vmax, 4); - vmax = _mm_max_epu8(vmax, vtmp); - vtmp = _mm_srli_si128(vmax, 2); - vmax = _mm_max_epu8(vmax, vtmp); - vtmp = _mm_srli_si128(vmax, 1); - vmax = _mm_max_epu8(vmax, vtmp); - // Update metrics if(!debug) { size_t ninner = (rff_ - rfi_) * iter; @@ -934,8 +888,9 @@ TAlScore SwAligner::alignGatherLoc8(int& flag, bool debug) { met.fixup += nfixup; // DP fixup loop iters } - int score = _mm_extract_epi16(vmax, 0); - score = score & 0x00ff; + // Find largest score in vmax + int score = 0; + sse_max_score_u8(vmax, score); flag = 0; // Could we have saturated? @@ -994,8 +949,6 @@ TAlScore SwAligner::alignNucleotidesLocalSseU8(int& flag, bool debug) { size_t iter = (dpRows() + (NWORDS_PER_REG-1)) / NWORDS_PER_REG; // iter = segLen - int dup; - // Many thanks to Michael Farrar for releasing his striped Smith-Waterman // implementation: // @@ -1004,72 +957,56 @@ TAlScore SwAligner::alignNucleotidesLocalSseU8(int& flag, bool debug) { // Much of the implmentation below is adapted from Michael's code. // Set all elts to reference gap open penalty - __m128i rfgapo = _mm_setzero_si128(); - __m128i rfgape = _mm_setzero_si128(); - __m128i rdgapo = _mm_setzero_si128(); - __m128i rdgape = _mm_setzero_si128(); - __m128i vlo = _mm_setzero_si128(); - __m128i vhi = _mm_setzero_si128(); - __m128i vmax = _mm_setzero_si128(); - __m128i vcolmax = _mm_setzero_si128(); - __m128i vmaxtmp = _mm_setzero_si128(); - __m128i ve = _mm_setzero_si128(); - __m128i vf = _mm_setzero_si128(); - __m128i vh = _mm_setzero_si128(); - __m128i vtmp = _mm_setzero_si128(); - __m128i vzero = _mm_setzero_si128(); - __m128i vbias = _mm_setzero_si128(); + SSERegI rfgapo = sse_setzero_siall(); + SSERegI rfgape = sse_setzero_siall(); + SSERegI rdgapo = sse_setzero_siall(); + SSERegI rdgape = sse_setzero_siall(); + SSERegI vlo = sse_setzero_siall(); + SSERegI vhi = sse_setzero_siall(); + SSERegI vmax = sse_setzero_siall(); + SSERegI vcolmax = sse_setzero_siall(); + SSERegI ve = sse_setzero_siall(); + SSERegI vf = sse_setzero_siall(); + SSERegI vh = sse_setzero_siall(); + SSERegI vtmp = sse_setzero_siall(); + SSERegI vzero = sse_setzero_siall(); + SSERegI vbias = sse_setzero_siall(); assert_gt(sc_->refGapOpen(), 0); assert_leq(sc_->refGapOpen(), MAX_U8); - dup = (sc_->refGapOpen() << 8) | (sc_->refGapOpen() & 0x00ff); - rfgapo = _mm_insert_epi16(rfgapo, dup, 0); - rfgapo = _mm_shufflelo_epi16(rfgapo, 0); - rfgapo = _mm_shuffle_epi32(rfgapo, 0); + sse_fill_u8(sc_->refGapOpen(), rfgapo); // Set all elts to reference gap extension penalty assert_gt(sc_->refGapExtend(), 0); assert_leq(sc_->refGapExtend(), MAX_U8); assert_leq(sc_->refGapExtend(), sc_->refGapOpen()); - dup = (sc_->refGapExtend() << 8) | (sc_->refGapExtend() & 0x00ff); - rfgape = _mm_insert_epi16(rfgape, dup, 0); - rfgape = _mm_shufflelo_epi16(rfgape, 0); - rfgape = _mm_shuffle_epi32(rfgape, 0); + sse_fill_u8(sc_->refGapExtend(), rfgape); // Set all elts to read gap open penalty assert_gt(sc_->readGapOpen(), 0); assert_leq(sc_->readGapOpen(), MAX_U8); - dup = (sc_->readGapOpen() << 8) | (sc_->readGapOpen() & 0x00ff); - rdgapo = _mm_insert_epi16(rdgapo, dup, 0); - rdgapo = _mm_shufflelo_epi16(rdgapo, 0); - rdgapo = _mm_shuffle_epi32(rdgapo, 0); + sse_fill_u8(sc_->readGapOpen(), rdgapo); // Set all elts to read gap extension penalty assert_gt(sc_->readGapExtend(), 0); assert_leq(sc_->readGapExtend(), MAX_U8); assert_leq(sc_->readGapExtend(), sc_->readGapOpen()); - dup = (sc_->readGapExtend() << 8) | (sc_->readGapExtend() & 0x00ff); - rdgape = _mm_insert_epi16(rdgape, dup, 0); - rdgape = _mm_shufflelo_epi16(rdgape, 0); - rdgape = _mm_shuffle_epi32(rdgape, 0); + sse_fill_u8(sc_->readGapExtend(), rdgape); - vhi = _mm_cmpeq_epi16(vhi, vhi); // all elts = 0xffff - vlo = _mm_xor_si128(vlo, vlo); // all elts = 0 + sse_fill_u8_opt(0xff, vhi); + sse_fill_u8_opt(0, vlo); vmax = vlo; // Make a vector of bias offsets - dup = (d.bias_ << 8) | (d.bias_ & 0x00ff); - vbias = _mm_insert_epi16(vbias, dup, 0); - vbias = _mm_shufflelo_epi16(vbias, 0); - vbias = _mm_shuffle_epi32(vbias, 0); + sse_fill_u8(d.bias_, vbias); - // Points to a long vector of __m128i where each element is a block of + // Points to a long vector of SSERegI where each element is a block of // contiguous cells in the E, F or H matrix. If the index % 3 == 0, then // the block of cells is from the E matrix. If index % 3 == 1, they're // from the F matrix. If index % 3 == 2, then they're from the H matrix. // Blocks of cells are organized in the same interleaved manner as they are // calculated by the Farrar algorithm. - const __m128i *pvScore; // points into the query profile + const SSERegI *pvScore; // points into the query profile d.mat_.init(dpRows(), rff_ - rfi_, NWORDS_PER_REG); const size_t colstride = d.mat_.colstride(); @@ -1077,22 +1014,22 @@ TAlScore SwAligner::alignNucleotidesLocalSseU8(int& flag, bool debug) { assert_eq(ROWSTRIDE, colstride / iter); // Initialize the H and E vectors in the first matrix column - __m128i *pvHTmp = d.mat_.tmpvec(0, 0); - __m128i *pvETmp = d.mat_.evec(0, 0); + SSERegI *pvHTmp = d.mat_.tmpvec(0, 0); + SSERegI *pvETmp = d.mat_.evec(0, 0); for(size_t i = 0; i < iter; i++) { - _mm_store_si128(pvETmp, vlo); - _mm_store_si128(pvHTmp, vlo); // start low in local mode + sse_store_siall(pvETmp, vlo); + sse_store_siall(pvHTmp, vlo); // start low in local mode pvETmp += ROWSTRIDE; pvHTmp += ROWSTRIDE; } // These are swapped just before the innermost loop - __m128i *pvHStore = d.mat_.hvec(0, 0); - __m128i *pvHLoad = d.mat_.tmpvec(0, 0); - __m128i *pvELoad = d.mat_.evec(0, 0); - __m128i *pvEStore = d.mat_.evecUnsafe(0, 1); - __m128i *pvFStore = d.mat_.fvec(0, 0); - __m128i *pvFTmp = NULL; + SSERegI *pvHStore = d.mat_.hvec(0, 0); + SSERegI *pvHLoad = d.mat_.tmpvec(0, 0); + SSERegI *pvELoad = d.mat_.evec(0, 0); + SSERegI *pvEStore = d.mat_.evecUnsafe(0, 1); + SSERegI *pvFStore = d.mat_.fvec(0, 0); + SSERegI *pvFTmp = NULL; assert_gt(sc_->gapbar, 0); size_t nfixup = 0; @@ -1124,60 +1061,60 @@ TAlScore SwAligner::alignNucleotidesLocalSseU8(int& flag, bool debug) { pvScore = d.profbuf_.ptr() + off; // even elts = query profile, odd = gap barrier // Load H vector from the final row of the previous column - vh = _mm_load_si128(pvHLoad + colstride - ROWSTRIDE); + vh = sse_load_siall(pvHLoad + colstride - ROWSTRIDE); // Set all cells to low value - vf = _mm_xor_si128(vf, vf); + vf = sse_xor_siall(vf, vf); // Store cells in F, calculated previously // No need to veto ref gap extensions, they're all 0x00s - _mm_store_si128(pvFStore, vf); + sse_store_siall(pvFStore, vf); pvFStore += ROWSTRIDE; // Shift down so that topmost (least sig) cell gets 0 - vh = _mm_slli_si128(vh, NBYTES_PER_WORD); + vh = sse_slli_siall(vh, NBYTES_PER_WORD); // We pull out one loop iteration to make it easier to veto values in the top row // Load cells from E, calculated previously - ve = _mm_load_si128(pvELoad); + ve = sse_load_siall(pvELoad); assert_all_lt(ve, vhi); pvELoad += ROWSTRIDE; // Factor in query profile (matches and mismatches) - vh = _mm_adds_epu8(vh, pvScore[0]); - vh = _mm_subs_epu8(vh, vbias); + vh = sse_adds_epu8(vh, pvScore[0]); + vh = sse_subs_epu8(vh, vbias); // Update H, factoring in E and F - vh = _mm_max_epu8(vh, ve); - vh = _mm_max_epu8(vh, vf); + vh = sse_max_epu8(vh, ve); + vh = sse_max_epu8(vh, vf); // Update highest score so far - vcolmax = _mm_xor_si128(vcolmax, vcolmax); - vcolmax = _mm_max_epu8(vcolmax, vh); + vcolmax = sse_xor_siall(vcolmax, vcolmax); + vcolmax = sse_max_epu8(vcolmax, vh); // Save the new vH values - _mm_store_si128(pvHStore, vh); + sse_store_siall(pvHStore, vh); pvHStore += ROWSTRIDE; // Update vE value vf = vh; - vh = _mm_subs_epu8(vh, rdgapo); - vh = _mm_subs_epu8(vh, pvScore[1]); // veto some read gap opens - ve = _mm_subs_epu8(ve, rdgape); - ve = _mm_max_epu8(ve, vh); + vh = sse_subs_epu8(vh, rdgapo); + vh = sse_subs_epu8(vh, pvScore[1]); // veto some read gap opens + ve = sse_subs_epu8(ve, rdgape); + ve = sse_max_epu8(ve, vh); assert_all_lt(ve, vhi); // Load the next h value - vh = _mm_load_si128(pvHLoad); + vh = sse_load_siall(pvHLoad); pvHLoad += ROWSTRIDE; // Save E values - _mm_store_si128(pvEStore, ve); + sse_store_siall(pvEStore, ve); pvEStore += ROWSTRIDE; // Update vf value - vf = _mm_subs_epu8(vf, rfgapo); + vf = sse_subs_epu8(vf, rfgapo); assert_all_lt(vf, vhi); pvScore += 2; // move on to next query profile @@ -1186,127 +1123,127 @@ TAlScore SwAligner::alignNucleotidesLocalSseU8(int& flag, bool debug) { size_t j; for(j = 1; j < iter; j++) { // Load cells from E, calculated previously - ve = _mm_load_si128(pvELoad); + ve = sse_load_siall(pvELoad); assert_all_lt(ve, vhi); pvELoad += ROWSTRIDE; // Store cells in F, calculated previously - vf = _mm_subs_epu8(vf, pvScore[1]); // veto some ref gap extensions - _mm_store_si128(pvFStore, vf); + vf = sse_subs_epu8(vf, pvScore[1]); // veto some ref gap extensions + sse_store_siall(pvFStore, vf); pvFStore += ROWSTRIDE; // Factor in query profile (matches and mismatches) - vh = _mm_adds_epu8(vh, pvScore[0]); - vh = _mm_subs_epu8(vh, vbias); + vh = sse_adds_epu8(vh, pvScore[0]); + vh = sse_subs_epu8(vh, vbias); // Update H, factoring in E and F - vh = _mm_max_epu8(vh, ve); - vh = _mm_max_epu8(vh, vf); + vh = sse_max_epu8(vh, ve); + vh = sse_max_epu8(vh, vf); // Update highest score encountered this far - vcolmax = _mm_max_epu8(vcolmax, vh); + vcolmax = sse_max_epu8(vcolmax, vh); // Save the new vH values - _mm_store_si128(pvHStore, vh); + sse_store_siall(pvHStore, vh); pvHStore += ROWSTRIDE; // Update vE value vtmp = vh; - vh = _mm_subs_epu8(vh, rdgapo); - vh = _mm_subs_epu8(vh, pvScore[1]); // veto some read gap opens - ve = _mm_subs_epu8(ve, rdgape); - ve = _mm_max_epu8(ve, vh); + vh = sse_subs_epu8(vh, rdgapo); + vh = sse_subs_epu8(vh, pvScore[1]); // veto some read gap opens + ve = sse_subs_epu8(ve, rdgape); + ve = sse_max_epu8(ve, vh); assert_all_lt(ve, vhi); // Load the next h value - vh = _mm_load_si128(pvHLoad); + vh = sse_load_siall(pvHLoad); pvHLoad += ROWSTRIDE; // Save E values - _mm_store_si128(pvEStore, ve); + sse_store_siall(pvEStore, ve); pvEStore += ROWSTRIDE; // Update vf value - vtmp = _mm_subs_epu8(vtmp, rfgapo); - vf = _mm_subs_epu8(vf, rfgape); + vtmp = sse_subs_epu8(vtmp, rfgapo); + vf = sse_subs_epu8(vf, rfgape); assert_all_lt(vf, vhi); - vf = _mm_max_epu8(vf, vtmp); + vf = sse_max_epu8(vf, vtmp); pvScore += 2; // move on to next query profile / gap veto } // pvHStore, pvELoad, pvEStore have all rolled over to the next column pvFTmp = pvFStore; pvFStore -= colstride; // reset to start of column - vtmp = _mm_load_si128(pvFStore); + vtmp = sse_load_siall(pvFStore); pvHStore -= colstride; // reset to start of column - vh = _mm_load_si128(pvHStore); + vh = sse_load_siall(pvHStore); pvEStore -= colstride; // reset to start of column - ve = _mm_load_si128(pvEStore); + ve = sse_load_siall(pvEStore); pvHLoad = pvHStore; // new pvHLoad = pvHStore pvScore = d.profbuf_.ptr() + off + 1; // reset veto vector // vf from last row gets shifted down by one to overlay the first row // rfgape has already been subtracted from it. - vf = _mm_slli_si128(vf, NBYTES_PER_WORD); + vf = sse_slli_siall(vf, NBYTES_PER_WORD); - vf = _mm_subs_epu8(vf, *pvScore); // veto some ref gap extensions - vf = _mm_max_epu8(vtmp, vf); - vtmp = _mm_subs_epu8(vf, vtmp); - vtmp = _mm_cmpeq_epi8(vtmp, vzero); - int cmp = _mm_movemask_epi8(vtmp); + vf = sse_subs_epu8(vf, *pvScore); // veto some ref gap extensions + vf = sse_max_epu8(vtmp, vf); + vtmp = sse_subs_epu8(vf, vtmp); + vtmp = sse_cmpeq_epi8(vtmp, vzero); + int cmp = sse_movemask_epi8(vtmp); // If any element of vtmp is greater than H - gap-open... j = 0; - while(cmp != 0xffff) { + while(cmp != SSE_MASK_ALL) { // Store this vf - _mm_store_si128(pvFStore, vf); + sse_store_siall(pvFStore, vf); pvFStore += ROWSTRIDE; // Update vh w/r/t new vf - vh = _mm_max_epu8(vh, vf); + vh = sse_max_epu8(vh, vf); // Save vH values - _mm_store_si128(pvHStore, vh); + sse_store_siall(pvHStore, vh); pvHStore += ROWSTRIDE; // Update highest score encountered this far - vcolmax = _mm_max_epu8(vcolmax, vh); + vcolmax = sse_max_epu8(vcolmax, vh); // Update E in case it can be improved using our new vh - vh = _mm_subs_epu8(vh, rdgapo); - vh = _mm_subs_epu8(vh, *pvScore); // veto some read gap opens - ve = _mm_max_epu8(ve, vh); - _mm_store_si128(pvEStore, ve); + vh = sse_subs_epu8(vh, rdgapo); + vh = sse_subs_epu8(vh, *pvScore); // veto some read gap opens + ve = sse_max_epu8(ve, vh); + sse_store_siall(pvEStore, ve); pvEStore += ROWSTRIDE; pvScore += 2; assert_lt(j, iter); if(++j == iter) { pvFStore -= colstride; - vtmp = _mm_load_si128(pvFStore); // load next vf ASAP + vtmp = sse_load_siall(pvFStore); // load next vf ASAP pvHStore -= colstride; - vh = _mm_load_si128(pvHStore); // load next vh ASAP + vh = sse_load_siall(pvHStore); // load next vh ASAP pvEStore -= colstride; - ve = _mm_load_si128(pvEStore); // load next ve ASAP + ve = sse_load_siall(pvEStore); // load next ve ASAP pvScore = d.profbuf_.ptr() + off + 1; j = 0; - vf = _mm_slli_si128(vf, NBYTES_PER_WORD); + vf = sse_slli_siall(vf, NBYTES_PER_WORD); } else { - vtmp = _mm_load_si128(pvFStore); // load next vf ASAP - vh = _mm_load_si128(pvHStore); // load next vh ASAP - ve = _mm_load_si128(pvEStore); // load next vh ASAP + vtmp = sse_load_siall(pvFStore); // load next vf ASAP + vh = sse_load_siall(pvHStore); // load next vh ASAP + ve = sse_load_siall(pvEStore); // load next vh ASAP } // Update F with another gap extension - vf = _mm_subs_epu8(vf, rfgape); - vf = _mm_subs_epu8(vf, *pvScore); // veto some ref gap extensions - vf = _mm_max_epu8(vtmp, vf); - vtmp = _mm_subs_epu8(vf, vtmp); - vtmp = _mm_cmpeq_epi8(vtmp, vzero); - cmp = _mm_movemask_epi8(vtmp); + vf = sse_subs_epu8(vf, rfgape); + vf = sse_subs_epu8(vf, *pvScore); // veto some ref gap extensions + vf = sse_max_epu8(vtmp, vf); + vtmp = sse_subs_epu8(vf, vtmp); + vtmp = sse_cmpeq_epi8(vtmp, vzero); + cmp = sse_movemask_epi8(vtmp); nfixup++; } @@ -1328,23 +1265,14 @@ TAlScore SwAligner::alignNucleotidesLocalSseU8(int& flag, bool debug) { #endif // Store column maximum vector in first element of tmp - vmax = _mm_max_epu8(vmax, vcolmax); + vmax = sse_max_epu8(vmax, vcolmax); - _mm_store_si128(d.mat_.tmpvec(0, i - rfi_), vcolmax); + sse_store_siall(d.mat_.tmpvec(0, i - rfi_), vcolmax); { // Get single largest score in this column - vmaxtmp = vcolmax; - vtmp = _mm_srli_si128(vmaxtmp, 8); - vmaxtmp = _mm_max_epu8(vmaxtmp, vtmp); - vtmp = _mm_srli_si128(vmaxtmp, 4); - vmaxtmp = _mm_max_epu8(vmaxtmp, vtmp); - vtmp = _mm_srli_si128(vmaxtmp, 2); - vmaxtmp = _mm_max_epu8(vmaxtmp, vtmp); - vtmp = _mm_srli_si128(vmaxtmp, 1); - vmaxtmp = _mm_max_epu8(vmaxtmp, vtmp); - int score = _mm_extract_epi16(vmaxtmp, 0); - score = score & 0x00ff; + int score = 0; + sse_max_score_u8(vcolmax, score); // Could we have saturated? if(score + d.bias_ >= 255) { @@ -1374,16 +1302,6 @@ TAlScore SwAligner::alignNucleotidesLocalSseU8(int& flag, bool debug) { pvFStore = pvFTmp; } - // Find largest score in vmax - vtmp = _mm_srli_si128(vmax, 8); - vmax = _mm_max_epu8(vmax, vtmp); - vtmp = _mm_srli_si128(vmax, 4); - vmax = _mm_max_epu8(vmax, vtmp); - vtmp = _mm_srli_si128(vmax, 2); - vmax = _mm_max_epu8(vmax, vtmp); - vtmp = _mm_srli_si128(vmax, 1); - vmax = _mm_max_epu8(vmax, vtmp); - // Update metrics if(!debug) { size_t ninner = (rff_ - rfi_) * iter; @@ -1393,8 +1311,9 @@ TAlScore SwAligner::alignNucleotidesLocalSseU8(int& flag, bool debug) { met.fixup += nfixup; // DP fixup loop iters } - int score = _mm_extract_epi16(vmax, 0); - score = score & 0x00ff; + // Find largest score in vmax + int score = 0; + sse_max_score_u8(vmax, score); flag = 0; // Could we have saturated? @@ -1495,26 +1414,13 @@ bool SwAligner::gatherCellsNucleotidesLocalSseU8(TAlScore best) { size_t nrow_hi = nrow; // First, check if there is a cell in this column with a score // above the score threshold - __m128i vmax = *d.mat_.tmpvec(0, j); - __m128i vtmp = _mm_srli_si128(vmax, 8); - vmax = _mm_max_epu8(vmax, vtmp); - vtmp = _mm_srli_si128(vmax, 4); - vmax = _mm_max_epu8(vmax, vtmp); - vtmp = _mm_srli_si128(vmax, 2); - vmax = _mm_max_epu8(vmax, vtmp); - vtmp = _mm_srli_si128(vmax, 1); - vmax = _mm_max_epu8(vmax, vtmp); - int score = _mm_extract_epi16(vmax, 0); -#if defined(SIMDE_ENDIAN_ORDER) && SIMDE_ENDIAN_ORDER == SIMDE_ENDIAN_BIG - score = (score >> 8) & 0x00ff; -#else - score = score & 0x00ff; -#endif + int score = 0; + sse_max_score_u8(*d.mat_.tmpvec(0, j), score); #ifndef NDEBUG { // Start in upper vector row and move down TAlScore max = 0; - __m128i *pvH = d.mat_.hvec(0, j); + SSERegI *pvH = d.mat_.hvec(0, j); for(size_t i = 0; i < iter; i++) { for(size_t k = 0; k < NWORDS_PER_REG; k++) { TAlScore sc = (TAlScore)((TCScore*)pvH)[k]; @@ -1532,11 +1438,11 @@ bool SwAligner::gatherCellsNucleotidesLocalSseU8(TAlScore best) { continue; } // Get pointer to first cell in column to examine: - __m128i *pvHorig = d.mat_.hvec(0, j); - __m128i *pvH = pvHorig; + SSERegI *pvHorig = d.mat_.hvec(0, j); + SSERegI *pvH = pvHorig; // Get pointer to the vector in the following column that corresponds // to the cells diagonally down and to the right from the cells in pvH - __m128i *pvHSucc = (j < ncol-1) ? d.mat_.hvec(0, j+1) : NULL; + SSERegI *pvHSucc = (j < ncol-1) ? d.mat_.hvec(0, j+1) : NULL; // Start in upper vector row and move down for(size_t i = 0; i < iter; i++) { if(pvHSucc != NULL) { @@ -1712,7 +1618,7 @@ bool SwAligner::backtraceNucleotidesLocalSseU8( size_t rowelt, rowvec, eltvec; size_t left_rowelt, up_rowelt, upleft_rowelt; size_t left_rowvec, up_rowvec, upleft_rowvec; - __m128i *cur_vec, *left_vec, *up_vec, *upleft_vec; + SSERegI *cur_vec, *left_vec, *up_vec, *upleft_vec; NEW_ROW_COL(row, col); while((int)row >= 0) { met.btcell++; diff --git a/aln_sink.cpp b/aln_sink.cpp index 793d245..4256bf8 100644 --- a/aln_sink.cpp +++ b/aln_sink.cpp @@ -834,7 +834,9 @@ void AlnSinkWrap::finishRead( } else { met.nconcord_uni++; assert(!rs1_.empty()); - if(rs1_.size() == 1) { + AlnScore sc1 = concordSumm.bestUnchosenP1Score(); + AlnScore sc2 = concordSumm.bestUnchosenP2Score(); + if(!sc1.valid() && !sc2.valid()) { met.nconcord_uni1++; } else { met.nconcord_uni2++; @@ -46,7 +46,7 @@ public: protected: - EList_m128i mat_; + EList_sse mat_; }; #endif diff --git a/blockwise_sa.h b/blockwise_sa.h index a10d889..90be524 100644 --- a/blockwise_sa.h +++ b/blockwise_sa.h @@ -28,6 +28,15 @@ #include <thread> #include <memory> #include <stdexcept> + +#ifdef USE_SAIS +#ifdef BOWTIE_64BIT_INDEX +#include <libsais64.h> +#else +#include <libsais.h> +#endif +#endif + #include "assert_helpers.h" #include "diff_sample.h" #include "multikey_qsort.h" @@ -185,9 +194,52 @@ public: BlockwiseSA<TStr>(__text, __bucketSz, __sanityCheck, __passMemExc, __verbose, __logger) { } }; +#ifdef USE_SAIS +template<typename TStr> +class SAISBlockwiseSA : public InorderBlockwiseSA<TStr> { +public: + SAISBlockwiseSA(TStr& __text, TIndexOffU __bucketSz, int __nthreads) : + InorderBlockwiseSA<TStr>(__text, __bucketSz, false, false, false, cout), _i(0), _nthreads(__nthreads) + { + __text.wbuf()[__text.length()] = (char)127; // $ is larger than any character in the suffix array + _suffixes.resize(__text.length() + 1); +#ifdef BOWTIE_64BIT_INDEX + libsais64_omp((const uint8_t *)__text.buf(), (int64_t *)(_suffixes.ptr()), (int64_t)_suffixes.size(), 0, NULL, _nthreads); +#else + libsais_omp((const uint8_t *)__text.buf(), (int32_t *)(_suffixes.ptr()), (int32_t)_suffixes.size(), 0, NULL, _nthreads); + +#endif + _suffixes[__text.length()] = __text.length(); + } + inline TIndexOffU nextSuffix() { + return _suffixes[_i++]; + } + + void reset() { + _i = 0; + } + + bool isReset() { + return _i == 0; + } + + void nextBlock(int a, int b) { + return; + } + + bool hasMoreBlocks() const { + return _i == _suffixes.size(); + } + +private: + TIndexOffU _i; + EList<TIndexOffU> _suffixes; + int _nthreads; +}; +#endif /** * Build the SA a block at a time according to the scheme outlined in * Karkkainen's "Fast BWT" paper. diff --git a/bt2_build.cpp b/bt2_build.cpp index 9615c38..3a2e52d 100644 --- a/bt2_build.cpp +++ b/bt2_build.cpp @@ -362,7 +362,8 @@ static void deleteIdxFiles( static void renameIdxFiles() { for (size_t i = 0; i < filesWritten.size(); i++) { std::string oldName = filesWritten[i] + ".tmp"; - std::cerr << "Renaming " << oldName << " to " << filesWritten[i] << std::endl; + if (verbose) + std::cerr << "Renaming " << oldName << " to " << filesWritten[i] << std::endl; std::rename(oldName.c_str(), filesWritten[i].c_str()); } } @@ -381,8 +381,9 @@ struct SideLocus { const TIndexOffU sByteOff = sideNum * sideSz; if (prefetch) { __builtin_prefetch(ebwt + sByteOff); -#if (OFF_SIZE>4) __builtin_prefetch(ebwt + sByteOff + 64); //64 byte cache lines +#if (OFF_SIZE>4) + __builtin_prefetch(ebwt + sByteOff + 2*64); #endif } @@ -405,15 +406,17 @@ struct SideLocus { const TIndexOffU sideNum = row / (48*OFF_SIZE); const TIndexOffU sideByteOff = sideNum * sideSz; __builtin_prefetch(ebwt + sideByteOff); -#if (OFF_SIZE>4) __builtin_prefetch(ebwt + sideByteOff + 64); //64 byte cache lines +#if (OFF_SIZE>4) + __builtin_prefetch(ebwt + sideByteOff + 2*64); #endif } void prefetch(const uint8_t* ebwt) const { __builtin_prefetch(ebwt + _sideByteOff); -#if (OFF_SIZE>4) __builtin_prefetch(ebwt + _sideByteOff + 64); //64 byte cache lines +#if (OFF_SIZE>4) + __builtin_prefetch(ebwt + _sideByteOff + 2*64); #endif } @@ -1133,7 +1136,11 @@ public: VMSG_NL(""); } VMSG_NL("Constructing suffix-array element generator"); +#ifdef USE_SAIS + SAISBlockwiseSA<TStr> bsa(s, bmax, nthreads); +#else KarkkainenBlockwiseSA<TStr> bsa(s, bmax, nthreads, pool, dcv, seed, _sanity, _passMemExc, _verbose, outfile); +#endif assert(bsa.suffixItrIsReset()); assert_eq(bsa.size(), s.length()+1); VMSG_NL("Converting suffix-array elements to index image"); @@ -1384,6 +1391,12 @@ public: i); } + void ftabHiPrefetch(TIndexOffU i) const { + Ebwt::ftabHiPrefetch( + ftab(), + i); + } + /** * Get "high interpretation" of ftab entry at index i. The high * interpretation of a regular ftab entry is just the entry @@ -1411,6 +1424,13 @@ public: } } + static void ftabHiPrefetch( + const TIndexOffU *ftab, + TIndexOffU i) + { + __builtin_prefetch(&(ftab[i])); + } + /** * Non-static facade for static function ftabLo. */ @@ -1424,6 +1444,32 @@ public: i); } + void ftabLoPrefetch(TIndexOffU i) const { + Ebwt::ftabLoPrefetch( + ftab(), + i); + } + + /** + * Get low and high bound of ftab range. + */ + void + ftabLoHi( + TIndexOffU i, + TIndexOffU& top, + TIndexOffU& bot) const + { + top = ftabHi(i); + bot = ftabLo(i+1); + assert_geq(bot, top); + } + + void ftabLoHiPrefetch(TIndexOffU i) const + { + ftabHiPrefetch(i); + ftabLoPrefetch(i+1); + } + /** * Get low bound of ftab range. */ @@ -1457,9 +1503,7 @@ public: if(fi == std::numeric_limits<TIndexOffU>::max()) { return false; } - top = ftabHi(fi); - bot = ftabLo(fi+1); - assert_geq(bot, top); + ftabLoHi(fi, top, bot); return true; } @@ -1490,6 +1534,13 @@ public: } } + static void ftabLoPrefetch( + const TIndexOffU *ftab, + TIndexOffU i) + { + __builtin_prefetch(&(ftab[i])); + } + /** * Try to resolve the reference offset of the BW element 'elt'. If * it can be resolved immediately, return the reference offset. If @@ -1852,11 +1903,13 @@ public: * Counts the number of occurrences of character 'c' in the given Ebwt * side up to (but not including) the given byte/bitpair (by/bp). * - * This is a performance-critical function. This is the top search- + * This is a performance-critical function. This used to be the top search- * related hit in the time profile. - * The bottleneck seems to be cache misses due to random memory access pattern. + * The bottleneck was due to cache misses due to random memory access pattern. + * + * The use of prefetch instructions in initFromRow, when applied enough in advance, + * mostly eliminate the cache misses. * - * Function gets 11.09% in profile */ inline TIndexOffU countUpTo(const SideLocus& l, int c) const { // @double-check // Count occurrences of c in each 64-bit (using bit trickery); @@ -1956,9 +2009,12 @@ public: * given side up to (but not including) the given byte/bitpair (by/bp). * Count for 'a' goes in arrs[0], 'c' in arrs[1], etc. * - * This is a performance-critical function. This is the top search- + * This is a performance-critical function. This used to be the top search- * related hit in the time profile. - * The bottleneck seems to be cache misses due to random memory access pattern. + * The bottleneck was due to cache misses due to random memory access pattern. + * + * The use of prefetch instructions in initFromRow, when applied enough in advance, + * mostly eliminate the cache misses. */ inline void countUpToEx(const SideLocus& l, TIndexOffU* arrs) const { int i = 0; @@ -2676,7 +2732,6 @@ void Ebwt::joinToDisk( for(unsigned int i = 0; i < l.size(); i++) { assert(!l[i]->eof()); bool first = true; - TIndexOffU patoff = 0; // For each *fragment* (not necessary an entire sequence) we // can pull out of istream l[i]... while(!l[i]->eof()) { @@ -2711,17 +2766,9 @@ void Ebwt::joinToDisk( if(rec.first) seqsRead++; if(bases == 0) continue; assert_leq(bases, this->plen()[seqsRead-1]); - // Reset the patoff if this is the first fragment - if(rec.first) patoff = 0; - patoff += rec.off; // add fragment's offset from end of last frag. // Adjust rpcps //uint32_t seq = seqsRead-1; ASSERT_ONLY(entsWritten++); - // This is where rstarts elements are written to the output stream - //writeU32(out1, oldRetLen, this->toBe()); // offset from beginning of joined string - //writeU32(out1, seq, this->toBe()); // sequence id - //writeU32(out1, patoff, this->toBe()); // offset into sequence - patoff += bases; } assert_gt(szsi, 0); l[i]->reset(); diff --git a/bt2_search.cpp b/bt2_search.cpp index e4c2b5e..6643bc2 100644 --- a/bt2_search.cpp +++ b/bt2_search.cpp @@ -266,6 +266,17 @@ static EList<string> sra_accs; #define DMAX std::numeric_limits<double>::max() +static void set_format(int ¤t_format, file_format format) { + if (current_format == UNKNOWN) + current_format = format; + else { + std::cerr << file_format_names[current_format] << " and " + << file_format_names[format] << " formats are " + << "mutually exclusive." << std::endl; + exit(1); + } +} + static void resetOptions() { mates1.clear(); mates2.clear(); @@ -275,7 +286,7 @@ static void resetOptions() { startVerbose = 0; gQuiet = false; sanityCheck = 0; // enable expensive sanity checks - format = FASTQ; // default read format is FASTQ + format = UNKNOWN; // default read format is FASTQ interleaved = false; // reads are not interleaved by default origString = ""; // reference text, or filename(s) seed = 0; // srandom() seed @@ -467,199 +478,199 @@ static void resetOptions() { static const char *short_options = "bfF:qbzhcu:rv:s:aP:t3:5:w:p:k:M:1:2:I:X:CQ:N:i:L:U:x:S:g:O:D:R:"; static struct option long_options[] = { -{(char*)"verbose", no_argument, 0, ARG_VERBOSE}, -{(char*)"startverbose", no_argument, 0, ARG_STARTVERBOSE}, -{(char*)"quiet", no_argument, 0, ARG_QUIET}, -{(char*)"sanity", no_argument, 0, ARG_SANITY}, -{(char*)"pause", no_argument, &ipause, 1}, -{(char*)"orig", required_argument, 0, ARG_ORIG}, -{(char*)"all", no_argument, 0, 'a'}, -{(char*)"solexa-quals", no_argument, 0, ARG_SOLEXA_QUALS}, -{(char*)"integer-quals", no_argument, 0, ARG_INTEGER_QUALS}, -{(char*)"int-quals", no_argument, 0, ARG_INTEGER_QUALS}, -{(char*)"metrics", required_argument, 0, ARG_METRIC_IVAL}, -{(char*)"metrics-file", required_argument, 0, ARG_METRIC_FILE}, -{(char*)"metrics-stderr", no_argument, 0, ARG_METRIC_STDERR}, -{(char*)"metrics-per-read", no_argument, 0, ARG_METRIC_PER_READ}, -{(char*)"met-read", no_argument, 0, ARG_METRIC_PER_READ}, -{(char*)"met", required_argument, 0, ARG_METRIC_IVAL}, -{(char*)"met-file", required_argument, 0, ARG_METRIC_FILE}, -{(char*)"met-stderr", no_argument, 0, ARG_METRIC_STDERR}, -{(char*)"time", no_argument, 0, 't'}, -{(char*)"trim3", required_argument, 0, '3'}, -{(char*)"trim5", required_argument, 0, '5'}, -{(char*)"seed", required_argument, 0, ARG_SEED}, -{(char*)"qupto", required_argument, 0, 'u'}, -{(char*)"upto", required_argument, 0, 'u'}, -{(char*)"version", no_argument, 0, ARG_VERSION}, -{(char*)"reads-per-batch", required_argument, 0, ARG_READS_PER_BATCH}, -{(char*)"filepar", no_argument, 0, ARG_FILEPAR}, -{(char*)"help", no_argument, 0, 'h'}, -{(char*)"threads", required_argument, 0, 'p'}, -{(char*)"khits", required_argument, 0, 'k'}, -{(char*)"minins", required_argument, 0, 'I'}, -{(char*)"maxins", required_argument, 0, 'X'}, -{(char*)"quals", required_argument, 0, 'Q'}, -{(char*)"Q1", required_argument, 0, ARG_QUALS1}, -{(char*)"Q2", required_argument, 0, ARG_QUALS2}, -{(char*)"refidx", no_argument, 0, ARG_REFIDX}, -{(char*)"partition", required_argument, 0, ARG_PARTITION}, -{(char*)"ff", no_argument, 0, ARG_FF}, -{(char*)"fr", no_argument, 0, ARG_FR}, -{(char*)"rf", no_argument, 0, ARG_RF}, -{(char*)"cachelim", required_argument, 0, ARG_CACHE_LIM}, -{(char*)"cachesz", required_argument, 0, ARG_CACHE_SZ}, -{(char*)"nofw", no_argument, 0, ARG_NO_FW}, -{(char*)"norc", no_argument, 0, ARG_NO_RC}, -{(char*)"skip", required_argument, 0, 's'}, -{(char*)"12", required_argument, 0, ARG_ONETWO}, -{(char*)"tab5", required_argument, 0, ARG_TAB5}, -{(char*)"tab6", required_argument, 0, ARG_TAB6}, -{(char*)"interleaved", required_argument, 0, ARG_INTERLEAVED}, -{(char*)"phred33-quals", no_argument, 0, ARG_PHRED33}, -{(char*)"phred64-quals", no_argument, 0, ARG_PHRED64}, -{(char*)"phred33", no_argument, 0, ARG_PHRED33}, -{(char*)"phred64", no_argument, 0, ARG_PHRED64}, -{(char*)"solexa1.3-quals", no_argument, 0, ARG_PHRED64}, -{(char*)"mm", no_argument, 0, ARG_MM}, -{(char*)"shmem", no_argument, 0, ARG_SHMEM}, -{(char*)"mmsweep", no_argument, 0, ARG_MMSWEEP}, -{(char*)"hadoopout", no_argument, 0, ARG_HADOOPOUT}, -{(char*)"fullref", no_argument, 0, ARG_FULLREF}, -{(char*)"usage", no_argument, 0, ARG_USAGE}, -{(char*)"sam-no-qname-trunc", no_argument, 0, ARG_SAM_NO_QNAME_TRUNC}, -{(char*)"sam-omit-sec-seq", no_argument, 0, ARG_SAM_OMIT_SEC_SEQ}, -{(char*)"omit-sec-seq", no_argument, 0, ARG_SAM_OMIT_SEC_SEQ}, -{(char*)"sam-no-head", no_argument, 0, ARG_SAM_NOHEAD}, -{(char*)"sam-nohead", no_argument, 0, ARG_SAM_NOHEAD}, -{(char*)"sam-noHD", no_argument, 0, ARG_SAM_NOHEAD}, -{(char*)"sam-no-hd", no_argument, 0, ARG_SAM_NOHEAD}, -{(char*)"sam-nosq", no_argument, 0, ARG_SAM_NOSQ}, -{(char*)"sam-no-sq", no_argument, 0, ARG_SAM_NOSQ}, -{(char*)"sam-noSQ", no_argument, 0, ARG_SAM_NOSQ}, -{(char*)"no-head", no_argument, 0, ARG_SAM_NOHEAD}, -{(char*)"no-hd", no_argument, 0, ARG_SAM_NOHEAD}, -{(char*)"no-sq", no_argument, 0, ARG_SAM_NOSQ}, -{(char*)"no-HD", no_argument, 0, ARG_SAM_NOHEAD}, -{(char*)"no-SQ", no_argument, 0, ARG_SAM_NOSQ}, -{(char*)"no-unal", no_argument, 0, ARG_SAM_NO_UNAL}, -{(char*)"sam-RG", required_argument, 0, ARG_SAM_RG}, -{(char*)"sam-rg", required_argument, 0, ARG_SAM_RG}, -{(char*)"sam-rg-id", required_argument, 0, ARG_SAM_RGID}, -{(char*)"RG", required_argument, 0, ARG_SAM_RG}, -{(char*)"rg", required_argument, 0, ARG_SAM_RG}, -{(char*)"rg-id", required_argument, 0, ARG_SAM_RGID}, -{(char*)"snpphred", required_argument, 0, ARG_SNPPHRED}, -{(char*)"snpfrac", required_argument, 0, ARG_SNPFRAC}, -{(char*)"gbar", required_argument, 0, ARG_GAP_BAR}, -{(char*)"qseq", no_argument, 0, ARG_QSEQ}, -{(char*)"policy", required_argument, 0, ARG_ALIGN_POLICY}, -{(char*)"preset", required_argument, 0, 'P'}, -{(char*)"seed-summ", no_argument, 0, ARG_SEED_SUMM}, -{(char*)"seed-summary", no_argument, 0, ARG_SEED_SUMM}, -{(char*)"overhang", no_argument, 0, ARG_OVERHANG}, -{(char*)"no-cache", no_argument, 0, ARG_NO_CACHE}, -{(char*)"cache", no_argument, 0, ARG_USE_CACHE}, -{(char*)"454", no_argument, 0, ARG_NOISY_HPOLY}, -{(char*)"ion-torrent", no_argument, 0, ARG_NOISY_HPOLY}, -{(char*)"no-mixed", no_argument, 0, ARG_NO_MIXED}, -{(char*)"no-discordant", no_argument, 0, ARG_NO_DISCORDANT}, -{(char*)"local", no_argument, 0, ARG_LOCAL}, -{(char*)"end-to-end", no_argument, 0, ARG_END_TO_END}, -{(char*)"ungapped", no_argument, 0, ARG_UNGAPPED}, -{(char*)"no-ungapped", no_argument, 0, ARG_UNGAPPED_NO}, -{(char*)"sse8", no_argument, 0, ARG_SSE8}, -{(char*)"no-sse8", no_argument, 0, ARG_SSE8_NO}, -{(char*)"scan-narrowed", no_argument, 0, ARG_SCAN_NARROWED}, -{(char*)"qc-filter", no_argument, 0, ARG_QC_FILTER}, -{(char*)"bwa-sw-like", no_argument, 0, ARG_BWA_SW_LIKE}, -{(char*)"multiseed", required_argument, 0, ARG_MULTISEED_IVAL}, -{(char*)"ma", required_argument, 0, ARG_SCORE_MA}, -{(char*)"mp", required_argument, 0, ARG_SCORE_MMP}, -{(char*)"np", required_argument, 0, ARG_SCORE_NP}, -{(char*)"rdg", required_argument, 0, ARG_SCORE_RDG}, -{(char*)"rfg", required_argument, 0, ARG_SCORE_RFG}, -{(char*)"score-min", required_argument, 0, ARG_SCORE_MIN}, -{(char*)"min-score", required_argument, 0, ARG_SCORE_MIN}, -{(char*)"n-ceil", required_argument, 0, ARG_N_CEIL}, -{(char*)"dpad", required_argument, 0, ARG_DPAD}, -{(char*)"mapq-print-inputs", no_argument, 0, ARG_SAM_PRINT_YI}, -{(char*)"very-fast", no_argument, 0, ARG_PRESET_VERY_FAST}, -{(char*)"fast", no_argument, 0, ARG_PRESET_FAST}, -{(char*)"sensitive", no_argument, 0, ARG_PRESET_SENSITIVE}, -{(char*)"very-sensitive", no_argument, 0, ARG_PRESET_VERY_SENSITIVE}, -{(char*)"very-fast-local", no_argument, 0, ARG_PRESET_VERY_FAST_LOCAL}, -{(char*)"fast-local", no_argument, 0, ARG_PRESET_FAST_LOCAL}, -{(char*)"sensitive-local", no_argument, 0, ARG_PRESET_SENSITIVE_LOCAL}, -{(char*)"very-sensitive-local", no_argument, 0, ARG_PRESET_VERY_SENSITIVE_LOCAL}, -{(char*)"seedlen", required_argument, 0, 'L'}, -{(char*)"seedmms", required_argument, 0, 'N'}, -{(char*)"seedival", required_argument, 0, 'i'}, -{(char*)"ignore-quals", no_argument, 0, ARG_IGNORE_QUALS}, -{(char*)"index", required_argument, 0, 'x'}, -{(char*)"arg-desc", no_argument, 0, ARG_DESC}, -{(char*)"wrapper", required_argument, 0, ARG_WRAPPER}, -{(char*)"unpaired", required_argument, 0, 'U'}, -{(char*)"output", required_argument, 0, 'S'}, -{(char*)"mapq-v", required_argument, 0, ARG_MAPQ_V}, -{(char*)"dovetail", no_argument, 0, ARG_DOVETAIL}, -{(char*)"no-dovetail", no_argument, 0, ARG_NO_DOVETAIL}, -{(char*)"contain", no_argument, 0, ARG_CONTAIN}, -{(char*)"no-contain", no_argument, 0, ARG_NO_CONTAIN}, -{(char*)"overlap", no_argument, 0, ARG_OVERLAP}, -{(char*)"no-overlap", no_argument, 0, ARG_NO_OVERLAP}, -{(char*)"tighten", required_argument, 0, ARG_TIGHTEN}, -{(char*)"exact-upfront", no_argument, 0, ARG_EXACT_UPFRONT}, -{(char*)"1mm-upfront", no_argument, 0, ARG_1MM_UPFRONT}, -{(char*)"no-exact-upfront", no_argument, 0, ARG_EXACT_UPFRONT_NO}, -{(char*)"no-1mm-upfront", no_argument, 0, ARG_1MM_UPFRONT_NO}, -{(char*)"1mm-minlen", required_argument, 0, ARG_1MM_MINLEN}, -{(char*)"seed-off", required_argument, 0, 'O'}, -{(char*)"seed-boost", required_argument, 0, ARG_SEED_BOOST_THRESH}, -{(char*)"read-times", no_argument, 0, ARG_READ_TIMES}, -{(char*)"show-rand-seed", no_argument, 0, ARG_SHOW_RAND_SEED}, -{(char*)"dp-fail-streak", required_argument, 0, ARG_DP_FAIL_STREAK_THRESH}, -{(char*)"ee-fail-streak", required_argument, 0, ARG_EE_FAIL_STREAK_THRESH}, -{(char*)"ug-fail-streak", required_argument, 0, ARG_UG_FAIL_STREAK_THRESH}, -{(char*)"fail-streak", required_argument, 0, 'D'}, -{(char*)"dp-fails", required_argument, 0, ARG_DP_FAIL_THRESH}, -{(char*)"ug-fails", required_argument, 0, ARG_UG_FAIL_THRESH}, -{(char*)"extends", required_argument, 0, ARG_EXTEND_ITERS}, -{(char*)"no-extend", no_argument, 0, ARG_NO_EXTEND}, -{(char*)"mapq-extra", no_argument, 0, ARG_MAPQ_EX}, -{(char*)"seed-rounds", required_argument, 0, 'R'}, -{(char*)"reorder", no_argument, 0, ARG_REORDER}, -{(char*)"passthrough", no_argument, 0, ARG_READ_PASSTHRU}, -{(char*)"sample", required_argument, 0, ARG_SAMPLE}, -{(char*)"cp-min", required_argument, 0, ARG_CP_MIN}, -{(char*)"cp-ival", required_argument, 0, ARG_CP_IVAL}, -{(char*)"tri", no_argument, 0, ARG_TRI}, -{(char*)"nondeterministic", no_argument, 0, ARG_NON_DETERMINISTIC}, -{(char*)"non-deterministic", no_argument, 0, ARG_NON_DETERMINISTIC}, -{(char*)"local-seed-cache-sz", required_argument, 0, ARG_LOCAL_SEED_CACHE_SZ}, -{(char*)"seed-cache-sz", required_argument, 0, ARG_CURRENT_SEED_CACHE_SZ}, -{(char*)"no-unal", no_argument, 0, ARG_SAM_NO_UNAL}, -{(char*)"test-25", no_argument, 0, ARG_TEST_25}, + {(char*)"verbose", no_argument, 0, ARG_VERBOSE}, + {(char*)"startverbose", no_argument, 0, ARG_STARTVERBOSE}, + {(char*)"quiet", no_argument, 0, ARG_QUIET}, + {(char*)"sanity", no_argument, 0, ARG_SANITY}, + {(char*)"pause", no_argument, &ipause, 1}, + {(char*)"orig", required_argument, 0, ARG_ORIG}, + {(char*)"all", no_argument, 0, 'a'}, + {(char*)"solexa-quals", no_argument, 0, ARG_SOLEXA_QUALS}, + {(char*)"integer-quals", no_argument, 0, ARG_INTEGER_QUALS}, + {(char*)"int-quals", no_argument, 0, ARG_INTEGER_QUALS}, + {(char*)"metrics", required_argument, 0, ARG_METRIC_IVAL}, + {(char*)"metrics-file", required_argument, 0, ARG_METRIC_FILE}, + {(char*)"metrics-stderr", no_argument, 0, ARG_METRIC_STDERR}, + {(char*)"metrics-per-read", no_argument, 0, ARG_METRIC_PER_READ}, + {(char*)"met-read", no_argument, 0, ARG_METRIC_PER_READ}, + {(char*)"met", required_argument, 0, ARG_METRIC_IVAL}, + {(char*)"met-file", required_argument, 0, ARG_METRIC_FILE}, + {(char*)"met-stderr", no_argument, 0, ARG_METRIC_STDERR}, + {(char*)"time", no_argument, 0, 't'}, + {(char*)"trim3", required_argument, 0, '3'}, + {(char*)"trim5", required_argument, 0, '5'}, + {(char*)"seed", required_argument, 0, ARG_SEED}, + {(char*)"qupto", required_argument, 0, 'u'}, + {(char*)"upto", required_argument, 0, 'u'}, + {(char*)"version", no_argument, 0, ARG_VERSION}, + {(char*)"reads-per-batch", required_argument, 0, ARG_READS_PER_BATCH}, + {(char*)"filepar", no_argument, 0, ARG_FILEPAR}, + {(char*)"help", no_argument, 0, 'h'}, + {(char*)"threads", required_argument, 0, 'p'}, + {(char*)"khits", required_argument, 0, 'k'}, + {(char*)"minins", required_argument, 0, 'I'}, + {(char*)"maxins", required_argument, 0, 'X'}, + {(char*)"quals", required_argument, 0, 'Q'}, + {(char*)"Q1", required_argument, 0, ARG_QUALS1}, + {(char*)"Q2", required_argument, 0, ARG_QUALS2}, + {(char*)"refidx", no_argument, 0, ARG_REFIDX}, + {(char*)"partition", required_argument, 0, ARG_PARTITION}, + {(char*)"ff", no_argument, 0, ARG_FF}, + {(char*)"fr", no_argument, 0, ARG_FR}, + {(char*)"rf", no_argument, 0, ARG_RF}, + {(char*)"cachelim", required_argument, 0, ARG_CACHE_LIM}, + {(char*)"cachesz", required_argument, 0, ARG_CACHE_SZ}, + {(char*)"nofw", no_argument, 0, ARG_NO_FW}, + {(char*)"norc", no_argument, 0, ARG_NO_RC}, + {(char*)"skip", required_argument, 0, 's'}, + {(char*)"12", required_argument, 0, ARG_ONETWO}, + {(char*)"tab5", required_argument, 0, ARG_TAB5}, + {(char*)"tab6", required_argument, 0, ARG_TAB6}, + {(char*)"interleaved", required_argument, 0, ARG_INTERLEAVED}, + {(char*)"phred33-quals", no_argument, 0, ARG_PHRED33}, + {(char*)"phred64-quals", no_argument, 0, ARG_PHRED64}, + {(char*)"phred33", no_argument, 0, ARG_PHRED33}, + {(char*)"phred64", no_argument, 0, ARG_PHRED64}, + {(char*)"solexa1.3-quals", no_argument, 0, ARG_PHRED64}, + {(char*)"mm", no_argument, 0, ARG_MM}, + {(char*)"shmem", no_argument, 0, ARG_SHMEM}, + {(char*)"mmsweep", no_argument, 0, ARG_MMSWEEP}, + {(char*)"hadoopout", no_argument, 0, ARG_HADOOPOUT}, + {(char*)"fullref", no_argument, 0, ARG_FULLREF}, + {(char*)"usage", no_argument, 0, ARG_USAGE}, + {(char*)"sam-no-qname-trunc", no_argument, 0, ARG_SAM_NO_QNAME_TRUNC}, + {(char*)"sam-omit-sec-seq", no_argument, 0, ARG_SAM_OMIT_SEC_SEQ}, + {(char*)"omit-sec-seq", no_argument, 0, ARG_SAM_OMIT_SEC_SEQ}, + {(char*)"sam-no-head", no_argument, 0, ARG_SAM_NOHEAD}, + {(char*)"sam-nohead", no_argument, 0, ARG_SAM_NOHEAD}, + {(char*)"sam-noHD", no_argument, 0, ARG_SAM_NOHEAD}, + {(char*)"sam-no-hd", no_argument, 0, ARG_SAM_NOHEAD}, + {(char*)"sam-nosq", no_argument, 0, ARG_SAM_NOSQ}, + {(char*)"sam-no-sq", no_argument, 0, ARG_SAM_NOSQ}, + {(char*)"sam-noSQ", no_argument, 0, ARG_SAM_NOSQ}, + {(char*)"no-head", no_argument, 0, ARG_SAM_NOHEAD}, + {(char*)"no-hd", no_argument, 0, ARG_SAM_NOHEAD}, + {(char*)"no-sq", no_argument, 0, ARG_SAM_NOSQ}, + {(char*)"no-HD", no_argument, 0, ARG_SAM_NOHEAD}, + {(char*)"no-SQ", no_argument, 0, ARG_SAM_NOSQ}, + {(char*)"no-unal", no_argument, 0, ARG_SAM_NO_UNAL}, + {(char*)"sam-RG", required_argument, 0, ARG_SAM_RG}, + {(char*)"sam-rg", required_argument, 0, ARG_SAM_RG}, + {(char*)"sam-rg-id", required_argument, 0, ARG_SAM_RGID}, + {(char*)"RG", required_argument, 0, ARG_SAM_RG}, + {(char*)"rg", required_argument, 0, ARG_SAM_RG}, + {(char*)"rg-id", required_argument, 0, ARG_SAM_RGID}, + {(char*)"snpphred", required_argument, 0, ARG_SNPPHRED}, + {(char*)"snpfrac", required_argument, 0, ARG_SNPFRAC}, + {(char*)"gbar", required_argument, 0, ARG_GAP_BAR}, + {(char*)"qseq", no_argument, 0, ARG_QSEQ}, + {(char*)"policy", required_argument, 0, ARG_ALIGN_POLICY}, + {(char*)"preset", required_argument, 0, 'P'}, + {(char*)"seed-summ", no_argument, 0, ARG_SEED_SUMM}, + {(char*)"seed-summary", no_argument, 0, ARG_SEED_SUMM}, + {(char*)"overhang", no_argument, 0, ARG_OVERHANG}, + {(char*)"no-cache", no_argument, 0, ARG_NO_CACHE}, + {(char*)"cache", no_argument, 0, ARG_USE_CACHE}, + {(char*)"454", no_argument, 0, ARG_NOISY_HPOLY}, + {(char*)"ion-torrent", no_argument, 0, ARG_NOISY_HPOLY}, + {(char*)"no-mixed", no_argument, 0, ARG_NO_MIXED}, + {(char*)"no-discordant", no_argument, 0, ARG_NO_DISCORDANT}, + {(char*)"local", no_argument, 0, ARG_LOCAL}, + {(char*)"end-to-end", no_argument, 0, ARG_END_TO_END}, + {(char*)"ungapped", no_argument, 0, ARG_UNGAPPED}, + {(char*)"no-ungapped", no_argument, 0, ARG_UNGAPPED_NO}, + {(char*)"sse8", no_argument, 0, ARG_SSE8}, + {(char*)"no-sse8", no_argument, 0, ARG_SSE8_NO}, + {(char*)"scan-narrowed", no_argument, 0, ARG_SCAN_NARROWED}, + {(char*)"qc-filter", no_argument, 0, ARG_QC_FILTER}, + {(char*)"bwa-sw-like", no_argument, 0, ARG_BWA_SW_LIKE}, + {(char*)"multiseed", required_argument, 0, ARG_MULTISEED_IVAL}, + {(char*)"ma", required_argument, 0, ARG_SCORE_MA}, + {(char*)"mp", required_argument, 0, ARG_SCORE_MMP}, + {(char*)"np", required_argument, 0, ARG_SCORE_NP}, + {(char*)"rdg", required_argument, 0, ARG_SCORE_RDG}, + {(char*)"rfg", required_argument, 0, ARG_SCORE_RFG}, + {(char*)"score-min", required_argument, 0, ARG_SCORE_MIN}, + {(char*)"min-score", required_argument, 0, ARG_SCORE_MIN}, + {(char*)"n-ceil", required_argument, 0, ARG_N_CEIL}, + {(char*)"dpad", required_argument, 0, ARG_DPAD}, + {(char*)"mapq-print-inputs", no_argument, 0, ARG_SAM_PRINT_YI}, + {(char*)"very-fast", no_argument, 0, ARG_PRESET_VERY_FAST}, + {(char*)"fast", no_argument, 0, ARG_PRESET_FAST}, + {(char*)"sensitive", no_argument, 0, ARG_PRESET_SENSITIVE}, + {(char*)"very-sensitive", no_argument, 0, ARG_PRESET_VERY_SENSITIVE}, + {(char*)"very-fast-local", no_argument, 0, ARG_PRESET_VERY_FAST_LOCAL}, + {(char*)"fast-local", no_argument, 0, ARG_PRESET_FAST_LOCAL}, + {(char*)"sensitive-local", no_argument, 0, ARG_PRESET_SENSITIVE_LOCAL}, + {(char*)"very-sensitive-local", no_argument, 0, ARG_PRESET_VERY_SENSITIVE_LOCAL}, + {(char*)"seedlen", required_argument, 0, 'L'}, + {(char*)"seedmms", required_argument, 0, 'N'}, + {(char*)"seedival", required_argument, 0, 'i'}, + {(char*)"ignore-quals", no_argument, 0, ARG_IGNORE_QUALS}, + {(char*)"index", required_argument, 0, 'x'}, + {(char*)"arg-desc", no_argument, 0, ARG_DESC}, + {(char*)"wrapper", required_argument, 0, ARG_WRAPPER}, + {(char*)"unpaired", required_argument, 0, 'U'}, + {(char*)"output", required_argument, 0, 'S'}, + {(char*)"mapq-v", required_argument, 0, ARG_MAPQ_V}, + {(char*)"dovetail", no_argument, 0, ARG_DOVETAIL}, + {(char*)"no-dovetail", no_argument, 0, ARG_NO_DOVETAIL}, + {(char*)"contain", no_argument, 0, ARG_CONTAIN}, + {(char*)"no-contain", no_argument, 0, ARG_NO_CONTAIN}, + {(char*)"overlap", no_argument, 0, ARG_OVERLAP}, + {(char*)"no-overlap", no_argument, 0, ARG_NO_OVERLAP}, + {(char*)"tighten", required_argument, 0, ARG_TIGHTEN}, + {(char*)"exact-upfront", no_argument, 0, ARG_EXACT_UPFRONT}, + {(char*)"1mm-upfront", no_argument, 0, ARG_1MM_UPFRONT}, + {(char*)"no-exact-upfront", no_argument, 0, ARG_EXACT_UPFRONT_NO}, + {(char*)"no-1mm-upfront", no_argument, 0, ARG_1MM_UPFRONT_NO}, + {(char*)"1mm-minlen", required_argument, 0, ARG_1MM_MINLEN}, + {(char*)"seed-off", required_argument, 0, 'O'}, + {(char*)"seed-boost", required_argument, 0, ARG_SEED_BOOST_THRESH}, + {(char*)"read-times", no_argument, 0, ARG_READ_TIMES}, + {(char*)"show-rand-seed", no_argument, 0, ARG_SHOW_RAND_SEED}, + {(char*)"dp-fail-streak", required_argument, 0, ARG_DP_FAIL_STREAK_THRESH}, + {(char*)"ee-fail-streak", required_argument, 0, ARG_EE_FAIL_STREAK_THRESH}, + {(char*)"ug-fail-streak", required_argument, 0, ARG_UG_FAIL_STREAK_THRESH}, + {(char*)"fail-streak", required_argument, 0, 'D'}, + {(char*)"dp-fails", required_argument, 0, ARG_DP_FAIL_THRESH}, + {(char*)"ug-fails", required_argument, 0, ARG_UG_FAIL_THRESH}, + {(char*)"extends", required_argument, 0, ARG_EXTEND_ITERS}, + {(char*)"no-extend", no_argument, 0, ARG_NO_EXTEND}, + {(char*)"mapq-extra", no_argument, 0, ARG_MAPQ_EX}, + {(char*)"seed-rounds", required_argument, 0, 'R'}, + {(char*)"reorder", no_argument, 0, ARG_REORDER}, + {(char*)"passthrough", no_argument, 0, ARG_READ_PASSTHRU}, + {(char*)"sample", required_argument, 0, ARG_SAMPLE}, + {(char*)"cp-min", required_argument, 0, ARG_CP_MIN}, + {(char*)"cp-ival", required_argument, 0, ARG_CP_IVAL}, + {(char*)"tri", no_argument, 0, ARG_TRI}, + {(char*)"nondeterministic", no_argument, 0, ARG_NON_DETERMINISTIC}, + {(char*)"non-deterministic", no_argument, 0, ARG_NON_DETERMINISTIC}, + {(char*)"local-seed-cache-sz", required_argument, 0, ARG_LOCAL_SEED_CACHE_SZ}, + {(char*)"seed-cache-sz", required_argument, 0, ARG_CURRENT_SEED_CACHE_SZ}, + {(char*)"no-unal", no_argument, 0, ARG_SAM_NO_UNAL}, + {(char*)"test-25", no_argument, 0, ARG_TEST_25}, // TODO: following should be a function of read length? -{(char*)"desc-kb", required_argument, 0, ARG_DESC_KB}, -{(char*)"desc-landing", required_argument, 0, ARG_DESC_LANDING}, -{(char*)"desc-exp", required_argument, 0, ARG_DESC_EXP}, -{(char*)"desc-prioritize", no_argument, 0, ARG_DESC_PRIORITIZE}, -{(char*)"desc-fmops", required_argument, 0, ARG_DESC_FMOPS}, -{(char*)"log-dp", required_argument, 0, ARG_LOG_DP}, -{(char*)"log-dp-opp", required_argument, 0, ARG_LOG_DP_OPP}, -{(char*)"soft-clipped-unmapped-tlen", no_argument, 0, ARG_SC_UNMAPPED}, -{(char*)"xeq", no_argument, 0, ARG_XEQ}, -{(char*)"thread-ceiling", required_argument, 0, ARG_THREAD_CEILING}, -{(char*)"thread-piddir", required_argument, 0, ARG_THREAD_PIDDIR}, -{(char*)"trim-to", required_argument, 0, ARG_TRIM_TO}, -{(char*)"preserve-tags", no_argument, 0, ARG_PRESERVE_TAGS}, -{(char*)"align-paired-reads", no_argument, 0, ARG_ALIGN_PAIRED_READS}, + {(char*)"desc-kb", required_argument, 0, ARG_DESC_KB}, + {(char*)"desc-landing", required_argument, 0, ARG_DESC_LANDING}, + {(char*)"desc-exp", required_argument, 0, ARG_DESC_EXP}, + {(char*)"desc-prioritize", no_argument, 0, ARG_DESC_PRIORITIZE}, + {(char*)"desc-fmops", required_argument, 0, ARG_DESC_FMOPS}, + {(char*)"log-dp", required_argument, 0, ARG_LOG_DP}, + {(char*)"log-dp-opp", required_argument, 0, ARG_LOG_DP_OPP}, + {(char*)"soft-clipped-unmapped-tlen", no_argument, 0, ARG_SC_UNMAPPED}, + {(char*)"xeq", no_argument, 0, ARG_XEQ}, + {(char*)"thread-ceiling", required_argument, 0, ARG_THREAD_CEILING}, + {(char*)"thread-piddir", required_argument, 0, ARG_THREAD_PIDDIR}, + {(char*)"trim-to", required_argument, 0, ARG_TRIM_TO}, + {(char*)"preserve-tags", no_argument, 0, ARG_PRESERVE_TAGS}, + {(char*)"align-paired-reads", no_argument, 0, ARG_ALIGN_PAIRED_READS}, #ifdef USE_SRA -{(char*)"sra-acc", required_argument, 0, ARG_SRA_ACC}, + {(char*)"sra-acc", required_argument, 0, ARG_SRA_ACC}, #endif -{(char*)"sam-append-comment", no_argument, 0, ARG_SAM_APPEND_COMMENT}, -{(char*)0, 0, 0, 0} // terminator + {(char*)"sam-append-comment", no_argument, 0, ARG_SAM_APPEND_COMMENT}, + {(char*)0, 0, 0, 0} // terminator }; /** @@ -715,8 +726,8 @@ static void printUsage(ostream& out) { << " " << tool_name.c_str() << " [options]* -x <bt2-idx> {-1 <m1> -2 <m2> | -U <r> | --interleaved <i> | -b <bam>} [-S <sam>]" << endl #endif << endl - << " <bt2-idx> Index filename prefix (minus trailing .X." + gEbwt_ext + ")." << endl - << " NOTE: Bowtie 1 and Bowtie 2 indexes are not compatible." << endl + << " <bt2-idx> Index filename prefix (minus trailing .X." + gEbwt_ext + ")." << endl + << " NOTE: Bowtie 1 and Bowtie 2 indexes are not compatible." << endl << " <m1> Files with #1 mates, paired with files in <m2>." << endl; if(wrapper == "basic-0") { out << " Could be gzip'ed (extension: .gz) or bzip2'ed (extension: .bz2)." << endl; @@ -741,9 +752,9 @@ static void printUsage(ostream& out) { out << " <sam> File for SAM output (default: stdout)" << endl << endl << " <m1>, <m2>, <r> can be comma-separated lists (no whitespace) and can be" << endl - << " specified many times. E.g. '-U file1.fq,file2.fq -U file3.fq'." << endl + << " specified many times. E.g. '-U file1.fq,file2.fq -U file3.fq'." << endl // Wrapper script should write <bam> line next - << endl + << endl << "Options (defaults in parentheses):" << endl << endl << " Input:" << endl @@ -840,19 +851,19 @@ static void printUsage(ostream& out) { //} out << " -t/--time print wall-clock time taken by search phases" << endl; if(wrapper == "basic-0") { - out << " --un <path> write unpaired reads that didn't align to <path>" << endl - << " --al <path> write unpaired reads that aligned at least once to <path>" << endl - << " --un-conc <path> write pairs that didn't align concordantly to <path>" << endl - << " --al-conc <path> write pairs that aligned concordantly at least once to <path>" << endl - << " (Note: for --un, --al, --un-conc, or --al-conc, add '-gz' to the option name, e.g." << endl - << " --un-gz <path>, to gzip compress output, or add '-bz2' to bzip2 compress output.)" << endl; + out << " --un <path> write unpaired reads that didn't align to <path>" << endl + << " --al <path> write unpaired reads that aligned at least once to <path>" << endl + << " --un-conc <path> write pairs that didn't align concordantly to <path>" << endl + << " --al-conc <path> write pairs that aligned concordantly at least once to <path>" << endl + << " (Note: for --un, --al, --un-conc, or --al-conc, add '-gz' to the option name, e.g." << endl + << " --un-gz <path>, to gzip compress output, or add '-bz2' to bzip2 compress output.)" << endl; } out << " --quiet print nothing to stderr except serious errors" << endl - // << " --refidx refer to ref. seqs by 0-based index rather than name" << endl + // << " --refidx refer to ref. seqs by 0-based index rather than name" << endl << " --met-file <path> send metrics to file at <path> (off)" << endl << " --met-stderr send metrics to stderr (off)" << endl << " --met <int> report internal counters & metrics every <int> secs (1)" << endl - // Following is supported in the wrapper instead + // Following is supported in the wrapper instead << " --no-unal suppress SAM records for unaligned reads" << endl << " --no-head suppress header lines, i.e. lines starting with @" << endl << " --no-sq suppress @SQ header lines" << endl @@ -870,7 +881,7 @@ static void printUsage(ostream& out) { << " Append FASTA/FASTQ comment to SAM record" << endl << endl << " Performance:" << endl - // << " -o/--offrate <int> override offrate of index; must be >= index's offrate" << endl + // << " -o/--offrate <int> override offrate of index; must be >= index's offrate" << endl << " -p/--threads <int> number of alignment threads to launch (1)" << endl << " --reorder force SAM output order to match order of input reads" << endl #ifdef BOWTIE_MM @@ -885,7 +896,7 @@ static void printUsage(ostream& out) { << " --seed <int> seed for random number generator (0)" << endl << " --non-deterministic" << endl << " seed rand. gen. arbitrarily instead of using read attributes" << endl - // << " --verbose verbose output for debugging" << endl + // << " --verbose verbose output for debugging" << endl << " --version print version information and quit" << endl << " -h/--help print this usage message" << endl; if(wrapper.empty()) { @@ -972,7 +983,7 @@ static string applyPreset(const string& sorig, Presets& presets) { } if(gVerbose) { cerr << "Applying preset: '" << s.c_str() << "' using preset menu '" - << presets.name() << "'" << endl; + << presets.name() << "'" << endl; } string pol; presets.apply(s, pol, extra_opts); @@ -999,565 +1010,565 @@ static EList<string> presetList; */ static void parseOption(int next_option, const char *arg) { switch (next_option) { - case ARG_TEST_25: bowtie2p5 = true; break; - case ARG_DESC_KB: descentTotSz = SimpleFunc::parse(arg, 0.0, 1024.0, 1024.0, DMAX); break; - case ARG_DESC_FMOPS: descentTotFmops = SimpleFunc::parse(arg, 0.0, 10.0, 100.0, DMAX); break; - case ARG_LOG_DP: logDps = arg; break; - case ARG_LOG_DP_OPP: logDpsOpp = arg; break; - case ARG_DESC_LANDING: { - descLanding = parse<int>(arg); - if(descLanding < 1) { - cerr << "Error: --desc-landing must be greater than or equal to 1" << endl; - throw 1; - } - break; - } - case ARG_DESC_EXP: { - descConsExp = parse<double>(arg); - if(descConsExp < 0.0) { - cerr << "Error: --desc-exp must be greater than or equal to 0" << endl; - throw 1; - } - break; - } - case ARG_DESC_PRIORITIZE: descPrioritizeRoots = true; break; - case '1': tokenize(arg, ",", mates1); break; - case '2': tokenize(arg, ",", mates2); break; - case ARG_ONETWO: tokenize(arg, ",", mates12); format = TAB_MATE5; break; - case ARG_TAB5: tokenize(arg, ",", mates12); format = TAB_MATE5; break; - case ARG_TAB6: tokenize(arg, ",", mates12); format = TAB_MATE6; break; - case ARG_INTERLEAVED: { - tokenize(arg, ",", mates12); - interleaved = true; - break; - } - case 'b': { - format = BAM; - saw_bam = true; - break; - } - case 'f': format = FASTA; break; - case 'F': { - format = FASTA_CONT; - pair<uint32_t, uint32_t> p = parsePair<uint32_t>(arg, ','); - fastaContLen = p.first; - fastaContFreq = p.second; - break; - } - case ARG_BWA_SW_LIKE: { - bwaSwLikeC = 5.5f; - bwaSwLikeT = 30; - bwaSwLike = true; - localAlign = true; - // -a INT Score of a match [1] - // -b INT Mismatch penalty [3] - // -q INT Gap open penalty [5] - // -r INT Gap extension penalty. The penalty for a contiguous - // gap of size k is q+k*r. [2] - polstr += ";MA=1;MMP=C3;RDG=5,2;RFG=5,2"; - break; - } - case 'q': format = FASTQ; break; - case 'r': format = RAW; break; - case 'c': format = CMDLINE; break; - case ARG_QSEQ: format = QSEQ; break; - case 'I': - gMinInsert = parseInt(0, "-I arg must be positive", arg); - break; - case 'X': - gMaxInsert = parseInt(1, "-X arg must be at least 1", arg); - break; - case ARG_NO_DISCORDANT: gReportDiscordant = false; break; - case ARG_NO_MIXED: gReportMixed = false; break; - case 's': - skipReads = (uint32_t)parseInt(0, "-s arg must be positive", arg); - break; - case ARG_FF: gMate1fw = true; gMate2fw = true; break; - case ARG_RF: gMate1fw = false; gMate2fw = true; break; - case ARG_FR: gMate1fw = true; gMate2fw = false; break; - case ARG_SHMEM: useShmem = true; break; - case ARG_SEED_SUMM: seedSumm = true; break; - case ARG_SC_UNMAPPED: scUnMapped = true; break; - case ARG_XEQ: xeq = true; break; - case ARG_PRESERVE_TAGS: { - preserve_tags = true; - saw_preserve_tags = true; - break; + case ARG_TEST_25: bowtie2p5 = true; break; + case ARG_DESC_KB: descentTotSz = SimpleFunc::parse(arg, 0.0, 1024.0, 1024.0, DMAX); break; + case ARG_DESC_FMOPS: descentTotFmops = SimpleFunc::parse(arg, 0.0, 10.0, 100.0, DMAX); break; + case ARG_LOG_DP: logDps = arg; break; + case ARG_LOG_DP_OPP: logDpsOpp = arg; break; + case ARG_DESC_LANDING: { + descLanding = parse<int>(arg); + if(descLanding < 1) { + cerr << "Error: --desc-landing must be greater than or equal to 1" << endl; + throw 1; } - case ARG_ALIGN_PAIRED_READS: { - align_paired_reads = true; - saw_align_paired_reads = true; - break; + break; + } + case ARG_DESC_EXP: { + descConsExp = parse<double>(arg); + if(descConsExp < 0.0) { + cerr << "Error: --desc-exp must be greater than or equal to 0" << endl; + throw 1; } - case ARG_MM: { + break; + } + case ARG_DESC_PRIORITIZE: descPrioritizeRoots = true; break; + case '1': tokenize(arg, ",", mates1); break; + case '2': tokenize(arg, ",", mates2); break; + case ARG_ONETWO: tokenize(arg, ",", mates12); set_format(format, TAB_MATE5); break; + case ARG_TAB5: tokenize(arg, ",", mates12); set_format(format, TAB_MATE5); break; + case ARG_TAB6: tokenize(arg, ",", mates12); set_format(format, TAB_MATE6); break; + case ARG_INTERLEAVED: { + tokenize(arg, ",", mates12); + interleaved = true; + break; + } + case 'b': { + set_format(format, BAM); + saw_bam = true; + break; + } + case 'f': set_format(format, FASTA); break; + case 'F': { + set_format(format, FASTA_CONT); + pair<uint32_t, uint32_t> p = parsePair<uint32_t>(arg, ','); + fastaContLen = p.first; + fastaContFreq = p.second; + break; + } + case ARG_BWA_SW_LIKE: { + bwaSwLikeC = 5.5f; + bwaSwLikeT = 30; + bwaSwLike = true; + localAlign = true; + // -a INT Score of a match [1] + // -b INT Mismatch penalty [3] + // -q INT Gap open penalty [5] + // -r INT Gap extension penalty. The penalty for a contiguous + // gap of size k is q+k*r. [2] + polstr += ";MA=1;MMP=C3;RDG=5,2;RFG=5,2"; + break; + } + case 'q': set_format(format, FASTQ); break; + case 'r': set_format(format, RAW); break; + case 'c': set_format(format, CMDLINE); break; + case ARG_QSEQ: set_format(format, QSEQ); break; + case 'I': + gMinInsert = parseInt(0, "-I arg must be positive", arg); + break; + case 'X': + gMaxInsert = parseInt(1, "-X arg must be at least 1", arg); + break; + case ARG_NO_DISCORDANT: gReportDiscordant = false; break; + case ARG_NO_MIXED: gReportMixed = false; break; + case 's': + skipReads = (uint32_t)parseInt(0, "-s arg must be positive", arg); + break; + case ARG_FF: gMate1fw = true; gMate2fw = true; break; + case ARG_RF: gMate1fw = false; gMate2fw = true; break; + case ARG_FR: gMate1fw = true; gMate2fw = false; break; + case ARG_SHMEM: useShmem = true; break; + case ARG_SEED_SUMM: seedSumm = true; break; + case ARG_SC_UNMAPPED: scUnMapped = true; break; + case ARG_XEQ: xeq = true; break; + case ARG_PRESERVE_TAGS: { + preserve_tags = true; + saw_preserve_tags = true; + break; + } + case ARG_ALIGN_PAIRED_READS: { + align_paired_reads = true; + saw_align_paired_reads = true; + break; + } + case ARG_MM: { #ifdef BOWTIE_MM - useMm = true; - break; + useMm = true; + break; #else - cerr << "Memory-mapped I/O mode is disabled because bowtie was not compiled with" << endl - << "BOWTIE_MM defined. Memory-mapped I/O is not supported under Windows. If you" << endl - << "would like to use memory-mapped I/O on a platform that supports it, please" << endl - << "refrain from specifying BOWTIE_MM=0 when compiling Bowtie." << endl; - throw 1; + cerr << "Memory-mapped I/O mode is disabled because bowtie was not compiled with" << endl + << "BOWTIE_MM defined. Memory-mapped I/O is not supported under Windows. If you" << endl + << "would like to use memory-mapped I/O on a platform that supports it, please" << endl + << "refrain from specifying BOWTIE_MM=0 when compiling Bowtie." << endl; + throw 1; #endif - } - case ARG_MMSWEEP: mmSweep = true; break; - case ARG_HADOOPOUT: hadoopOut = true; break; - case ARG_SOLEXA_QUALS: solexaQuals = true; break; - case ARG_INTEGER_QUALS: integerQuals = true; break; - case ARG_PHRED64: phred64Quals = true; break; - case ARG_PHRED33: solexaQuals = false; phred64Quals = false; break; - case ARG_OVERHANG: gReportOverhangs = true; break; - case ARG_NO_CACHE: msNoCache = true; break; - case ARG_USE_CACHE: msNoCache = false; break; - case ARG_LOCAL_SEED_CACHE_SZ: - seedCacheLocalMB = (uint32_t)parseInt(1, "--local-seed-cache-sz arg must be at least 1", arg); - break; - case ARG_CURRENT_SEED_CACHE_SZ: - seedCacheCurrentMB = (uint32_t)parseInt(1, "--seed-cache-sz arg must be at least 1", arg); - break; - case ARG_REFIDX: noRefNames = true; break; - case ARG_FULLREF: fullRef = true; break; - case ARG_GAP_BAR: - gGapBarrier = parseInt(1, "--gbar must be no less than 1", arg); - break; - case ARG_SEED: - seed = parseInt(0, "--seed arg must be at least 0", arg); - break; - case ARG_NON_DETERMINISTIC: - arbitraryRandom = true; - break; - case 'u': - qUpto = (uint32_t)parseInt(1, "-u/--qupto arg must be at least 1", arg); - break; - case 'Q': - tokenize(arg, ",", qualities); - integerQuals = true; - break; - case ARG_QUALS1: - tokenize(arg, ",", qualities1); - integerQuals = true; - break; - case ARG_QUALS2: - tokenize(arg, ",", qualities2); - integerQuals = true; - break; - case ARG_CACHE_LIM: - cacheLimit = (uint32_t)parseInt(1, "--cachelim arg must be at least 1", arg); - break; - case ARG_CACHE_SZ: - cacheSize = (uint32_t)parseInt(1, "--cachesz arg must be at least 1", arg); - cacheSize *= (1024 * 1024); // convert from MB to B - break; - case ARG_WRAPPER: wrapper = arg; break; - case 'p': - nthreads = parseInt(1, "-p/--threads arg must be at least 1", arg); - break; - case ARG_THREAD_CEILING: - thread_ceiling = parseInt(0, "--thread-ceiling must be at least 0", arg); - break; - case ARG_THREAD_PIDDIR: - thread_stealing_dir = arg; - break; - case ARG_FILEPAR: - fileParallel = true; - break; - case '3': gTrim3 = parseInt(0, "-3/--trim3 arg must be at least 0", arg); break; - case '5': gTrim5 = parseInt(0, "-5/--trim5 arg must be at least 0", arg); break; - case ARG_TRIM_TO: { - if (strlen(arg) > 1 && arg[1] != ':') { - trimTo.first = 3; - trimTo.second = parseInt(0, "--trim-to: the number of bases to trim must be at least 0", arg); - break; - } - pair<int, int> res = parsePair<int>(arg, ':'); - if (res.first != 3 && res.first != 5) { - cerr << "--trim-to: trim position must be either 3 or 5" << endl; - printUsage(cerr); - throw 1; - } - if(res.second < 0) { - cerr << "--trim-to: the number bases to trim must be at least 0" << endl; - printUsage(cerr); - throw 1; - } - trimTo = static_cast<pair<short, size_t> >(res); + } + case ARG_MMSWEEP: mmSweep = true; break; + case ARG_HADOOPOUT: hadoopOut = true; break; + case ARG_SOLEXA_QUALS: solexaQuals = true; break; + case ARG_INTEGER_QUALS: integerQuals = true; break; + case ARG_PHRED64: phred64Quals = true; break; + case ARG_PHRED33: solexaQuals = false; phred64Quals = false; break; + case ARG_OVERHANG: gReportOverhangs = true; break; + case ARG_NO_CACHE: msNoCache = true; break; + case ARG_USE_CACHE: msNoCache = false; break; + case ARG_LOCAL_SEED_CACHE_SZ: + seedCacheLocalMB = (uint32_t)parseInt(1, "--local-seed-cache-sz arg must be at least 1", arg); + break; + case ARG_CURRENT_SEED_CACHE_SZ: + seedCacheCurrentMB = (uint32_t)parseInt(1, "--seed-cache-sz arg must be at least 1", arg); + break; + case ARG_REFIDX: noRefNames = true; break; + case ARG_FULLREF: fullRef = true; break; + case ARG_GAP_BAR: + gGapBarrier = parseInt(1, "--gbar must be no less than 1", arg); + break; + case ARG_SEED: + seed = parseInt(0, "--seed arg must be at least 0", arg); + break; + case ARG_NON_DETERMINISTIC: + arbitraryRandom = true; + break; + case 'u': + qUpto = (uint32_t)parseInt(1, "-u/--qupto arg must be at least 1", arg); + break; + case 'Q': + tokenize(arg, ",", qualities); + integerQuals = true; + break; + case ARG_QUALS1: + tokenize(arg, ",", qualities1); + integerQuals = true; + break; + case ARG_QUALS2: + tokenize(arg, ",", qualities2); + integerQuals = true; + break; + case ARG_CACHE_LIM: + cacheLimit = (uint32_t)parseInt(1, "--cachelim arg must be at least 1", arg); + break; + case ARG_CACHE_SZ: + cacheSize = (uint32_t)parseInt(1, "--cachesz arg must be at least 1", arg); + cacheSize *= (1024 * 1024); // convert from MB to B + break; + case ARG_WRAPPER: wrapper = arg; break; + case 'p': + nthreads = parseInt(1, "-p/--threads arg must be at least 1", arg); + break; + case ARG_THREAD_CEILING: + thread_ceiling = parseInt(0, "--thread-ceiling must be at least 0", arg); + break; + case ARG_THREAD_PIDDIR: + thread_stealing_dir = arg; + break; + case ARG_FILEPAR: + fileParallel = true; + break; + case '3': gTrim3 = parseInt(0, "-3/--trim3 arg must be at least 0", arg); break; + case '5': gTrim5 = parseInt(0, "-5/--trim5 arg must be at least 0", arg); break; + case ARG_TRIM_TO: { + if (strlen(arg) > 1 && arg[1] != ':') { + trimTo.first = 3; + trimTo.second = parseInt(0, "--trim-to: the number of bases to trim must be at least 0", arg); break; } - case 'h': printUsage(cout); throw 0; break; - case ARG_USAGE: printUsage(cout); throw 0; break; + pair<int, int> res = parsePair<int>(arg, ':'); + if (res.first != 3 && res.first != 5) { + cerr << "--trim-to: trim position must be either 3 or 5" << endl; + printUsage(cerr); + throw 1; + } + if(res.second < 0) { + cerr << "--trim-to: the number bases to trim must be at least 0" << endl; + printUsage(cerr); + throw 1; + } + trimTo = static_cast<pair<short, size_t> >(res); + break; + } + case 'h': printUsage(cout); throw 0; break; + case ARG_USAGE: printUsage(cout); throw 0; break; // // NOTE that unlike in Bowtie 1, -M, -a and -k are mutually // exclusive here. // - case 'M': { - msample = true; - mhits = parse<uint32_t>(arg); - if(saw_a || saw_k) { - cerr << "Warning: -M, -k and -a are mutually exclusive. " - << "-M will override" << endl; - khits = 1; - } - assert_eq(1, khits); - saw_M = true; - cerr << "Warning: -M is deprecated. Use -D and -R to adjust " << - "effort instead." << endl; - break; - } - case ARG_EXTEND_ITERS: { - maxIters = parse<size_t>(arg); - break; - } - case ARG_NO_EXTEND: { - doExtend = false; - break; - } - case 'R': { polstr += ";ROUNDS="; polstr += arg; break; } - case 'D': { polstr += ";DPS="; polstr += arg; break; } - case ARG_DP_MATE_STREAK_THRESH: { - maxMateStreak = parse<size_t>(arg); - break; - } - case ARG_DP_FAIL_STREAK_THRESH: { - maxDpStreak = parse<size_t>(arg); - break; - } - case ARG_EE_FAIL_STREAK_THRESH: { - maxEeStreak = parse<size_t>(arg); - break; - } - case ARG_UG_FAIL_STREAK_THRESH: { - maxUgStreak = parse<size_t>(arg); - break; - } - case ARG_DP_FAIL_THRESH: { - maxDp = parse<size_t>(arg); - break; - } - case ARG_UG_FAIL_THRESH: { - maxUg = parse<size_t>(arg); - break; - } - case ARG_SEED_BOOST_THRESH: { - seedBoostThresh = parse<size_t>(arg); - break; - } - case 'a': { - msample = false; - allHits = true; - mhits = 0; // disable -M - if(saw_M || saw_k) { - cerr << "Warning: -M, -k and -a are mutually exclusive. " - << "-a will override" << endl; - } - saw_a = true; - break; - } - case 'k': { - msample = false; - khits = (uint32_t)parseInt(1, "-k arg must be at least 1", arg); - mhits = 0; // disable -M - if(saw_M || saw_a) { - cerr << "Warning: -M, -k and -a are mutually exclusive. " - << "-k will override" << endl; - } - saw_k = true; - break; - } - case ARG_VERBOSE: gVerbose = 1; break; - case ARG_STARTVERBOSE: startVerbose = true; break; - case ARG_QUIET: gQuiet = true; break; - case ARG_SANITY: sanityCheck = true; break; - case 't': timing = true; break; - case ARG_METRIC_IVAL: { - metricsIval = parseInt(1, "--metrics arg must be at least 1", arg); - break; - } - case ARG_METRIC_FILE: metricsFile = arg; break; - case ARG_METRIC_STDERR: metricsStderr = true; break; - case ARG_METRIC_PER_READ: metricsPerRead = true; break; - case ARG_NO_FW: gNofw = true; break; - case ARG_NO_RC: gNorc = true; break; - case ARG_SAM_NO_QNAME_TRUNC: samTruncQname = false; break; - case ARG_SAM_APPEND_COMMENT: samAppendComment = true; break; - case ARG_SAM_OMIT_SEC_SEQ: samOmitSecSeqQual = true; break; - case ARG_SAM_NO_UNAL: samNoUnal = true; break; - case ARG_SAM_NOHEAD: samNoHead = true; break; - case ARG_SAM_NOSQ: samNoSQ = true; break; - case ARG_SAM_PRINT_YI: sam_print_yi = true; break; - case ARG_REORDER: reorder = true; break; - case ARG_MAPQ_EX: { - sam_print_zt = true; - break; + case 'M': { + msample = true; + mhits = parse<uint32_t>(arg); + if(saw_a || saw_k) { + cerr << "Warning: -M, -k and -a are mutually exclusive. " + << "-M will override" << endl; + khits = 1; } - case ARG_SHOW_RAND_SEED: { - sam_print_zs = true; - break; + assert_eq(1, khits); + saw_M = true; + cerr << "Warning: -M is deprecated. Use -D and -R to adjust " << + "effort instead." << endl; + break; + } + case ARG_EXTEND_ITERS: { + maxIters = parse<size_t>(arg); + break; + } + case ARG_NO_EXTEND: { + doExtend = false; + break; + } + case 'R': { polstr += ";ROUNDS="; polstr += arg; break; } + case 'D': { polstr += ";DPS="; polstr += arg; break; } + case ARG_DP_MATE_STREAK_THRESH: { + maxMateStreak = parse<size_t>(arg); + break; + } + case ARG_DP_FAIL_STREAK_THRESH: { + maxDpStreak = parse<size_t>(arg); + break; + } + case ARG_EE_FAIL_STREAK_THRESH: { + maxEeStreak = parse<size_t>(arg); + break; + } + case ARG_UG_FAIL_STREAK_THRESH: { + maxUgStreak = parse<size_t>(arg); + break; + } + case ARG_DP_FAIL_THRESH: { + maxDp = parse<size_t>(arg); + break; + } + case ARG_UG_FAIL_THRESH: { + maxUg = parse<size_t>(arg); + break; + } + case ARG_SEED_BOOST_THRESH: { + seedBoostThresh = parse<size_t>(arg); + break; + } + case 'a': { + msample = false; + allHits = true; + mhits = 0; // disable -M + if(saw_M || saw_k) { + cerr << "Warning: -M, -k and -a are mutually exclusive. " + << "-a will override" << endl; } - case ARG_SAMPLE: - sampleFrac = parse<float>(arg); - break; - case ARG_CP_MIN: - cminlen = parse<size_t>(arg); - break; - case ARG_CP_IVAL: - cpow2 = parse<size_t>(arg); - break; - case ARG_TRI: - doTri = true; - break; - case ARG_READ_PASSTHRU: { - sam_print_xr = true; - break; + saw_a = true; + break; + } + case 'k': { + msample = false; + khits = (uint32_t)parseInt(1, "-k arg must be at least 1", arg); + mhits = 0; // disable -M + if(saw_M || saw_a) { + cerr << "Warning: -M, -k and -a are mutually exclusive. " + << "-k will override" << endl; } - case ARG_READ_TIMES: { - sam_print_xt = true; - sam_print_xd = true; - sam_print_xu = true; - sam_print_yl = true; - sam_print_ye = true; - sam_print_yu = true; - sam_print_yr = true; - sam_print_zb = true; - sam_print_zr = true; - sam_print_zf = true; - sam_print_zm = true; - sam_print_zi = true; - break; + saw_k = true; + break; + } + case ARG_VERBOSE: gVerbose = 1; break; + case ARG_STARTVERBOSE: startVerbose = true; break; + case ARG_QUIET: gQuiet = true; break; + case ARG_SANITY: sanityCheck = true; break; + case 't': timing = true; break; + case ARG_METRIC_IVAL: { + metricsIval = parseInt(1, "--metrics arg must be at least 1", arg); + break; + } + case ARG_METRIC_FILE: metricsFile = arg; break; + case ARG_METRIC_STDERR: metricsStderr = true; break; + case ARG_METRIC_PER_READ: metricsPerRead = true; break; + case ARG_NO_FW: gNofw = true; break; + case ARG_NO_RC: gNorc = true; break; + case ARG_SAM_NO_QNAME_TRUNC: samTruncQname = false; break; + case ARG_SAM_APPEND_COMMENT: samAppendComment = true; break; + case ARG_SAM_OMIT_SEC_SEQ: samOmitSecSeqQual = true; break; + case ARG_SAM_NO_UNAL: samNoUnal = true; break; + case ARG_SAM_NOHEAD: samNoHead = true; break; + case ARG_SAM_NOSQ: samNoSQ = true; break; + case ARG_SAM_PRINT_YI: sam_print_yi = true; break; + case ARG_REORDER: reorder = true; break; + case ARG_MAPQ_EX: { + sam_print_zt = true; + break; + } + case ARG_SHOW_RAND_SEED: { + sam_print_zs = true; + break; + } + case ARG_SAMPLE: + sampleFrac = parse<float>(arg); + break; + case ARG_CP_MIN: + cminlen = parse<size_t>(arg); + break; + case ARG_CP_IVAL: + cpow2 = parse<size_t>(arg); + break; + case ARG_TRI: + doTri = true; + break; + case ARG_READ_PASSTHRU: { + sam_print_xr = true; + break; + } + case ARG_READ_TIMES: { + sam_print_xt = true; + sam_print_xd = true; + sam_print_xu = true; + sam_print_yl = true; + sam_print_ye = true; + sam_print_yu = true; + sam_print_yr = true; + sam_print_zb = true; + sam_print_zr = true; + sam_print_zf = true; + sam_print_zm = true; + sam_print_zi = true; + break; + } + case ARG_SAM_RG: { + string argstr = arg; + if(argstr.substr(0, 3) == "ID:") { + rgid = "\t"; + rgid += argstr; + rgs_optflag = "RG:Z:" + argstr.substr(3); + } else { + rgs += '\t'; + rgs += argstr; } - case ARG_SAM_RG: { - string argstr = arg; - if(argstr.substr(0, 3) == "ID:") { - rgid = "\t"; - rgid += argstr; - rgs_optflag = "RG:Z:" + argstr.substr(3); - } else { - rgs += '\t'; - rgs += argstr; - } - break; + break; + } + case ARG_SAM_RGID: { + string argstr = arg; + rgid = "\t"; + rgid = "\tID:" + argstr; + rgs_optflag = "RG:Z:" + argstr; + break; + } + case ARG_PARTITION: partitionSz = parse<int>(arg); break; + case ARG_READS_PER_BATCH: + readsPerBatch = parseInt(1, "--reads-per-batch arg must be at least 1", arg); + break; + case ARG_DPAD: + maxhalf = parseInt(0, "--dpad must be no less than 0", arg); + break; + case ARG_ORIG: + if(arg == NULL || strlen(arg) == 0) { + cerr << "--orig arg must be followed by a string" << endl; + printUsage(cerr); + throw 1; } - case ARG_SAM_RGID: { - string argstr = arg; - rgid = "\t"; - rgid = "\tID:" + argstr; - rgs_optflag = "RG:Z:" + argstr; - break; + origString = arg; + break; + case ARG_LOCAL: { + localAlign = true; + gDefaultSeedLen = DEFAULT_LOCAL_SEEDLEN; + break; + } + case ARG_END_TO_END: localAlign = false; break; + case ARG_SSE8: enable8 = true; break; + case ARG_SSE8_NO: enable8 = false; break; + case ARG_UNGAPPED: doUngapped = true; break; + case ARG_UNGAPPED_NO: doUngapped = false; break; + case ARG_NO_DOVETAIL: gDovetailMatesOK = false; break; + case ARG_NO_CONTAIN: gContainMatesOK = false; break; + case ARG_NO_OVERLAP: gOlapMatesOK = false; break; + case ARG_DOVETAIL: gDovetailMatesOK = true; break; + case ARG_CONTAIN: gContainMatesOK = true; break; + case ARG_OVERLAP: gOlapMatesOK = true; break; + case ARG_QC_FILTER: qcFilter = true; break; + case ARG_IGNORE_QUALS: ignoreQuals = true; break; + case ARG_MAPQ_V: mapqv = parse<int>(arg); break; + case ARG_TIGHTEN: tighten = parse<int>(arg); break; + case ARG_EXACT_UPFRONT: doExactUpFront = true; break; + case ARG_1MM_UPFRONT: do1mmUpFront = true; break; + case ARG_EXACT_UPFRONT_NO: doExactUpFront = false; break; + case ARG_1MM_UPFRONT_NO: do1mmUpFront = false; break; + case ARG_1MM_MINLEN: do1mmMinLen = parse<size_t>(arg); break; + case ARG_NOISY_HPOLY: noisyHpolymer = true; break; + case 'x': bt2index = arg; break; + case ARG_PRESET_VERY_FAST_LOCAL: localAlign = true; + case ARG_PRESET_VERY_FAST: { + presetList.push_back("very-fast%LOCAL%"); break; + } + case ARG_PRESET_FAST_LOCAL: localAlign = true; + case ARG_PRESET_FAST: { + presetList.push_back("fast%LOCAL%"); break; + } + case ARG_PRESET_SENSITIVE_LOCAL: localAlign = true; + case ARG_PRESET_SENSITIVE: { + presetList.push_back("sensitive%LOCAL%"); break; + } + case ARG_PRESET_VERY_SENSITIVE_LOCAL: localAlign = true; + case ARG_PRESET_VERY_SENSITIVE: { + presetList.push_back("very-sensitive%LOCAL%"); break; + } + case 'P': { presetList.push_back(arg); break; } + case ARG_ALIGN_POLICY: { + if(strlen(arg) > 0) { + polstr += ";"; polstr += arg; } - case ARG_PARTITION: partitionSz = parse<int>(arg); break; - case ARG_READS_PER_BATCH: - readsPerBatch = parseInt(1, "--reads-per-batch arg must be at least 1", arg); - break; - case ARG_DPAD: - maxhalf = parseInt(0, "--dpad must be no less than 0", arg); - break; - case ARG_ORIG: - if(arg == NULL || strlen(arg) == 0) { - cerr << "--orig arg must be followed by a string" << endl; - printUsage(cerr); - throw 1; - } - origString = arg; - break; - case ARG_LOCAL: { - localAlign = true; - gDefaultSeedLen = DEFAULT_LOCAL_SEEDLEN; - break; + break; + } + case 'N': { + int64_t len = parse<size_t>(arg); + if (len < 0 || len > 1) { + cerr << "Error: -N argument must be within the interval [0,1]; was " << arg << endl; + throw 1; } - case ARG_END_TO_END: localAlign = false; break; - case ARG_SSE8: enable8 = true; break; - case ARG_SSE8_NO: enable8 = false; break; - case ARG_UNGAPPED: doUngapped = true; break; - case ARG_UNGAPPED_NO: doUngapped = false; break; - case ARG_NO_DOVETAIL: gDovetailMatesOK = false; break; - case ARG_NO_CONTAIN: gContainMatesOK = false; break; - case ARG_NO_OVERLAP: gOlapMatesOK = false; break; - case ARG_DOVETAIL: gDovetailMatesOK = true; break; - case ARG_CONTAIN: gContainMatesOK = true; break; - case ARG_OVERLAP: gOlapMatesOK = true; break; - case ARG_QC_FILTER: qcFilter = true; break; - case ARG_IGNORE_QUALS: ignoreQuals = true; break; - case ARG_MAPQ_V: mapqv = parse<int>(arg); break; - case ARG_TIGHTEN: tighten = parse<int>(arg); break; - case ARG_EXACT_UPFRONT: doExactUpFront = true; break; - case ARG_1MM_UPFRONT: do1mmUpFront = true; break; - case ARG_EXACT_UPFRONT_NO: doExactUpFront = false; break; - case ARG_1MM_UPFRONT_NO: do1mmUpFront = false; break; - case ARG_1MM_MINLEN: do1mmMinLen = parse<size_t>(arg); break; - case ARG_NOISY_HPOLY: noisyHpolymer = true; break; - case 'x': bt2index = arg; break; - case ARG_PRESET_VERY_FAST_LOCAL: localAlign = true; - case ARG_PRESET_VERY_FAST: { - presetList.push_back("very-fast%LOCAL%"); break; + polstr += ";SEED="; + polstr += arg; + break; + } + case 'L': { + int64_t len = parse<size_t>(arg); + if(len < 1 || len > 32) { + cerr << "Error: -L argument must be within the interval [1,32]; was " << arg << endl; + throw 1; } - case ARG_PRESET_FAST_LOCAL: localAlign = true; - case ARG_PRESET_FAST: { - presetList.push_back("fast%LOCAL%"); break; + polstr += ";SEEDLEN="; + polstr += arg; + break; + } + case 'O': + multiseedOff = parse<size_t>(arg); + break; + case 'i': { + EList<string> args; + tokenize(arg, ",", args); + if(args.size() > 3 || args.size() == 0) { + cerr << "Error: expected 3 or fewer comma-separated " + << "arguments to -i option, got " + << args.size() << endl; + throw 1; } - case ARG_PRESET_SENSITIVE_LOCAL: localAlign = true; - case ARG_PRESET_SENSITIVE: { - presetList.push_back("sensitive%LOCAL%"); break; + // Interval-settings arguments + polstr += (";IVAL=" + args[0]); // Function type + if(args.size() > 1) { + polstr += ("," + args[1]); // Constant term } - case ARG_PRESET_VERY_SENSITIVE_LOCAL: localAlign = true; - case ARG_PRESET_VERY_SENSITIVE: { - presetList.push_back("very-sensitive%LOCAL%"); break; + if(args.size() > 2) { + polstr += ("," + args[2]); // Coefficient } - case 'P': { presetList.push_back(arg); break; } - case ARG_ALIGN_POLICY: { - if(strlen(arg) > 0) { - polstr += ";"; polstr += arg; - } - break; + break; + } + case ARG_MULTISEED_IVAL: { + polstr += ";"; + // Split argument by comma + EList<string> args; + tokenize(arg, ",", args); + if(args.size() > 5 || args.size() == 0) { + cerr << "Error: expected 5 or fewer comma-separated " + << "arguments to --multiseed option, got " + << args.size() << endl; + throw 1; } - case 'N': { - int64_t len = parse<size_t>(arg); - if (len < 0 || len > 1) { - cerr << "Error: -N argument must be within the interval [0,1]; was " << arg << endl; - throw 1; - } - polstr += ";SEED="; - polstr += arg; - break; + // Seed mm and length arguments + polstr += "SEED="; + polstr += (args[0]); // # mismatches + if(args.size() > 1) polstr += (";SEEDLEN=" + args[1]); // length + if(args.size() > 2) polstr += (";IVAL=" + args[2]); // Func type + if(args.size() > 3) polstr += ("," + args[ 3]); // Constant term + if(args.size() > 4) polstr += ("," + args[ 4]); // Coefficient + break; + } + case ARG_N_CEIL: { + // Split argument by comma + EList<string> args; + tokenize(arg, ",", args); + if(args.size() > 3) { + cerr << "Error: expected 3 or fewer comma-separated " + << "arguments to --n-ceil option, got " + << args.size() << endl; + throw 1; } - case 'L': { - int64_t len = parse<size_t>(arg); - if(len < 1 || len > 32) { - cerr << "Error: -L argument must be within the interval [1,32]; was " << arg << endl; - throw 1; - } - polstr += ";SEEDLEN="; - polstr += arg; - break; + if(args.size() == 0) { + cerr << "Error: expected at least one argument to --n-ceil option" << endl; + throw 1; } - case 'O': - multiseedOff = parse<size_t>(arg); - break; - case 'i': { - EList<string> args; - tokenize(arg, ",", args); - if(args.size() > 3 || args.size() == 0) { - cerr << "Error: expected 3 or fewer comma-separated " - << "arguments to -i option, got " - << args.size() << endl; - throw 1; - } - // Interval-settings arguments - polstr += (";IVAL=" + args[0]); // Function type + polstr += ";NCEIL="; + if(args.size() == 3) { + polstr += (args[0] + "," + args[1] + "," + args[2]); + } else { + polstr += ("L," + args[0]); if(args.size() > 1) { - polstr += ("," + args[1]); // Constant term + polstr += ("," + (args[1])); } - if(args.size() > 2) { - polstr += ("," + args[2]); // Coefficient - } - break; } - case ARG_MULTISEED_IVAL: { - polstr += ";"; - // Split argument by comma - EList<string> args; - tokenize(arg, ",", args); - if(args.size() > 5 || args.size() == 0) { - cerr << "Error: expected 5 or fewer comma-separated " - << "arguments to --multiseed option, got " - << args.size() << endl; - throw 1; - } - // Seed mm and length arguments - polstr += "SEED="; - polstr += (args[0]); // # mismatches - if(args.size() > 1) polstr += (";SEEDLEN=" + args[1]); // length - if(args.size() > 2) polstr += (";IVAL=" + args[2]); // Func type - if(args.size() > 3) polstr += ("," + args[ 3]); // Constant term - if(args.size() > 4) polstr += ("," + args[ 4]); // Coefficient - break; + break; + } + case ARG_SCORE_MA: polstr += ";MA="; polstr += arg; break; + case ARG_SCORE_MMP: { + EList<string> args; + tokenize(arg, ",", args); + if(args.size() > 2 || args.size() == 0) { + cerr << "Error: expected 1 or 2 comma-separated " + << "arguments to --mmp option, got " << args.size() << endl; + throw 1; } - case ARG_N_CEIL: { - // Split argument by comma - EList<string> args; - tokenize(arg, ",", args); - if(args.size() > 3) { - cerr << "Error: expected 3 or fewer comma-separated " - << "arguments to --n-ceil option, got " - << args.size() << endl; - throw 1; - } - if(args.size() == 0) { - cerr << "Error: expected at least one argument to --n-ceil option" << endl; - throw 1; - } - polstr += ";NCEIL="; - if(args.size() == 3) { - polstr += (args[0] + "," + args[1] + "," + args[2]); - } else { - polstr += ("L," + args[0]); - if(args.size() > 1) { - polstr += ("," + (args[1])); - } + if(args.size() >= 1) { + polstr += ";MMP=Q,"; + polstr += args[0]; + if(args.size() >= 2) { + polstr += ","; + polstr += args[1]; } - break; } - case ARG_SCORE_MA: polstr += ";MA="; polstr += arg; break; - case ARG_SCORE_MMP: { - EList<string> args; - tokenize(arg, ",", args); - if(args.size() > 2 || args.size() == 0) { - cerr << "Error: expected 1 or 2 comma-separated " - << "arguments to --mmp option, got " << args.size() << endl; - throw 1; - } - if(args.size() >= 1) { - polstr += ";MMP=Q,"; - polstr += args[0]; - if(args.size() >= 2) { - polstr += ","; - polstr += args[1]; - } - } - break; + break; + } + case ARG_SCORE_NP: polstr += ";NP=C"; polstr += arg; break; + case ARG_SCORE_RDG: polstr += ";RDG="; polstr += arg; break; + case ARG_SCORE_RFG: polstr += ";RFG="; polstr += arg; break; + case ARG_SCORE_MIN: { + polstr += ";"; + EList<string> args; + tokenize(arg, ",", args); + if(args.size() > 3 || args.size() == 0) { + cerr << "Error: expected 3 or fewer comma-separated " + << "arguments to --n-ceil option, got " + << args.size() << endl; + throw 1; } - case ARG_SCORE_NP: polstr += ";NP=C"; polstr += arg; break; - case ARG_SCORE_RDG: polstr += ";RDG="; polstr += arg; break; - case ARG_SCORE_RFG: polstr += ";RFG="; polstr += arg; break; - case ARG_SCORE_MIN: { - polstr += ";"; - EList<string> args; - tokenize(arg, ",", args); - if(args.size() > 3 || args.size() == 0) { - cerr << "Error: expected 3 or fewer comma-separated " - << "arguments to --n-ceil option, got " - << args.size() << endl; - throw 1; - } - polstr += ("MIN=" + args[0]); - if(args.size() > 1) { - polstr += ("," + args[1]); - } - if(args.size() > 2) { - polstr += ("," + args[2]); - } - break; + polstr += ("MIN=" + args[0]); + if(args.size() > 1) { + polstr += ("," + args[1]); } - case ARG_DESC: printArgDesc(cout); throw 0; - case 'S': outfile = arg; break; - case 'U': { - EList<string> args; - tokenize(arg, ",", args); - for(size_t i = 0; i < args.size(); i++) { - queries.push_back(args[i]); - } - break; + if(args.size() > 2) { + polstr += ("," + args[2]); + } + break; + } + case ARG_DESC: printArgDesc(cout); throw 0; + case 'S': outfile = arg; break; + case 'U': { + EList<string> args; + tokenize(arg, ",", args); + for(size_t i = 0; i < args.size(); i++) { + queries.push_back(args[i]); } + break; + } #ifdef USE_SRA case ARG_SRA_ACC: { - tokenize(arg, ",", sra_accs); - format = SRA_FASTA; - break; + tokenize(arg, ",", sra_accs); + set_format(format, SRA_FASTA); + break; } #endif case ARG_VERSION: showVersion = 1; break; - default: - printUsage(cerr); - throw 1; + default: + printUsage(cerr); + throw 1; } } @@ -1602,7 +1613,7 @@ static void parseOptions(int argc, const char **argv) { if ((saw_trim3 || saw_trim5) && saw_trim_to) { cerr << "ERROR: --trim5/--trim3 and --trim-to are mutually exclusive " - << "options." << endl; + << "options." << endl; exit(1); } @@ -1625,10 +1636,6 @@ static void parseOptions(int argc, const char **argv) { for (size_t i = presetList.size(); i != 0; i--) polstr = applyPreset(presetList[i-1], *presets.get()) + polstr; } - // Apply specified presets - for(size_t i = 0; i < presetList.size(); i++) { - polstr += applyPreset(presetList[i], *presets.get()); - } for(size_t i = 0; i < extra_opts.size(); i++) { next_option = extra_opts[extra_opts_cur].first; const char *arg = extra_opts[extra_opts_cur].second.c_str(); @@ -1678,6 +1685,8 @@ static void parseOptions(int argc, const char **argv) { assert_gt(mhits, 0); msample = true; } + if (format == UNKNOWN) + set_format(format, FASTQ); if(mates1.size() != mates2.size()) { cerr << "Error: " << mates1.size() << " mate files/sequences were specified with -1, but " << mates2.size() << endl << "mate files/sequences were specified with -2. The same number of mate files/" << endl @@ -1722,7 +1731,7 @@ static void parseOptions(int argc, const char **argv) { if(!rgs.empty() && rgid.empty()) { cerr << "Warning: --rg was specified without --rg-id also " << "being specified. @RG line is not printed unless --rg-id " - << "is specified." << endl; + << "is specified." << endl; } // Check for duplicate mate input files if(format != CMDLINE) { @@ -1752,30 +1761,31 @@ static void parseOptions(int argc, const char **argv) { if(bonusMatch > 0 && !scoreMin.alwaysPositive()) { cerr << "Error: the match penalty is greater than 0 (" << bonusMatch << ") but the --score-min function can be less than or equal to " - << "zero. Either let the match penalty be 0 or make --score-min " - << "always positive." << endl; + << "zero. Either let the match penalty be 0 or make --score-min " + << "always positive." << endl; throw 1; } if(multiseedMms >= multiseedLen) { assert_gt(multiseedLen, 0); cerr << "Warning: seed mismatches (" << multiseedMms << ") is less than seed length (" << multiseedLen - << "); setting mismatches to " << (multiseedMms-1) - << " instead" << endl; + << "); setting mismatches to " << (multiseedMms-1) + << " instead" << endl; multiseedMms = multiseedLen-1; } sam_print_zm = sam_print_zm && bowtie2p5; #ifndef NDEBUG if(!gQuiet) { cerr << "Warning: Running in debug mode. Please use debug mode only " - << "for diagnosing errors, and not for typical use of Bowtie 2." - << endl; + << "for diagnosing errors, and not for typical use of Bowtie 2." + << endl; } #endif } static const char *argv0 = NULL; +#if 0 /// Create a PatternSourcePerThread for the current thread according /// to the global params and return a pointer to it static PatternSourcePerThreadFactory* @@ -1789,11 +1799,11 @@ createPatsrcFactory( assert(patsrcFact != NULL); return patsrcFact; } +#endif #define PTHREAD_ATTRS (PTHREAD_CREATE_JOINABLE | PTHREAD_CREATE_DETACHED) -static PatternComposer* multiseed_patsrc; -static PatternParams multiseed_pp; +static PatternSourceReadAheadFactory* multiseed_readahead_factory; static Ebwt* multiseed_ebwtFw; static Ebwt* multiseed_ebwtBw; static Scoring* multiseed_sc; @@ -1809,7 +1819,7 @@ static OutFileBuf* multiseed_metricsOfb; struct OuterLoopMetrics { OuterLoopMetrics() { - reset(); + reset(); } /** @@ -1817,7 +1827,7 @@ struct OuterLoopMetrics { */ void reset() { reads = bases = srreads = srbases = - freads = fbases = ureads = ubases = 0; + freads = fbases = ureads = ubases = 0; } /** @@ -1905,42 +1915,42 @@ struct PerfMetrics { uint64_t nbtfiltst_, uint64_t nbtfiltsc_, uint64_t nbtfiltdo_) - { - ThreadSafe ts(mutex_m); - if(ol != NULL) { - olmu.merge(*ol); - } - if(sd != NULL) { - sdmu.merge(*sd); - } - if(wl != NULL) { - wlmu.merge(*wl); - } - if(swSeed != NULL) { - swmuSeed.merge(*swSeed); - } - if(swMate != NULL) { - swmuMate.merge(*swMate); - } - if(rm != NULL) { - rpmu.merge(*rm); - } - if(dpSse8Ex != NULL) { - dpSse8uSeed.merge(*dpSse8Ex); - } - if(dpSse8Ma != NULL) { - dpSse8uMate.merge(*dpSse8Ma); - } - if(dpSse16Ex != NULL) { - dpSse16uSeed.merge(*dpSse16Ex); - } - if(dpSse16Ma != NULL) { - dpSse16uMate.merge(*dpSse16Ma); + { + ThreadSafe ts(mutex_m); + if(ol != NULL) { + olmu.merge(*ol); + } + if(sd != NULL) { + sdmu.merge(*sd); + } + if(wl != NULL) { + wlmu.merge(*wl); + } + if(swSeed != NULL) { + swmuSeed.merge(*swSeed); + } + if(swMate != NULL) { + swmuMate.merge(*swMate); + } + if(rm != NULL) { + rpmu.merge(*rm); + } + if(dpSse8Ex != NULL) { + dpSse8uSeed.merge(*dpSse8Ex); + } + if(dpSse8Ma != NULL) { + dpSse8uMate.merge(*dpSse8Ma); + } + if(dpSse16Ex != NULL) { + dpSse16uSeed.merge(*dpSse16Ex); + } + if(dpSse16Ma != NULL) { + dpSse16uMate.merge(*dpSse16Ma); + } + nbtfiltst_u += nbtfiltst_; + nbtfiltsc_u += nbtfiltsc_; + nbtfiltdo_u += nbtfiltdo_; } - nbtfiltst_u += nbtfiltst_; - nbtfiltsc_u += nbtfiltsc_; - nbtfiltdo_u += nbtfiltdo_; - } /** * Reports a matrix of results, incl. column labels, to an OutFileBuf. @@ -1952,738 +1962,738 @@ struct PerfMetrics { bool metricsStderr, // additionally output to stderr? bool total, // true -> report total, otherwise incremental const BTString *name) // non-NULL name pointer if is per-read record - { - ThreadSafe ts(mutex_m); - ostringstream stderrSs; - time_t curtime = time(0); - char buf[1024]; - if(first) { - const char *str = - /* 1 */ "Time" "\t" - /* 2 */ "Read" "\t" - /* 3 */ "Base" "\t" - /* 4 */ "SameRead" "\t" - /* 5 */ "SameReadBase" "\t" - /* 6 */ "UnfilteredRead" "\t" - /* 7 */ "UnfilteredBase" "\t" - - /* 8 */ "Paired" "\t" - /* 9 */ "Unpaired" "\t" - /* 10 */ "AlConUni" "\t" - /* 11 */ "AlConRep" "\t" - /* 12 */ "AlConFail" "\t" - /* 13 */ "AlDis" "\t" - /* 14 */ "AlConFailUni" "\t" - /* 15 */ "AlConFailRep" "\t" - /* 16 */ "AlConFailFail" "\t" - /* 17 */ "AlConRepUni" "\t" - /* 18 */ "AlConRepRep" "\t" - /* 19 */ "AlConRepFail" "\t" - /* 20 */ "AlUnpUni" "\t" - /* 21 */ "AlUnpRep" "\t" - /* 22 */ "AlUnpFail" "\t" - - /* 23 */ "SeedSearch" "\t" - /* 24 */ "NRange" "\t" - /* 25 */ "NElt" "\t" - /* 26 */ "IntraSCacheHit" "\t" - /* 27 */ "InterSCacheHit" "\t" - /* 28 */ "OutOfMemory" "\t" - /* 29 */ "AlBWOp" "\t" - /* 30 */ "AlBWBranch" "\t" - /* 31 */ "ResBWOp" "\t" - /* 32 */ "ResBWBranch" "\t" - /* 33 */ "ResResolve" "\t" - /* 34 */ "ResReport" "\t" - /* 35 */ "RedundantSHit" "\t" - - /* 36 */ "BestMinEdit0" "\t" - /* 37 */ "BestMinEdit1" "\t" - /* 38 */ "BestMinEdit2" "\t" - - /* 39 */ "ExactAttempts" "\t" - /* 40 */ "ExactSucc" "\t" - /* 41 */ "ExactRanges" "\t" - /* 42 */ "ExactRows" "\t" - /* 43 */ "ExactOOMs" "\t" - - /* 44 */ "1mmAttempts" "\t" - /* 45 */ "1mmSucc" "\t" - /* 46 */ "1mmRanges" "\t" - /* 47 */ "1mmRows" "\t" - /* 48 */ "1mmOOMs" "\t" - - /* 49 */ "UngappedSucc" "\t" - /* 50 */ "UngappedFail" "\t" - /* 51 */ "UngappedNoDec" "\t" - - /* 52 */ "DPExLt10Gaps" "\t" - /* 53 */ "DPExLt5Gaps" "\t" - /* 54 */ "DPExLt3Gaps" "\t" - - /* 55 */ "DPMateLt10Gaps" "\t" - /* 56 */ "DPMateLt5Gaps" "\t" - /* 57 */ "DPMateLt3Gaps" "\t" - - /* 58 */ "DP16ExDps" "\t" - /* 59 */ "DP16ExDpSat" "\t" - /* 60 */ "DP16ExDpFail" "\t" - /* 61 */ "DP16ExDpSucc" "\t" - /* 62 */ "DP16ExCol" "\t" - /* 63 */ "DP16ExCell" "\t" - /* 64 */ "DP16ExInner" "\t" - /* 65 */ "DP16ExFixup" "\t" - /* 66 */ "DP16ExGathSol" "\t" - /* 67 */ "DP16ExBt" "\t" - /* 68 */ "DP16ExBtFail" "\t" - /* 69 */ "DP16ExBtSucc" "\t" - /* 70 */ "DP16ExBtCell" "\t" - /* 71 */ "DP16ExCoreRej" "\t" - /* 72 */ "DP16ExNRej" "\t" - - /* 73 */ "DP8ExDps" "\t" - /* 74 */ "DP8ExDpSat" "\t" - /* 75 */ "DP8ExDpFail" "\t" - /* 76 */ "DP8ExDpSucc" "\t" - /* 77 */ "DP8ExCol" "\t" - /* 78 */ "DP8ExCell" "\t" - /* 79 */ "DP8ExInner" "\t" - /* 80 */ "DP8ExFixup" "\t" - /* 81 */ "DP8ExGathSol" "\t" - /* 82 */ "DP8ExBt" "\t" - /* 83 */ "DP8ExBtFail" "\t" - /* 84 */ "DP8ExBtSucc" "\t" - /* 85 */ "DP8ExBtCell" "\t" - /* 86 */ "DP8ExCoreRej" "\t" - /* 87 */ "DP8ExNRej" "\t" - - /* 88 */ "DP16MateDps" "\t" - /* 89 */ "DP16MateDpSat" "\t" - /* 90 */ "DP16MateDpFail" "\t" - /* 91 */ "DP16MateDpSucc" "\t" - /* 92 */ "DP16MateCol" "\t" - /* 93 */ "DP16MateCell" "\t" - /* 94 */ "DP16MateInner" "\t" - /* 95 */ "DP16MateFixup" "\t" - /* 96 */ "DP16MateGathSol" "\t" - /* 97 */ "DP16MateBt" "\t" - /* 98 */ "DP16MateBtFail" "\t" - /* 99 */ "DP16MateBtSucc" "\t" - /* 100 */ "DP16MateBtCell" "\t" - /* 101 */ "DP16MateCoreRej" "\t" - /* 102 */ "DP16MateNRej" "\t" - - /* 103 */ "DP8MateDps" "\t" - /* 104 */ "DP8MateDpSat" "\t" - /* 105 */ "DP8MateDpFail" "\t" - /* 106 */ "DP8MateDpSucc" "\t" - /* 107 */ "DP8MateCol" "\t" - /* 108 */ "DP8MateCell" "\t" - /* 109 */ "DP8MateInner" "\t" - /* 110 */ "DP8MateFixup" "\t" - /* 111 */ "DP8MateGathSol" "\t" - /* 112 */ "DP8MateBt" "\t" - /* 113 */ "DP8MateBtFail" "\t" - /* 114 */ "DP8MateBtSucc" "\t" - /* 115 */ "DP8MateBtCell" "\t" - /* 116 */ "DP8MateCoreRej" "\t" - /* 117 */ "DP8MateNRej" "\t" - - /* 118 */ "DPBtFiltStart" "\t" - /* 119 */ "DPBtFiltScore" "\t" - /* 120 */ "DpBtFiltDom" "\t" + { + ThreadSafe ts(mutex_m); + ostringstream stderrSs; + time_t curtime = time(0); + char buf[1024]; + if(first) { + const char *str = + /* 1 */ "Time" "\t" + /* 2 */ "Read" "\t" + /* 3 */ "Base" "\t" + /* 4 */ "SameRead" "\t" + /* 5 */ "SameReadBase" "\t" + /* 6 */ "UnfilteredRead" "\t" + /* 7 */ "UnfilteredBase" "\t" + + /* 8 */ "Paired" "\t" + /* 9 */ "Unpaired" "\t" + /* 10 */ "AlConUni" "\t" + /* 11 */ "AlConRep" "\t" + /* 12 */ "AlConFail" "\t" + /* 13 */ "AlDis" "\t" + /* 14 */ "AlConFailUni" "\t" + /* 15 */ "AlConFailRep" "\t" + /* 16 */ "AlConFailFail" "\t" + /* 17 */ "AlConRepUni" "\t" + /* 18 */ "AlConRepRep" "\t" + /* 19 */ "AlConRepFail" "\t" + /* 20 */ "AlUnpUni" "\t" + /* 21 */ "AlUnpRep" "\t" + /* 22 */ "AlUnpFail" "\t" + + /* 23 */ "SeedSearch" "\t" + /* 24 */ "NRange" "\t" + /* 25 */ "NElt" "\t" + /* 26 */ "IntraSCacheHit" "\t" + /* 27 */ "InterSCacheHit" "\t" + /* 28 */ "OutOfMemory" "\t" + /* 29 */ "AlBWOp" "\t" + /* 30 */ "AlBWBranch" "\t" + /* 31 */ "ResBWOp" "\t" + /* 32 */ "ResBWBranch" "\t" + /* 33 */ "ResResolve" "\t" + /* 34 */ "ResReport" "\t" + /* 35 */ "RedundantSHit" "\t" + + /* 36 */ "BestMinEdit0" "\t" + /* 37 */ "BestMinEdit1" "\t" + /* 38 */ "BestMinEdit2" "\t" + + /* 39 */ "ExactAttempts" "\t" + /* 40 */ "ExactSucc" "\t" + /* 41 */ "ExactRanges" "\t" + /* 42 */ "ExactRows" "\t" + /* 43 */ "ExactOOMs" "\t" + + /* 44 */ "1mmAttempts" "\t" + /* 45 */ "1mmSucc" "\t" + /* 46 */ "1mmRanges" "\t" + /* 47 */ "1mmRows" "\t" + /* 48 */ "1mmOOMs" "\t" + + /* 49 */ "UngappedSucc" "\t" + /* 50 */ "UngappedFail" "\t" + /* 51 */ "UngappedNoDec" "\t" + + /* 52 */ "DPExLt10Gaps" "\t" + /* 53 */ "DPExLt5Gaps" "\t" + /* 54 */ "DPExLt3Gaps" "\t" + + /* 55 */ "DPMateLt10Gaps" "\t" + /* 56 */ "DPMateLt5Gaps" "\t" + /* 57 */ "DPMateLt3Gaps" "\t" + + /* 58 */ "DP16ExDps" "\t" + /* 59 */ "DP16ExDpSat" "\t" + /* 60 */ "DP16ExDpFail" "\t" + /* 61 */ "DP16ExDpSucc" "\t" + /* 62 */ "DP16ExCol" "\t" + /* 63 */ "DP16ExCell" "\t" + /* 64 */ "DP16ExInner" "\t" + /* 65 */ "DP16ExFixup" "\t" + /* 66 */ "DP16ExGathSol" "\t" + /* 67 */ "DP16ExBt" "\t" + /* 68 */ "DP16ExBtFail" "\t" + /* 69 */ "DP16ExBtSucc" "\t" + /* 70 */ "DP16ExBtCell" "\t" + /* 71 */ "DP16ExCoreRej" "\t" + /* 72 */ "DP16ExNRej" "\t" + + /* 73 */ "DP8ExDps" "\t" + /* 74 */ "DP8ExDpSat" "\t" + /* 75 */ "DP8ExDpFail" "\t" + /* 76 */ "DP8ExDpSucc" "\t" + /* 77 */ "DP8ExCol" "\t" + /* 78 */ "DP8ExCell" "\t" + /* 79 */ "DP8ExInner" "\t" + /* 80 */ "DP8ExFixup" "\t" + /* 81 */ "DP8ExGathSol" "\t" + /* 82 */ "DP8ExBt" "\t" + /* 83 */ "DP8ExBtFail" "\t" + /* 84 */ "DP8ExBtSucc" "\t" + /* 85 */ "DP8ExBtCell" "\t" + /* 86 */ "DP8ExCoreRej" "\t" + /* 87 */ "DP8ExNRej" "\t" + + /* 88 */ "DP16MateDps" "\t" + /* 89 */ "DP16MateDpSat" "\t" + /* 90 */ "DP16MateDpFail" "\t" + /* 91 */ "DP16MateDpSucc" "\t" + /* 92 */ "DP16MateCol" "\t" + /* 93 */ "DP16MateCell" "\t" + /* 94 */ "DP16MateInner" "\t" + /* 95 */ "DP16MateFixup" "\t" + /* 96 */ "DP16MateGathSol" "\t" + /* 97 */ "DP16MateBt" "\t" + /* 98 */ "DP16MateBtFail" "\t" + /* 99 */ "DP16MateBtSucc" "\t" + /* 100 */ "DP16MateBtCell" "\t" + /* 101 */ "DP16MateCoreRej" "\t" + /* 102 */ "DP16MateNRej" "\t" + + /* 103 */ "DP8MateDps" "\t" + /* 104 */ "DP8MateDpSat" "\t" + /* 105 */ "DP8MateDpFail" "\t" + /* 106 */ "DP8MateDpSucc" "\t" + /* 107 */ "DP8MateCol" "\t" + /* 108 */ "DP8MateCell" "\t" + /* 109 */ "DP8MateInner" "\t" + /* 110 */ "DP8MateFixup" "\t" + /* 111 */ "DP8MateGathSol" "\t" + /* 112 */ "DP8MateBt" "\t" + /* 113 */ "DP8MateBtFail" "\t" + /* 114 */ "DP8MateBtSucc" "\t" + /* 115 */ "DP8MateBtCell" "\t" + /* 116 */ "DP8MateCoreRej" "\t" + /* 117 */ "DP8MateNRej" "\t" + + /* 118 */ "DPBtFiltStart" "\t" + /* 119 */ "DPBtFiltScore" "\t" + /* 120 */ "DpBtFiltDom" "\t" #ifdef USE_MEM_TALLY - /* 121 */ "MemPeak" "\t" - /* 122 */ "UncatMemPeak" "\t" // 0 - /* 123 */ "EbwtMemPeak" "\t" // EBWT_CAT - /* 124 */ "CacheMemPeak" "\t" // CA_CAT - /* 125 */ "ResolveMemPeak" "\t" // GW_CAT - /* 126 */ "AlignMemPeak" "\t" // AL_CAT - /* 127 */ "DPMemPeak" "\t" // DP_CAT - /* 128 */ "MiscMemPeak" "\t" // MISC_CAT - /* 129 */ "DebugMemPeak" "\t" // DEBUG_CAT + /* 121 */ "MemPeak" "\t" + /* 122 */ "UncatMemPeak" "\t" // 0 + /* 123 */ "EbwtMemPeak" "\t" // EBWT_CAT + /* 124 */ "CacheMemPeak" "\t" // CA_CAT + /* 125 */ "ResolveMemPeak" "\t" // GW_CAT + /* 126 */ "AlignMemPeak" "\t" // AL_CAT + /* 127 */ "DPMemPeak" "\t" // DP_CAT + /* 128 */ "MiscMemPeak" "\t" // MISC_CAT + /* 129 */ "DebugMemPeak" "\t" // DEBUG_CAT #endif - "\n"; + "\n"; - if(name != NULL) { - if(o != NULL) o->writeChars("Name\t"); - if(metricsStderr) stderrSs << "Name\t"; - } + if(name != NULL) { + if(o != NULL) o->writeChars("Name\t"); + if(metricsStderr) stderrSs << "Name\t"; + } - if(o != NULL) o->writeChars(str); - if(metricsStderr) stderrSs << str; - first = false; - } + if(o != NULL) o->writeChars(str); + if(metricsStderr) stderrSs << str; + first = false; + } - if(total) mergeIncrementals(); + if(total) mergeIncrementals(); - // 0. Read name, if needed - if(name != NULL) { - if(o != NULL) { - o->writeChars(name->toZBuf()); - o->write('\t'); - } - if(metricsStderr) { - stderrSs << (*name) << '\t'; + // 0. Read name, if needed + if(name != NULL) { + if(o != NULL) { + o->writeChars(name->toZBuf()); + o->write('\t'); + } + if(metricsStderr) { + stderrSs << (*name) << '\t'; + } } - } - // 1. Current time in secs - itoa10<time_t>(curtime, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - - const OuterLoopMetrics& ol = total ? olm : olmu; - - // 2. Reads - itoa10<uint64_t>(ol.reads, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 3. Bases - itoa10<uint64_t>(ol.bases, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 4. Same-read reads - itoa10<uint64_t>(ol.srreads, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 5. Same-read bases - itoa10<uint64_t>(ol.srbases, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 6. Unfiltered reads - itoa10<uint64_t>(ol.ureads, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 7. Unfiltered bases - itoa10<uint64_t>(ol.ubases, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - - const ReportingMetrics& rp = total ? rpm : rpmu; - - // 8. Paired reads - itoa10<uint64_t>(rp.npaired, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 9. Unpaired reads - itoa10<uint64_t>(rp.nunpaired, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 10. Pairs with unique concordant alignments - itoa10<uint64_t>(rp.nconcord_uni, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 11. Pairs with repetitive concordant alignments - itoa10<uint64_t>(rp.nconcord_rep, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 12. Pairs with 0 concordant alignments - itoa10<uint64_t>(rp.nconcord_0, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 13. Pairs with 1 discordant alignment - itoa10<uint64_t>(rp.ndiscord, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 14. Mates from unaligned pairs that align uniquely - itoa10<uint64_t>(rp.nunp_0_uni, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 15. Mates from unaligned pairs that align repetitively - itoa10<uint64_t>(rp.nunp_0_rep, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 16. Mates from unaligned pairs that fail to align - itoa10<uint64_t>(rp.nunp_0_0, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 17. Mates from repetitive pairs that align uniquely - itoa10<uint64_t>(rp.nunp_rep_uni, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 18. Mates from repetitive pairs that align repetitively - itoa10<uint64_t>(rp.nunp_rep_rep, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 19. Mates from repetitive pairs that fail to align - itoa10<uint64_t>(rp.nunp_rep_0, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 20. Unpaired reads that align uniquely - itoa10<uint64_t>(rp.nunp_uni, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 21. Unpaired reads that align repetitively - itoa10<uint64_t>(rp.nunp_rep, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 22. Unpaired reads that fail to align - itoa10<uint64_t>(rp.nunp_0, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - - const SeedSearchMetrics& sd = total ? sdm : sdmu; - - // 23. Seed searches - itoa10<uint64_t>(sd.seedsearch, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 24. Seed ranges found - itoa10<uint64_t>(sd.nrange, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 25. Seed elements found - itoa10<uint64_t>(sd.nelt, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 26. Hits in 'current' cache - itoa10<uint64_t>(sd.intrahit, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 27. Hits in 'local' cache - itoa10<uint64_t>(sd.interhit, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 28. Out of memory - itoa10<uint64_t>(sd.ooms, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 29. Burrows-Wheeler ops in aligner - itoa10<uint64_t>(sd.bwops, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 30. Burrows-Wheeler branches (edits) in aligner - itoa10<uint64_t>(sd.bweds, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - - const WalkMetrics& wl = total ? wlm : wlmu; - - // 31. Burrows-Wheeler ops in resolver - itoa10<uint64_t>(wl.bwops, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 32. Burrows-Wheeler branches in resolver - itoa10<uint64_t>(wl.branches, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 33. Burrows-Wheeler offset resolutions - itoa10<uint64_t>(wl.resolves, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 34. Offset reports - itoa10<uint64_t>(wl.reports, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - - // 35. Redundant seed hit - itoa10<uint64_t>(total ? swmSeed.rshit : swmuSeed.rshit, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - - // 36. # times the best (out of fw/rc) minimum # edits was 0 - itoa10<uint64_t>(total ? sdm.bestmin0 : sdmu.bestmin0, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 37. # times the best (out of fw/rc) minimum # edits was 1 - itoa10<uint64_t>(total ? sdm.bestmin1 : sdmu.bestmin1, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 38. # times the best (out of fw/rc) minimum # edits was 2 - itoa10<uint64_t>(total ? sdm.bestmin2 : sdmu.bestmin2, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - - // 39. Exact aligner attempts - itoa10<uint64_t>(total ? swmSeed.exatts : swmuSeed.exatts, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 40. Exact aligner successes - itoa10<uint64_t>(total ? swmSeed.exsucc : swmuSeed.exsucc, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 41. Exact aligner ranges - itoa10<uint64_t>(total ? swmSeed.exranges : swmuSeed.exranges, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 42. Exact aligner rows - itoa10<uint64_t>(total ? swmSeed.exrows : swmuSeed.exrows, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 43. Exact aligner OOMs - itoa10<uint64_t>(total ? swmSeed.exooms : swmuSeed.exooms, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - - // 44. 1mm aligner attempts - itoa10<uint64_t>(total ? swmSeed.mm1atts : swmuSeed.mm1atts, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 45. 1mm aligner successes - itoa10<uint64_t>(total ? swmSeed.mm1succ : swmuSeed.mm1succ, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 46. 1mm aligner ranges - itoa10<uint64_t>(total ? swmSeed.mm1ranges : swmuSeed.mm1ranges, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 47. 1mm aligner rows - itoa10<uint64_t>(total ? swmSeed.mm1rows : swmuSeed.mm1rows, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 48. 1mm aligner OOMs - itoa10<uint64_t>(total ? swmSeed.mm1ooms : swmuSeed.mm1ooms, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - - // 49 Ungapped aligner success - itoa10<uint64_t>(total ? swmSeed.ungapsucc : swmuSeed.ungapsucc, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 50. Ungapped aligner fail - itoa10<uint64_t>(total ? swmSeed.ungapfail : swmuSeed.ungapfail, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 51. Ungapped aligner no decision - itoa10<uint64_t>(total ? swmSeed.ungapnodec : swmuSeed.ungapnodec, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - - // 52. # seed-extend DPs with < 10 gaps - itoa10<uint64_t>(total ? swmSeed.sws10 : swmuSeed.sws10, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 53. # seed-extend DPs with < 5 gaps - itoa10<uint64_t>(total ? swmSeed.sws5 : swmuSeed.sws5, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 54. # seed-extend DPs with < 3 gaps - itoa10<uint64_t>(total ? swmSeed.sws3 : swmuSeed.sws3, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - - // 55. # seed-extend DPs with < 10 gaps - itoa10<uint64_t>(total ? swmMate.sws10 : swmuMate.sws10, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 56. # seed-extend DPs with < 5 gaps - itoa10<uint64_t>(total ? swmMate.sws5 : swmuMate.sws5, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 57. # seed-extend DPs with < 3 gaps - itoa10<uint64_t>(total ? swmMate.sws3 : swmuMate.sws3, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - - const SSEMetrics& dpSse16s = total ? dpSse16Seed : dpSse16uSeed; - - // 58. 16-bit SSE seed-extend DPs tried - itoa10<uint64_t>(dpSse16s.dp, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 59. 16-bit SSE seed-extend DPs saturated - itoa10<uint64_t>(dpSse16s.dpsat, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 60. 16-bit SSE seed-extend DPs failed - itoa10<uint64_t>(dpSse16s.dpfail, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 61. 16-bit SSE seed-extend DPs succeeded - itoa10<uint64_t>(dpSse16s.dpsucc, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 62. 16-bit SSE seed-extend DP columns completed - itoa10<uint64_t>(dpSse16s.col, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 63. 16-bit SSE seed-extend DP cells completed - itoa10<uint64_t>(dpSse16s.cell, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 64. 16-bit SSE seed-extend DP inner loop iters completed - itoa10<uint64_t>(dpSse16s.inner, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 65. 16-bit SSE seed-extend DP fixup loop iters completed - itoa10<uint64_t>(dpSse16s.fixup, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 66. 16-bit SSE seed-extend DP gather, cells with potential solutions - itoa10<uint64_t>(dpSse16s.gathsol, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 67. 16-bit SSE seed-extend DP backtrace attempts - itoa10<uint64_t>(dpSse16s.bt, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 68. 16-bit SSE seed-extend DP failed backtrace attempts - itoa10<uint64_t>(dpSse16s.btfail, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 69. 16-bit SSE seed-extend DP succesful backtrace attempts - itoa10<uint64_t>(dpSse16s.btsucc, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 70. 16-bit SSE seed-extend DP backtrace cells - itoa10<uint64_t>(dpSse16s.btcell, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 71. 16-bit SSE seed-extend DP core-diag rejections - itoa10<uint64_t>(dpSse16s.corerej, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 72. 16-bit SSE seed-extend DP N rejections - itoa10<uint64_t>(dpSse16s.nrej, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - - const SSEMetrics& dpSse8s = total ? dpSse8Seed : dpSse8uSeed; - - // 73. 8-bit SSE seed-extend DPs tried - itoa10<uint64_t>(dpSse8s.dp, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 74. 8-bit SSE seed-extend DPs saturated - itoa10<uint64_t>(dpSse8s.dpsat, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 75. 8-bit SSE seed-extend DPs failed - itoa10<uint64_t>(dpSse8s.dpfail, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 76. 8-bit SSE seed-extend DPs succeeded - itoa10<uint64_t>(dpSse8s.dpsucc, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 77. 8-bit SSE seed-extend DP columns completed - itoa10<uint64_t>(dpSse8s.col, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 78. 8-bit SSE seed-extend DP cells completed - itoa10<uint64_t>(dpSse8s.cell, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 79. 8-bit SSE seed-extend DP inner loop iters completed - itoa10<uint64_t>(dpSse8s.inner, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 80. 8-bit SSE seed-extend DP fixup loop iters completed - itoa10<uint64_t>(dpSse8s.fixup, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 81. 16-bit SSE seed-extend DP gather, cells with potential solutions - itoa10<uint64_t>(dpSse8s.gathsol, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 82. 16-bit SSE seed-extend DP backtrace attempts - itoa10<uint64_t>(dpSse8s.bt, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 83. 16-bit SSE seed-extend DP failed backtrace attempts - itoa10<uint64_t>(dpSse8s.btfail, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 84. 16-bit SSE seed-extend DP succesful backtrace attempts - itoa10<uint64_t>(dpSse8s.btsucc, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 85. 16-bit SSE seed-extend DP backtrace cells - itoa10<uint64_t>(dpSse8s.btcell, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 86. 16-bit SSE seed-extend DP core-diag rejections - itoa10<uint64_t>(dpSse8s.corerej, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 87. 16-bit SSE seed-extend DP N rejections - itoa10<uint64_t>(dpSse8s.nrej, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - - const SSEMetrics& dpSse16m = total ? dpSse16Mate : dpSse16uMate; - - // 88. 16-bit SSE mate-finding DPs tried - itoa10<uint64_t>(dpSse16m.dp, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 89. 16-bit SSE mate-finding DPs saturated - itoa10<uint64_t>(dpSse16m.dpsat, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 90. 16-bit SSE mate-finding DPs failed - itoa10<uint64_t>(dpSse16m.dpfail, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 91. 16-bit SSE mate-finding DPs succeeded - itoa10<uint64_t>(dpSse16m.dpsucc, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 92. 16-bit SSE mate-finding DP columns completed - itoa10<uint64_t>(dpSse16m.col, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 93. 16-bit SSE mate-finding DP cells completed - itoa10<uint64_t>(dpSse16m.cell, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 94. 16-bit SSE mate-finding DP inner loop iters completed - itoa10<uint64_t>(dpSse16m.inner, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 95. 16-bit SSE mate-finding DP fixup loop iters completed - itoa10<uint64_t>(dpSse16m.fixup, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 96. 16-bit SSE mate-finding DP gather, cells with potential solutions - itoa10<uint64_t>(dpSse16m.gathsol, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 97. 16-bit SSE mate-finding DP backtrace attempts - itoa10<uint64_t>(dpSse16m.bt, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 98. 16-bit SSE mate-finding DP failed backtrace attempts - itoa10<uint64_t>(dpSse16m.btfail, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 99. 16-bit SSE mate-finding DP succesful backtrace attempts - itoa10<uint64_t>(dpSse16m.btsucc, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 100. 16-bit SSE mate-finding DP backtrace cells - itoa10<uint64_t>(dpSse16m.btcell, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 101. 16-bit SSE mate-finding DP core-diag rejections - itoa10<uint64_t>(dpSse16m.corerej, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 102. 16-bit SSE mate-finding DP N rejections - itoa10<uint64_t>(dpSse16m.nrej, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - - const SSEMetrics& dpSse8m = total ? dpSse8Mate : dpSse8uMate; - - // 103. 8-bit SSE mate-finding DPs tried - itoa10<uint64_t>(dpSse8m.dp, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 104. 8-bit SSE mate-finding DPs saturated - itoa10<uint64_t>(dpSse8m.dpsat, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 105. 8-bit SSE mate-finding DPs failed - itoa10<uint64_t>(dpSse8m.dpfail, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 106. 8-bit SSE mate-finding DPs succeeded - itoa10<uint64_t>(dpSse8m.dpsucc, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 107. 8-bit SSE mate-finding DP columns completed - itoa10<uint64_t>(dpSse8m.col, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 108. 8-bit SSE mate-finding DP cells completed - itoa10<uint64_t>(dpSse8m.cell, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 109. 8-bit SSE mate-finding DP inner loop iters completed - itoa10<uint64_t>(dpSse8m.inner, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 110. 8-bit SSE mate-finding DP fixup loop iters completed - itoa10<uint64_t>(dpSse8m.fixup, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 111. 16-bit SSE mate-finding DP gather, cells with potential solutions - itoa10<uint64_t>(dpSse8m.gathsol, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 112. 16-bit SSE mate-finding DP backtrace attempts - itoa10<uint64_t>(dpSse8m.bt, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 113. 16-bit SSE mate-finding DP failed backtrace attempts - itoa10<uint64_t>(dpSse8m.btfail, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 114. 16-bit SSE mate-finding DP succesful backtrace attempts - itoa10<uint64_t>(dpSse8m.btsucc, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 115. 16-bit SSE mate-finding DP backtrace cells - itoa10<uint64_t>(dpSse8m.btcell, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 116. 16-bit SSE mate-finding DP core rejections - itoa10<uint64_t>(dpSse8m.corerej, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 117. 16-bit SSE mate-finding N rejections - itoa10<uint64_t>(dpSse8m.nrej, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - - // 118. Backtrace candidates filtered due to starting cell - itoa10<uint64_t>(total ? nbtfiltst : nbtfiltst_u, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 119. Backtrace candidates filtered due to low score - itoa10<uint64_t>(total ? nbtfiltsc : nbtfiltsc_u, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 120. Backtrace candidates filtered due to domination - itoa10<uint64_t>(total ? nbtfiltdo : nbtfiltdo_u, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 1. Current time in secs + itoa10<time_t>(curtime, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + + const OuterLoopMetrics& ol = total ? olm : olmu; + + // 2. Reads + itoa10<uint64_t>(ol.reads, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 3. Bases + itoa10<uint64_t>(ol.bases, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 4. Same-read reads + itoa10<uint64_t>(ol.srreads, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 5. Same-read bases + itoa10<uint64_t>(ol.srbases, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 6. Unfiltered reads + itoa10<uint64_t>(ol.ureads, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 7. Unfiltered bases + itoa10<uint64_t>(ol.ubases, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + + const ReportingMetrics& rp = total ? rpm : rpmu; + + // 8. Paired reads + itoa10<uint64_t>(rp.npaired, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 9. Unpaired reads + itoa10<uint64_t>(rp.nunpaired, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 10. Pairs with unique concordant alignments + itoa10<uint64_t>(rp.nconcord_uni, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 11. Pairs with repetitive concordant alignments + itoa10<uint64_t>(rp.nconcord_rep, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 12. Pairs with 0 concordant alignments + itoa10<uint64_t>(rp.nconcord_0, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 13. Pairs with 1 discordant alignment + itoa10<uint64_t>(rp.ndiscord, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 14. Mates from unaligned pairs that align uniquely + itoa10<uint64_t>(rp.nunp_0_uni, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 15. Mates from unaligned pairs that align repetitively + itoa10<uint64_t>(rp.nunp_0_rep, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 16. Mates from unaligned pairs that fail to align + itoa10<uint64_t>(rp.nunp_0_0, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 17. Mates from repetitive pairs that align uniquely + itoa10<uint64_t>(rp.nunp_rep_uni, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 18. Mates from repetitive pairs that align repetitively + itoa10<uint64_t>(rp.nunp_rep_rep, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 19. Mates from repetitive pairs that fail to align + itoa10<uint64_t>(rp.nunp_rep_0, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 20. Unpaired reads that align uniquely + itoa10<uint64_t>(rp.nunp_uni, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 21. Unpaired reads that align repetitively + itoa10<uint64_t>(rp.nunp_rep, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 22. Unpaired reads that fail to align + itoa10<uint64_t>(rp.nunp_0, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + + const SeedSearchMetrics& sd = total ? sdm : sdmu; + + // 23. Seed searches + itoa10<uint64_t>(sd.seedsearch, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 24. Seed ranges found + itoa10<uint64_t>(sd.nrange, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 25. Seed elements found + itoa10<uint64_t>(sd.nelt, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 26. Hits in 'current' cache + itoa10<uint64_t>(sd.intrahit, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 27. Hits in 'local' cache + itoa10<uint64_t>(sd.interhit, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 28. Out of memory + itoa10<uint64_t>(sd.ooms, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 29. Burrows-Wheeler ops in aligner + itoa10<uint64_t>(sd.bwops, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 30. Burrows-Wheeler branches (edits) in aligner + itoa10<uint64_t>(sd.bweds, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + + const WalkMetrics& wl = total ? wlm : wlmu; + + // 31. Burrows-Wheeler ops in resolver + itoa10<uint64_t>(wl.bwops, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 32. Burrows-Wheeler branches in resolver + itoa10<uint64_t>(wl.branches, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 33. Burrows-Wheeler offset resolutions + itoa10<uint64_t>(wl.resolves, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 34. Offset reports + itoa10<uint64_t>(wl.reports, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + + // 35. Redundant seed hit + itoa10<uint64_t>(total ? swmSeed.rshit : swmuSeed.rshit, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + + // 36. # times the best (out of fw/rc) minimum # edits was 0 + itoa10<uint64_t>(total ? sdm.bestmin0 : sdmu.bestmin0, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 37. # times the best (out of fw/rc) minimum # edits was 1 + itoa10<uint64_t>(total ? sdm.bestmin1 : sdmu.bestmin1, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 38. # times the best (out of fw/rc) minimum # edits was 2 + itoa10<uint64_t>(total ? sdm.bestmin2 : sdmu.bestmin2, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + + // 39. Exact aligner attempts + itoa10<uint64_t>(total ? swmSeed.exatts : swmuSeed.exatts, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 40. Exact aligner successes + itoa10<uint64_t>(total ? swmSeed.exsucc : swmuSeed.exsucc, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 41. Exact aligner ranges + itoa10<uint64_t>(total ? swmSeed.exranges : swmuSeed.exranges, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 42. Exact aligner rows + itoa10<uint64_t>(total ? swmSeed.exrows : swmuSeed.exrows, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 43. Exact aligner OOMs + itoa10<uint64_t>(total ? swmSeed.exooms : swmuSeed.exooms, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + + // 44. 1mm aligner attempts + itoa10<uint64_t>(total ? swmSeed.mm1atts : swmuSeed.mm1atts, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 45. 1mm aligner successes + itoa10<uint64_t>(total ? swmSeed.mm1succ : swmuSeed.mm1succ, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 46. 1mm aligner ranges + itoa10<uint64_t>(total ? swmSeed.mm1ranges : swmuSeed.mm1ranges, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 47. 1mm aligner rows + itoa10<uint64_t>(total ? swmSeed.mm1rows : swmuSeed.mm1rows, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 48. 1mm aligner OOMs + itoa10<uint64_t>(total ? swmSeed.mm1ooms : swmuSeed.mm1ooms, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + + // 49 Ungapped aligner success + itoa10<uint64_t>(total ? swmSeed.ungapsucc : swmuSeed.ungapsucc, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 50. Ungapped aligner fail + itoa10<uint64_t>(total ? swmSeed.ungapfail : swmuSeed.ungapfail, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 51. Ungapped aligner no decision + itoa10<uint64_t>(total ? swmSeed.ungapnodec : swmuSeed.ungapnodec, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + + // 52. # seed-extend DPs with < 10 gaps + itoa10<uint64_t>(total ? swmSeed.sws10 : swmuSeed.sws10, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 53. # seed-extend DPs with < 5 gaps + itoa10<uint64_t>(total ? swmSeed.sws5 : swmuSeed.sws5, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 54. # seed-extend DPs with < 3 gaps + itoa10<uint64_t>(total ? swmSeed.sws3 : swmuSeed.sws3, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + + // 55. # seed-extend DPs with < 10 gaps + itoa10<uint64_t>(total ? swmMate.sws10 : swmuMate.sws10, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 56. # seed-extend DPs with < 5 gaps + itoa10<uint64_t>(total ? swmMate.sws5 : swmuMate.sws5, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 57. # seed-extend DPs with < 3 gaps + itoa10<uint64_t>(total ? swmMate.sws3 : swmuMate.sws3, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + + const SSEMetrics& dpSse16s = total ? dpSse16Seed : dpSse16uSeed; + + // 58. 16-bit SSE seed-extend DPs tried + itoa10<uint64_t>(dpSse16s.dp, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 59. 16-bit SSE seed-extend DPs saturated + itoa10<uint64_t>(dpSse16s.dpsat, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 60. 16-bit SSE seed-extend DPs failed + itoa10<uint64_t>(dpSse16s.dpfail, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 61. 16-bit SSE seed-extend DPs succeeded + itoa10<uint64_t>(dpSse16s.dpsucc, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 62. 16-bit SSE seed-extend DP columns completed + itoa10<uint64_t>(dpSse16s.col, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 63. 16-bit SSE seed-extend DP cells completed + itoa10<uint64_t>(dpSse16s.cell, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 64. 16-bit SSE seed-extend DP inner loop iters completed + itoa10<uint64_t>(dpSse16s.inner, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 65. 16-bit SSE seed-extend DP fixup loop iters completed + itoa10<uint64_t>(dpSse16s.fixup, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 66. 16-bit SSE seed-extend DP gather, cells with potential solutions + itoa10<uint64_t>(dpSse16s.gathsol, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 67. 16-bit SSE seed-extend DP backtrace attempts + itoa10<uint64_t>(dpSse16s.bt, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 68. 16-bit SSE seed-extend DP failed backtrace attempts + itoa10<uint64_t>(dpSse16s.btfail, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 69. 16-bit SSE seed-extend DP succesful backtrace attempts + itoa10<uint64_t>(dpSse16s.btsucc, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 70. 16-bit SSE seed-extend DP backtrace cells + itoa10<uint64_t>(dpSse16s.btcell, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 71. 16-bit SSE seed-extend DP core-diag rejections + itoa10<uint64_t>(dpSse16s.corerej, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 72. 16-bit SSE seed-extend DP N rejections + itoa10<uint64_t>(dpSse16s.nrej, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + + const SSEMetrics& dpSse8s = total ? dpSse8Seed : dpSse8uSeed; + + // 73. 8-bit SSE seed-extend DPs tried + itoa10<uint64_t>(dpSse8s.dp, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 74. 8-bit SSE seed-extend DPs saturated + itoa10<uint64_t>(dpSse8s.dpsat, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 75. 8-bit SSE seed-extend DPs failed + itoa10<uint64_t>(dpSse8s.dpfail, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 76. 8-bit SSE seed-extend DPs succeeded + itoa10<uint64_t>(dpSse8s.dpsucc, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 77. 8-bit SSE seed-extend DP columns completed + itoa10<uint64_t>(dpSse8s.col, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 78. 8-bit SSE seed-extend DP cells completed + itoa10<uint64_t>(dpSse8s.cell, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 79. 8-bit SSE seed-extend DP inner loop iters completed + itoa10<uint64_t>(dpSse8s.inner, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 80. 8-bit SSE seed-extend DP fixup loop iters completed + itoa10<uint64_t>(dpSse8s.fixup, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 81. 16-bit SSE seed-extend DP gather, cells with potential solutions + itoa10<uint64_t>(dpSse8s.gathsol, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 82. 16-bit SSE seed-extend DP backtrace attempts + itoa10<uint64_t>(dpSse8s.bt, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 83. 16-bit SSE seed-extend DP failed backtrace attempts + itoa10<uint64_t>(dpSse8s.btfail, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 84. 16-bit SSE seed-extend DP succesful backtrace attempts + itoa10<uint64_t>(dpSse8s.btsucc, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 85. 16-bit SSE seed-extend DP backtrace cells + itoa10<uint64_t>(dpSse8s.btcell, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 86. 16-bit SSE seed-extend DP core-diag rejections + itoa10<uint64_t>(dpSse8s.corerej, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 87. 16-bit SSE seed-extend DP N rejections + itoa10<uint64_t>(dpSse8s.nrej, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + + const SSEMetrics& dpSse16m = total ? dpSse16Mate : dpSse16uMate; + + // 88. 16-bit SSE mate-finding DPs tried + itoa10<uint64_t>(dpSse16m.dp, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 89. 16-bit SSE mate-finding DPs saturated + itoa10<uint64_t>(dpSse16m.dpsat, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 90. 16-bit SSE mate-finding DPs failed + itoa10<uint64_t>(dpSse16m.dpfail, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 91. 16-bit SSE mate-finding DPs succeeded + itoa10<uint64_t>(dpSse16m.dpsucc, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 92. 16-bit SSE mate-finding DP columns completed + itoa10<uint64_t>(dpSse16m.col, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 93. 16-bit SSE mate-finding DP cells completed + itoa10<uint64_t>(dpSse16m.cell, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 94. 16-bit SSE mate-finding DP inner loop iters completed + itoa10<uint64_t>(dpSse16m.inner, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 95. 16-bit SSE mate-finding DP fixup loop iters completed + itoa10<uint64_t>(dpSse16m.fixup, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 96. 16-bit SSE mate-finding DP gather, cells with potential solutions + itoa10<uint64_t>(dpSse16m.gathsol, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 97. 16-bit SSE mate-finding DP backtrace attempts + itoa10<uint64_t>(dpSse16m.bt, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 98. 16-bit SSE mate-finding DP failed backtrace attempts + itoa10<uint64_t>(dpSse16m.btfail, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 99. 16-bit SSE mate-finding DP succesful backtrace attempts + itoa10<uint64_t>(dpSse16m.btsucc, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 100. 16-bit SSE mate-finding DP backtrace cells + itoa10<uint64_t>(dpSse16m.btcell, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 101. 16-bit SSE mate-finding DP core-diag rejections + itoa10<uint64_t>(dpSse16m.corerej, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 102. 16-bit SSE mate-finding DP N rejections + itoa10<uint64_t>(dpSse16m.nrej, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + + const SSEMetrics& dpSse8m = total ? dpSse8Mate : dpSse8uMate; + + // 103. 8-bit SSE mate-finding DPs tried + itoa10<uint64_t>(dpSse8m.dp, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 104. 8-bit SSE mate-finding DPs saturated + itoa10<uint64_t>(dpSse8m.dpsat, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 105. 8-bit SSE mate-finding DPs failed + itoa10<uint64_t>(dpSse8m.dpfail, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 106. 8-bit SSE mate-finding DPs succeeded + itoa10<uint64_t>(dpSse8m.dpsucc, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 107. 8-bit SSE mate-finding DP columns completed + itoa10<uint64_t>(dpSse8m.col, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 108. 8-bit SSE mate-finding DP cells completed + itoa10<uint64_t>(dpSse8m.cell, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 109. 8-bit SSE mate-finding DP inner loop iters completed + itoa10<uint64_t>(dpSse8m.inner, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 110. 8-bit SSE mate-finding DP fixup loop iters completed + itoa10<uint64_t>(dpSse8m.fixup, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 111. 16-bit SSE mate-finding DP gather, cells with potential solutions + itoa10<uint64_t>(dpSse8m.gathsol, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 112. 16-bit SSE mate-finding DP backtrace attempts + itoa10<uint64_t>(dpSse8m.bt, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 113. 16-bit SSE mate-finding DP failed backtrace attempts + itoa10<uint64_t>(dpSse8m.btfail, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 114. 16-bit SSE mate-finding DP succesful backtrace attempts + itoa10<uint64_t>(dpSse8m.btsucc, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 115. 16-bit SSE mate-finding DP backtrace cells + itoa10<uint64_t>(dpSse8m.btcell, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 116. 16-bit SSE mate-finding DP core rejections + itoa10<uint64_t>(dpSse8m.corerej, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 117. 16-bit SSE mate-finding N rejections + itoa10<uint64_t>(dpSse8m.nrej, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + + // 118. Backtrace candidates filtered due to starting cell + itoa10<uint64_t>(total ? nbtfiltst : nbtfiltst_u, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 119. Backtrace candidates filtered due to low score + itoa10<uint64_t>(total ? nbtfiltsc : nbtfiltsc_u, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 120. Backtrace candidates filtered due to domination + itoa10<uint64_t>(total ? nbtfiltdo : nbtfiltdo_u, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } #ifdef USE_MEM_TALLY - // 121. Overall memory peak - itoa10<size_t>(gMemTally.peak() >> 20, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 122. Uncategorized memory peak - itoa10<size_t>(gMemTally.peak(0) >> 20, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 123. Ebwt memory peak - itoa10<size_t>(gMemTally.peak(EBWT_CAT) >> 20, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 124. Cache memory peak - itoa10<size_t>(gMemTally.peak(CA_CAT) >> 20, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 125. Resolver memory peak - itoa10<size_t>(gMemTally.peak(GW_CAT) >> 20, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 126. Seed aligner memory peak - itoa10<size_t>(gMemTally.peak(AL_CAT) >> 20, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 127. Dynamic programming aligner memory peak - itoa10<size_t>(gMemTally.peak(DP_CAT) >> 20, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 128. Miscellaneous memory peak - itoa10<size_t>(gMemTally.peak(MISC_CAT) >> 20, buf); - if(metricsStderr) stderrSs << buf << '\t'; - if(o != NULL) { o->writeChars(buf); o->write('\t'); } - // 129. Debug memory peak - itoa10<size_t>(gMemTally.peak(DEBUG_CAT) >> 20, buf); - if(metricsStderr) stderrSs << buf; - if(o != NULL) { o->writeChars(buf); } + // 121. Overall memory peak + itoa10<size_t>(gMemTally.peak() >> 20, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 122. Uncategorized memory peak + itoa10<size_t>(gMemTally.peak(0) >> 20, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 123. Ebwt memory peak + itoa10<size_t>(gMemTally.peak(EBWT_CAT) >> 20, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 124. Cache memory peak + itoa10<size_t>(gMemTally.peak(CA_CAT) >> 20, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 125. Resolver memory peak + itoa10<size_t>(gMemTally.peak(GW_CAT) >> 20, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 126. Seed aligner memory peak + itoa10<size_t>(gMemTally.peak(AL_CAT) >> 20, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 127. Dynamic programming aligner memory peak + itoa10<size_t>(gMemTally.peak(DP_CAT) >> 20, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 128. Miscellaneous memory peak + itoa10<size_t>(gMemTally.peak(MISC_CAT) >> 20, buf); + if(metricsStderr) stderrSs << buf << '\t'; + if(o != NULL) { o->writeChars(buf); o->write('\t'); } + // 129. Debug memory peak + itoa10<size_t>(gMemTally.peak(DEBUG_CAT) >> 20, buf); + if(metricsStderr) stderrSs << buf; + if(o != NULL) { o->writeChars(buf); } #endif - if(o != NULL) { o->write('\n'); } - if(metricsStderr) cerr << stderrSs.str().c_str() << endl; - if(!total) mergeIncrementals(); - } + if(o != NULL) { o->write('\n'); } + if(metricsStderr) cerr << stderrSs.str().c_str() << endl; + if(!total) mergeIncrementals(); + } void mergeIncrementals() { olm.merge(olmu); @@ -2900,45 +2910,45 @@ static void setupMinScores( } } -#define MERGE_METRICS(met) { \ - msink.mergeMetrics(rpm); \ - met.merge( \ - &olm, \ - &sdm, \ - &wlm, \ - &swmSeed, \ - &swmMate, \ - &rpm, \ - &sseU8ExtendMet, \ - &sseU8MateMet, \ - &sseI16ExtendMet, \ - &sseI16MateMet, \ - nbtfiltst, \ - nbtfiltsc, \ - nbtfiltdo); \ - olm.reset(); \ - sdm.reset(); \ - wlm.reset(); \ - swmSeed.reset(); \ - swmMate.reset(); \ - rpm.reset(); \ - sseU8ExtendMet.reset(); \ - sseU8MateMet.reset(); \ - sseI16ExtendMet.reset(); \ - sseI16MateMet.reset(); \ -} +#define MERGE_METRICS(met) { \ + msink.mergeMetrics(rpm); \ + met.merge( \ + &olm, \ + &sdm, \ + &wlm, \ + &swmSeed, \ + &swmMate, \ + &rpm, \ + &sseU8ExtendMet, \ + &sseU8MateMet, \ + &sseI16ExtendMet, \ + &sseI16MateMet, \ + nbtfiltst, \ + nbtfiltsc, \ + nbtfiltdo); \ + olm.reset(); \ + sdm.reset(); \ + wlm.reset(); \ + swmSeed.reset(); \ + swmMate.reset(); \ + rpm.reset(); \ + sseU8ExtendMet.reset(); \ + sseU8MateMet.reset(); \ + sseI16ExtendMet.reset(); \ + sseI16MateMet.reset(); \ + } -#define MERGE_SW(x) { \ - x.merge( \ - sseU8ExtendMet, \ - sseU8MateMet, \ - sseI16ExtendMet, \ - sseI16MateMet, \ - nbtfiltst, \ - nbtfiltsc, \ - nbtfiltdo); \ - x.resetCounters(); \ -} +#define MERGE_SW(x) { \ + x.merge( \ + sseU8ExtendMet, \ + sseU8MateMet, \ + sseI16ExtendMet, \ + sseI16MateMet, \ + nbtfiltst, \ + nbtfiltsc, \ + nbtfiltdo); \ + x.resetCounters(); \ + } #ifdef PER_THREAD_TIMING /// Based on http://stackoverflow.com/questions/16862620/numa-get-current-node-core @@ -2983,8 +2993,7 @@ static void multiseedSearchWorker(void *vp) { int tid = p->tid; assert(multiseed_ebwtFw != NULL); assert(multiseedMms == 0 || multiseed_ebwtBw != NULL); - PatternComposer& patsrc = *multiseed_patsrc; - PatternParams pp = multiseed_pp; + PatternSourceReadAheadFactory& readahead_factory = *multiseed_readahead_factory; const Ebwt& ebwtFw = *multiseed_ebwtFw; const Ebwt* ebwtBw = multiseed_ebwtBw; const Scoring& sc = *multiseed_sc; @@ -3013,8 +3022,6 @@ static void multiseedSearchWorker(void *vp) { // problems, or generally characterize performance. //const BitPairReference& refs = *multiseed_refs; - unique_ptr<PatternSourcePerThreadFactory> patsrcFact(createPatsrcFactory(patsrc, pp, tid)); - unique_ptr<PatternSourcePerThread> ps(patsrcFact->create()); // Thread-local cache for seed alignments PtrWrap<AlignmentCache> scLocal; @@ -3133,7 +3140,14 @@ static void multiseedSearchWorker(void *vp) { int mergeival = 16; bool done = false; while(!done) { - pair<bool, bool> ret = ps->nextReadPair(); + PatternSourceReadAhead psrah(readahead_factory); + PatternSourcePerThread* const ps = psrah.ptr(); + bool firstPS = true; + do { + pair<bool, bool> ret = firstPS ? + psrah.readResult() : // nextReadPair was already called in the psrah constructor + ps->nextReadPair(); + firstPS = false; bool success = ret.first; done = ret.second; if(!success && done) { @@ -3309,7 +3323,7 @@ static void multiseedSearchWorker(void *vp) { exhaustive[0] = exhaustive[1] = false; size_t matemap[2] = { 0, 1 }; bool pairPostFilt = filt[0] && filt[1]; - if(pairPostFilt) { + if(false && pairPostFilt) { rnd.init(ps->read_a().seed ^ ps->read_b().seed); } else { rnd.init(ps->read_a().seed); @@ -3379,241 +3393,517 @@ static void multiseedSearchWorker(void *vp) { bool done[2] = { !filt[0], !filt[1] }; size_t nelt[2] = {0, 0}; - // Find end-to-end exact alignments for each read - if(doExactUpFront) { - for(size_t matei = 0; matei < (paired ? 2:1); matei++) { - size_t mate = matemap[matei]; - if(!filt[mate] || done[mate] || msinkwrap.state().doneWithMate(mate == 0)) { - continue; - } - swmSeed.exatts++; - nelt[mate] = al.exactSweep( - ebwtFw, // index - *rds[mate], // read - sc, // scoring scheme - nofw[mate], // nofw? - norc[mate], // norc? - 2, // max # edits we care about - minedfw[mate], // minimum # edits for fw mate - minedrc[mate], // minimum # edits for rc mate - true, // report 0mm hits - shs[mate], // put end-to-end results here - sdm); // metrics - size_t bestmin = min(minedfw[mate], minedrc[mate]); - if(bestmin == 0) { - sdm.bestmin0++; - } else if(bestmin == 1) { - sdm.bestmin1++; - } else { - assert_eq(2, bestmin); - sdm.bestmin2++; - } + // Find end-to-end exact alignments for each read + if(doExactUpFront) { + for(size_t matei = 0; matei < (paired ? 2:1); matei++) { + size_t mate = matemap[matei]; + if(!filt[mate] || done[mate] || msinkwrap.state().doneWithMate(mate == 0)) { + continue; } - matemap[0] = 0; matemap[1] = 1; - if(nelt[0] > 0 && nelt[1] > 0 && nelt[0] > nelt[1]) { - // Do the mate with fewer exact hits first - // TODO: Consider mates & orientations separately? - matemap[0] = 1; matemap[1] = 0; + swmSeed.exatts++; + nelt[mate] = al.exactSweep( + ebwtFw, // index + *rds[mate], // read + sc, // scoring scheme + nofw[mate], // nofw? + norc[mate], // norc? + 2, // max # edits we care about + minedfw[mate], // minimum # edits for fw mate + minedrc[mate], // minimum # edits for rc mate + true, // report 0mm hits + shs[mate], // put end-to-end results here + sdm); // metrics + size_t bestmin = min(minedfw[mate], minedrc[mate]); + if(bestmin == 0) { + sdm.bestmin0++; + } else if(bestmin == 1) { + sdm.bestmin1++; + } else { + assert_eq(2, bestmin); + sdm.bestmin2++; } - for(size_t matei = 0; matei < (seedSumm ? 0:2); matei++) { - size_t mate = matemap[matei]; - if(nelt[mate] == 0 || nelt[mate] > eePeEeltLimit) { - shs[mate].clearExactE2eHits(); - continue; - } + } + matemap[0] = 0; matemap[1] = 1; + if(nelt[0] > 0 && nelt[1] > 0 && nelt[0] > nelt[1]) { + // Do the mate with fewer exact hits first + // TODO: Consider mates & orientations separately? + matemap[0] = 1; matemap[1] = 0; + } + for(size_t matei = 0; matei < (seedSumm ? 0:2); matei++) { + size_t mate = matemap[matei]; + if(nelt[mate] == 0 || nelt[mate] > eePeEeltLimit) { + shs[mate].clearExactE2eHits(); + continue; + } + if(msinkwrap.state().doneWithMate(mate == 0)) { + shs[mate].clearExactE2eHits(); + done[mate] = true; + continue; + } + assert(filt[mate]); + assert(matei == 0 || paired); + assert(!msinkwrap.maxed()); + assert(msinkwrap.repOk()); + int ret = 0; + if(paired) { + // Paired-end dynamic programming driver + ret = sd.extendSeedsPaired( + *rds[mate], // mate to align as anchor + *rds[mate ^ 1], // mate to align as opp. + mate == 0, // anchor is mate 1? + !filt[mate ^ 1],// opposite mate filtered out? + shs[mate], // seed hits for anchor + ebwtFw, // bowtie index + ebwtBw, // rev bowtie index + ref, // packed reference strings + sw, // dyn prog aligner, anchor + osw, // dyn prog aligner, opposite + sc, // scoring scheme + pepol, // paired-end policy + -1, // # mms allowed in a seed + 0, // length of a seed + 0, // interval between seeds + minsc[mate], // min score for anchor + minsc[mate^1], // min score for opp. + nceil[mate], // N ceil for anchor + nceil[mate^1], // N ceil for opp. + nofw[mate], // don't align forward read + norc[mate], // don't align revcomp read + maxhalf, // max width on one DP side + doUngapped, // do ungapped alignment + mxIter[mate], // max extend loop iters + mxUg[mate], // max # ungapped extends + mxDp[mate], // max # DPs + streak[mate], // stop after streak of this many end-to-end fails + streak[mate], // stop after streak of this many ungap fails + streak[mate], // stop after streak of this many dp fails + mtStreak[mate], // max mate fails per seed range + doExtend, // extend seed hits + enable8, // use 8-bit SSE where possible + cminlen, // checkpoint if read is longer + cpow2, // checkpointer interval, log2 + doTri, // triangular mini-fills? + tighten, // -M score tightening mode + ca, // seed alignment cache + rnd, // pseudo-random source + wlm, // group walk left metrics + swmSeed, // DP metrics, seed extend + swmMate, // DP metrics, mate finding + prm, // per-read metrics + &msinkwrap, // for organizing hits + true, // seek mate immediately + true, // report hits once found + gReportDiscordant,// look for discordant alns? + gReportMixed, // look for unpaired alns? + exhaustive[mate]); + // Might be done, but just with this mate + } else { + // Unpaired dynamic programming driver + ret = sd.extendSeeds( + *rds[mate], // read + mate == 0, // mate #1? + shs[mate], // seed hits + ebwtFw, // bowtie index + ebwtBw, // rev bowtie index + ref, // packed reference strings + sw, // dynamic prog aligner + sc, // scoring scheme + -1, // # mms allowed in a seed + 0, // length of a seed + 0, // interval between seeds + minsc[mate], // minimum score for valid + nceil[mate], // N ceil for anchor + maxhalf, // max width on one DP side + doUngapped, // do ungapped alignment + mxIter[mate], // max extend loop iters + mxUg[mate], // max # ungapped extends + mxDp[mate], // max # DPs + streak[mate], // stop after streak of this many end-to-end fails + streak[mate], // stop after streak of this many ungap fails + doExtend, // extend seed hits + enable8, // use 8-bit SSE where possible + cminlen, // checkpoint if read is longer + cpow2, // checkpointer interval, log2 + doTri, // triangular mini-fills + tighten, // -M score tightening mode + ca, // seed alignment cache + rnd, // pseudo-random source + wlm, // group walk left metrics + swmSeed, // DP metrics, seed extend + prm, // per-read metrics + &msinkwrap, // for organizing hits + true, // report hits once found + exhaustive[mate]); + } + assert_gt(ret, 0); + MERGE_SW(sw); + MERGE_SW(osw); + // Clear out the exact hits so that we don't try to + // extend them again later! + shs[mate].clearExactE2eHits(); + if(ret == EXTEND_EXHAUSTED_CANDIDATES) { + // Not done yet + } else if(ret == EXTEND_POLICY_FULFILLED) { + // Policy is satisfied for this mate at least if(msinkwrap.state().doneWithMate(mate == 0)) { - shs[mate].clearExactE2eHits(); done[mate] = true; - continue; } - assert(filt[mate]); - assert(matei == 0 || paired); - assert(!msinkwrap.maxed()); - assert(msinkwrap.repOk()); - int ret = 0; - if(paired) { - // Paired-end dynamic programming driver - ret = sd.extendSeedsPaired( - *rds[mate], // mate to align as anchor - *rds[mate ^ 1], // mate to align as opp. - mate == 0, // anchor is mate 1? - !filt[mate ^ 1],// opposite mate filtered out? - shs[mate], // seed hits for anchor - ebwtFw, // bowtie index - ebwtBw, // rev bowtie index - ref, // packed reference strings - sw, // dyn prog aligner, anchor - osw, // dyn prog aligner, opposite - sc, // scoring scheme - pepol, // paired-end policy - -1, // # mms allowed in a seed - 0, // length of a seed - 0, // interval between seeds - minsc[mate], // min score for anchor - minsc[mate^1], // min score for opp. - nceil[mate], // N ceil for anchor - nceil[mate^1], // N ceil for opp. - nofw[mate], // don't align forward read - norc[mate], // don't align revcomp read - maxhalf, // max width on one DP side - doUngapped, // do ungapped alignment - mxIter[mate], // max extend loop iters - mxUg[mate], // max # ungapped extends - mxDp[mate], // max # DPs - streak[mate], // stop after streak of this many end-to-end fails - streak[mate], // stop after streak of this many ungap fails - streak[mate], // stop after streak of this many dp fails - mtStreak[mate], // max mate fails per seed range - doExtend, // extend seed hits - enable8, // use 8-bit SSE where possible - cminlen, // checkpoint if read is longer - cpow2, // checkpointer interval, log2 - doTri, // triangular mini-fills? - tighten, // -M score tightening mode - ca, // seed alignment cache - rnd, // pseudo-random source - wlm, // group walk left metrics - swmSeed, // DP metrics, seed extend - swmMate, // DP metrics, mate finding - prm, // per-read metrics - &msinkwrap, // for organizing hits - true, // seek mate immediately - true, // report hits once found - gReportDiscordant,// look for discordant alns? - gReportMixed, // look for unpaired alns? - exhaustive[mate]); - // Might be done, but just with this mate - } else { - // Unpaired dynamic programming driver - ret = sd.extendSeeds( - *rds[mate], // read - mate == 0, // mate #1? - shs[mate], // seed hits - ebwtFw, // bowtie index - ebwtBw, // rev bowtie index - ref, // packed reference strings - sw, // dynamic prog aligner - sc, // scoring scheme - -1, // # mms allowed in a seed - 0, // length of a seed - 0, // interval between seeds - minsc[mate], // minimum score for valid - nceil[mate], // N ceil for anchor - maxhalf, // max width on one DP side - doUngapped, // do ungapped alignment - mxIter[mate], // max extend loop iters - mxUg[mate], // max # ungapped extends - mxDp[mate], // max # DPs - streak[mate], // stop after streak of this many end-to-end fails - streak[mate], // stop after streak of this many ungap fails - doExtend, // extend seed hits - enable8, // use 8-bit SSE where possible - cminlen, // checkpoint if read is longer - cpow2, // checkpointer interval, log2 - doTri, // triangular mini-fills - tighten, // -M score tightening mode - ca, // seed alignment cache - rnd, // pseudo-random source - wlm, // group walk left metrics - swmSeed, // DP metrics, seed extend - prm, // per-read metrics - &msinkwrap, // for organizing hits - true, // report hits once found - exhaustive[mate]); + if(msinkwrap.state().doneWithMate(mate == 1)) { + done[mate^1] = true; } - assert_gt(ret, 0); - MERGE_SW(sw); - MERGE_SW(osw); - // Clear out the exact hits so that we don't try to - // extend them again later! - shs[mate].clearExactE2eHits(); - if(ret == EXTEND_EXHAUSTED_CANDIDATES) { - // Not done yet - } else if(ret == EXTEND_POLICY_FULFILLED) { - // Policy is satisfied for this mate at least - if(msinkwrap.state().doneWithMate(mate == 0)) { - done[mate] = true; - } - if(msinkwrap.state().doneWithMate(mate == 1)) { - done[mate^1] = true; - } - } else if(ret == EXTEND_PERFECT_SCORE) { - // We exhausted this mode at least - done[mate] = true; - } else if(ret == EXTEND_EXCEEDED_HARD_LIMIT) { - // We exceeded a per-read limit + } else if(ret == EXTEND_PERFECT_SCORE) { + // We exhausted this mode at least + done[mate] = true; + } else if(ret == EXTEND_EXCEEDED_HARD_LIMIT) { + // We exceeded a per-read limit + done[mate] = true; + } else if(ret == EXTEND_EXCEEDED_SOFT_LIMIT) { + // Not done yet + } else { + // + cerr << "Bad return value: " << ret << endl; + throw 1; + } + if(!done[mate]) { + TAlScore perfectScore = sc.perfectScore(rdlens[mate]); + if(!done[mate] && minsc[mate] == perfectScore) { done[mate] = true; - } else if(ret == EXTEND_EXCEEDED_SOFT_LIMIT) { - // Not done yet - } else { - // - cerr << "Bad return value: " << ret << endl; - throw 1; - } - if(!done[mate]) { - TAlScore perfectScore = sc.perfectScore(rdlens[mate]); - if(!done[mate] && minsc[mate] == perfectScore) { - done[mate] = true; - } } } } + } - // 1-mismatch - if(do1mmUpFront && !seedSumm) { - for(size_t matei = 0; matei < (paired ? 2:1); matei++) { - size_t mate = matemap[matei]; - if(!filt[mate] || done[mate] || nelt[mate] > eePeEeltLimit) { - // Done with this mate - shs[mate].clear1mmE2eHits(); - nelt[mate] = 0; - continue; - } + // 1-mismatch + if(do1mmUpFront && !seedSumm) { + for(size_t matei = 0; matei < (paired ? 2:1); matei++) { + size_t mate = matemap[matei]; + if(!filt[mate] || done[mate] || nelt[mate] > eePeEeltLimit) { + // Done with this mate + shs[mate].clear1mmE2eHits(); nelt[mate] = 0; - assert(!msinkwrap.maxed()); - assert(msinkwrap.repOk()); - //rnd.init(ROTL(rds[mate]->seed, 10)); - assert(shs[mate].empty()); - assert(shs[mate].repOk(&ca.current())); - bool yfw = minedfw[mate] <= 1 && !nofw[mate]; - bool yrc = minedrc[mate] <= 1 && !norc[mate]; - if(yfw || yrc) { - // Clear out the exact hits - swmSeed.mm1atts++; - al.oneMmSearch( - &ebwtFw, // BWT index - ebwtBw, // BWT' index - *rds[mate], // read - sc, // scoring scheme - minsc[mate], // minimum score - !yfw, // don't align forward read - !yrc, // don't align revcomp read - localAlign, // must be legal local alns? - false, // do exact match - true, // do 1mm - shs[mate], // seed hits (hits installed here) - sdm); // metrics - nelt[mate] = shs[mate].num1mmE2eHits(); - } + continue; } - // Possibly reorder the mates - matemap[0] = 0; matemap[1] = 1; - if(nelt[0] > 0 && nelt[1] > 0 && nelt[0] > nelt[1]) { - // Do the mate with fewer exact hits first - // TODO: Consider mates & orientations separately? - matemap[0] = 1; matemap[1] = 0; + nelt[mate] = 0; + assert(!msinkwrap.maxed()); + assert(msinkwrap.repOk()); + //rnd.init(ROTL(rds[mate]->seed, 10)); + assert(shs[mate].empty()); + assert(shs[mate].repOk(&ca.current())); + bool yfw = minedfw[mate] <= 1 && !nofw[mate]; + bool yrc = minedrc[mate] <= 1 && !norc[mate]; + if(yfw || yrc) { + // Clear out the exact hits + swmSeed.mm1atts++; + al.oneMmSearch( + &ebwtFw, // BWT index + ebwtBw, // BWT' index + *rds[mate], // read + sc, // scoring scheme + minsc[mate], // minimum score + !yfw, // don't align forward read + !yrc, // don't align revcomp read + localAlign, // must be legal local alns? + false, // do exact match + true, // do 1mm + shs[mate], // seed hits (hits installed here) + sdm); // metrics + nelt[mate] = shs[mate].num1mmE2eHits(); } - for(size_t matei = 0; matei < (seedSumm ? 0:2); matei++) { - size_t mate = matemap[matei]; - if(nelt[mate] == 0 || nelt[mate] > eePeEeltLimit) { - continue; - } + } + // Possibly reorder the mates + matemap[0] = 0; matemap[1] = 1; + if(nelt[0] > 0 && nelt[1] > 0 && nelt[0] > nelt[1]) { + // Do the mate with fewer exact hits first + // TODO: Consider mates & orientations separately? + matemap[0] = 1; matemap[1] = 0; + } + for(size_t matei = 0; matei < (seedSumm ? 0:2); matei++) { + size_t mate = matemap[matei]; + if(nelt[mate] == 0 || nelt[mate] > eePeEeltLimit) { + continue; + } + if(msinkwrap.state().doneWithMate(mate == 0)) { + done[mate] = true; + continue; + } + int ret = 0; + if(paired) { + // Paired-end dynamic programming driver + ret = sd.extendSeedsPaired( + *rds[mate], // mate to align as anchor + *rds[mate ^ 1], // mate to align as opp. + mate == 0, // anchor is mate 1? + !filt[mate ^ 1],// opposite mate filtered out? + shs[mate], // seed hits for anchor + ebwtFw, // bowtie index + ebwtBw, // rev bowtie index + ref, // packed reference strings + sw, // dyn prog aligner, anchor + osw, // dyn prog aligner, opposite + sc, // scoring scheme + pepol, // paired-end policy + -1, // # mms allowed in a seed + 0, // length of a seed + 0, // interval between seeds + minsc[mate], // min score for anchor + minsc[mate^1], // min score for opp. + nceil[mate], // N ceil for anchor + nceil[mate^1], // N ceil for opp. + nofw[mate], // don't align forward read + norc[mate], // don't align revcomp read + maxhalf, // max width on one DP side + doUngapped, // do ungapped alignment + mxIter[mate], // max extend loop iters + mxUg[mate], // max # ungapped extends + mxDp[mate], // max # DPs + streak[mate], // stop after streak of this many end-to-end fails + streak[mate], // stop after streak of this many ungap fails + streak[mate], // stop after streak of this many dp fails + mtStreak[mate], // max mate fails per seed range + doExtend, // extend seed hits + enable8, // use 8-bit SSE where possible + cminlen, // checkpoint if read is longer + cpow2, // checkpointer interval, log2 + doTri, // triangular mini-fills? + tighten, // -M score tightening mode + ca, // seed alignment cache + rnd, // pseudo-random source + wlm, // group walk left metrics + swmSeed, // DP metrics, seed extend + swmMate, // DP metrics, mate finding + prm, // per-read metrics + &msinkwrap, // for organizing hits + true, // seek mate immediately + true, // report hits once found + gReportDiscordant,// look for discordant alns? + gReportMixed, // look for unpaired alns? + exhaustive[mate]); + // Might be done, but just with this mate + } else { + // Unpaired dynamic programming driver + ret = sd.extendSeeds( + *rds[mate], // read + mate == 0, // mate #1? + shs[mate], // seed hits + ebwtFw, // bowtie index + ebwtBw, // rev bowtie index + ref, // packed reference strings + sw, // dynamic prog aligner + sc, // scoring scheme + -1, // # mms allowed in a seed + 0, // length of a seed + 0, // interval between seeds + minsc[mate], // minimum score for valid + nceil[mate], // N ceil for anchor + maxhalf, // max width on one DP side + doUngapped, // do ungapped alignment + mxIter[mate], // max extend loop iters + mxUg[mate], // max # ungapped extends + mxDp[mate], // max # DPs + streak[mate], // stop after streak of this many end-to-end fails + streak[mate], // stop after streak of this many ungap fails + doExtend, // extend seed hits + enable8, // use 8-bit SSE where possible + cminlen, // checkpoint if read is longer + cpow2, // checkpointer interval, log2 + doTri, // triangular mini-fills? + tighten, // -M score tightening mode + ca, // seed alignment cache + rnd, // pseudo-random source + wlm, // group walk left metrics + swmSeed, // DP metrics, seed extend + prm, // per-read metrics + &msinkwrap, // for organizing hits + true, // report hits once found + exhaustive[mate]); + } + assert_gt(ret, 0); + MERGE_SW(sw); + MERGE_SW(osw); + // Clear out the 1mm hits so that we don't try to + // extend them again later! + shs[mate].clear1mmE2eHits(); + if(ret == EXTEND_EXHAUSTED_CANDIDATES) { + // Not done yet + } else if(ret == EXTEND_POLICY_FULFILLED) { + // Policy is satisfied for this mate at least if(msinkwrap.state().doneWithMate(mate == 0)) { done[mate] = true; - continue; } + if(msinkwrap.state().doneWithMate(mate == 1)) { + done[mate^1] = true; + } + } else if(ret == EXTEND_PERFECT_SCORE) { + // We exhausted this mode at least + done[mate] = true; + } else if(ret == EXTEND_EXCEEDED_HARD_LIMIT) { + // We exceeded a per-read limit + done[mate] = true; + } else if(ret == EXTEND_EXCEEDED_SOFT_LIMIT) { + // Not done yet + } else { + // + cerr << "Bad return value: " << ret << endl; + throw 1; + } + if(!done[mate]) { + TAlScore perfectScore = sc.perfectScore(rdlens[mate]); + if(!done[mate] && minsc[mate] == perfectScore) { + done[mate] = true; + } + } + } + } + int seedlens[2] = { multiseedLen, multiseedLen }; + nrounds[0] = min<size_t>(nrounds[0], interval[0]); + nrounds[1] = min<size_t>(nrounds[1], interval[1]); + Constraint gc = Constraint::penaltyFuncBased(scoreMin); + size_t seedsTried = 0; + size_t seedsTriedMS[] = {0, 0, 0, 0}; + size_t nUniqueSeeds = 0, nRepeatSeeds = 0, seedHitTot = 0; + size_t nUniqueSeedsMS[] = {0, 0, 0, 0}; + size_t nRepeatSeedsMS[] = {0, 0, 0, 0}; + size_t seedHitTotMS[] = {0, 0, 0, 0}; + for(size_t roundi = 0; roundi < nSeedRounds; roundi++) { + ca.nextRead(); // Clear cache in preparation for new search + shs[0].clearSeeds(); + shs[1].clearSeeds(); + assert(shs[0].empty()); + assert(shs[1].empty()); + assert(shs[0].repOk(&ca.current())); + assert(shs[1].repOk(&ca.current())); + //if(roundi > 0) { + // if(seedlens[0] > 8) seedlens[0]--; + // if(seedlens[1] > 8) seedlens[1]--; + //} + for(size_t matei = 0; matei < (paired ? 2:1); matei++) { + size_t mate = matemap[matei]; + if(done[mate] || msinkwrap.state().doneWithMate(mate == 0)) { + // Done with this mate + done[mate] = true; + continue; + } + if(roundi >= nrounds[mate]) { + // Not doing this round for this mate + continue; + } + // Figure out the seed offset + if(interval[mate] <= (int)roundi) { + // Can't do this round, seeds already packed as + // tight as possible + continue; + } + size_t offset = (interval[mate] * roundi) / nrounds[mate]; + assert(roundi == 0 || offset > 0); + assert(!msinkwrap.maxed()); + assert(msinkwrap.repOk()); + //rnd.init(ROTL(rds[mate]->seed, 10)); + assert(shs[mate].repOk(&ca.current())); + swmSeed.sdatts++; + // Set up seeds + seeds[mate]->clear(); + Seed::mmSeeds( + multiseedMms, // max # mms per seed + seedlens[mate], // length of a multiseed seed + *seeds[mate], // seeds + gc); // global constraint + // Check whether the offset would drive the first seed + // off the end + if(offset > 0 && (*seeds[mate])[0].len + offset > rds[mate]->length()) { + continue; + } + // Instantiate the seeds + std::pair<int, int> instFw, instRc; + std::pair<int, int> inst = al.instantiateSeeds( + *seeds[mate], // search seeds + offset, // offset to begin extracting + interval[mate], // interval between seeds + *rds[mate], // read to align + sc, // scoring scheme + nofw[mate], // don't align forward read + norc[mate], // don't align revcomp read + ca, // holds some seed hits from previous reads + shs[mate], // holds all the seed hits + sdm, // metrics + instFw, + instRc); + assert(shs[mate].repOk(&ca.current())); + if(inst.first + inst.second == 0) { + // No seed hits! Done with this mate. + assert(shs[mate].empty()); + done[mate] = true; + break; + } + seedsTried += (inst.first + inst.second); + seedsTriedMS[mate * 2 + 0] = instFw.first + instFw.second; + seedsTriedMS[mate * 2 + 1] = instRc.first + instRc.second; + // Align seeds + al.searchAllSeeds( + *seeds[mate], // search seeds + &ebwtFw, // BWT index + ebwtBw, // BWT' index + *rds[mate], // read + sc, // scoring scheme + ca, // alignment cache + shs[mate], // store seed hits here + sdm, // metrics + prm); // per-read metrics + assert(shs[mate].repOk(&ca.current())); + if(shs[mate].empty()) { + // No seed alignments! Done with this mate. + done[mate] = true; + break; + } + } + // shs contain what we need to know to update our seed + // summaries for this seeding + for(size_t mate = 0; mate < 2; mate++) { + if(!shs[mate].empty()) { + nUniqueSeeds += shs[mate].numUniqueSeeds(); + nUniqueSeedsMS[mate * 2 + 0] += shs[mate].numUniqueSeedsStrand(true); + nUniqueSeedsMS[mate * 2 + 1] += shs[mate].numUniqueSeedsStrand(false); + nRepeatSeeds += shs[mate].numRepeatSeeds(); + nRepeatSeedsMS[mate * 2 + 0] += shs[mate].numRepeatSeedsStrand(true); + nRepeatSeedsMS[mate * 2 + 1] += shs[mate].numRepeatSeedsStrand(false); + seedHitTot += shs[mate].numElts(); + seedHitTotMS[mate * 2 + 0] += shs[mate].numEltsFw(); + seedHitTotMS[mate * 2 + 1] += shs[mate].numEltsRc(); + } + } + double uniqFactor[2] = { 0.0f, 0.0f }; + for(size_t i = 0; i < 2; i++) { + if(!shs[i].empty()) { + swmSeed.sdsucc++; + uniqFactor[i] = shs[i].uniquenessFactor(); + } + } + // Possibly reorder the mates + matemap[0] = 0; matemap[1] = 1; + if(!shs[0].empty() && !shs[1].empty() && uniqFactor[1] > uniqFactor[0]) { + // Do the mate with fewer exact hits first + // TODO: Consider mates & orientations separately? + matemap[0] = 1; matemap[1] = 0; + } + for(size_t matei = 0; matei < (paired ? 2:1); matei++) { + size_t mate = matemap[matei]; + if(done[mate] || msinkwrap.state().doneWithMate(mate == 0)) { + // Done with this mate + done[mate] = true; + continue; + } + assert(!msinkwrap.maxed()); + assert(msinkwrap.repOk()); + //rnd.init(ROTL(rds[mate]->seed, 10)); + assert(shs[mate].repOk(&ca.current())); + if(!seedSumm) { + // If there aren't any seed hits... + if(shs[mate].empty()) { + continue; // on to the next mate + } + // Sort seed hits into ranks + shs[mate].rankSeedHits(rnd, msinkwrap.allHits()); int ret = 0; - if(paired) { + if(false && paired) { // Paired-end dynamic programming driver ret = sd.extendSeedsPaired( *rds[mate], // mate to align as anchor @@ -3628,9 +3918,9 @@ static void multiseedSearchWorker(void *vp) { osw, // dyn prog aligner, opposite sc, // scoring scheme pepol, // paired-end policy - -1, // # mms allowed in a seed - 0, // length of a seed - 0, // interval between seeds + multiseedMms, // # mms allowed in a seed + seedlens[mate], // length of a seed + interval[mate], // interval between seeds minsc[mate], // min score for anchor minsc[mate^1], // min score for opp. nceil[mate], // N ceil for anchor @@ -3676,18 +3966,18 @@ static void multiseedSearchWorker(void *vp) { ref, // packed reference strings sw, // dynamic prog aligner sc, // scoring scheme - -1, // # mms allowed in a seed - 0, // length of a seed - 0, // interval between seeds + multiseedMms, // # mms allowed in a seed + seedlens[mate], // length of a seed + interval[mate], // interval between seeds minsc[mate], // minimum score for valid nceil[mate], // N ceil for anchor maxhalf, // max width on one DP side doUngapped, // do ungapped alignment - mxIter[mate], // max extend loop iters - mxUg[mate], // max # ungapped extends - mxDp[mate], // max # DPs - streak[mate], // stop after streak of this many end-to-end fails - streak[mate], // stop after streak of this many ungap fails + mxIter[mate] * 2, // max extend loop iters + mxUg[mate] * 2, // max # ungapped extends + mxDp[mate] * 2, // max # DPs + streak[mate] * 2, // stop after streak of this many end-to-end fails + streak[mate] * 2, // stop after streak of this many ungap fails doExtend, // extend seed hits enable8, // use 8-bit SSE where possible cminlen, // checkpoint if read is longer @@ -3706,9 +3996,6 @@ static void multiseedSearchWorker(void *vp) { assert_gt(ret, 0); MERGE_SW(sw); MERGE_SW(osw); - // Clear out the 1mm hits so that we don't try to - // extend them again later! - shs[mate].clear1mmE2eHits(); if(ret == EXTEND_EXHAUSTED_CANDIDATES) { // Not done yet } else if(ret == EXTEND_POLICY_FULFILLED) { @@ -3720,7 +4007,7 @@ static void multiseedSearchWorker(void *vp) { done[mate^1] = true; } } else if(ret == EXTEND_PERFECT_SCORE) { - // We exhausted this mode at least + // We exhausted this made at least done[mate] = true; } else if(ret == EXTEND_EXCEEDED_HARD_LIMIT) { // We exceeded a per-read limit @@ -3732,295 +4019,22 @@ static void multiseedSearchWorker(void *vp) { cerr << "Bad return value: " << ret << endl; throw 1; } - if(!done[mate]) { - TAlScore perfectScore = sc.perfectScore(rdlens[mate]); - if(!done[mate] && minsc[mate] == perfectScore) { - done[mate] = true; - } - } + } // if(!seedSumm) + } // for(size_t matei = 0; matei < 2; matei++) + + // We don't necessarily have to continue investigating both + // mates. We continue on a mate only if its average + // interval length is high (> 1000) + for(size_t mate = 0; mate < 2; mate++) { + if(!done[mate] && shs[mate].averageHitsPerSeed() < seedBoostThresh) { + done[mate] = true; } } - int seedlens[2] = { multiseedLen, multiseedLen }; - nrounds[0] = min<size_t>(nrounds[0], interval[0]); - nrounds[1] = min<size_t>(nrounds[1], interval[1]); - Constraint gc = Constraint::penaltyFuncBased(scoreMin); - size_t seedsTried = 0; - size_t seedsTriedMS[] = {0, 0, 0, 0}; - size_t nUniqueSeeds = 0, nRepeatSeeds = 0, seedHitTot = 0; - size_t nUniqueSeedsMS[] = {0, 0, 0, 0}; - size_t nRepeatSeedsMS[] = {0, 0, 0, 0}; - size_t seedHitTotMS[] = {0, 0, 0, 0}; - for(size_t roundi = 0; roundi < nSeedRounds; roundi++) { - ca.nextRead(); // Clear cache in preparation for new search - shs[0].clearSeeds(); - shs[1].clearSeeds(); - assert(shs[0].empty()); - assert(shs[1].empty()); - assert(shs[0].repOk(&ca.current())); - assert(shs[1].repOk(&ca.current())); - //if(roundi > 0) { - // if(seedlens[0] > 8) seedlens[0]--; - // if(seedlens[1] > 8) seedlens[1]--; - //} - for(size_t matei = 0; matei < (paired ? 2:1); matei++) { - size_t mate = matemap[matei]; - if(done[mate] || msinkwrap.state().doneWithMate(mate == 0)) { - // Done with this mate - done[mate] = true; - continue; - } - if(roundi >= nrounds[mate]) { - // Not doing this round for this mate - continue; - } - // Figure out the seed offset - if(interval[mate] <= (int)roundi) { - // Can't do this round, seeds already packed as - // tight as possible - continue; - } - size_t offset = (interval[mate] * roundi) / nrounds[mate]; - assert(roundi == 0 || offset > 0); - assert(!msinkwrap.maxed()); - assert(msinkwrap.repOk()); - //rnd.init(ROTL(rds[mate]->seed, 10)); - assert(shs[mate].repOk(&ca.current())); - swmSeed.sdatts++; - // Set up seeds - seeds[mate]->clear(); - Seed::mmSeeds( - multiseedMms, // max # mms per seed - seedlens[mate], // length of a multiseed seed - *seeds[mate], // seeds - gc); // global constraint - // Check whether the offset would drive the first seed - // off the end - if(offset > 0 && (*seeds[mate])[0].len + offset > rds[mate]->length()) { - continue; - } - // Instantiate the seeds - std::pair<int, int> instFw, instRc; - std::pair<int, int> inst = al.instantiateSeeds( - *seeds[mate], // search seeds - offset, // offset to begin extracting - interval[mate], // interval between seeds - *rds[mate], // read to align - sc, // scoring scheme - nofw[mate], // don't align forward read - norc[mate], // don't align revcomp read - ca, // holds some seed hits from previous reads - shs[mate], // holds all the seed hits - sdm, // metrics - instFw, - instRc); - assert(shs[mate].repOk(&ca.current())); - if(inst.first + inst.second == 0) { - // No seed hits! Done with this mate. - assert(shs[mate].empty()); - done[mate] = true; - break; - } - seedsTried += (inst.first + inst.second); - seedsTriedMS[mate * 2 + 0] = instFw.first + instFw.second; - seedsTriedMS[mate * 2 + 1] = instRc.first + instRc.second; - // Align seeds - al.searchAllSeeds( - *seeds[mate], // search seeds - &ebwtFw, // BWT index - ebwtBw, // BWT' index - *rds[mate], // read - sc, // scoring scheme - ca, // alignment cache - shs[mate], // store seed hits here - sdm, // metrics - prm); // per-read metrics - assert(shs[mate].repOk(&ca.current())); - if(shs[mate].empty()) { - // No seed alignments! Done with this mate. - done[mate] = true; - break; - } - } - // shs contain what we need to know to update our seed - // summaries for this seeding - for(size_t mate = 0; mate < 2; mate++) { - if(!shs[mate].empty()) { - nUniqueSeeds += shs[mate].numUniqueSeeds(); - nUniqueSeedsMS[mate * 2 + 0] += shs[mate].numUniqueSeedsStrand(true); - nUniqueSeedsMS[mate * 2 + 1] += shs[mate].numUniqueSeedsStrand(false); - nRepeatSeeds += shs[mate].numRepeatSeeds(); - nRepeatSeedsMS[mate * 2 + 0] += shs[mate].numRepeatSeedsStrand(true); - nRepeatSeedsMS[mate * 2 + 1] += shs[mate].numRepeatSeedsStrand(false); - seedHitTot += shs[mate].numElts(); - seedHitTotMS[mate * 2 + 0] += shs[mate].numEltsFw(); - seedHitTotMS[mate * 2 + 1] += shs[mate].numEltsRc(); - } - } - double uniqFactor[2] = { 0.0f, 0.0f }; - for(size_t i = 0; i < 2; i++) { - if(!shs[i].empty()) { - swmSeed.sdsucc++; - uniqFactor[i] = shs[i].uniquenessFactor(); - } - } - // Possibly reorder the mates - matemap[0] = 0; matemap[1] = 1; - if(!shs[0].empty() && !shs[1].empty() && uniqFactor[1] > uniqFactor[0]) { - // Do the mate with fewer exact hits first - // TODO: Consider mates & orientations separately? - matemap[0] = 1; matemap[1] = 0; - } - for(size_t matei = 0; matei < (paired ? 2:1); matei++) { - size_t mate = matemap[matei]; - if(done[mate] || msinkwrap.state().doneWithMate(mate == 0)) { - // Done with this mate - done[mate] = true; - continue; - } - assert(!msinkwrap.maxed()); - assert(msinkwrap.repOk()); - //rnd.init(ROTL(rds[mate]->seed, 10)); - assert(shs[mate].repOk(&ca.current())); - if(!seedSumm) { - // If there aren't any seed hits... - if(shs[mate].empty()) { - continue; // on to the next mate - } - // Sort seed hits into ranks - shs[mate].rankSeedHits(rnd, msinkwrap.allHits()); - int ret = 0; - if(paired) { - // Paired-end dynamic programming driver - ret = sd.extendSeedsPaired( - *rds[mate], // mate to align as anchor - *rds[mate ^ 1], // mate to align as opp. - mate == 0, // anchor is mate 1? - !filt[mate ^ 1],// opposite mate filtered out? - shs[mate], // seed hits for anchor - ebwtFw, // bowtie index - ebwtBw, // rev bowtie index - ref, // packed reference strings - sw, // dyn prog aligner, anchor - osw, // dyn prog aligner, opposite - sc, // scoring scheme - pepol, // paired-end policy - multiseedMms, // # mms allowed in a seed - seedlens[mate], // length of a seed - interval[mate], // interval between seeds - minsc[mate], // min score for anchor - minsc[mate^1], // min score for opp. - nceil[mate], // N ceil for anchor - nceil[mate^1], // N ceil for opp. - nofw[mate], // don't align forward read - norc[mate], // don't align revcomp read - maxhalf, // max width on one DP side - doUngapped, // do ungapped alignment - mxIter[mate], // max extend loop iters - mxUg[mate], // max # ungapped extends - mxDp[mate], // max # DPs - streak[mate], // stop after streak of this many end-to-end fails - streak[mate], // stop after streak of this many ungap fails - streak[mate], // stop after streak of this many dp fails - mtStreak[mate], // max mate fails per seed range - doExtend, // extend seed hits - enable8, // use 8-bit SSE where possible - cminlen, // checkpoint if read is longer - cpow2, // checkpointer interval, log2 - doTri, // triangular mini-fills? - tighten, // -M score tightening mode - ca, // seed alignment cache - rnd, // pseudo-random source - wlm, // group walk left metrics - swmSeed, // DP metrics, seed extend - swmMate, // DP metrics, mate finding - prm, // per-read metrics - &msinkwrap, // for organizing hits - true, // seek mate immediately - true, // report hits once found - gReportDiscordant,// look for discordant alns? - gReportMixed, // look for unpaired alns? - exhaustive[mate]); - // Might be done, but just with this mate - } else { - // Unpaired dynamic programming driver - ret = sd.extendSeeds( - *rds[mate], // read - mate == 0, // mate #1? - shs[mate], // seed hits - ebwtFw, // bowtie index - ebwtBw, // rev bowtie index - ref, // packed reference strings - sw, // dynamic prog aligner - sc, // scoring scheme - multiseedMms, // # mms allowed in a seed - seedlens[mate], // length of a seed - interval[mate], // interval between seeds - minsc[mate], // minimum score for valid - nceil[mate], // N ceil for anchor - maxhalf, // max width on one DP side - doUngapped, // do ungapped alignment - mxIter[mate], // max extend loop iters - mxUg[mate], // max # ungapped extends - mxDp[mate], // max # DPs - streak[mate], // stop after streak of this many end-to-end fails - streak[mate], // stop after streak of this many ungap fails - doExtend, // extend seed hits - enable8, // use 8-bit SSE where possible - cminlen, // checkpoint if read is longer - cpow2, // checkpointer interval, log2 - doTri, // triangular mini-fills? - tighten, // -M score tightening mode - ca, // seed alignment cache - rnd, // pseudo-random source - wlm, // group walk left metrics - swmSeed, // DP metrics, seed extend - prm, // per-read metrics - &msinkwrap, // for organizing hits - true, // report hits once found - exhaustive[mate]); - } - assert_gt(ret, 0); - MERGE_SW(sw); - MERGE_SW(osw); - if(ret == EXTEND_EXHAUSTED_CANDIDATES) { - // Not done yet - } else if(ret == EXTEND_POLICY_FULFILLED) { - // Policy is satisfied for this mate at least - if(msinkwrap.state().doneWithMate(mate == 0)) { - done[mate] = true; - } - if(msinkwrap.state().doneWithMate(mate == 1)) { - done[mate^1] = true; - } - } else if(ret == EXTEND_PERFECT_SCORE) { - // We exhausted this made at least - done[mate] = true; - } else if(ret == EXTEND_EXCEEDED_HARD_LIMIT) { - // We exceeded a per-read limit - done[mate] = true; - } else if(ret == EXTEND_EXCEEDED_SOFT_LIMIT) { - // Not done yet - } else { - // - cerr << "Bad return value: " << ret << endl; - throw 1; - } - } // if(!seedSumm) - } // for(size_t matei = 0; matei < 2; matei++) - - // We don't necessarily have to continue investigating both - // mates. We continue on a mate only if its average - // interval length is high (> 1000) - for(size_t mate = 0; mate < 2; mate++) { - if(!done[mate] && shs[mate].averageHitsPerSeed() < seedBoostThresh) { - done[mate] = true; - } - } - } // end loop over reseeding rounds + } // end loop over reseeding rounds if(seedsTried > 0) { - prm.seedPctUnique = (float)nUniqueSeeds / seedsTried; - prm.seedPctRep = (float)nRepeatSeeds / seedsTried; - prm.seedHitAvg = (float)seedHitTot / seedsTried; + prm.seedPctUnique = (float)nUniqueSeeds / seedsTried; + prm.seedPctRep = (float)nRepeatSeeds / seedsTried; + prm.seedHitAvg = (float)seedHitTot / seedsTried; } else { prm.seedPctUnique = -1.0f; prm.seedPctRep = -1.0f; @@ -4036,31 +4050,31 @@ static void multiseedSearchWorker(void *vp) { prm.seedPctRepMS[i] = -1.0f; prm.seedHitAvgMS[i] = -1.0f; } - } - size_t totnucs = 0; - for(size_t mate = 0; mate < (paired ? 2:1); mate++) { - if(filt[mate]) { - size_t len = rdlens[mate]; - if(!nofw[mate] && !norc[mate]) { - len *= 2; - } - totnucs += len; + } + size_t totnucs = 0; + for(size_t mate = 0; mate < (paired ? 2:1); mate++) { + if(filt[mate]) { + size_t len = rdlens[mate]; + if(!nofw[mate] && !norc[mate]) { + len *= 2; } + totnucs += len; } + } prm.seedsPerNuc = totnucs > 0 ? ((float)seedsTried / totnucs) : -1; for(int i = 0; i < 4; i++) { prm.seedsPerNucMS[i] = totnucs > 0 ? ((float)seedsTriedMS[i] / totnucs) : -1; } - for(size_t i = 0; i < 2; i++) { - assert_leq(prm.nExIters, mxIter[i]); - assert_leq(prm.nExDps, mxDp[i]); - assert_leq(prm.nMateDps, mxDp[i]); - assert_leq(prm.nExUgs, mxUg[i]); - assert_leq(prm.nMateUgs, mxUg[i]); - assert_leq(prm.nDpFail, streak[i]); - assert_leq(prm.nUgFail, streak[i]); - assert_leq(prm.nEeFail, streak[i]); - } + for(size_t i = 0; i < 2; i++) { + assert_leq(prm.nExIters, mxIter[i]); + assert_leq(prm.nExDps, mxDp[i]); + assert_leq(prm.nMateDps, mxDp[i]); + assert_leq(prm.nExUgs, mxUg[i]); + assert_leq(prm.nMateUgs, mxUg[i]); + // assert_leq(prm.nDpFail, streak[i]); + // assert_leq(prm.nUgFail, streak[i]); + // assert_leq(prm.nEeFail, streak[i]); + } // Commit and report paired-end/unpaired alignments //uint32_t sd = rds[0]->seed ^ rds[1]->seed; @@ -4090,6 +4104,7 @@ static void multiseedSearchWorker(void *vp) { } // while(retry) } // if(rdid >= skipReads && rdid < qUpto) else if(rdid >= qUpto) { + done = true; break; } if(metricsPerRead) { @@ -4099,13 +4114,14 @@ static void multiseedSearchWorker(void *vp) { metricsOfb, metricsStderr, true, &nametmp); metricsPt.reset(); } + } while (ps->nextReadPairReady()); // must read the whole cached buffer } // while(true) - // One last metrics merge - MERGE_METRICS(metrics); + // One last metrics merge + MERGE_METRICS(metrics); - if(dpLog != NULL) dpLog->close(); - if(dpLogOpp != NULL) dpLogOpp->close(); + if(dpLog != NULL) dpLog->close(); + if(dpLogOpp != NULL) dpLogOpp->close(); #ifdef PER_THREAD_TIMING ss.str(""); @@ -4127,8 +4143,7 @@ static void multiseedSearchWorker_2p5(void *vp) { int tid = p->tid; assert(multiseed_ebwtFw != NULL); assert(multiseedMms == 0 || multiseed_ebwtBw != NULL); - PatternComposer& patsrc = *multiseed_patsrc; - PatternParams pp = multiseed_pp; + PatternSourceReadAheadFactory& readahead_factory = *multiseed_readahead_factory; const Ebwt& ebwtFw = *multiseed_ebwtFw; const Ebwt& ebwtBw = *multiseed_ebwtBw; const Scoring& sc = *multiseed_sc; @@ -4142,8 +4157,6 @@ static void multiseedSearchWorker_2p5(void *vp) { // problems, or generally characterize performance. ThreadCounter tc; - unique_ptr<PatternSourcePerThreadFactory> patsrcFact(createPatsrcFactory(patsrc, pp, tid)); - unique_ptr<PatternSourcePerThread> ps(patsrcFact->create()); // Instantiate an object for holding reporting-related parameters. ReportingParams rp( @@ -4240,6 +4253,9 @@ static void multiseedSearchWorker_2p5(void *vp) { int mergei = 0; int mergeival = 16; while(true) { + PatternSourceReadAhead psrah(readahead_factory); + PatternSourcePerThread* const ps = psrah.ptr(); + do { pair<bool, bool> ret = ps->nextReadPair(); bool success = ret.first; bool done = ret.second; @@ -4364,7 +4380,7 @@ static void multiseedSearchWorker_2p5(void *vp) { } exhaustive[0] = exhaustive[1] = false; bool pairPostFilt = filt[0] && filt[1]; - if(pairPostFilt) { + if(false && pairPostFilt) { rnd.init(ROTL((rds[0]->seed ^ rds[1]->seed), 10)); } // Calculate streak length @@ -4451,6 +4467,7 @@ static void multiseedSearchWorker_2p5(void *vp) { metricsOfb, metricsStderr, true, &nametmp); metricsPt.reset(); } + } while (ps->nextReadPairReady()); // must read the whole cached buffer } // while(true) // One last metrics merge @@ -4646,8 +4663,6 @@ static void multiseedSearch( Ebwt* ebwtBw, // index of mirror text OutFileBuf *metricsOfb) { - multiseed_patsrc = &patsrc; - multiseed_pp = pp; multiseed_msink = &msink; multiseed_ebwtFw = &ebwtFw; multiseed_ebwtBw = ebwtBw; @@ -4667,7 +4682,7 @@ static void multiseedSearch( mmSweep, gVerbose, startVerbose) - ); + ); delete _t; if(!refs->loaded()) throw 1; multiseed_refs = refs.get(); @@ -4711,6 +4726,11 @@ static void multiseedSearch( !noRefNames, // load names? startVerbose); } + + // Important: Need at least nthreads+1 elements, more is OK + PatternSourceReadAheadFactory readahead_factory(patsrc,pp,2*nthreads+1); + multiseed_readahead_factory = &readahead_factory; + // Start the metrics thread std::atomic<int> all_threads_done; @@ -4807,7 +4827,7 @@ static void driver( outType != OUTPUT_SAM, // whether to fix mate names preserve_tags, // keep existing tags when aligning BAM files align_paired_reads // Align only the paired reads in BAM file - ); + ); if(gVerbose || startVerbose) { cerr << "Creating PatternSource: "; logTime(cerr, true); } @@ -4967,23 +4987,23 @@ static void driver( // memory so that we can easily sanity check them later on AlnSink *mssink = NULL; switch(outType) { - case OUTPUT_SAM: { - mssink = new AlnSinkSam( - oq, // output queue - samc, // settings & routines for SAM output - refnames, // reference names - gQuiet); // don't print alignment summary at end - if(!samNoHead) { - bool printHd = true, printSq = true; - BTString buf; - samc.printHeader(buf, rgid, rgs, printHd, !samNoSQ, printSq); - fout->writeString(buf); - } - break; + case OUTPUT_SAM: { + mssink = new AlnSinkSam( + oq, // output queue + samc, // settings & routines for SAM output + refnames, // reference names + gQuiet); // don't print alignment summary at end + if(!samNoHead) { + bool printHd = true, printSq = true; + BTString buf; + samc.printHeader(buf, rgid, rgs, printHd, !samNoSQ, printSq); + fout->writeString(buf); } - default: - cerr << "Invalid output type: " << outType << endl; - throw 1; + break; + } + default: + cerr << "Invalid output type: " << outType << endl; + throw 1; } if(gVerbose || startVerbose) { cerr << "Dispatching to search driver: "; logTime(cerr, true); @@ -5079,11 +5099,11 @@ extern "C" { */ int bowtie(int argc, const char **argv) { try { - #ifdef WITH_AFFINITY +#ifdef WITH_AFFINITY //CWILKS: adjust this depending on # of hyperthreads per core pinning_observer pinner( 2 /* the number of hyper threads on each core */ ); pinner.observe( true ); - #endif +#endif // Reset all global state, including getopt state opterr = optind = 1; resetOptions(); @@ -5108,10 +5128,10 @@ int bowtie(int argc, const char **argv) { cout << "Compiler: " << COMPILER_VERSION << endl; cout << "Options: " << COMPILER_OPTIONS << endl; cout << "Sizeof {int, long, long long, void*, size_t, off_t}: {" - << sizeof(int) - << ", " << sizeof(long) << ", " << sizeof(long long) - << ", " << sizeof(void *) << ", " << sizeof(size_t) - << ", " << sizeof(off_t) << "}" << endl; + << sizeof(int) + << ", " << sizeof(long) << ", " << sizeof(long long) + << ", " << sizeof(void *) << ", " << sizeof(size_t) + << ", " << sizeof(off_t) << "}" << endl; return 0; } { @@ -5148,7 +5168,7 @@ int bowtie(int argc, const char **argv) { #ifdef USE_SRA << "Error: Must specify at least one read input with -U/-1/-2/--sra-acc" << endl; #else - << "Error: Must specify at least one read input with -U/-1/-2" << endl; + << "Error: Must specify at least one read input with -U/-1/-2" << endl; #endif return 1; } @@ -5167,8 +5187,8 @@ int bowtie(int argc, const char **argv) { outfile = argv[optind++]; cerr << "Warning: Output file '" << outfile.c_str() << "' was specified without -S. This will not work in " - << "future Bowtie 2 versions. Please use -S instead." - << endl; + << "future Bowtie 2 versions. Please use -S instead." + << endl; } // Extra parametesr? @@ -5181,7 +5201,7 @@ int bowtie(int argc, const char **argv) { cerr << endl; if(mates1.size() > 0) { cerr << "Note that if <mates> files are specified using -1/-2, a <singles> file cannot" << endl - << "also be specified. Please run bowtie separately for mates and singles." << endl; + << "also be specified. Please run bowtie separately for mates and singles." << endl; } throw 1; } @@ -5200,11 +5220,11 @@ int bowtie(int argc, const char **argv) { cout << "Output file: \"" << outfile.c_str() << "\"" << endl; cout << "Local endianness: " << (currentlyBigEndian()? "big":"little") << endl; cout << "Sanity checking: " << (sanityCheck? "enabled":"disabled") << endl; - #ifdef NDEBUG +#ifdef NDEBUG cout << "Assertions: disabled" << endl; - #else +#else cout << "Assertions: enabled" << endl; - #endif +#endif } if(ipause) { cout << "Press key to continue..." << endl; diff --git a/doc/manual.html b/doc/manual.html index f313cb6..498ebb9 100644 --- a/doc/manual.html +++ b/doc/manual.html @@ -8,10 +8,15 @@ <style> code{white-space: pre-wrap;} span.smallcaps{font-variant: small-caps;} - span.underline{text-decoration: underline;} - div.column{display: inline-block; vertical-align: top; width: 50%;} + div.columns{display: flex; gap: min(4vw, 1.5em);} + div.column{flex: auto; overflow-x: auto;} div.hanging-indent{margin-left: 1.5em; text-indent: -1.5em;} ul.task-list{list-style: none;} + ul.task-list li input[type="checkbox"] { + width: 0.8em; + margin: 0 0.8em 0.2em -1.6em; + vertical-align: middle; + } .display.math{display: block; text-align: center; margin: 0.5rem auto;} </style> <link rel="stylesheet" href="doc/style.css" /> @@ -26,91 +31,163 @@ </header> <nav id="TOC" role="doc-toc"> <ul> -<li><a href="#introduction">Introduction</a> +<li><a href="#introduction" id="toc-introduction">Introduction</a> <ul> -<li><a href="#how-is-bowtie-2-different-from-bowtie-1">How is Bowtie 2 different from Bowtie 1?</a></li> -<li><a href="#what-isnt-bowtie-2">What isn't Bowtie 2?</a></li> +<li><a href="#how-is-bowtie-2-different-from-bowtie-1" +id="toc-how-is-bowtie-2-different-from-bowtie-1">How is Bowtie 2 +different from Bowtie 1?</a></li> +<li><a href="#what-isnt-bowtie-2" id="toc-what-isnt-bowtie-2">What isn't +Bowtie 2?</a></li> </ul></li> -<li><a href="#obtaining-bowtie-2">Obtaining Bowtie 2</a> +<li><a href="#obtaining-bowtie-2" id="toc-obtaining-bowtie-2">Obtaining +Bowtie 2</a> <ul> -<li><a href="#building-from-source">Building from source</a></li> -<li><a href="#adding-to-path">Adding to PATH</a></li> +<li><a href="#building-from-source" +id="toc-building-from-source">Building from source</a></li> +<li><a href="#adding-to-path" id="toc-adding-to-path">Adding to +PATH</a></li> </ul></li> -<li><a href="#the-bowtie2-aligner">The <code>bowtie2</code> aligner</a> +<li><a href="#the-bowtie2-aligner" id="toc-the-bowtie2-aligner">The +<code>bowtie2</code> aligner</a> <ul> -<li><a href="#end-to-end-alignment-versus-local-alignment">End-to-end alignment versus local alignment</a> +<li><a href="#end-to-end-alignment-versus-local-alignment" +id="toc-end-to-end-alignment-versus-local-alignment">End-to-end +alignment versus local alignment</a> <ul> -<li><a href="#end-to-end-alignment-example">End-to-end alignment example</a></li> -<li><a href="#local-alignment-example">Local alignment example</a></li> +<li><a href="#end-to-end-alignment-example" +id="toc-end-to-end-alignment-example">End-to-end alignment +example</a></li> +<li><a href="#local-alignment-example" +id="toc-local-alignment-example">Local alignment example</a></li> </ul></li> -<li><a href="#scores-higher-more-similar">Scores: higher = more similar</a> +<li><a href="#scores-higher-more-similar" +id="toc-scores-higher-more-similar">Scores: higher = more similar</a> <ul> -<li><a href="#end-to-end-alignment-score-example">End-to-end alignment score example</a></li> -<li><a href="#local-alignment-score-example">Local alignment score example</a></li> -<li><a href="#valid-alignments-meet-or-exceed-the-minimum-score-threshold">Valid alignments meet or exceed the minimum score threshold</a></li> +<li><a href="#end-to-end-alignment-score-example" +id="toc-end-to-end-alignment-score-example">End-to-end alignment score +example</a></li> +<li><a href="#local-alignment-score-example" +id="toc-local-alignment-score-example">Local alignment score +example</a></li> +<li><a +href="#valid-alignments-meet-or-exceed-the-minimum-score-threshold" +id="toc-valid-alignments-meet-or-exceed-the-minimum-score-threshold">Valid +alignments meet or exceed the minimum score threshold</a></li> </ul></li> -<li><a href="#mapping-quality-higher-more-unique">Mapping quality: higher = more unique</a></li> -<li><a href="#aligning-pairs">Aligning pairs</a> +<li><a href="#mapping-quality-higher-more-unique" +id="toc-mapping-quality-higher-more-unique">Mapping quality: higher = +more unique</a></li> +<li><a href="#aligning-pairs" id="toc-aligning-pairs">Aligning pairs</a> <ul> -<li><a href="#paired-inputs">Paired inputs</a></li> -<li><a href="#paired-sam-output">Paired SAM output</a></li> -<li><a href="#concordant-pairs-match-pair-expectations-discordant-pairs-dont">Concordant pairs match pair expectations, discordant pairs don't</a></li> -<li><a href="#mixed-mode-paired-where-possible-unpaired-otherwise">Mixed mode: paired where possible, unpaired otherwise</a></li> -<li><a href="#some-sam-flags-describe-paired-end-properties">Some SAM FLAGS describe paired-end properties</a></li> -<li><a href="#some-sam-optional-fields-describe-more-paired-end-properties">Some SAM optional fields describe more paired-end properties</a></li> -<li><a href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can overlap, contain, or dovetail each other</a></li> +<li><a href="#paired-inputs" id="toc-paired-inputs">Paired +inputs</a></li> +<li><a href="#paired-sam-output" id="toc-paired-sam-output">Paired SAM +output</a></li> +<li><a +href="#concordant-pairs-match-pair-expectations-discordant-pairs-dont" +id="toc-concordant-pairs-match-pair-expectations-discordant-pairs-dont">Concordant +pairs match pair expectations, discordant pairs don't</a></li> +<li><a href="#mixed-mode-paired-where-possible-unpaired-otherwise" +id="toc-mixed-mode-paired-where-possible-unpaired-otherwise">Mixed mode: +paired where possible, unpaired otherwise</a></li> +<li><a href="#some-sam-flags-describe-paired-end-properties" +id="toc-some-sam-flags-describe-paired-end-properties">Some SAM FLAGS +describe paired-end properties</a></li> +<li><a +href="#some-sam-optional-fields-describe-more-paired-end-properties" +id="toc-some-sam-optional-fields-describe-more-paired-end-properties">Some +SAM optional fields describe more paired-end properties</a></li> +<li><a href="#mates-can-overlap-contain-or-dovetail-each-other" +id="toc-mates-can-overlap-contain-or-dovetail-each-other">Mates can +overlap, contain, or dovetail each other</a></li> </ul></li> -<li><a href="#reporting">Reporting</a> +<li><a href="#reporting" id="toc-reporting">Reporting</a> <ul> -<li><a href="#distinct-alignments-map-a-read-to-different-places">Distinct alignments map a read to different places</a></li> -<li><a href="#default-mode-search-for-multiple-alignments-report-the-best-one">Default mode: search for multiple alignments, report the best one</a></li> -<li><a href="#k-mode-search-for-one-or-more-alignments-report-each">-k mode: search for one or more alignments, report each</a></li> -<li><a href="#a-mode-search-for-and-report-all-alignments">-a mode: search for and report all alignments</a></li> -<li><a href="#randomness-in-bowtie-2">Randomness in Bowtie 2</a></li> +<li><a href="#distinct-alignments-map-a-read-to-different-places" +id="toc-distinct-alignments-map-a-read-to-different-places">Distinct +alignments map a read to different places</a></li> +<li><a +href="#default-mode-search-for-multiple-alignments-report-the-best-one" +id="toc-default-mode-search-for-multiple-alignments-report-the-best-one">Default +mode: search for multiple alignments, report the best one</a></li> +<li><a href="#k-mode-search-for-one-or-more-alignments-report-each" +id="toc-k-mode-search-for-one-or-more-alignments-report-each">-k mode: +search for one or more alignments, report each</a></li> +<li><a href="#a-mode-search-for-and-report-all-alignments" +id="toc-a-mode-search-for-and-report-all-alignments">-a mode: search for +and report all alignments</a></li> +<li><a href="#randomness-in-bowtie-2" +id="toc-randomness-in-bowtie-2">Randomness in Bowtie 2</a></li> </ul></li> -<li><a href="#multiseed-heuristic">Multiseed heuristic</a> +<li><a href="#multiseed-heuristic" +id="toc-multiseed-heuristic">Multiseed heuristic</a> <ul> -<li><a href="#fm-index-memory-footprint">FM Index memory footprint</a></li> +<li><a href="#fm-index-memory-footprint" +id="toc-fm-index-memory-footprint">FM Index memory footprint</a></li> </ul></li> -<li><a href="#ambiguous-characters">Ambiguous characters</a></li> -<li><a href="#presets-setting-many-settings-at-once">Presets: setting many settings at once</a></li> -<li><a href="#filtering">Filtering</a></li> -<li><a href="#alignment-summary">Alignment summary</a></li> -<li><a href="#wrapper-scripts">Wrapper scripts</a></li> -<li><a href="#small-and-large-indexes">Small and large indexes</a></li> -<li><a href="#performance-tuning">Performance tuning</a></li> -<li><a href="#command-line">Command Line</a> +<li><a href="#ambiguous-characters" +id="toc-ambiguous-characters">Ambiguous characters</a></li> +<li><a href="#presets-setting-many-settings-at-once" +id="toc-presets-setting-many-settings-at-once">Presets: setting many +settings at once</a></li> +<li><a href="#filtering" id="toc-filtering">Filtering</a></li> +<li><a href="#alignment-summary" id="toc-alignment-summary">Alignment +summary</a></li> +<li><a href="#wrapper-scripts" id="toc-wrapper-scripts">Wrapper +scripts</a></li> +<li><a href="#small-and-large-indexes" +id="toc-small-and-large-indexes">Small and large indexes</a></li> +<li><a href="#performance-tuning" +id="toc-performance-tuning">Performance tuning</a></li> +<li><a href="#command-line" id="toc-command-line">Command Line</a> <ul> -<li><a href="#setting-function-options">Setting function options</a></li> -<li><a href="#usage">Usage</a></li> -<li><a href="#main-arguments">Main arguments</a></li> -<li><a href="#options">Options</a></li> +<li><a href="#setting-function-options" +id="toc-setting-function-options">Setting function options</a></li> +<li><a href="#usage" id="toc-usage">Usage</a></li> +<li><a href="#main-arguments" id="toc-main-arguments">Main +arguments</a></li> +<li><a href="#options" id="toc-options">Options</a></li> </ul></li> -<li><a href="#sam-output">SAM output</a></li> +<li><a href="#sam-output" id="toc-sam-output">SAM output</a></li> </ul></li> -<li><a href="#the-bowtie2-build-indexer">The <code>bowtie2-build</code> indexer</a> +<li><a href="#the-bowtie2-build-indexer" +id="toc-the-bowtie2-build-indexer">The <code>bowtie2-build</code> +indexer</a> <ul> -<li><a href="#command-line-1">Command Line</a> +<li><a href="#command-line-1" id="toc-command-line-1">Command Line</a> <ul> -<li><a href="#main-arguments-1">Main arguments</a></li> -<li><a href="#options-1">Options</a></li> +<li><a href="#main-arguments-1" id="toc-main-arguments-1">Main +arguments</a></li> +<li><a href="#options-1" id="toc-options-1">Options</a></li> </ul></li> </ul></li> -<li><a href="#the-bowtie2-inspect-index-inspector">The <code>bowtie2-inspect</code> index inspector</a> +<li><a href="#the-bowtie2-inspect-index-inspector" +id="toc-the-bowtie2-inspect-index-inspector">The +<code>bowtie2-inspect</code> index inspector</a> <ul> -<li><a href="#command-line-2">Command Line</a> +<li><a href="#command-line-2" id="toc-command-line-2">Command Line</a> <ul> -<li><a href="#main-arguments-2">Main arguments</a></li> -<li><a href="#options-2">Options</a></li> +<li><a href="#main-arguments-2" id="toc-main-arguments-2">Main +arguments</a></li> +<li><a href="#options-2" id="toc-options-2">Options</a></li> </ul></li> </ul></li> -<li><a href="#getting-started-with-bowtie-2-lambda-phage-example">Getting started with Bowtie 2: Lambda phage example</a> +<li><a href="#getting-started-with-bowtie-2-lambda-phage-example" +id="toc-getting-started-with-bowtie-2-lambda-phage-example">Getting +started with Bowtie 2: Lambda phage example</a> <ul> -<li><a href="#indexing-a-reference-genome">Indexing a reference genome</a></li> -<li><a href="#aligning-example-reads">Aligning example reads</a></li> -<li><a href="#paired-end-example">Paired-end example</a></li> -<li><a href="#local-alignment-example-1">Local alignment example</a></li> -<li><a href="#using-samtoolsbcftools-downstream">Using SAMtools/BCFtools downstream</a></li> +<li><a href="#indexing-a-reference-genome" +id="toc-indexing-a-reference-genome">Indexing a reference +genome</a></li> +<li><a href="#aligning-example-reads" +id="toc-aligning-example-reads">Aligning example reads</a></li> +<li><a href="#paired-end-example" id="toc-paired-end-example">Paired-end +example</a></li> +<li><a href="#local-alignment-example-1" +id="toc-local-alignment-example-1">Local alignment example</a></li> +<li><a href="#using-samtoolsbcftools-downstream" +id="toc-using-samtoolsbcftools-downstream">Using SAMtools/BCFtools +downstream</a></li> </ul></li> </ul> </nav> @@ -120,58 +197,204 @@ ! of this text document, or see the HTML manual online. ! --> <h1 id="introduction">Introduction</h1> -<p><a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> is an ultrafast and memory-efficient tool for aligning sequencing reads to long reference sequences. It is particularly good at aligning reads of about 50 up to 100s of characters to relatively long (e.g. mammalian) genomes. Bowtie 2 indexes the genome with an <a href="http://en.wikipedia.org/wiki/FM-index">FM Index</a> (based on the <a href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler Transform</a> or <a href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">BWT</a>) to keep its memory footprint small: for the human genome, its memory footprint is typically around 3.2 gigabytes of RAM. Bowtie 2 supports gapped, local, and paired-end alignment modes. Multiple processors can be used simultaneously to achieve greater alignment speed.</p> -<p>Bowtie 2 outputs alignments in <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> format, enabling interoperation with a large number of other tools (e.g. <a href="http://samtools.sourceforge.net">SAMtools</a>, <a href="http://www.broadinstitute.org/gsa/wiki/index.php/The_Genome_Analysis_Toolkit">GATK</a>) that use SAM. Bowtie 2 is distributed under the <a href="http://www.gnu.org/licenses/gpl-3.0.html">GPLv3 license</a>, and it runs on the command line under Windows, Mac OS X and Linux and BSD.</p> -<p><a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> is often the first step in pipelines for comparative genomics, including for variation calling, ChIP-seq, RNA-seq, BS-seq. <a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> and <a href="http://bowtie-bio.sf.net">Bowtie</a> (also called "<a href="http://bowtie-bio.sf.net">Bowtie 1</a>" here) are also tightly integrated into many other tools, some of which <a href="http://bowtie-bio.sourceforge.net/bowtie2/other_tools.shtml">are listed here</a>.</p> -<p>If you use <a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> for your published research, please cite our work. Papers describing Bowtie 2 are:</p> +<p><a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> is an +ultrafast and memory-efficient tool for aligning sequencing reads to +long reference sequences. It is particularly good at aligning reads of +about 50 up to 100s of characters to relatively long (e.g. mammalian) +genomes. Bowtie 2 indexes the genome with an <a +href="http://en.wikipedia.org/wiki/FM-index">FM Index</a> (based on the +<a +href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler +Transform</a> or <a +href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">BWT</a>) +to keep its memory footprint small: for the human genome, its memory +footprint is typically around 3.2 gigabytes of RAM. Bowtie 2 supports +gapped, local, and paired-end alignment modes. Multiple processors can +be used simultaneously to achieve greater alignment speed.</p> +<p>Bowtie 2 outputs alignments in <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> format, enabling +interoperation with a large number of other tools (e.g. <a +href="http://samtools.sourceforge.net">SAMtools</a>, <a +href="http://www.broadinstitute.org/gsa/wiki/index.php/The_Genome_Analysis_Toolkit">GATK</a>) +that use SAM. Bowtie 2 is distributed under the <a +href="http://www.gnu.org/licenses/gpl-3.0.html">GPLv3 license</a>, and +it runs on the command line under Windows, Mac OS X and Linux and +BSD.</p> +<p><a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> is often the +first step in pipelines for comparative genomics, including for +variation calling, ChIP-seq, RNA-seq, BS-seq. <a +href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> and <a +href="http://bowtie-bio.sf.net">Bowtie</a> (also called "<a +href="http://bowtie-bio.sf.net">Bowtie 1</a>" here) are also tightly +integrated into many other tools, some of which <a +href="http://bowtie-bio.sourceforge.net/bowtie2/other_tools.shtml">are +listed here</a>.</p> +<p>If you use <a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> +for your published research, please cite our work. Papers describing +Bowtie 2 are:</p> <ul> -<li><p>Langmead B, Wilks C, Antonescu V, Charles R. <a href="https://doi.org/10.1093/bioinformatics/bty648">Scaling read aligners to hundreds of threads on general-purpose processors</a>. <em>Bioinformatics</em>. 2018 Jul 18. doi: 10.1093/bioinformatics/bty648.</p></li> -<li><p>Langmead B, Salzberg SL. <a href="https://www.nature.com/articles/nmeth.1923">Fast gapped-read alignment with Bowtie 2</a>. <em>Nature Methods</em>. 2012 Mar 4;9(4):357-9. doi: 10.1038/nmeth.1923.</p></li> +<li><p>Langmead B, Wilks C, Antonescu V, Charles R. <a +href="https://doi.org/10.1093/bioinformatics/bty648">Scaling read +aligners to hundreds of threads on general-purpose processors</a>. +<em>Bioinformatics</em>. 2018 Jul 18. doi: +10.1093/bioinformatics/bty648.</p></li> +<li><p>Langmead B, Salzberg SL. <a +href="https://www.nature.com/articles/nmeth.1923">Fast gapped-read +alignment with Bowtie 2</a>. <em>Nature Methods</em>. 2012 Mar +4;9(4):357-9. doi: 10.1038/nmeth.1923.</p></li> </ul> -<h2 id="how-is-bowtie-2-different-from-bowtie-1">How is Bowtie 2 different from Bowtie 1?</h2> -<p>Bowtie 1 was released in 2009 and was geared toward aligning the relatively short sequencing reads (up to 25-50 nucleotides) prevalent at the time. Since then, technology has improved both sequencing throughput (more nucleotides produced per sequencer per day) and read length (more nucleotides per read).</p> +<h2 id="how-is-bowtie-2-different-from-bowtie-1">How is Bowtie 2 +different from Bowtie 1?</h2> +<p>Bowtie 1 was released in 2009 and was geared toward aligning the +relatively short sequencing reads (up to 25-50 nucleotides) prevalent at +the time. Since then, technology has improved both sequencing throughput +(more nucleotides produced per sequencer per day) and read length (more +nucleotides per read).</p> <p>The chief differences between Bowtie 1 and Bowtie 2 are:</p> <ol type="1"> -<li><p>For reads longer than about 50 bp Bowtie 2 is generally faster, more sensitive, and uses less memory than Bowtie 1. For relatively short reads (e.g. less than 50 bp) Bowtie 1 is sometimes faster and/or more sensitive.</p></li> -<li><p>Bowtie 2 supports gapped alignment with affine gap penalties. Number of gaps and gap lengths are not restricted, except by way of the configurable scoring scheme. Bowtie 1 finds just ungapped alignments.</p></li> -<li><p>Bowtie 2 supports <a href="#end-to-end-alignment-versus-local-alignment">local alignment</a>, which doesn't require reads to align end-to-end. Local alignments might be "trimmed" ("soft clipped") at one or both extremes in a way that optimizes alignment score. Bowtie 2 also supports <a href="#end-to-end-alignment-versus-local-alignment">end-to-end alignment</a> which, like Bowtie 1, requires that the read align entirely.</p></li> -<li><p>There is no upper limit on read length in Bowtie 2. Bowtie 1 had an upper limit of around 1000 bp.</p></li> -<li><p>Bowtie 2 allows alignments to <a href="#ambiguous-characters">overlap ambiguous characters</a> (e.g. <code>N</code>s) in the reference. Bowtie 1 does not.</p></li> -<li><p>Bowtie 2 does away with Bowtie 1's notion of alignment "stratum", and its distinction between "Maq-like" and "end-to-end" modes. In Bowtie 2 all alignments lie along a continuous spectrum of alignment scores where the <a href="#scores-higher-more-similar">scoring scheme</a>, similar to <a href="http://en.wikipedia.org/wiki/Needleman-Wunsch_algorithm">Needleman-Wunsch</a> and <a href="http://en.wikipedia.org/wiki/Smith_waterman">Smith-Waterman</a>.</p></li> -<li><p>Bowtie 2's <a href="#aligning-pairs">paired-end alignment</a> is more flexible. E.g. for pairs that do not align in a paired fashion, Bowtie 2 attempts to find unpaired alignments for each mate.</p></li> -<li><p>Bowtie 2 reports a spectrum of mapping qualities, in contrast for Bowtie 1 which reports either 0 or high.</p></li> +<li><p>For reads longer than about 50 bp Bowtie 2 is generally faster, +more sensitive, and uses less memory than Bowtie 1. For relatively short +reads (e.g. less than 50 bp) Bowtie 1 is sometimes faster and/or more +sensitive.</p></li> +<li><p>Bowtie 2 supports gapped alignment with affine gap penalties. +Number of gaps and gap lengths are not restricted, except by way of the +configurable scoring scheme. Bowtie 1 finds just ungapped +alignments.</p></li> +<li><p>Bowtie 2 supports <a +href="#end-to-end-alignment-versus-local-alignment">local alignment</a>, +which doesn't require reads to align end-to-end. Local alignments might +be "trimmed" ("soft clipped") at one or both extremes in a way that +optimizes alignment score. Bowtie 2 also supports <a +href="#end-to-end-alignment-versus-local-alignment">end-to-end +alignment</a> which, like Bowtie 1, requires that the read align +entirely.</p></li> +<li><p>There is no upper limit on read length in Bowtie 2. Bowtie 1 had +an upper limit of around 1000 bp.</p></li> +<li><p>Bowtie 2 allows alignments to <a +href="#ambiguous-characters">overlap ambiguous characters</a> (e.g. +<code>N</code>s) in the reference. Bowtie 1 does not.</p></li> +<li><p>Bowtie 2 does away with Bowtie 1's notion of alignment "stratum", +and its distinction between "Maq-like" and "end-to-end" modes. In Bowtie +2 all alignments lie along a continuous spectrum of alignment scores +where the <a href="#scores-higher-more-similar">scoring scheme</a>, +similar to <a +href="http://en.wikipedia.org/wiki/Needleman-Wunsch_algorithm">Needleman-Wunsch</a> +and <a +href="http://en.wikipedia.org/wiki/Smith_waterman">Smith-Waterman</a>.</p></li> +<li><p>Bowtie 2's <a href="#aligning-pairs">paired-end alignment</a> is +more flexible. E.g. for pairs that do not align in a paired fashion, +Bowtie 2 attempts to find unpaired alignments for each mate.</p></li> +<li><p>Bowtie 2 reports a spectrum of mapping qualities, in contrast for +Bowtie 1 which reports either 0 or high.</p></li> <li><p>Bowtie 2 does not align colorspace reads.</p></li> </ol> -<p>Bowtie 2 is not a "drop-in" replacement for Bowtie 1. Bowtie 2's command-line arguments and genome index format are both different from Bowtie 1's.</p> +<p>Bowtie 2 is not a "drop-in" replacement for Bowtie 1. Bowtie 2's +command-line arguments and genome index format are both different from +Bowtie 1's.</p> <h2 id="what-isnt-bowtie-2">What isn't Bowtie 2?</h2> -<p>Bowtie 2 is geared toward aligning relatively short sequencing reads to long genomes. That said, it handles arbitrarily small reference sequences (e.g. amplicons) and very long reads (i.e. upwards of 10s or 100s of kilobases), though it is slower in those settings. It is optimized for the read lengths and error modes yielded by typical Illumina sequencers.</p> -<p>Bowtie 2 does not support alignment of colorspace reads. (Bowtie 1 does.)</p> +<p>Bowtie 2 is geared toward aligning relatively short sequencing reads +to long genomes. That said, it handles arbitrarily small reference +sequences (e.g. amplicons) and very long reads (i.e. upwards of 10s or +100s of kilobases), though it is slower in those settings. It is +optimized for the read lengths and error modes yielded by typical +Illumina sequencers.</p> +<p>Bowtie 2 does not support alignment of colorspace reads. (Bowtie 1 +does.)</p> <h1 id="obtaining-bowtie-2">Obtaining Bowtie 2</h1> -<p>Bowtie 2 is available from various package managers, notably <a href="https://anaconda.org/bioconda/bowtie2">Bioconda</a>. With Bioconda installed, you should be able to install Bowtie 2 with <code>conda install bowtie2</code>.</p> -<p>Containerized versions of Bowtie 2 are also available via the <a href="https://BioContainers.pro">Biocontainers</a> project (e.g. <a href="https://hub.docker.com/r/biocontainers/bowtie2/">via Docker Hub</a>).</p> -<p>You can also download Bowtie 2 sources and binaries from the <a href="https://sourceforge.net/projects/bowtie-bio/files/bowtie2/">Download</a> section of the Sourceforge site. Binaries are available for the <code>x86_64</code> architecture running Linux, Mac OS X, and Windows. FreeBSD users can obtain the latest version of Bowtie 2 from ports using <code>pkg install bowtie2</code>. If you plan to compile Bowtie 2 yourself, make sure to get the source package, i.e., the filename that ends in "-source.zip".</p> +<p>Bowtie 2 is available from various package managers, notably <a +href="https://anaconda.org/bioconda/bowtie2">Bioconda</a>. With Bioconda +installed, you should be able to install Bowtie 2 with +<code>conda install bowtie2</code>.</p> +<p>You can also download Bowtie 2 sources and binaries from the <a +href="https://sourceforge.net/projects/bowtie-bio/files/bowtie2/">Download</a> +section of the Sourceforge site. Binaries are available for the +<code>x86_64</code> architecture running Linux, Mac OS X, and Windows. +FreeBSD users can obtain the latest version of Bowtie 2 from ports using +<code>pkg install bowtie2</code>. If you plan to compile Bowtie 2 +yourself, make sure to get the source package, i.e., the filename that +ends in "-source.zip".</p> <h2 id="building-from-source">Building from source</h2> <p>Building from source</p> -<p>Building Bowtie 2 from source requires a GNU-like environment with Clang/GCC, GNU Make and other basics. It should be possible to build Bowtie 2 on most vanilla *NIX installations or on a Mac installation with Xcode installed. Bowtie 2 can also be built on Windows using a 64-bit MinGW distribution and MSYS. In order to simplify the MinGW setup it might be worth investigating popular MinGW personal builds since these are coming already prepared with most of the toolchains needed.</p> -<p>First, download the source package from the <a href="https://sourceforge.net/projects/bowtie-bio/files/bowtie2/">sourceforge site</a>. Make sure you're getting the source package; the file downloaded should end in <code>-source.zip</code>. Unzip the file, change to the unzipped directory, and build the Bowtie 2 tools by running GNU <code>make</code> (usually with the command <code>make</code>, but sometimes with <code>gmake</code>) with no arguments. If building with MinGW, run <code>make</code> from the MSYS environment.</p> -<p>The Bowtie 2 Makefile also includes recipes for basic automatic dependency management. Running <code>make static-libs && make STATIC_BUILD=1</code> will issue a series of commands that will: 1. download zstd and zlib 2. compile them as static libraries 3. link the resulting libraries to the compiled Bowtie 2 binaries</p> -<p>As of version 2.3.5 bowtie2 now supports aligning SRA reads. Prepackaged builds will include a package that supports SRA. If you're building bowtie2 from source please make sure that the Java runtime is available on your system. You can then proceed with the build by running <code>make sra-deps && make USE_SRA=1</code>.</p> +<p>Building Bowtie 2 from source requires a GNU-like environment with +Clang/GCC, GNU Make and other basics. It should be possible to build +Bowtie 2 on most vanilla *NIX installations or on a Mac installation +with Xcode installed. Bowtie 2 can also be built on Windows using a +64-bit MinGW distribution and MSYS. In order to simplify the MinGW setup +it might be worth investigating popular MinGW personal builds since +these are coming already prepared with most of the toolchains +needed.</p> +<p>First, download the source package from the <a +href="https://sourceforge.net/projects/bowtie-bio/files/bowtie2/">sourceforge +site</a>. Make sure you're getting the source package; the file +downloaded should end in <code>-source.zip</code>. Unzip the file, +change to the unzipped directory, and build the Bowtie 2 tools by +running GNU <code>make</code> (usually with the command +<code>make</code>, but sometimes with <code>gmake</code>) with no +arguments. If building with MinGW, run <code>make</code> from the MSYS +environment.</p> +<p>The Bowtie 2 Makefile also includes recipes for basic automatic +dependency management. Running +<code>make static-libs && make STATIC_BUILD=1</code> will issue +a series of commands that will: 1. download zstd and zlib 2. compile +them as static libraries 3. link the resulting libraries to the compiled +Bowtie 2 binaries</p> +<p>As of version 2.3.5 bowtie2 now supports aligning SRA reads. +Prepackaged builds will include a package that supports SRA. If you're +building bowtie2 from source please make sure that the Java runtime is +available on your system. You can then proceed with the build by running +<code>make sra-deps && make USE_SRA=1</code>.</p> <h2 id="adding-to-path">Adding to PATH</h2> -<p>By adding your new Bowtie 2 directory to your <a href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH environment variable</a>, you ensure that whenever you run <code>bowtie2</code>, <code>bowtie2-build</code> or <code>bowtie2-inspect</code> from the command line, you will get the version you just installed without having to specify the entire path. This is recommended for most users. To do this, follow your operating system's instructions for adding the directory to your <a href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH</a>.</p> -<p>If you would like to install Bowtie 2 by copying the Bowtie 2 executable files to an existing directory in your <a href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH</a>, make sure that you copy all the executables, including <code>bowtie2</code>, <code>bowtie2-align-s</code>, <code>bowtie2-align-l</code>, <code>bowtie2-build</code>, <code>bowtie2-build-s</code>, <code>bowtie2-build-l</code>, <code>bowtie2-inspect</code>, <code>bowtie2-inspect-s</code> and <code>bowtie2-inspect-l</code>.</p> +<p>By adding your new Bowtie 2 directory to your <a +href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH environment +variable</a>, you ensure that whenever you run <code>bowtie2</code>, +<code>bowtie2-build</code> or <code>bowtie2-inspect</code> from the +command line, you will get the version you just installed without having +to specify the entire path. This is recommended for most users. To do +this, follow your operating system's instructions for adding the +directory to your <a +href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH</a>.</p> +<p>If you would like to install Bowtie 2 by copying the Bowtie 2 +executable files to an existing directory in your <a +href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH</a>, make sure +that you copy all the executables, including <code>bowtie2</code>, +<code>bowtie2-align-s</code>, <code>bowtie2-align-l</code>, +<code>bowtie2-build</code>, <code>bowtie2-build-s</code>, +<code>bowtie2-build-l</code>, <code>bowtie2-inspect</code>, +<code>bowtie2-inspect-s</code> and <code>bowtie2-inspect-l</code>.</p> <h1 id="the-bowtie2-aligner">The <code>bowtie2</code> aligner</h1> -<p><code>bowtie2</code> takes a Bowtie 2 index and a set of sequencing read files and outputs a set of alignments in SAM format.</p> -<p>"Alignment" is the process by which we discover how and where the read sequences are similar to the reference sequence. An "alignment" is a result from this process, specifically: an alignment is a way of "lining up" some or all of the characters in the read with some characters from the reference in a way that reveals how they're similar. For example:</p> +<p><code>bowtie2</code> takes a Bowtie 2 index and a set of sequencing +read files and outputs a set of alignments in SAM format.</p> +<p>"Alignment" is the process by which we discover how and where the +read sequences are similar to the reference sequence. An "alignment" is +a result from this process, specifically: an alignment is a way of +"lining up" some or all of the characters in the read with some +characters from the reference in a way that reveals how they're similar. +For example:</p> <pre><code> Read: GACTGGGCGATCTCGACTTCG ||||| |||||||||| ||| Reference: GACTG--CGATCTCGACATCG</code></pre> -<p>Where dash symbols represent gaps and vertical bars show where aligned characters match.</p> -<p>We use alignment to make an educated guess as to where a read originated with respect to the reference genome. It's not always possible to determine this with certainty. For instance, if the reference genome contains several long stretches of As (<code>AAAAAAAAA</code> etc.) and the read sequence is a short stretch of As (<code>AAAAAAA</code>), we cannot know for certain exactly where in the sea of <code>A</code>s the read originated.</p> -<h2 id="end-to-end-alignment-versus-local-alignment">End-to-end alignment versus local alignment</h2> -<p>By default, Bowtie 2 performs end-to-end read alignment. That is, it searches for alignments involving all of the read characters. This is also called an "untrimmed" or "unclipped" alignment.</p> -<p>When the --local option is specified, Bowtie 2 performs local read alignment. In this mode, Bowtie 2 might "trim" or "clip" some read characters from one or both ends of the alignment if doing so maximizes the alignment score.</p> +<p>Where dash symbols represent gaps and vertical bars show where +aligned characters match.</p> +<p>We use alignment to make an educated guess as to where a read +originated with respect to the reference genome. It's not always +possible to determine this with certainty. For instance, if the +reference genome contains several long stretches of As +(<code>AAAAAAAAA</code> etc.) and the read sequence is a short stretch +of As (<code>AAAAAAA</code>), we cannot know for certain exactly where +in the sea of <code>A</code>s the read originated.</p> +<h2 id="end-to-end-alignment-versus-local-alignment">End-to-end +alignment versus local alignment</h2> +<p>By default, Bowtie 2 performs end-to-end read alignment. That is, it +searches for alignments involving all of the read characters. This is +also called an "untrimmed" or "unclipped" alignment.</p> +<p>When the --local option is specified, Bowtie 2 performs local read +alignment. In this mode, Bowtie 2 might "trim" or "clip" some read +characters from one or both ends of the alignment if doing so maximizes +the alignment score.</p> <h3 id="end-to-end-alignment-example">End-to-end alignment example</h3> -<p>The following is an "end-to-end" alignment because it involves all the characters in the read. Such an alignment can be produced by Bowtie 2 in either end-to-end mode or in local mode.</p> +<p>The following is an "end-to-end" alignment because it involves all +the characters in the read. Such an alignment can be produced by Bowtie +2 in either end-to-end mode or in local mode.</p> <pre><code>Read: GACTGGGCGATCTCGACTTCG Reference: GACTGCGATCTCGACATCG @@ -180,7 +403,11 @@ Alignment: ||||| |||||||||| ||| Reference: GACTG--CGATCTCGACATCG</code></pre> <h3 id="local-alignment-example">Local alignment example</h3> -<p>The following is a "local" alignment because some of the characters at the ends of the read do not participate. In this case, 4 characters are omitted (or "soft trimmed" or "soft clipped") from the beginning and 3 characters are omitted from the end. This sort of alignment can be produced by Bowtie 2 only in local mode.</p> +<p>The following is a "local" alignment because some of the characters +at the ends of the read do not participate. In this case, 4 characters +are omitted (or "soft trimmed" or "soft clipped") from the beginning and +3 characters are omitted from the end. This sort of alignment can be +produced by Bowtie 2 only in local mode.</p> <pre><code>Read: ACGGTTGCGTTAATCCGCCACG Reference: TAACTTGCGTTAAATCCGCCTGG @@ -189,47 +416,204 @@ Alignment: ||||||||| |||||| Reference: TAACTTGCGTTAAATCCGCCTGG</code></pre> <h2 id="scores-higher-more-similar">Scores: higher = more similar</h2> -<p>An alignment score quantifies how similar the read sequence is to the reference sequence aligned to. The higher the score, the more similar they are. A score is calculated by subtracting penalties for each difference (mismatch, gap, etc.) and, in local alignment mode, adding bonuses for each match.</p> -<p>The scores can be configured with the <a href="#bowtie2-options-ma"><code>--ma</code></a> (match bonus), <a href="#bowtie2-options-mp"><code>--mp</code></a> (mismatch penalty), <a href="#bowtie2-options-np"><code>--np</code></a> (penalty for having an N in either the read or the reference), <a href="#bowtie2-options-rdg"><code>--rdg</code></a> (affine read gap penalty) and <a href="#bowtie2-options-rfg"><code>--rfg</code></a> (affine reference gap penalty) options.</p> -<h3 id="end-to-end-alignment-score-example">End-to-end alignment score example</h3> -<p>A mismatched base at a high-quality position in the read receives a penalty of -6 by default. A length-2 read gap receives a penalty of -11 by default (-5 for the gap open, -3 for the first extension, -3 for the second extension). Thus, in end-to-end alignment mode, if the read is 50 bp long and it matches the reference exactly except for one mismatch at a high-quality position and one length-2 read gap, then the overall score is -(6 + 11) = -17.</p> -<p>The best possible alignment score in end-to-end mode is 0, which happens when there are no differences between the read and the reference.</p> -<h3 id="local-alignment-score-example">Local alignment score example</h3> -<p>A mismatched base at a high-quality position in the read receives a penalty of -6 by default. A length-2 read gap receives a penalty of -11 by default (-5 for the gap open, -3 for the first extension, -3 for the second extension). A base that matches receives a bonus of +2 be default. Thus, in local alignment mode, if the read is 50 bp long and it matches the reference exactly except for one mismatch at a high-quality position and one length-2 read gap, then the overall score equals the total bonus, 2 * 49, minus the total penalty, 6 + 11, = 81.</p> -<p>The best possible score in local mode equals the match bonus times the length of the read. This happens when there are no differences between the read and the reference.</p> -<h3 id="valid-alignments-meet-or-exceed-the-minimum-score-threshold">Valid alignments meet or exceed the minimum score threshold</h3> -<p>For an alignment to be considered "valid" (i.e. "good enough") by Bowtie 2, it must have an alignment score no less than the minimum score threshold. The threshold is configurable and is expressed as a function of the read length. In end-to-end alignment mode, the default minimum score threshold is <code>-0.6 + -0.6 * L</code>, where <code>L</code> is the read length. In local alignment mode, the default minimum score threshold is <code>20 + 8.0 * ln(L)</code>, where L is the read length. This can be configured with the <a href="#bowtie2-options-score-min"><code>--score-min</code></a> option. For details on how to set options like <code>--score-min</code> that correspond to functions, see the section on <a href="#setting-function-options">setting function options</a>.</p> -<h2 id="mapping-quality-higher-more-unique">Mapping quality: higher = more unique</h2> -<p>The aligner cannot always assign a read to its point of origin with high confidence. For instance, a read that originated inside a repeat element might align equally well to many occurrences of the element throughout the genome, leaving the aligner with no basis for preferring one over the others.</p> -<p>Aligners characterize their degree of confidence in the point of origin by reporting a mapping quality: a non-negative integer Q = -10 log10 p, where p is an estimate of the probability that the alignment does not correspond to the read's true point of origin. Mapping quality is sometimes abbreviated MAPQ, and is recorded in the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> <code>MAPQ</code> field.</p> -<p>Mapping quality is related to "uniqueness." We say an alignment is unique if it has a much higher alignment score than all the other possible alignments. The bigger the gap between the best alignment's score and the second-best alignment's score, the more unique the best alignment, and the higher its mapping quality should be.</p> -<p>Accurate mapping qualities are useful for downstream tools like variant callers. For instance, a variant caller might choose to ignore evidence from alignments with mapping quality less than, say, 10. A mapping quality of 10 or less indicates that there is at least a 1 in 10 chance that the read truly originated elsewhere.</p> +<p>An alignment score quantifies how similar the read sequence is to the +reference sequence aligned to. The higher the score, the more similar +they are. A score is calculated by subtracting penalties for each +difference (mismatch, gap, etc.) and, in local alignment mode, adding +bonuses for each match.</p> +<p>The scores can be configured with the <a +href="#bowtie2-options-ma"><code>--ma</code></a> (match bonus), <a +href="#bowtie2-options-mp"><code>--mp</code></a> (mismatch penalty), <a +href="#bowtie2-options-np"><code>--np</code></a> (penalty for having an +N in either the read or the reference), <a +href="#bowtie2-options-rdg"><code>--rdg</code></a> (affine read gap +penalty) and <a href="#bowtie2-options-rfg"><code>--rfg</code></a> +(affine reference gap penalty) options.</p> +<h3 id="end-to-end-alignment-score-example">End-to-end alignment score +example</h3> +<p>A mismatched base at a high-quality position in the read receives a +penalty of -6 by default. A length-2 read gap receives a penalty of -11 +by default (-5 for the gap open, -3 for the first extension, -3 for the +second extension). Thus, in end-to-end alignment mode, if the read is 50 +bp long and it matches the reference exactly except for one mismatch at +a high-quality position and one length-2 read gap, then the overall +score is -(6 + 11) = -17.</p> +<p>The best possible alignment score in end-to-end mode is 0, which +happens when there are no differences between the read and the +reference.</p> +<h3 id="local-alignment-score-example">Local alignment score +example</h3> +<p>A mismatched base at a high-quality position in the read receives a +penalty of -6 by default. A length-2 read gap receives a penalty of -11 +by default (-5 for the gap open, -3 for the first extension, -3 for the +second extension). A base that matches receives a bonus of +2 be +default. Thus, in local alignment mode, if the read is 50 bp long and it +matches the reference exactly except for one mismatch at a high-quality +position and one length-2 read gap, then the overall score equals the +total bonus, 2 * 49, minus the total penalty, 6 + 11, = 81.</p> +<p>The best possible score in local mode equals the match bonus times +the length of the read. This happens when there are no differences +between the read and the reference.</p> +<h3 +id="valid-alignments-meet-or-exceed-the-minimum-score-threshold">Valid +alignments meet or exceed the minimum score threshold</h3> +<p>For an alignment to be considered "valid" (i.e. "good enough") by +Bowtie 2, it must have an alignment score no less than the minimum score +threshold. The threshold is configurable and is expressed as a function +of the read length. In end-to-end alignment mode, the default minimum +score threshold is <code>-0.6 + -0.6 * L</code>, where <code>L</code> is +the read length. In local alignment mode, the default minimum score +threshold is <code>20 + 8.0 * ln(L)</code>, where L is the read length. +This can be configured with the <a +href="#bowtie2-options-score-min"><code>--score-min</code></a> option. +For details on how to set options like <code>--score-min</code> that +correspond to functions, see the section on <a +href="#setting-function-options">setting function options</a>.</p> +<h2 id="mapping-quality-higher-more-unique">Mapping quality: higher = +more unique</h2> +<p>The aligner cannot always assign a read to its point of origin with +high confidence. For instance, a read that originated inside a repeat +element might align equally well to many occurrences of the element +throughout the genome, leaving the aligner with no basis for preferring +one over the others.</p> +<p>Aligners characterize their degree of confidence in the point of +origin by reporting a mapping quality: a non-negative integer Q = -10 +log10 p, where p is an estimate of the probability that the alignment +does not correspond to the read's true point of origin. Mapping quality +is sometimes abbreviated MAPQ, and is recorded in the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> +<code>MAPQ</code> field.</p> +<p>Mapping quality is related to "uniqueness." We say an alignment is +unique if it has a much higher alignment score than all the other +possible alignments. The bigger the gap between the best alignment's +score and the second-best alignment's score, the more unique the best +alignment, and the higher its mapping quality should be.</p> +<p>Accurate mapping qualities are useful for downstream tools like +variant callers. For instance, a variant caller might choose to ignore +evidence from alignments with mapping quality less than, say, 10. A +mapping quality of 10 or less indicates that there is at least a 1 in 10 +chance that the read truly originated elsewhere.</p> <h2 id="aligning-pairs">Aligning pairs</h2> -<p>A "paired-end" or "mate-pair" read consists of pair of mates, called mate 1 and mate 2. Pairs come with a prior expectation about (a) the relative orientation of the mates, and (b) the distance separating them on the original DNA molecule. Exactly what expectations hold for a given dataset depends on the lab procedures used to generate the data. For example, a common lab procedure for producing pairs is Illumina's Paired-end Sequencing Assay, which yields pairs with a relative orientation of FR ("forward, reverse") meaning that if mate 1 came from the Watson strand, mate 2 very likely came from the Crick strand and vice versa. Also, this protocol yields pairs where the expected genomic distance from end to end is about 200-500 base pairs.</p> -<p>For simplicity, this manual uses the term "paired-end" to refer to any pair of reads with some expected relative orientation and distance. Depending on the protocol, these might actually be referred to as "paired-end" or "mate-paired." Also, we always refer to the individual sequences making up the pair as "mates."</p> +<p>A "paired-end" or "mate-pair" read consists of pair of mates, called +mate 1 and mate 2. Pairs come with a prior expectation about (a) the +relative orientation of the mates, and (b) the distance separating them +on the original DNA molecule. Exactly what expectations hold for a given +dataset depends on the lab procedures used to generate the data. For +example, a common lab procedure for producing pairs is Illumina's +Paired-end Sequencing Assay, which yields pairs with a relative +orientation of FR ("forward, reverse") meaning that if mate 1 came from +the Watson strand, mate 2 very likely came from the Crick strand and +vice versa. Also, this protocol yields pairs where the expected genomic +distance from end to end is about 200-500 base pairs.</p> +<p>For simplicity, this manual uses the term "paired-end" to refer to +any pair of reads with some expected relative orientation and distance. +Depending on the protocol, these might actually be referred to as +"paired-end" or "mate-paired." Also, we always refer to the individual +sequences making up the pair as "mates."</p> <h3 id="paired-inputs">Paired inputs</h3> -<p>Pairs are often stored in a pair of files, one file containing the mate 1s and the other containing the mates 2s. The first mate in the file for mate 1 forms a pair with the first mate in the file for mate 2, the second with the second, and so on. When aligning pairs with Bowtie 2, specify the file with the mate 1s mates using the <a href="#bowtie2-options-1"><code>-1</code></a> argument and the file with the mate 2s using the <a href="#bowtie2-options-2"><code>-2</code></a> argument. This causes Bowtie 2 to take the paired nature of the reads into account when aligning them.</p> +<p>Pairs are often stored in a pair of files, one file containing the +mate 1s and the other containing the mates 2s. The first mate in the +file for mate 1 forms a pair with the first mate in the file for mate 2, +the second with the second, and so on. When aligning pairs with Bowtie +2, specify the file with the mate 1s mates using the <a +href="#bowtie2-options-1"><code>-1</code></a> argument and the file with +the mate 2s using the <a href="#bowtie2-options-2"><code>-2</code></a> +argument. This causes Bowtie 2 to take the paired nature of the reads +into account when aligning them.</p> <h3 id="paired-sam-output">Paired SAM output</h3> -<p>When Bowtie 2 prints a SAM alignment for a pair, it prints two records (i.e. two lines of output), one for each mate. The first record describes the alignment for mate 1 and the second record describes the alignment for mate 2. In both records, some of the fields of the SAM record describe various properties of the alignment; for instance, the 7th and 8th fields (<code>RNEXT</code> and <code>PNEXT</code> respectively) indicate the reference name and position where the other mate aligned, and the 9th field indicates the inferred length of the DNA fragment from which the two mates were sequenced. See the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> for more details regarding these fields.</p> -<h3 id="concordant-pairs-match-pair-expectations-discordant-pairs-dont">Concordant pairs match pair expectations, discordant pairs don't</h3> -<p>A pair that aligns with the expected relative mate orientation and with the expected range of distances between mates is said to align "concordantly". If both mates have unique alignments, but the alignments do not match paired-end expectations (i.e. the mates aren't in the expected relative orientation, or aren't within the expected distance range, or both), the pair is said to align "discordantly". Discordant alignments may be of particular interest, for instance, when seeking <a href="http://www.ncbi.nlm.nih.gov/dbvar/content/overview/">structural variants</a>.</p> -<p>The expected relative orientation of the mates is set using the <a href="#bowtie2-options-fr"><code>--ff</code></a>, <a href="#bowtie2-options-fr"><code>--fr</code></a>, or <a href="#bowtie2-options-fr"><code>--rf</code></a> options. The expected range of inter-mates distances (as measured from the furthest extremes of the mates; also called "outer distance") is set with the <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a> options. Note that setting <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a> far apart makes Bowtie 2 slower. See documentation for <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a>.</p> -<p>To declare that a pair aligns discordantly, Bowtie 2 requires that both mates align uniquely. This is a conservative threshold, but this is often desirable when seeking structural variants.</p> -<p>By default, Bowtie 2 searches for both concordant and discordant alignments, though searching for discordant alignments can be disabled with the <a href="#bowtie2-options-no-discordant"><code>--no-discordant</code></a> option.</p> -<h3 id="mixed-mode-paired-where-possible-unpaired-otherwise">Mixed mode: paired where possible, unpaired otherwise</h3> -<p>If Bowtie 2 cannot find a paired-end alignment for a pair, by default it will go on to look for unpaired alignments for the constituent mates. This is called "mixed mode." To disable mixed mode, set the <a href="#bowtie2-options-no-mixed"><code>--no-mixed</code></a> option.</p> -<p>Bowtie 2 runs a little faster in <code>--no-mixed</code> mode, but will only consider alignment status of pairs per se, not individual mates.</p> -<h3 id="some-sam-flags-describe-paired-end-properties">Some SAM FLAGS describe paired-end properties</h3> -<p>The SAM <code>FLAGS</code> field, the second field in a SAM record, has multiple bits that describe the paired-end nature of the read and alignment. The first (least significant) bit (1 in decimal, 0x1 in hexadecimal) is set if the read is part of a pair. The second bit (2 in decimal, 0x2 in hexadecimal) is set if the read is part of a pair that aligned in a paired-end fashion. The fourth bit (8 in decimal, 0x8 in hexadecimal) is set if the read is part of a pair and the other mate in the pair had at least one valid alignment. The sixth bit (32 in decimal, 0x20 in hexadecimal) is set if the read is part of a pair and the other mate in the pair aligned to the Crick strand (or, equivalently, if the reverse complement of the other mate aligned to the Watson strand). The seventh bit (64 in decimal, 0x40 in hexadecimal) is set if the read is mate 1 in a pair. The eighth bit (128 in decimal, 0x80 in hexadecimal) is set if the read is mate 2 in a pair. See the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> for a more detailed description of the <code>FLAGS</code> field.</p> -<h3 id="some-sam-optional-fields-describe-more-paired-end-properties">Some SAM optional fields describe more paired-end properties</h3> -<p>The last several fields of each SAM record usually contain SAM optional fields, which are simply tab-separated strings conveying additional information about the reads and alignments. A SAM optional field is formatted like this: "XP:i:1" where "XP" is the <code>TAG</code>, "i" is the <code>TYPE</code> ("integer" in this case), and "1" is the <code>VALUE</code>. See the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> for details regarding SAM optional fields.</p> -<h3 id="mates-can-overlap-contain-or-dovetail-each-other">Mates can overlap, contain, or dovetail each other</h3> -<p>The fragment and read lengths might be such that alignments for the two mates from a pair overlap each other. Consider this example:</p> -<p>(For these examples, assume we expect mate 1 to align to the left of mate 2.)</p> +<p>When Bowtie 2 prints a SAM alignment for a pair, it prints two +records (i.e. two lines of output), one for each mate. The first record +describes the alignment for mate 1 and the second record describes the +alignment for mate 2. In both records, some of the fields of the SAM +record describe various properties of the alignment; for instance, the +7th and 8th fields (<code>RNEXT</code> and <code>PNEXT</code> +respectively) indicate the reference name and position where the other +mate aligned, and the 9th field indicates the inferred length of the DNA +fragment from which the two mates were sequenced. See the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> +for more details regarding these fields.</p> +<h3 +id="concordant-pairs-match-pair-expectations-discordant-pairs-dont">Concordant +pairs match pair expectations, discordant pairs don't</h3> +<p>A pair that aligns with the expected relative mate orientation and +with the expected range of distances between mates is said to align +"concordantly". If both mates have unique alignments, but the alignments +do not match paired-end expectations (i.e. the mates aren't in the +expected relative orientation, or aren't within the expected distance +range, or both), the pair is said to align "discordantly". Discordant +alignments may be of particular interest, for instance, when seeking <a +href="http://www.ncbi.nlm.nih.gov/dbvar/content/overview/">structural +variants</a>.</p> +<p>The expected relative orientation of the mates is set using the <a +href="#bowtie2-options-fr"><code>--ff</code></a>, <a +href="#bowtie2-options-fr"><code>--fr</code></a>, or <a +href="#bowtie2-options-fr"><code>--rf</code></a> options. The expected +range of inter-mates distances (as measured from the furthest extremes +of the mates; also called "outer distance") is set with the <a +href="#bowtie2-options-I"><code>-I</code></a> and <a +href="#bowtie2-options-X"><code>-X</code></a> options. Note that setting +<a href="#bowtie2-options-I"><code>-I</code></a> and <a +href="#bowtie2-options-X"><code>-X</code></a> far apart makes Bowtie 2 +slower. See documentation for <a +href="#bowtie2-options-I"><code>-I</code></a> and <a +href="#bowtie2-options-X"><code>-X</code></a>.</p> +<p>To declare that a pair aligns discordantly, Bowtie 2 requires that +both mates align uniquely. This is a conservative threshold, but this is +often desirable when seeking structural variants.</p> +<p>By default, Bowtie 2 searches for both concordant and discordant +alignments, though searching for discordant alignments can be disabled +with the <a +href="#bowtie2-options-no-discordant"><code>--no-discordant</code></a> +option.</p> +<h3 id="mixed-mode-paired-where-possible-unpaired-otherwise">Mixed mode: +paired where possible, unpaired otherwise</h3> +<p>If Bowtie 2 cannot find a paired-end alignment for a pair, by default +it will go on to look for unpaired alignments for the constituent mates. +This is called "mixed mode." To disable mixed mode, set the <a +href="#bowtie2-options-no-mixed"><code>--no-mixed</code></a> option.</p> +<p>Bowtie 2 runs a little faster in <code>--no-mixed</code> mode, but +will only consider alignment status of pairs per se, not individual +mates.</p> +<h3 id="some-sam-flags-describe-paired-end-properties">Some SAM FLAGS +describe paired-end properties</h3> +<p>The SAM <code>FLAGS</code> field, the second field in a SAM record, +has multiple bits that describe the paired-end nature of the read and +alignment. The first (least significant) bit (1 in decimal, 0x1 in +hexadecimal) is set if the read is part of a pair. The second bit (2 in +decimal, 0x2 in hexadecimal) is set if the read is part of a pair that +aligned in a paired-end fashion. The fourth bit (8 in decimal, 0x8 in +hexadecimal) is set if the read is part of a pair and the other mate in +the pair had at least one valid alignment. The sixth bit (32 in decimal, +0x20 in hexadecimal) is set if the read is part of a pair and the other +mate in the pair aligned to the Crick strand (or, equivalently, if the +reverse complement of the other mate aligned to the Watson strand). The +seventh bit (64 in decimal, 0x40 in hexadecimal) is set if the read is +mate 1 in a pair. The eighth bit (128 in decimal, 0x80 in hexadecimal) +is set if the read is mate 2 in a pair. See the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> +for a more detailed description of the <code>FLAGS</code> field.</p> +<h3 +id="some-sam-optional-fields-describe-more-paired-end-properties">Some +SAM optional fields describe more paired-end properties</h3> +<p>The last several fields of each SAM record usually contain SAM +optional fields, which are simply tab-separated strings conveying +additional information about the reads and alignments. A SAM optional +field is formatted like this: "XP:i:1" where "XP" is the +<code>TAG</code>, "i" is the <code>TYPE</code> ("integer" in this case), +and "1" is the <code>VALUE</code>. See the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> +for details regarding SAM optional fields.</p> +<h3 id="mates-can-overlap-contain-or-dovetail-each-other">Mates can +overlap, contain, or dovetail each other</h3> +<p>The fragment and read lengths might be such that alignments for the +two mates from a pair overlap each other. Consider this example:</p> +<p>(For these examples, assume we expect mate 1 to align to the left of +mate 2.)</p> <pre><code>Mate 1: GCAGATTATATGAGTCAGCTACGATATTGTT Mate 2: TGTTTGGGGTGACACATTACGCGTCTTTGAC Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pre> -<p>It's also possible, though unusual, for one mate alignment to contain the other, as in these examples:</p> +<p>It's also possible, though unusual, for one mate alignment to contain +the other, as in these examples:</p> <pre><code>Mate 1: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGC Mate 2: TGTTTGGGGTGACACATTACGC Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC @@ -237,64 +621,269 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC Mate 1: CAGCTACGATATTGTTTGGGGTGACACATTACGC Mate 2: CTACGATATTGTTTGGGGTGAC Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pre> -<p>And it's also possible, though unusual, for the mates to "dovetail", with the mates seemingly extending "past" each other as in this example:</p> +<p>And it's also possible, though unusual, for the mates to "dovetail", +with the mates seemingly extending "past" each other as in this +example:</p> <pre><code>Mate 1: GTCAGCTACGATATTGTTTGGGGTGACACATTACGC Mate 2: TATGAGTCAGCTACGATATTGTTTGGGGTGACACAT Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pre> -<p>In some situations, it's desirable for the aligner to consider all these cases as "concordant" as long as other paired-end constraints are not violated. Bowtie 2's default behavior is to consider overlapping and containing as being consistent with concordant alignment. By default, dovetailing is considered inconsistent with concordant alignment.</p> -<p>These defaults can be overridden. Setting <a href="#bowtie2-options-no-overlap"><code>--no-overlap</code></a> causes Bowtie 2 to consider overlapping mates as non-concordant. Setting <a href="#bowtie2-options-no-contain"><code>--no-contain</code></a> causes Bowtie 2 to consider cases where one mate alignment contains the other as non-concordant. Setting <a href="#bowtie2-options-dovetail"><code>--dovetail</code></a> causes Bowtie 2 to consider cases where the mate alignments dovetail as concordant.</p> +<p>In some situations, it's desirable for the aligner to consider all +these cases as "concordant" as long as other paired-end constraints are +not violated. Bowtie 2's default behavior is to consider overlapping and +containing as being consistent with concordant alignment. By default, +dovetailing is considered inconsistent with concordant alignment.</p> +<p>These defaults can be overridden. Setting <a +href="#bowtie2-options-no-overlap"><code>--no-overlap</code></a> causes +Bowtie 2 to consider overlapping mates as non-concordant. Setting <a +href="#bowtie2-options-no-contain"><code>--no-contain</code></a> causes +Bowtie 2 to consider cases where one mate alignment contains the other +as non-concordant. Setting <a +href="#bowtie2-options-dovetail"><code>--dovetail</code></a> causes +Bowtie 2 to consider cases where the mate alignments dovetail as +concordant.</p> <h2 id="reporting">Reporting</h2> -<p>The reporting mode governs how many alignments Bowtie 2 looks for, and how to report them. Bowtie 2 has three distinct reporting modes. The default reporting mode is similar to the default reporting mode of many other read alignment tools, including <a href="http://bio-bwa.sourceforge.net/">BWA</a>. It is also similar to Bowtie 1's <code>-M</code> alignment mode.</p> -<p>In general, when we say that a read has an alignment, we mean that it has a <a href="#valid-alignments-meet-or-exceed-the-minimum-score-threshold">valid alignment</a>. When we say that a read has multiple alignments, we mean that it has multiple alignments that are valid and distinct from one another.</p> -<h3 id="distinct-alignments-map-a-read-to-different-places">Distinct alignments map a read to different places</h3> -<p>Two alignments for the same individual read are "distinct" if they map the same read to different places. Specifically, we say that two alignments are distinct if there are no alignment positions where a particular read offset is aligned opposite a particular reference offset in both alignments with the same orientation. E.g. if the first alignment is in the forward orientation and aligns the read character at read offset 10 to the reference character at chromosome 3, offset 3,445,245, and the second alignment is also in the forward orientation and also aligns the read character at read offset 10 to the reference character at chromosome 3, offset 3,445,245, they are not distinct alignments.</p> -<p>Two alignments for the same pair are distinct if either the mate 1s in the two paired-end alignments are distinct or the mate 2s in the two alignments are distinct or both.</p> -<h3 id="default-mode-search-for-multiple-alignments-report-the-best-one">Default mode: search for multiple alignments, report the best one</h3> -<p>By default, Bowtie 2 searches for distinct, valid alignments for each read. When it finds a valid alignment, it generally will continue to look for alignments that are nearly as good or better. It will eventually stop looking, either because it exceeded a limit placed on search effort (see <a href="#bowtie2-options-D"><code>-D</code></a> and <a href="#bowtie2-options-R"><code>-R</code></a>) or because it already knows all it needs to know to report an alignment. Information from the best alignments are used to estimate mapping quality (the <code>MAPQ</code> <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> field) and to set SAM optional fields, such as <a href="#bowtie2-build-opt-fields-as"><code>AS:i</code></a> and <a href="#bowtie2-build-opt-fields-xs"><code>XS:i</code></a>. Bowtie 2 does not guarantee that the alignment reported is the best possible in terms of alignment score.</p> -<p>See also: <a href="#bowtie2-options-D"><code>-D</code></a>, which puts an upper limit on the number of dynamic programming problems (i.e. seed extensions) that can "fail" in a row before Bowtie 2 stops searching. Increasing <a href="#bowtie2-options-D"><code>-D</code></a> makes Bowtie 2 slower, but increases the likelihood that it will report the correct alignment for a read that aligns many places.</p> -<p>See also: <a href="#bowtie2-options-R"><code>-R</code></a>, which sets the maximum number of times Bowtie 2 will "re-seed" when attempting to align a read with repetitive seeds. Increasing <a href="#bowtie2-options-R"><code>-R</code></a> makes Bowtie 2 slower, but increases the likelihood that it will report the correct alignment for a read that aligns many places.</p> -<h3 id="k-mode-search-for-one-or-more-alignments-report-each">-k mode: search for one or more alignments, report each</h3> -<p>In <a href="#bowtie2-options-k"><code>-k</code></a> mode, Bowtie 2 searches for up to N distinct, valid alignments for each read, where N equals the integer specified with the <code>-k</code> parameter. That is, if <code>-k 2</code> is specified, Bowtie 2 will search for at most 2 distinct alignments. It reports all alignments found, in descending order by alignment score. The alignment score for a paired-end alignment equals the sum of the alignment scores of the individual mates. Each reported read or pair alignment beyond the first has the SAM 'secondary' bit (which equals 256) set in its FLAGS field. Supplementary alignments will also be assigned a MAPQ of 255. See the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> for details.</p> -<p>Bowtie 2 does not "find" alignments in any specific order, so for reads that have more than N distinct, valid alignments, Bowtie 2 does not guarantee that the N alignments reported are the best possible in terms of alignment score. Still, this mode can be effective and fast in situations where the user cares more about whether a read aligns (or aligns a certain number of times) than where exactly it originated.</p> -<h3 id="a-mode-search-for-and-report-all-alignments">-a mode: search for and report all alignments</h3> -<p><a href="#bowtie2-options-a"><code>-a</code></a> mode is similar to <a href="#bowtie2-options-k"><code>-k</code></a> mode except that there is no upper limit on the number of alignments Bowtie 2 should report. Alignments are reported in descending order by alignment score. The alignment score for a paired-end alignment equals the sum of the alignment scores of the individual mates. Each reported read or pair alignment beyond the first has the SAM 'secondary' bit (which equals 256) set in its FLAGS field. Supplementary alignments will be assigned a MAPQ of 255. See the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> for details.</p> -<p>Some tools are designed with this reporting mode in mind. Bowtie 2 is not! For very large genomes, this mode is very slow.</p> +<p>The reporting mode governs how many alignments Bowtie 2 looks for, +and how to report them. Bowtie 2 has three distinct reporting modes. The +default reporting mode is similar to the default reporting mode of many +other read alignment tools, including <a +href="http://bio-bwa.sourceforge.net/">BWA</a>. It is also similar to +Bowtie 1's <code>-M</code> alignment mode.</p> +<p>In general, when we say that a read has an alignment, we mean that it +has a <a +href="#valid-alignments-meet-or-exceed-the-minimum-score-threshold">valid +alignment</a>. When we say that a read has multiple alignments, we mean +that it has multiple alignments that are valid and distinct from one +another.</p> +<h3 id="distinct-alignments-map-a-read-to-different-places">Distinct +alignments map a read to different places</h3> +<p>Two alignments for the same individual read are "distinct" if they +map the same read to different places. Specifically, we say that two +alignments are distinct if there are no alignment positions where a +particular read offset is aligned opposite a particular reference offset +in both alignments with the same orientation. E.g. if the first +alignment is in the forward orientation and aligns the read character at +read offset 10 to the reference character at chromosome 3, offset +3,445,245, and the second alignment is also in the forward orientation +and also aligns the read character at read offset 10 to the reference +character at chromosome 3, offset 3,445,245, they are not distinct +alignments.</p> +<p>Two alignments for the same pair are distinct if either the mate 1s +in the two paired-end alignments are distinct or the mate 2s in the two +alignments are distinct or both.</p> +<h3 +id="default-mode-search-for-multiple-alignments-report-the-best-one">Default +mode: search for multiple alignments, report the best one</h3> +<p>By default, Bowtie 2 searches for distinct, valid alignments for each +read. When it finds a valid alignment, it generally will continue to +look for alignments that are nearly as good or better. It will +eventually stop looking, either because it exceeded a limit placed on +search effort (see <a href="#bowtie2-options-D"><code>-D</code></a> and +<a href="#bowtie2-options-R"><code>-R</code></a>) or because it already +knows all it needs to know to report an alignment. Information from the +best alignments are used to estimate mapping quality (the +<code>MAPQ</code> <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> field) and to +set SAM optional fields, such as <a +href="#bowtie2-build-opt-fields-as"><code>AS:i</code></a> and <a +href="#bowtie2-build-opt-fields-xs"><code>XS:i</code></a>. Bowtie 2 does +not guarantee that the alignment reported is the best possible in terms +of alignment score.</p> +<p>See also: <a href="#bowtie2-options-D"><code>-D</code></a>, which +puts an upper limit on the number of dynamic programming problems (i.e. +seed extensions) that can "fail" in a row before Bowtie 2 stops +searching. Increasing <a href="#bowtie2-options-D"><code>-D</code></a> +makes Bowtie 2 slower, but increases the likelihood that it will report +the correct alignment for a read that aligns many places.</p> +<p>See also: <a href="#bowtie2-options-R"><code>-R</code></a>, which +sets the maximum number of times Bowtie 2 will "re-seed" when attempting +to align a read with repetitive seeds. Increasing <a +href="#bowtie2-options-R"><code>-R</code></a> makes Bowtie 2 slower, but +increases the likelihood that it will report the correct alignment for a +read that aligns many places.</p> +<h3 id="k-mode-search-for-one-or-more-alignments-report-each">-k mode: +search for one or more alignments, report each</h3> +<p>In <a href="#bowtie2-options-k"><code>-k</code></a> mode, Bowtie 2 +searches for up to N distinct, valid alignments for each read, where N +equals the integer specified with the <code>-k</code> parameter. That +is, if <code>-k 2</code> is specified, Bowtie 2 will search for at most +2 distinct alignments. It reports all alignments found, in descending +order by alignment score. The alignment score for a paired-end alignment +equals the sum of the alignment scores of the individual mates. Each +reported read or pair alignment beyond the first has the SAM 'secondary' +bit (which equals 256) set in its FLAGS field. Supplementary alignments +will also be assigned a MAPQ of 255. See the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> +for details.</p> +<p>Bowtie 2 does not "find" alignments in any specific order, so for +reads that have more than N distinct, valid alignments, Bowtie 2 does +not guarantee that the N alignments reported are the best possible in +terms of alignment score. Still, this mode can be effective and fast in +situations where the user cares more about whether a read aligns (or +aligns a certain number of times) than where exactly it originated.</p> +<h3 id="a-mode-search-for-and-report-all-alignments">-a mode: search for +and report all alignments</h3> +<p><a href="#bowtie2-options-a"><code>-a</code></a> mode is similar to +<a href="#bowtie2-options-k"><code>-k</code></a> mode except that there +is no upper limit on the number of alignments Bowtie 2 should report. +Alignments are reported in descending order by alignment score. The +alignment score for a paired-end alignment equals the sum of the +alignment scores of the individual mates. Each reported read or pair +alignment beyond the first has the SAM 'secondary' bit (which equals +256) set in its FLAGS field. Supplementary alignments will be assigned a +MAPQ of 255. See the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> +for details.</p> +<p>Some tools are designed with this reporting mode in mind. Bowtie 2 is +not! For very large genomes, this mode is very slow.</p> <h3 id="randomness-in-bowtie-2">Randomness in Bowtie 2</h3> -<p>Bowtie 2's search for alignments for a given read is "randomized." That is, when Bowtie 2 encounters a set of equally-good choices, it uses a pseudo-random number to choose. For example, if Bowtie 2 discovers a set of 3 equally-good alignments and wants to decide which to report, it picks a pseudo-random integer 0, 1 or 2 and reports the corresponding alignment. Arbitrary choices can crop up at various points during alignment.</p> -<p>The pseudo-random number generator is re-initialized for every read, and the seed used to initialize it is a function of the read name, nucleotide string, quality string, and the value specified with <a href="#bowtie2-options-seed"><code>--seed</code></a>. If you run the same version of Bowtie 2 on two reads with identical names, nucleotide strings, and quality strings, and if <a href="#bowtie2-options-seed"><code>--seed</code></a> is set the same for both runs, Bowtie 2 will produce the same output; i.e., it will align the read to the same place, even if there are multiple equally good alignments. This is intuitive and desirable in most cases. Most users expect Bowtie to produce the same output when run twice on the same input.</p> -<p>However, when the user specifies the <a href="#bowtie2-options-non-deterministic"><code>--non-deterministic</code></a> option, Bowtie 2 will use the current time to re-initialize the pseudo-random number generator. When this is specified, Bowtie 2 might report different alignments for identical reads. This is counter-intuitive for some users, but might be more appropriate in situations where the input consists of many identical reads.</p> +<p>Bowtie 2's search for alignments for a given read is "randomized." +That is, when Bowtie 2 encounters a set of equally-good choices, it uses +a pseudo-random number to choose. For example, if Bowtie 2 discovers a +set of 3 equally-good alignments and wants to decide which to report, it +picks a pseudo-random integer 0, 1 or 2 and reports the corresponding +alignment. Arbitrary choices can crop up at various points during +alignment.</p> +<p>The pseudo-random number generator is re-initialized for every read, +and the seed used to initialize it is a function of the read name, +nucleotide string, quality string, and the value specified with <a +href="#bowtie2-options-seed"><code>--seed</code></a>. If you run the +same version of Bowtie 2 on two reads with identical names, nucleotide +strings, and quality strings, and if <a +href="#bowtie2-options-seed"><code>--seed</code></a> is set the same for +both runs, Bowtie 2 will produce the same output; i.e., it will align +the read to the same place, even if there are multiple equally good +alignments. This is intuitive and desirable in most cases. Most users +expect Bowtie to produce the same output when run twice on the same +input.</p> +<p>However, when the user specifies the <a +href="#bowtie2-options-non-deterministic"><code>--non-deterministic</code></a> +option, Bowtie 2 will use the current time to re-initialize the +pseudo-random number generator. When this is specified, Bowtie 2 might +report different alignments for identical reads. This is +counter-intuitive for some users, but might be more appropriate in +situations where the input consists of many identical reads.</p> <h2 id="multiseed-heuristic">Multiseed heuristic</h2> -<p>To rapidly narrow the number of possible alignments that must be considered, Bowtie 2 begins by extracting substrings ("seeds") from the read and its reverse complement and aligning them in an ungapped fashion with the help of the <a href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a>. This is "multiseed alignment" and it is similar to what <a href="http://genomebiology.com/2009/10/3/R25">Bowtie 1 does</a>, except Bowtie 1 attempts to align the entire read this way.</p> -<p>This initial step makes Bowtie 2 much faster than it would be without such a filter, but at the expense of missing some valid alignments. For instance, it is possible for a read to have a valid overall alignment but to have no valid seed alignments because each potential seed alignment is interrupted by too many mismatches or gaps.</p> -<p>The trade-off between speed and sensitivity/accuracy can be adjusted by setting the seed length (<a href="#bowtie2-options-L"><code>-L</code></a>), the interval between extracted seeds (<a href="#bowtie2-options-i"><code>-i</code></a>), and the number of mismatches permitted per seed (<a href="#bowtie2-options-N"><code>-N</code></a>). For more sensitive alignment, set these parameters to (a) make the seeds closer together, (b) make the seeds shorter, and/or (c) allow more mismatches. You can adjust these options one-by-one, though Bowtie 2 comes with some useful combinations of options prepackaged as "<a href="#presets-setting-many-settings-at-once">preset options</a>."</p> -<p><a href="#bowtie2-options-D"><code>-D</code></a> and <a href="#bowtie2-options-R"><code>-R</code></a> are also options that adjust the trade-off between speed and sensitivity/accuracy.</p> +<p>To rapidly narrow the number of possible alignments that must be +considered, Bowtie 2 begins by extracting substrings ("seeds") from the +read and its reverse complement and aligning them in an ungapped fashion +with the help of the <a +href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a>. This +is "multiseed alignment" and it is similar to what <a +href="http://genomebiology.com/2009/10/3/R25">Bowtie 1 does</a>, except +Bowtie 1 attempts to align the entire read this way.</p> +<p>This initial step makes Bowtie 2 much faster than it would be without +such a filter, but at the expense of missing some valid alignments. For +instance, it is possible for a read to have a valid overall alignment +but to have no valid seed alignments because each potential seed +alignment is interrupted by too many mismatches or gaps.</p> +<p>The trade-off between speed and sensitivity/accuracy can be adjusted +by setting the seed length (<a +href="#bowtie2-options-L"><code>-L</code></a>), the interval between +extracted seeds (<a href="#bowtie2-options-i"><code>-i</code></a>), and +the number of mismatches permitted per seed (<a +href="#bowtie2-options-N"><code>-N</code></a>). For more sensitive +alignment, set these parameters to (a) make the seeds closer together, +(b) make the seeds shorter, and/or (c) allow more mismatches. You can +adjust these options one-by-one, though Bowtie 2 comes with some useful +combinations of options prepackaged as "<a +href="#presets-setting-many-settings-at-once">preset options</a>."</p> +<p><a href="#bowtie2-options-D"><code>-D</code></a> and <a +href="#bowtie2-options-R"><code>-R</code></a> are also options that +adjust the trade-off between speed and sensitivity/accuracy.</p> <h3 id="fm-index-memory-footprint">FM Index memory footprint</h3> -<p>Bowtie 2 uses the <a href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> to find ungapped alignments for seeds. This step accounts for the bulk of Bowtie 2's memory footprint, as the <a href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> itself is typically the largest data structure used. For instance, the memory footprint of the <a href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> for the human genome is about 3.2 gigabytes of RAM.</p> +<p>Bowtie 2 uses the <a +href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> to find +ungapped alignments for seeds. This step accounts for the bulk of Bowtie +2's memory footprint, as the <a +href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> itself +is typically the largest data structure used. For instance, the memory +footprint of the <a +href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> for the +human genome is about 3.2 gigabytes of RAM.</p> <h2 id="ambiguous-characters">Ambiguous characters</h2> -<p>Non-whitespace characters besides A, C, G or T are considered "ambiguous." N is a common ambiguous character that appears in reference sequences. Bowtie 2 considers all ambiguous characters in the reference (including <a href="http://www.bioinformatics.org/sms/iupac.html">IUPAC nucleotide codes</a>) to be Ns.</p> -<p>Bowtie 2 allows alignments to overlap ambiguous characters in the reference. An alignment position that contains an ambiguous character in the read, reference, or both, is penalized according to <a href="#bowtie2-options-np"><code>--np</code></a>. <a href="#bowtie2-options-n-ceil"><code>--n-ceil</code></a> sets an upper limit on the number of positions that may contain ambiguous reference characters in a valid alignment. The optional field <a href="#bowtie2-build-opt-fields-xn"><code>XN:i</code></a> reports the number of ambiguous reference characters overlapped by an alignment.</p> -<p>Note that the <a href="#multiseed-heuristic">multiseed heuristic</a> cannot find <em>seed</em> alignments that overlap ambiguous reference characters. For an alignment overlapping an ambiguous reference character to be found, it must have one or more seed alignments that do not overlap ambiguous reference characters.</p> -<h2 id="presets-setting-many-settings-at-once">Presets: setting many settings at once</h2> -<p>Bowtie 2 comes with some useful combinations of parameters packaged into shorter "preset" parameters. For example, running Bowtie 2 with the <a href="#bowtie2-options-very-sensitive"><code>--very-sensitive</code></a> option is the same as running with options: <code>-D 20 -R 3 -N 0 -L 20 -i S,1,0.50</code>. The preset options that come with Bowtie 2 are designed to cover a wide area of the speed/sensitivity/accuracy trade-off space, with the presets ending in <code>fast</code> generally being faster but less sensitive and less accurate, and the presets ending in <code>sensitive</code> generally being slower but more sensitive and more accurate. See the <a href="#preset-options-in---end-to-end-mode">documentation for the preset options</a> for details.</p> -<p>As of Bowtie2 v2.4.0, individual preset values can be overridden by providing the specific options e.g. the configured seed length of 20 in the [<code>--very-senitive</code>] preset above can be changed to 25 by also specifying the <code>-L 25</code> parameter anywhere on the command line.</p> +<p>Non-whitespace characters besides A, C, G or T are considered +"ambiguous." N is a common ambiguous character that appears in reference +sequences. Bowtie 2 considers all ambiguous characters in the reference +(including <a href="http://www.bioinformatics.org/sms/iupac.html">IUPAC +nucleotide codes</a>) to be Ns.</p> +<p>Bowtie 2 allows alignments to overlap ambiguous characters in the +reference. An alignment position that contains an ambiguous character in +the read, reference, or both, is penalized according to <a +href="#bowtie2-options-np"><code>--np</code></a>. <a +href="#bowtie2-options-n-ceil"><code>--n-ceil</code></a> sets an upper +limit on the number of positions that may contain ambiguous reference +characters in a valid alignment. The optional field <a +href="#bowtie2-build-opt-fields-xn"><code>XN:i</code></a> reports the +number of ambiguous reference characters overlapped by an alignment.</p> +<p>Note that the <a href="#multiseed-heuristic">multiseed heuristic</a> +cannot find <em>seed</em> alignments that overlap ambiguous reference +characters. For an alignment overlapping an ambiguous reference +character to be found, it must have one or more seed alignments that do +not overlap ambiguous reference characters.</p> +<h2 id="presets-setting-many-settings-at-once">Presets: setting many +settings at once</h2> +<p>Bowtie 2 comes with some useful combinations of parameters packaged +into shorter "preset" parameters. For example, running Bowtie 2 with the +<a +href="#bowtie2-options-very-sensitive"><code>--very-sensitive</code></a> +option is the same as running with options: +<code>-D 20 -R 3 -N 0 -L 20 -i S,1,0.50</code>. The preset options that +come with Bowtie 2 are designed to cover a wide area of the +speed/sensitivity/accuracy trade-off space, with the presets ending in +<code>fast</code> generally being faster but less sensitive and less +accurate, and the presets ending in <code>sensitive</code> generally +being slower but more sensitive and more accurate. See the <a +href="#preset-options-in---end-to-end-mode">documentation for the preset +options</a> for details.</p> +<p>As of Bowtie2 v2.4.0, individual preset values can be overridden by +providing the specific options e.g. the configured seed length of 20 in +the [<code>--very-senitive</code>] preset above can be changed to 25 by +also specifying the <code>-L 25</code> parameter anywhere on the command +line.</p> <h2 id="filtering">Filtering</h2> -<p>Some reads are skipped or "filtered out" by Bowtie 2. For example, reads may be filtered out because they are extremely short or have a high proportion of ambiguous nucleotides. Bowtie 2 will still print a SAM record for such a read, but no alignment will be reported and the <code>YF:i</code> SAM optional field will be set to indicate the reason the read was filtered.</p> +<p>Some reads are skipped or "filtered out" by Bowtie 2. For example, +reads may be filtered out because they are extremely short or have a +high proportion of ambiguous nucleotides. Bowtie 2 will still print a +SAM record for such a read, but no alignment will be reported and the +<code>YF:i</code> SAM optional field will be set to indicate the reason +the read was filtered.</p> <ul> -<li><code>YF:Z:LN</code>: the read was filtered because it had length less than or equal to the number of seed mismatches set with the <a href="#bowtie2-options-N"><code>-N</code></a> option.</li> -<li><code>YF:Z:NS</code>: the read was filtered because it contains a number of ambiguous characters (usually <code>N</code> or <code>.</code>) greater than the ceiling specified with <a href="#bowtie2-options-n-ceil"><code>--n-ceil</code></a>.</li> -<li><code>YF:Z:SC</code>: the read was filtered because the read length and the match bonus (set with <a href="#bowtie2-options-ma"><code>--ma</code></a>) are such that the read can't possibly earn an alignment score greater than or equal to the threshold set with <a href="#bowtie2-options-score-min"><code>--score-min</code></a></li> -<li><code>YF:Z:QC</code>: the read was filtered because it was marked as failing quality control and the user specified the <a href="#bowtie2-options-qc-filter"><code>--qc-filter</code></a> option. This only happens when the input is in Illumina's QSEQ format (i.e. when <a href="#bowtie2-options-qseq"><code>--qseq</code></a> is specified) and the last (11th) field of the read's QSEQ record contains <code>1</code>.</li> +<li><code>YF:Z:LN</code>: the read was filtered because it had length +less than or equal to the number of seed mismatches set with the <a +href="#bowtie2-options-N"><code>-N</code></a> option.</li> +<li><code>YF:Z:NS</code>: the read was filtered because it contains a +number of ambiguous characters (usually <code>N</code> or +<code>.</code>) greater than the ceiling specified with <a +href="#bowtie2-options-n-ceil"><code>--n-ceil</code></a>.</li> +<li><code>YF:Z:SC</code>: the read was filtered because the read length +and the match bonus (set with <a +href="#bowtie2-options-ma"><code>--ma</code></a>) are such that the read +can't possibly earn an alignment score greater than or equal to the +threshold set with <a +href="#bowtie2-options-score-min"><code>--score-min</code></a></li> +<li><code>YF:Z:QC</code>: the read was filtered because it was marked as +failing quality control and the user specified the <a +href="#bowtie2-options-qc-filter"><code>--qc-filter</code></a> option. +This only happens when the input is in Illumina's QSEQ format (i.e. when +<a href="#bowtie2-options-qseq"><code>--qseq</code></a> is specified) +and the last (11th) field of the read's QSEQ record contains +<code>1</code>.</li> </ul> -<p>If a read could be filtered for more than one reason, the value <code>YF:Z</code> flag will reflect only one of those reasons.</p> +<p>If a read could be filtered for more than one reason, the value +<code>YF:Z</code> flag will reflect only one of those reasons.</p> <h2 id="alignment-summary">Alignment summary</h2> -<p>When Bowtie 2 finishes running, it prints messages summarizing what happened. These messages are printed to the "standard error" ("stderr") filehandle. For datasets consisting of unpaired reads, the summary might look like this:</p> +<p>When Bowtie 2 finishes running, it prints messages summarizing what +happened. These messages are printed to the "standard error" ("stderr") +filehandle. For datasets consisting of unpaired reads, the summary might +look like this:</p> <pre><code>20000 reads; of these: 20000 (100.00%) were unpaired; of these: 1247 (6.24%) aligned 0 times 18739 (93.69%) aligned exactly 1 time 14 (0.07%) aligned >1 times 93.77% overall alignment rate</code></pre> -<p>For datasets consisting of pairs, the summary might look like this:</p> +<p>For datasets consisting of pairs, the summary might look like +this:</p> <pre><code>10000 reads; of these: 10000 (100.00%) were paired; of these: 650 (6.50%) aligned concordantly 0 times @@ -312,27 +901,82 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr 96.70% overall alignment rate</code></pre> <p>The indentation indicates how subtotals relate to totals.</p> <h2 id="wrapper-scripts">Wrapper scripts</h2> -<p>The <code>bowtie2</code>, <code>bowtie2-build</code> and <code>bowtie2-inspect</code> executables are actually wrapper scripts that call binary programs as appropriate. The wrappers shield users from having to distinguish between "small" and "large" index formats, discussed briefly in the following section. Also, the <code>bowtie2</code> wrapper provides some key functionality, like the ability to handle compressed inputs, and the functionality for <a href="#bowtie2-options-un"><code>--un</code></a>, <a href="#bowtie2-options-al"><code>--al</code></a> and related options.</p> -<p>It is recommended that you always run the bowtie2 wrappers and not run the binaries directly.</p> +<p>The <code>bowtie2</code>, <code>bowtie2-build</code> and +<code>bowtie2-inspect</code> executables are actually wrapper scripts +that call binary programs as appropriate. The wrappers shield users from +having to distinguish between "small" and "large" index formats, +discussed briefly in the following section. Also, the +<code>bowtie2</code> wrapper provides some key functionality, like the +ability to handle compressed inputs, and the functionality for <a +href="#bowtie2-options-un"><code>--un</code></a>, <a +href="#bowtie2-options-al"><code>--al</code></a> and related +options.</p> +<p>It is recommended that you always run the bowtie2 wrappers and not +run the binaries directly.</p> <h2 id="small-and-large-indexes">Small and large indexes</h2> -<p><code>bowtie2-build</code> can index reference genomes of any size. For genomes less than about 4 billion nucleotides in length, <code>bowtie2-build</code> builds a "small" index using 32-bit numbers in various parts of the index. When the genome is longer, <code>bowtie2-build</code> builds a "large" index using 64-bit numbers. Small indexes are stored in files with the <code>.bt2</code> extension, and large indexes are stored in files with the <code>.bt2l</code> extension. The user need not worry about whether a particular index is small or large; the wrapper scripts will automatically build and use the appropriate index.</p> +<p><code>bowtie2-build</code> can index reference genomes of any size. +For genomes less than about 4 billion nucleotides in length, +<code>bowtie2-build</code> builds a "small" index using 32-bit numbers +in various parts of the index. When the genome is longer, +<code>bowtie2-build</code> builds a "large" index using 64-bit numbers. +Small indexes are stored in files with the <code>.bt2</code> extension, +and large indexes are stored in files with the <code>.bt2l</code> +extension. The user need not worry about whether a particular index is +small or large; the wrapper scripts will automatically build and use the +appropriate index.</p> <h2 id="performance-tuning">Performance tuning</h2> <ol type="1"> -<li><p>If your computer has multiple processors/cores, use <code>-p</code></p> -<p>The <a href="#bowtie2-options-p"><code>-p</code></a> option causes Bowtie 2 to launch a specified number of parallel search threads. Each thread runs on a different processor/core and all threads find alignments in parallel, increasing alignment throughput by approximately a multiple of the number of threads (though in practice, speedup is somewhat worse than linear).</p></li> -<li><p>If reporting many alignments per read, try reducing <code>bowtie2-build --offrate</code></p> -<p>If you are using <a href="#bowtie2-options-k"><code>-k</code></a> or <a href="#bowtie2-options-a"><code>-a</code></a> options and Bowtie 2 is reporting many alignments per read, using an index with a denser SA sample can speed things up considerably. To do this, specify a smaller-than-default <a href="#bowtie2-options-o"><code>-o</code>/<code>--offrate</code></a> value when running <code>bowtie2-build</code>. A denser SA sample yields a larger index, but is also particularly effective at speeding up alignment when many alignments are reported per read.</p></li> -<li><p>If <code>bowtie2</code> "thrashes", try increasing <code>bowtie2-build --offrate</code></p> -<p>If <code>bowtie2</code> runs very slowly on a relatively low-memory computer, try setting <a href="#bowtie2-options-o"><code>-o</code>/<code>--offrate</code></a> to a <em>larger</em> value when building the index. This decreases the memory footprint of the index.</p></li> +<li><p>If your computer has multiple processors/cores, use +<code>-p</code></p> +<p>The <a href="#bowtie2-options-p"><code>-p</code></a> option causes +Bowtie 2 to launch a specified number of parallel search threads. Each +thread runs on a different processor/core and all threads find +alignments in parallel, increasing alignment throughput by approximately +a multiple of the number of threads (though in practice, speedup is +somewhat worse than linear).</p></li> +<li><p>If reporting many alignments per read, try reducing +<code>bowtie2-build --offrate</code></p> +<p>If you are using <a href="#bowtie2-options-k"><code>-k</code></a> or +<a href="#bowtie2-options-a"><code>-a</code></a> options and Bowtie 2 is +reporting many alignments per read, using an index with a denser SA +sample can speed things up considerably. To do this, specify a +smaller-than-default <a +href="#bowtie2-options-o"><code>-o</code>/<code>--offrate</code></a> +value when running <code>bowtie2-build</code>. A denser SA sample yields +a larger index, but is also particularly effective at speeding up +alignment when many alignments are reported per read.</p></li> +<li><p>If <code>bowtie2</code> "thrashes", try increasing +<code>bowtie2-build --offrate</code></p> +<p>If <code>bowtie2</code> runs very slowly on a relatively low-memory +computer, try setting <a +href="#bowtie2-options-o"><code>-o</code>/<code>--offrate</code></a> to +a <em>larger</em> value when building the index. This decreases the +memory footprint of the index.</p></li> </ol> <h2 id="command-line">Command Line</h2> <h3 id="setting-function-options">Setting function options</h3> -<p>Some Bowtie 2 options specify a function rather than an individual number or setting. In these cases the user specifies three parameters: (a) a function type <code>F</code>, (b) a constant term <code>B</code>, and (c) a coefficient <code>A</code>. The available function types are constant (<code>C</code>), linear (<code>L</code>), square-root (<code>S</code>), and natural log (<code>G</code>). The parameters are specified as <code>F,B,A</code> - that is, the function type, the constant term, and the coefficient are separated by commas with no whitespace. The constant term and coefficient may be negative and/or floating-point numbers.</p> -<p>For example, if the function specification is <code>L,-0.4,-0.6</code>, then the function defined is:</p> +<p>Some Bowtie 2 options specify a function rather than an individual +number or setting. In these cases the user specifies three parameters: +(a) a function type <code>F</code>, (b) a constant term <code>B</code>, +and (c) a coefficient <code>A</code>. The available function types are +constant (<code>C</code>), linear (<code>L</code>), square-root +(<code>S</code>), and natural log (<code>G</code>). The parameters are +specified as <code>F,B,A</code> - that is, the function type, the +constant term, and the coefficient are separated by commas with no +whitespace. The constant term and coefficient may be negative and/or +floating-point numbers.</p> +<p>For example, if the function specification is +<code>L,-0.4,-0.6</code>, then the function defined is:</p> <pre><code>f(x) = -0.4 + -0.6 * x</code></pre> -<p>If the function specification is <code>G,1,5.4</code>, then the function defined is:</p> +<p>If the function specification is <code>G,1,5.4</code>, then the +function defined is:</p> <pre><code>f(x) = 1.0 + 5.4 * ln(x)</code></pre> -<p>See the documentation for the option in question to learn what the parameter <code>x</code> is for. For example, in the case if the <a href="#bowtie2-options-score-min"><code>--score-min</code></a> option, the function <code>f(x)</code> sets the minimum alignment score necessary for an alignment to be considered valid, and <code>x</code> is the read length.</p> +<p>See the documentation for the option in question to learn what the +parameter <code>x</code> is for. For example, in the case if the <a +href="#bowtie2-options-score-min"><code>--score-min</code></a> option, +the function <code>f(x)</code> sets the minimum alignment score +necessary for an alignment to be considered valid, and <code>x</code> is +the read length.</p> <h3 id="usage">Usage</h3> <pre><code>bowtie2 [options]* -x <bt2-idx> {-1 <m1> -2 <m2> | -U <r> | --interleaved <i> | --sra-acc <acc> | b <bam>} -S [<sam>]</code></pre> <h3 id="main-arguments">Main arguments</h3> @@ -342,7 +986,12 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-x <bt2-idx></code></pre> </td> <td> -<p>The basename of the index for the reference genome. The basename is the name of any of the index files up to but not including the final <code>.1.bt2</code> / <code>.rev.1.bt2</code> / etc. <code>bowtie2</code> looks for the specified index first in the current directory, then in the directory specified in the <code>BOWTIE2_INDEXES</code> environment variable.</p> +<p>The basename of the index for the reference genome. The basename is +the name of any of the index files up to but not including the final +<code>.1.bt2</code> / <code>.rev.1.bt2</code> / etc. +<code>bowtie2</code> looks for the specified index first in the current +directory, then in the directory specified in the +<code>BOWTIE2_INDEXES</code> environment variable.</p> </td> </tr> <tr> @@ -350,7 +999,13 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-1 <m1></code></pre> </td> <td> -<p>Comma-separated list of files containing mate 1s (filename usually includes <code>_1</code>), e.g. <code>-1 flyA_1.fq,flyB_1.fq</code>. Sequences specified with this option must correspond file-for-file and read-for-read with those specified in <code><m2></code>. Reads may be a mix of different lengths. If <code>-</code> is specified, <code>bowtie2</code> will read the mate 1s from the "standard in" or "stdin" filehandle.</p> +<p>Comma-separated list of files containing mate 1s (filename usually +includes <code>_1</code>), e.g. <code>-1 flyA_1.fq,flyB_1.fq</code>. +Sequences specified with this option must correspond file-for-file and +read-for-read with those specified in <code><m2></code>. Reads may +be a mix of different lengths. If <code>-</code> is specified, +<code>bowtie2</code> will read the mate 1s from the "standard in" or +"stdin" filehandle.</p> </td> </tr> <tr> @@ -358,7 +1013,13 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-2 <m2></code></pre> </td> <td> -<p>Comma-separated list of files containing mate 2s (filename usually includes <code>_2</code>), e.g. <code>-2 flyA_2.fq,flyB_2.fq</code>. Sequences specified with this option must correspond file-for-file and read-for-read with those specified in <code><m1></code>. Reads may be a mix of different lengths. If <code>-</code> is specified, <code>bowtie2</code> will read the mate 2s from the "standard in" or "stdin" filehandle.</p> +<p>Comma-separated list of files containing mate 2s (filename usually +includes <code>_2</code>), e.g. <code>-2 flyA_2.fq,flyB_2.fq</code>. +Sequences specified with this option must correspond file-for-file and +read-for-read with those specified in <code><m1></code>. Reads may +be a mix of different lengths. If <code>-</code> is specified, +<code>bowtie2</code> will read the mate 2s from the "standard in" or +"stdin" filehandle.</p> </td> </tr> <tr> @@ -366,7 +1027,11 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-U <r></code></pre> </td> <td> -<p>Comma-separated list of files containing unpaired reads to be aligned, e.g. <code>lane1.fq,lane2.fq,lane3.fq,lane4.fq</code>. Reads may be a mix of different lengths. If <code>-</code> is specified, <code>bowtie2</code> gets the reads from the "standard in" or "stdin" filehandle.</p> +<p>Comma-separated list of files containing unpaired reads to be +aligned, e.g. <code>lane1.fq,lane2.fq,lane3.fq,lane4.fq</code>. Reads +may be a mix of different lengths. If <code>-</code> is specified, +<code>bowtie2</code> gets the reads from the "standard in" or "stdin" +filehandle.</p> </td> </tr> <tr> @@ -374,7 +1039,8 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>--interleaved</code></pre> </td> <td> -<p>Reads interleaved FASTQ files where the first two records (8 lines) represent a mate pair.</p> +<p>Reads interleaved FASTQ files where the first two records (8 lines) +represent a mate pair.</p> </td> </tr> <tr> @@ -382,8 +1048,14 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>--sra-acc</code></pre> </td> <td> -<p>Reads are SRA accessions. If the accession provided cannot be found in local storage it will be fetched from the NCBI database. If you find that SRA alignments are long running please rerun your command with the <a href="#bowtie2-options-p"><code>-p</code>/<code>--threads</code></a> parameter set to desired number of threads.</p> -<p>NB: this option is only available if bowtie 2 is compiled with the necessary SRA libraries. See <a href="#obtaining-bowtie-2">Obtaining Bowtie 2</a> for details.</p> +<p>Reads are SRA accessions. If the accession provided cannot be found +in local storage it will be fetched from the NCBI database. If you find +that SRA alignments are long running please rerun your command with the +<a href="#bowtie2-options-p"><code>-p</code>/<code>--threads</code></a> +parameter set to desired number of threads.</p> +<p>NB: this option is only available if bowtie 2 is compiled with the +necessary SRA libraries. See <a href="#obtaining-bowtie-2">Obtaining +Bowtie 2</a> for details.</p> </td> </tr> <tr> @@ -391,7 +1063,11 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-b <bam></code></pre> </td> <td> -<p>Reads are unaligned BAM records sorted by read name. The <a href="#bowtie2-options-align-paired-reads"><code>--align-paired-reads</code></a> and <a href="#bowtie2-options-preserve-tags"><code>--preserve-tags</code></a> options affect the way Bowtie 2 processes records.</p> +<p>Reads are unaligned BAM records sorted by read name. The <a +href="#bowtie2-options-align-paired-reads"><code>--align-paired-reads</code></a> +and <a +href="#bowtie2-options-preserve-tags"><code>--preserve-tags</code></a> +options affect the way Bowtie 2 processes records.</p> </td> </tr> <tr> @@ -399,7 +1075,8 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-S <sam></code></pre> </td> <td> -<p>File to write SAM alignments to. By default, alignments are written to the "standard out" or "stdout" filehandle (i.e. the console).</p> +<p>File to write SAM alignments to. By default, alignments are written +to the "standard out" or "stdout" filehandle (i.e. the console).</p> </td> </tr> </table> @@ -411,7 +1088,12 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-q</code></pre> </td> <td> -<p>Reads (specified with <code><m1></code>, <code><m2></code>, <code><s></code>) are FASTQ files. FASTQ files usually have extension <code>.fq</code> or <code>.fastq</code>. FASTQ is the default format. See also: <a href="#bowtie2-options-solexa-quals"><code>--solexa-quals</code></a> and <a href="#bowtie2-options-int-quals"><code>--int-quals</code></a>.</p> +<p>Reads (specified with <code><m1></code>, +<code><m2></code>, <code><s></code>) are FASTQ files. FASTQ +files usually have extension <code>.fq</code> or <code>.fastq</code>. +FASTQ is the default format. See also: <a +href="#bowtie2-options-solexa-quals"><code>--solexa-quals</code></a> and +<a href="#bowtie2-options-int-quals"><code>--int-quals</code></a>.</p> </td> </tr> <tr> @@ -419,7 +1101,11 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>--tab5</code></pre> </td> <td> -<p>Each read or pair is on a single line. An unpaired read line is <code>[name]\t[seq]\t[qual]\n</code>. A paired-end read line is <code>[name]\t[seq1]\t[qual1]\t[seq2]\t[qual2]\n</code>. An input file can be a mix of unpaired and paired-end reads and Bowtie 2 recognizes each according to the number of fields, handling each as it should.</p> +<p>Each read or pair is on a single line. An unpaired read line is +<code>[name]\t[seq]\t[qual]\n</code>. A paired-end read line is +<code>[name]\t[seq1]\t[qual1]\t[seq2]\t[qual2]\n</code>. An input file +can be a mix of unpaired and paired-end reads and Bowtie 2 recognizes +each according to the number of fields, handling each as it should.</p> </td> </tr> <tr> @@ -427,7 +1113,10 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>--tab6</code></pre> </td> <td> -<p>Similar to <a href="#bowtie2-options-tab5"><code>--tab5</code></a> except, for paired-end reads, the second end can have a different name from the first: <code>[name1]\t[seq1]\t[qual1]\t[name2]\t[seq2]\t[qual2]\n</code></p> +<p>Similar to <a href="#bowtie2-options-tab5"><code>--tab5</code></a> +except, for paired-end reads, the second end can have a different name +from the first: +<code>[name1]\t[seq1]\t[qual1]\t[name2]\t[seq2]\t[qual2]\n</code></p> </td> </tr> <tr> @@ -435,7 +1124,11 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>--qseq</code></pre> </td> <td> -<p>Reads (specified with <code><m1></code>, <code><m2></code>, <code><s></code>) are QSEQ files. QSEQ files usually end in <code>_qseq.txt</code>. See also: <a href="#bowtie2-options-solexa-quals"><code>--solexa-quals</code></a> and <a href="#bowtie2-options-int-quals"><code>--int-quals</code></a>.</p> +<p>Reads (specified with <code><m1></code>, +<code><m2></code>, <code><s></code>) are QSEQ files. QSEQ +files usually end in <code>_qseq.txt</code>. See also: <a +href="#bowtie2-options-solexa-quals"><code>--solexa-quals</code></a> and +<a href="#bowtie2-options-int-quals"><code>--int-quals</code></a>.</p> </td> </tr> <tr> @@ -443,7 +1136,17 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-f</code></pre> </td> <td> -<p>Reads (specified with <code><m1></code>, <code><m2></code>, <code><s></code>) are <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files. <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files usually have extension <code>.fa</code>, <code>.fasta</code>, <code>.mfa</code>, <code>.fna</code> or similar. <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files do not have a way of specifying quality values, so when <code>-f</code> is set, the result is as if <a href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a> is also set.</p> +<p>Reads (specified with <code><m1></code>, +<code><m2></code>, <code><s></code>) are <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files. +<a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> +files usually have extension <code>.fa</code>, <code>.fasta</code>, +<code>.mfa</code>, <code>.fna</code> or similar. <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files +do not have a way of specifying quality values, so when <code>-f</code> +is set, the result is as if <a +href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a> is +also set.</p> </td> </tr> <tr> @@ -451,7 +1154,12 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-r</code></pre> </td> <td> -<p>Reads (specified with <code><m1></code>, <code><m2></code>, <code><s></code>) are files with one input sequence per line, without any other information (no read names, no qualities). When <code>-r</code> is set, the result is as if <a href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a> is also set.</p> +<p>Reads (specified with <code><m1></code>, +<code><m2></code>, <code><s></code>) are files with one +input sequence per line, without any other information (no read names, +no qualities). When <code>-r</code> is set, the result is as if <a +href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a> is +also set.</p> </td> </tr> <tr> @@ -459,7 +1167,17 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-F k:<int>,i:<int></code></pre> </td> <td> -Reads are substrings (k-mers) extracted from a FASTA file <code><s></code>. Specifically, for every reference sequence in FASTA file <code><s></code>, Bowtie 2 aligns the k-mers at offsets 1, 1+i, 1+2i, ... until reaching the end of the reference. Each k-mer is aligned as a separate read. Quality values are set to all Is (40 on Phred scale). Each k-mer (read) is given a name like <code><sequence>_<offset></code>, where <code><sequence></code> is the name of the FASTA sequence it was drawn from and <code><offset></code> is its 0-based offset of origin with respect to the sequence. Only single k-mers, i.e. unpaired reads, can be aligned in this way. +Reads are substrings (k-mers) extracted from a FASTA file +<code><s></code>. Specifically, for every reference sequence in +FASTA file <code><s></code>, Bowtie 2 aligns the k-mers at offsets +1, 1+i, 1+2i, ... until reaching the end of the reference. Each k-mer is +aligned as a separate read. Quality values are set to all Is (40 on +Phred scale). Each k-mer (read) is given a name like +<code><sequence>_<offset></code>, where +<code><sequence></code> is the name of the FASTA sequence it was +drawn from and <code><offset></code> is its 0-based offset of +origin with respect to the sequence. Only single k-mers, i.e. unpaired +reads, can be aligned in this way. </td> </tr> <tr> @@ -467,7 +1185,12 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>-c</code></pre> </td> <td> -<p>The read sequences are given on command line. I.e. <code><m1></code>, <code><m2></code> and <code><singles></code> are comma-separated lists of reads rather than lists of read files. There is no way to specify read names or qualities, so <code>-c</code> also implies <a href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a>.</p> +<p>The read sequences are given on command line. I.e. +<code><m1></code>, <code><m2></code> and +<code><singles></code> are comma-separated lists of reads rather +than lists of read files. There is no way to specify read names or +qualities, so <code>-c</code> also implies <a +href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a>.</p> </td> </tr> <tr> @@ -475,7 +1198,8 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>-s/--skip <int></code></pre> </td> <td> -<p>Skip (i.e. do not align) the first <code><int></code> reads or pairs in the input.</p> +<p>Skip (i.e. do not align) the first <code><int></code> reads or +pairs in the input.</p> </td> </tr> <tr> @@ -483,7 +1207,10 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>-u/--qupto <int></code></pre> </td> <td> -<p>Align the first <code><int></code> reads or read pairs from the input (after the <a href="#bowtie2-options-s"><code>-s</code>/<code>--skip</code></a> reads or pairs have been skipped), then stop. Default: no limit.</p> +<p>Align the first <code><int></code> reads or read pairs from the +input (after the <a +href="#bowtie2-options-s"><code>-s</code>/<code>--skip</code></a> reads +or pairs have been skipped), then stop. Default: no limit.</p> </td> </tr> <tr> @@ -491,7 +1218,8 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>-5/--trim5 <int></code></pre> </td> <td> -<p>Trim <code><int></code> bases from 5' (left) end of each read before alignment (default: 0).</p> +<p>Trim <code><int></code> bases from 5' (left) end of each read +before alignment (default: 0).</p> </td> </tr> <tr> @@ -499,7 +1227,8 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>-3/--trim3 <int></code></pre> </td> <td> -<p>Trim <code><int></code> bases from 3' (right) end of each read before alignment (default: 0).</p> +<p>Trim <code><int></code> bases from 3' (right) end of each read +before alignment (default: 0).</p> </td> </tr> <tr> @@ -507,7 +1236,14 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>--trim-to [3:|5:]<int></code></pre> </td> <td> -<p>Trim reads exceeding <code><int></code> bases. Bases will be trimmed from either the 3' (right) or 5' (left) end of the read. If the read end if not specified, bowtie 2 will default to trimming from the 3' (right) end of the read. <a href="#bowtie2-options-trim-to"><code>--trim-to</code></a> and <a href="#bowtie2-options-3"><code>-3</code></a>/<a href="#bowtie2-options-5"><code>-5</code></a> are mutually exclusive.</p> +<p>Trim reads exceeding <code><int></code> bases. Bases will be +trimmed from either the 3' (right) or 5' (left) end of the read. If the +read end if not specified, bowtie 2 will default to trimming from the 3' +(right) end of the read. <a +href="#bowtie2-options-trim-to"><code>--trim-to</code></a> and <a +href="#bowtie2-options-3"><code>-3</code></a>/<a +href="#bowtie2-options-5"><code>-5</code></a> are mutually +exclusive.</p> </td> </tr> <tr> @@ -515,7 +1251,10 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>--phred33</code></pre> </td> <td> -<p>Input qualities are ASCII chars equal to the <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred quality</a> plus 33. This is also called the "Phred+33" encoding, which is used by the very latest Illumina pipelines.</p> +<p>Input qualities are ASCII chars equal to the <a +href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred +quality</a> plus 33. This is also called the "Phred+33" encoding, which +is used by the very latest Illumina pipelines.</p> </td> </tr> <tr> @@ -523,7 +1262,9 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>--phred64</code></pre> </td> <td> -<p>Input qualities are ASCII chars equal to the <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred quality</a> plus 64. This is also called the "Phred+64" encoding.</p> +<p>Input qualities are ASCII chars equal to the <a +href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred +quality</a> plus 64. This is also called the "Phred+64" encoding.</p> </td> </tr> <tr> @@ -531,7 +1272,12 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>--solexa-quals</code></pre> </td> <td> -<p>Convert input qualities from <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Solexa</a> (which can be negative) to <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred</a> (which can't). This scheme was used in older Illumina GA Pipeline versions (prior to 1.3). Default: off.</p> +<p>Convert input qualities from <a +href="http://en.wikipedia.org/wiki/Phred_quality_score">Solexa</a> +(which can be negative) to <a +href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred</a> (which +can't). This scheme was used in older Illumina GA Pipeline versions +(prior to 1.3). Default: off.</p> </td> </tr> <tr> @@ -539,11 +1285,19 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>--int-quals</code></pre> </td> <td> -<p>Quality values are represented in the read input file as space-separated ASCII integers, e.g., <code>40 40 30 40</code>..., rather than ASCII characters, e.g., <code>II?I</code>.... Integers are treated as being on the <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred quality</a> scale unless <a href="#bowtie2-options-solexa-quals"><code>--solexa-quals</code></a> is also specified. Default: off.</p> +<p>Quality values are represented in the read input file as +space-separated ASCII integers, e.g., <code>40 40 30 40</code>..., +rather than ASCII characters, e.g., <code>II?I</code>.... Integers are +treated as being on the <a +href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred +quality</a> scale unless <a +href="#bowtie2-options-solexa-quals"><code>--solexa-quals</code></a> is +also specified. Default: off.</p> </td> </tr> </table> -<h4 id="preset-options-in---end-to-end-mode">Preset options in <code>--end-to-end</code> mode</h4> +<h4 id="preset-options-in---end-to-end-mode">Preset options in +<code>--end-to-end</code> mode</h4> <table> <tr> <td id="bowtie2-options-very-fast"> @@ -566,7 +1320,9 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>--sensitive</code></pre> </td> <td> -<p>Same as: <code>-D 15 -R 2 -N 0 -L 22 -i S,1,1.15</code> (default in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode)</p> +<p>Same as: <code>-D 15 -R 2 -N 0 -L 22 -i S,1,1.15</code> (default in +<a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> +mode)</p> </td> </tr> <tr> @@ -578,7 +1334,8 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> </td> </tr> </table> -<h4 id="preset-options-in---local-mode">Preset options in <code>--local</code> mode</h4> +<h4 id="preset-options-in---local-mode">Preset options in +<code>--local</code> mode</h4> <table> <tr> <td id="bowtie2-options-very-fast-local"> @@ -601,7 +1358,8 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>--sensitive-local</code></pre> </td> <td> -<p>Same as: <code>-D 15 -R 2 -N 0 -L 20 -i S,1,0.75</code> (default in <a href="#bowtie2-options-local"><code>--local</code></a> mode)</p> +<p>Same as: <code>-D 15 -R 2 -N 0 -L 20 -i S,1,0.75</code> (default in +<a href="#bowtie2-options-local"><code>--local</code></a> mode)</p> </td> </tr> <tr> @@ -620,7 +1378,10 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>-N <int></code></pre> </td> <td> -<p>Sets the number of mismatches to allowed in a seed alignment during <a href="#multiseed-heuristic">multiseed alignment</a>. Can be set to 0 or 1. Setting this higher makes alignment slower (often much slower) but increases sensitivity. Default: 0.</p> +<p>Sets the number of mismatches to allowed in a seed alignment during +<a href="#multiseed-heuristic">multiseed alignment</a>. Can be set to 0 +or 1. Setting this higher makes alignment slower (often much slower) but +increases sensitivity. Default: 0.</p> </td> </tr> <tr> @@ -628,7 +1389,14 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>-L <int></code></pre> </td> <td> -<p>Sets the length of the seed substrings to align during <a href="#multiseed-heuristic">multiseed alignment</a>. Smaller values make alignment slower but more sensitive. Default: the <a href="#bowtie2-options-sensitive"><code>--sensitive</code></a> preset is used by default, which sets <code>-L</code> to 22 and 20 in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode and in <a href="#bowtie2-options-local"><code>--local</code></a> mode.</p> +<p>Sets the length of the seed substrings to align during <a +href="#multiseed-heuristic">multiseed alignment</a>. Smaller values make +alignment slower but more sensitive. Default: the <a +href="#bowtie2-options-sensitive"><code>--sensitive</code></a> preset is +used by default, which sets <code>-L</code> to 22 and 20 in <a +href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode +and in <a href="#bowtie2-options-local"><code>--local</code></a> +mode.</p> </td> </tr> <tr> @@ -636,7 +1404,10 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>-i <func></code></pre> </td> <td> -<p>Sets a function governing the interval between seed substrings to use during <a href="#multiseed-heuristic">multiseed alignment</a>. For instance, if the read has 30 characters, and seed length is 10, and the seed interval is 6, the seeds extracted will be:</p> +<p>Sets a function governing the interval between seed substrings to use +during <a href="#multiseed-heuristic">multiseed alignment</a>. For +instance, if the read has 30 characters, and seed length is 10, and the +seed interval is 6, the seeds extracted will be:</p> <pre><code>Read: TAGCTACGCTCTACGCTATCATGCATAAAC Seed 1 fw: TAGCTACGCT Seed 1 rc: AGCGTAGCTA @@ -646,7 +1417,19 @@ Seed 3 fw: ACGCTATCAT Seed 3 rc: ATGATAGCGT Seed 4 fw: TCATGCATAA Seed 4 rc: TTATGCATGA</code></pre> -<p>Since it's best to use longer intervals for longer reads, this parameter sets the interval as a function of the read length, rather than a single one-size-fits-all number. For instance, specifying <code>-i S,1,2.5</code> sets the interval function <code>f</code> to <code>f(x) = 1 + 2.5 * sqrt(x)</code>, where x is the read length. See also: <a href="#setting-function-options">setting function options</a>. If the function returns a result less than 1, it is rounded up to 1. Default: the <a href="#bowtie2-options-sensitive"><code>--sensitive</code></a> preset is used by default, which sets <code>-i</code> to <code>S,1,1.15</code> in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode to <code>-i S,1,0.75</code> in <a href="#bowtie2-options-local"><code>--local</code></a> mode.</p> +<p>Since it's best to use longer intervals for longer reads, this +parameter sets the interval as a function of the read length, rather +than a single one-size-fits-all number. For instance, specifying +<code>-i S,1,2.5</code> sets the interval function <code>f</code> to +<code>f(x) = 1 + 2.5 * sqrt(x)</code>, where x is the read length. See +also: <a href="#setting-function-options">setting function options</a>. +If the function returns a result less than 1, it is rounded up to 1. +Default: the <a +href="#bowtie2-options-sensitive"><code>--sensitive</code></a> preset is +used by default, which sets <code>-i</code> to <code>S,1,1.15</code> in +<a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode +to <code>-i S,1,0.75</code> in <a +href="#bowtie2-options-local"><code>--local</code></a> mode.</p> </td> </tr> <tr> @@ -654,7 +1437,14 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--n-ceil <func></code></pre> </td> <td> -<p>Sets a function governing the maximum number of ambiguous characters (usually <code>N</code>s and/or <code>.</code>s) allowed in a read as a function of read length. For instance, specifying <code>-L,0,0.15</code> sets the N-ceiling function <code>f</code> to <code>f(x) = 0 + 0.15 * x</code>, where x is the read length. See also: <a href="#setting-function-options">setting function options</a>. Reads exceeding this ceiling are <a href="#filtering">filtered out</a>. Default: <code>L,0,0.15</code>.</p> +<p>Sets a function governing the maximum number of ambiguous characters +(usually <code>N</code>s and/or <code>.</code>s) allowed in a read as a +function of read length. For instance, specifying <code>-L,0,0.15</code> +sets the N-ceiling function <code>f</code> to +<code>f(x) = 0 + 0.15 * x</code>, where x is the read length. See also: +<a href="#setting-function-options">setting function options</a>. Reads +exceeding this ceiling are <a href="#filtering">filtered out</a>. +Default: <code>L,0,0.15</code>.</p> </td> </tr> <tr> @@ -662,7 +1452,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--dpad <int></code></pre> </td> <td> -<p>"Pads" dynamic programming problems by <code><int></code> columns on either side to allow gaps. Default: 15.</p> +<p>"Pads" dynamic programming problems by <code><int></code> +columns on either side to allow gaps. Default: 15.</p> </td> </tr> <tr> @@ -670,7 +1461,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--gbar <int></code></pre> </td> <td> -<p>Disallow gaps within <code><int></code> positions of the beginning or end of the read. Default: 4.</p> +<p>Disallow gaps within <code><int></code> positions of the +beginning or end of the read. Default: 4.</p> </td> </tr> <tr> @@ -678,7 +1470,14 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--ignore-quals</code></pre> </td> <td> -<p>When calculating a mismatch penalty, always consider the quality value at the mismatched position to be the highest possible, regardless of the actual value. I.e. input is treated as though all quality values are high. This is also the default behavior when the input doesn't specify quality values (e.g. in <a href="#bowtie2-options-f"><code>-f</code></a>, <a href="#bowtie2-options-r"><code>-r</code></a>, or <a href="#bowtie2-options-c"><code>-c</code></a> modes).</p> +<p>When calculating a mismatch penalty, always consider the quality +value at the mismatched position to be the highest possible, regardless +of the actual value. I.e. input is treated as though all quality values +are high. This is also the default behavior when the input doesn't +specify quality values (e.g. in <a +href="#bowtie2-options-f"><code>-f</code></a>, <a +href="#bowtie2-options-r"><code>-r</code></a>, or <a +href="#bowtie2-options-c"><code>-c</code></a> modes).</p> </td> </tr> <tr> @@ -686,7 +1485,15 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--nofw/--norc</code></pre> </td> <td> -<p>If <code>--nofw</code> is specified, <code>bowtie2</code> will not attempt to align unpaired reads to the forward (Watson) reference strand. If <code>--norc</code> is specified, <code>bowtie2</code> will not attempt to align unpaired reads against the reverse-complement (Crick) reference strand. In paired-end mode, <code>--nofw</code> and <code>--norc</code> pertain to the fragments; i.e. specifying <code>--nofw</code> causes <code>bowtie2</code> to explore only those paired-end configurations corresponding to fragments from the reverse-complement (Crick) strand. Default: both strands enabled.</p> +<p>If <code>--nofw</code> is specified, <code>bowtie2</code> will not +attempt to align unpaired reads to the forward (Watson) reference +strand. If <code>--norc</code> is specified, <code>bowtie2</code> will +not attempt to align unpaired reads against the reverse-complement +(Crick) reference strand. In paired-end mode, <code>--nofw</code> and +<code>--norc</code> pertain to the fragments; i.e. specifying +<code>--nofw</code> causes <code>bowtie2</code> to explore only those +paired-end configurations corresponding to fragments from the +reverse-complement (Crick) strand. Default: both strands enabled.</p> </td> </tr> <tr> @@ -694,7 +1501,24 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--no-1mm-upfront</code></pre> </td> <td> -<p>By default, Bowtie 2 will attempt to find either an exact or a 1-mismatch end-to-end alignment for the read <em>before</em> trying the <a href="#multiseed-heuristic">multiseed heuristic</a>. Such alignments can be found very quickly, and many short read alignments have exact or near-exact end-to-end alignments. However, this can lead to unexpected alignments when the user also sets options governing the <a href="#multiseed-heuristic">multiseed heuristic</a>, like <a href="#bowtie2-options-L"><code>-L</code></a> and <a href="#bowtie2-options-N"><code>-N</code></a>. For instance, if the user specifies <code>-N 0</code> and <code>-L</code> equal to the length of the read, the user will be surprised to find 1-mismatch alignments reported. This option prevents Bowtie 2 from searching for 1-mismatch end-to-end alignments before using the <a href="#multiseed-heuristic">multiseed heuristic</a>, which leads to the expected behavior when combined with options such as <a href="#bowtie2-options-L"><code>-L</code></a> and <a href="#bowtie2-options-N"><code>-N</code></a>. This comes at the expense of speed.</p> +<p>By default, Bowtie 2 will attempt to find either an exact or a +1-mismatch end-to-end alignment for the read <em>before</em> trying the +<a href="#multiseed-heuristic">multiseed heuristic</a>. Such alignments +can be found very quickly, and many short read alignments have exact or +near-exact end-to-end alignments. However, this can lead to unexpected +alignments when the user also sets options governing the <a +href="#multiseed-heuristic">multiseed heuristic</a>, like <a +href="#bowtie2-options-L"><code>-L</code></a> and <a +href="#bowtie2-options-N"><code>-N</code></a>. For instance, if the user +specifies <code>-N 0</code> and <code>-L</code> equal to the length of +the read, the user will be surprised to find 1-mismatch alignments +reported. This option prevents Bowtie 2 from searching for 1-mismatch +end-to-end alignments before using the <a +href="#multiseed-heuristic">multiseed heuristic</a>, which leads to the +expected behavior when combined with options such as <a +href="#bowtie2-options-L"><code>-L</code></a> and <a +href="#bowtie2-options-N"><code>-N</code></a>. This comes at the expense +of speed.</p> </td> </tr> <tr> @@ -702,7 +1526,14 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--end-to-end</code></pre> </td> <td> -<p>In this mode, Bowtie 2 requires that the entire read align from one end to the other, without any trimming (or "soft clipping") of characters from either end. The match bonus <a href="#bowtie2-options-ma"><code>--ma</code></a> always equals 0 in this mode, so all alignment scores are less than or equal to 0, and the greatest possible alignment score is 0. This is mutually exclusive with <a href="#bowtie2-options-local"><code>--local</code></a>. <code>--end-to-end</code> is the default mode.</p> +<p>In this mode, Bowtie 2 requires that the entire read align from one +end to the other, without any trimming (or "soft clipping") of +characters from either end. The match bonus <a +href="#bowtie2-options-ma"><code>--ma</code></a> always equals 0 in this +mode, so all alignment scores are less than or equal to 0, and the +greatest possible alignment score is 0. This is mutually exclusive with +<a href="#bowtie2-options-local"><code>--local</code></a>. +<code>--end-to-end</code> is the default mode.</p> </td> </tr> <tr> @@ -710,7 +1541,19 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--local</code></pre> </td> <td> -<p>In this mode, Bowtie 2 does not require that the entire read align from one end to the other. Rather, some characters may be omitted ("soft clipped") from the ends in order to achieve the greatest possible alignment score. The match bonus <a href="#bowtie2-options-ma"><code>--ma</code></a> is used in this mode, and the best possible alignment score is equal to the match bonus (<a href="#bowtie2-options-ma"><code>--ma</code></a>) times the length of the read. Specifying <code>--local</code> and one of the presets (e.g. <code>--local --very-fast</code>) is equivalent to specifying the local version of the preset (<code>--very-fast-local</code>). This is mutually exclusive with <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a>. <code>--end-to-end</code> is the default mode.</p> +<p>In this mode, Bowtie 2 does not require that the entire read align +from one end to the other. Rather, some characters may be omitted ("soft +clipped") from the ends in order to achieve the greatest possible +alignment score. The match bonus <a +href="#bowtie2-options-ma"><code>--ma</code></a> is used in this mode, +and the best possible alignment score is equal to the match bonus (<a +href="#bowtie2-options-ma"><code>--ma</code></a>) times the length of +the read. Specifying <code>--local</code> and one of the presets (e.g. +<code>--local --very-fast</code>) is equivalent to specifying the local +version of the preset (<code>--very-fast-local</code>). This is mutually +exclusive with <a +href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a>. +<code>--end-to-end</code> is the default mode.</p> </td> </tr> </table> @@ -721,7 +1564,13 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--ma <int></code></pre> </td> <td> -<p>Sets the match bonus. In <a href="#bowtie2-options-local"><code>--local</code></a> mode <code><int></code> is added to the alignment score for each position where a read character aligns to a reference character and the characters match. Not used in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode. Default: 2.</p> +<p>Sets the match bonus. In <a +href="#bowtie2-options-local"><code>--local</code></a> mode +<code><int></code> is added to the alignment score for each +position where a read character aligns to a reference character and the +characters match. Not used in <a +href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode. +Default: 2.</p> </td> </tr> <tr> @@ -729,7 +1578,18 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--mp MX,MN</code></pre> </td> <td> -<p>Sets the maximum (<code>MX</code>) and minimum (<code>MN</code>) mismatch penalties, both integers. A number less than or equal to <code>MX</code> and greater than or equal to <code>MN</code> is subtracted from the alignment score for each position where a read character aligns to a reference character, the characters do not match, and neither is an <code>N</code>. If <a href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a> is specified, the number subtracted quals <code>MX</code>. Otherwise, the number subtracted is <code>MN + floor( (MX-MN)(MIN(Q, 40.0)/40.0) )</code> where Q is the Phred quality value. Default: <code>MX</code> = 6, <code>MN</code> = 2.</p> +<p>Sets the maximum (<code>MX</code>) and minimum (<code>MN</code>) +mismatch penalties, both integers. A number less than or equal to +<code>MX</code> and greater than or equal to <code>MN</code> is +subtracted from the alignment score for each position where a read +character aligns to a reference character, the characters do not match, +and neither is an <code>N</code>. If <a +href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a> is +specified, the number subtracted quals <code>MX</code>. Otherwise, the +number subtracted is +<code>MN + floor( (MX-MN)(MIN(Q, 40.0)/40.0) )</code> where Q is the +Phred quality value. Default: <code>MX</code> = 6, <code>MN</code> = +2.</p> </td> </tr> <tr> @@ -737,7 +1597,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--np <int></code></pre> </td> <td> -<p>Sets penalty for positions where the read, reference, or both, contain an ambiguous character such as <code>N</code>. Default: 1.</p> +<p>Sets penalty for positions where the read, reference, or both, +contain an ambiguous character such as <code>N</code>. Default: 1.</p> </td> </tr> <tr> @@ -745,7 +1606,10 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--rdg <int1>,<int2></code></pre> </td> <td> -<p>Sets the read gap open (<code><int1></code>) and extend (<code><int2></code>) penalties. A read gap of length N gets a penalty of <code><int1></code> + N * <code><int2></code>. Default: 5, 3.</p> +<p>Sets the read gap open (<code><int1></code>) and extend +(<code><int2></code>) penalties. A read gap of length N gets a +penalty of <code><int1></code> + N * <code><int2></code>. +Default: 5, 3.</p> </td> </tr> <tr> @@ -753,7 +1617,10 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--rfg <int1>,<int2></code></pre> </td> <td> -<p>Sets the reference gap open (<code><int1></code>) and extend (<code><int2></code>) penalties. A reference gap of length N gets a penalty of <code><int1></code> + N * <code><int2></code>. Default: 5, 3.</p> +<p>Sets the reference gap open (<code><int1></code>) and extend +(<code><int2></code>) penalties. A reference gap of length N gets +a penalty of <code><int1></code> + N * <code><int2></code>. +Default: 5, 3.</p> </td> </tr> <tr> @@ -761,7 +1628,17 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--score-min <func></code></pre> </td> <td> -<p>Sets a function governing the minimum alignment score needed for an alignment to be considered "valid" (i.e. good enough to report). This is a function of read length. For instance, specifying <code>L,0,-0.6</code> sets the minimum-score function <code>f</code> to <code>f(x) = 0 + -0.6 * x</code>, where <code>x</code> is the read length. See also: <a href="#setting-function-options">setting function options</a>. The default in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode is <code>L,-0.6,-0.6</code> and the default in <a href="#bowtie2-options-local"><code>--local</code></a> mode is <code>G,20,8</code>.</p> +<p>Sets a function governing the minimum alignment score needed for an +alignment to be considered "valid" (i.e. good enough to report). This is +a function of read length. For instance, specifying +<code>L,0,-0.6</code> sets the minimum-score function <code>f</code> to +<code>f(x) = 0 + -0.6 * x</code>, where <code>x</code> is the read +length. See also: <a href="#setting-function-options">setting function +options</a>. The default in <a +href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode is +<code>L,-0.6,-0.6</code> and the default in <a +href="#bowtie2-options-local"><code>--local</code></a> mode is +<code>G,20,8</code>.</p> </td> </tr> </table> @@ -772,9 +1649,32 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-k <int></code></pre> </td> <td> -<p>By default, <code>bowtie2</code> searches for distinct, valid alignments for each read. When it finds a valid alignment, it continues looking for alignments that are nearly as good or better. The best alignment found is reported (randomly selected from among best if tied). Information about the best alignments is used to estimate mapping quality and to set SAM optional fields, such as <a href="#bowtie2-build-opt-fields-as"><code>AS:i</code></a> and <a href="#bowtie2-build-opt-fields-xs"><code>XS:i</code></a>.</p> -<p>When <code>-k</code> is specified, however, <code>bowtie2</code> behaves differently. Instead, it searches for at most <code><int></code> distinct, valid alignments for each read. The search terminates when it can't find more distinct valid alignments, or when it finds <code><int></code>, whichever happens first. All alignments found are reported in descending order by alignment score. The alignment score for a paired-end alignment equals the sum of the alignment scores of the individual mates. Each reported read or pair alignment beyond the first has the SAM 'secondary' bit (which equals 256) set in its FLAGS field. For reads that have more than <code><int></code> distinct, valid alignments, <code>bowtie2</code> does not guarantee that the <code><int></code> alignments reported are the best possible in terms of alignment score. <code>-k</code> is mutually exclusive with <a href="#bowtie2-options-a"><code>-a</code></a>.</p> -<p>Note: Bowtie 2 is not designed with large values for <code>-k</code> in mind, and when aligning reads to long, repetitive genomes large <code>-k</code> can be very, very slow.</p> +<p>By default, <code>bowtie2</code> searches for distinct, valid +alignments for each read. When it finds a valid alignment, it continues +looking for alignments that are nearly as good or better. The best +alignment found is reported (randomly selected from among best if tied). +Information about the best alignments is used to estimate mapping +quality and to set SAM optional fields, such as <a +href="#bowtie2-build-opt-fields-as"><code>AS:i</code></a> and <a +href="#bowtie2-build-opt-fields-xs"><code>XS:i</code></a>.</p> +<p>When <code>-k</code> is specified, however, <code>bowtie2</code> +behaves differently. Instead, it searches for at most +<code><int></code> distinct, valid alignments for each read. The +search terminates when it can't find more distinct valid alignments, or +when it finds <code><int></code>, whichever happens first. All +alignments found are reported in descending order by alignment score. +The alignment score for a paired-end alignment equals the sum of the +alignment scores of the individual mates. Each reported read or pair +alignment beyond the first has the SAM 'secondary' bit (which equals +256) set in its FLAGS field. For reads that have more than +<code><int></code> distinct, valid alignments, +<code>bowtie2</code> does not guarantee that the +<code><int></code> alignments reported are the best possible in +terms of alignment score. <code>-k</code> is mutually exclusive with <a +href="#bowtie2-options-a"><code>-a</code></a>.</p> +<p>Note: Bowtie 2 is not designed with large values for <code>-k</code> +in mind, and when aligning reads to long, repetitive genomes large +<code>-k</code> can be very, very slow.</p> </td> </tr> <tr> @@ -782,8 +1682,13 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-a</code></pre> </td> <td> -<p>Like <a href="#bowtie2-options-k"><code>-k</code></a> but with no upper limit on number of alignments to search for. <code>-a</code> is mutually exclusive with <a href="#bowtie2-options-k"><code>-k</code></a>.</p> -<p>Note: Bowtie 2 is not designed with <code>-a</code> mode in mind, and when aligning reads to long, repetitive genomes this mode can be very, very slow.</p> +<p>Like <a href="#bowtie2-options-k"><code>-k</code></a> but with no +upper limit on number of alignments to search for. <code>-a</code> is +mutually exclusive with <a +href="#bowtie2-options-k"><code>-k</code></a>.</p> +<p>Note: Bowtie 2 is not designed with <code>-a</code> mode in mind, and +when aligning reads to long, repetitive genomes this mode can be very, +very slow.</p> </td> </tr> </table> @@ -794,7 +1699,11 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-D <int></code></pre> </td> <td> -<p>Up to <code><int></code> consecutive seed extension attempts can "fail" before Bowtie 2 moves on, using the alignments found so far. A seed extension "fails" if it does not yield a new best or a new second-best alignment. This limit is automatically adjusted up when -k or -a are specified. Default: 15.</p> +<p>Up to <code><int></code> consecutive seed extension attempts +can "fail" before Bowtie 2 moves on, using the alignments found so far. +A seed extension "fails" if it does not yield a new best or a new +second-best alignment. This limit is automatically adjusted up when -k +or -a are specified. Default: 15.</p> </td> </tr> <tr> @@ -802,7 +1711,13 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-R <int></code></pre> </td> <td> -<p><code><int></code> is the maximum number of times Bowtie 2 will "re-seed" reads with repetitive seeds. When "re-seeding," Bowtie 2 simply chooses a new set of reads (same length, same number of mismatches allowed) at different offsets and searches for more alignments. A read is considered to have repetitive seeds if the total number of seed hits divided by the number of seeds that aligned at least once is greater than 300. Default: 2.</p> +<p><code><int></code> is the maximum number of times Bowtie 2 will +"re-seed" reads with repetitive seeds. When "re-seeding," Bowtie 2 +simply chooses a new set of reads (same length, same number of +mismatches allowed) at different offsets and searches for more +alignments. A read is considered to have repetitive seeds if the total +number of seed hits divided by the number of seeds that aligned at least +once is greater than 300. Default: 2.</p> </td> </tr> </table> @@ -813,8 +1728,25 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-I/--minins <int></code></pre> </td> <td> -<p>The minimum fragment length for valid paired-end alignments. E.g. if <code>-I 60</code> is specified and a paired-end alignment consists of two 20-bp alignments in the appropriate orientation with a 20-bp gap between them, that alignment is considered valid (as long as <a href="#bowtie2-options-X"><code>-X</code></a> is also satisfied). A 19-bp gap would not be valid in that case. If trimming options <a href="#bowtie2-options-3"><code>-3</code></a> or <a href="#bowtie2-options-5"><code>-5</code></a> are also used, the <a href="#bowtie2-options-I"><code>-I</code></a> constraint is applied with respect to the untrimmed mates.</p> -<p>The larger the difference between <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a>, the slower Bowtie 2 will run. This is because larger differences between <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a> require that Bowtie 2 scan a larger window to determine if a concordant alignment exists. For typical fragment length ranges (200 to 400 nucleotides), Bowtie 2 is very efficient.</p> +<p>The minimum fragment length for valid paired-end alignments. E.g. if +<code>-I 60</code> is specified and a paired-end alignment consists of +two 20-bp alignments in the appropriate orientation with a 20-bp gap +between them, that alignment is considered valid (as long as <a +href="#bowtie2-options-X"><code>-X</code></a> is also satisfied). A +19-bp gap would not be valid in that case. If trimming options <a +href="#bowtie2-options-3"><code>-3</code></a> or <a +href="#bowtie2-options-5"><code>-5</code></a> are also used, the <a +href="#bowtie2-options-I"><code>-I</code></a> constraint is applied with +respect to the untrimmed mates.</p> +<p>The larger the difference between <a +href="#bowtie2-options-I"><code>-I</code></a> and <a +href="#bowtie2-options-X"><code>-X</code></a>, the slower Bowtie 2 will +run. This is because larger differences between <a +href="#bowtie2-options-I"><code>-I</code></a> and <a +href="#bowtie2-options-X"><code>-X</code></a> require that Bowtie 2 scan +a larger window to determine if a concordant alignment exists. For +typical fragment length ranges (200 to 400 nucleotides), Bowtie 2 is +very efficient.</p> <p>Default: 0 (essentially imposing no minimum)</p> </td> </tr> @@ -823,8 +1755,25 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-X/--maxins <int></code></pre> </td> <td> -<p>The maximum fragment length for valid paired-end alignments. E.g. if <code>-X 100</code> is specified and a paired-end alignment consists of two 20-bp alignments in the proper orientation with a 60-bp gap between them, that alignment is considered valid (as long as <a href="#bowtie2-options-I"><code>-I</code></a> is also satisfied). A 61-bp gap would not be valid in that case. If trimming options <a href="#bowtie2-options-3"><code>-3</code></a> or <a href="#bowtie2-options-5"><code>-5</code></a> are also used, the <code>-X</code> constraint is applied with respect to the untrimmed mates, not the trimmed mates.</p> -<p>The larger the difference between <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a>, the slower Bowtie 2 will run. This is because larger differences between <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a> require that Bowtie 2 scan a larger window to determine if a concordant alignment exists. For typical fragment length ranges (200 to 400 nucleotides), Bowtie 2 is very efficient.</p> +<p>The maximum fragment length for valid paired-end alignments. E.g. if +<code>-X 100</code> is specified and a paired-end alignment consists of +two 20-bp alignments in the proper orientation with a 60-bp gap between +them, that alignment is considered valid (as long as <a +href="#bowtie2-options-I"><code>-I</code></a> is also satisfied). A +61-bp gap would not be valid in that case. If trimming options <a +href="#bowtie2-options-3"><code>-3</code></a> or <a +href="#bowtie2-options-5"><code>-5</code></a> are also used, the +<code>-X</code> constraint is applied with respect to the untrimmed +mates, not the trimmed mates.</p> +<p>The larger the difference between <a +href="#bowtie2-options-I"><code>-I</code></a> and <a +href="#bowtie2-options-X"><code>-X</code></a>, the slower Bowtie 2 will +run. This is because larger differences between <a +href="#bowtie2-options-I"><code>-I</code></a> and <a +href="#bowtie2-options-X"><code>-X</code></a> require that Bowtie 2 scan +a larger window to determine if a concordant alignment exists. For +typical fragment length ranges (200 to 400 nucleotides), Bowtie 2 is +very efficient.</p> <p>Default: 500.</p> </td> </tr> @@ -833,7 +1782,20 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--fr/--rf/--ff</code></pre> </td> <td> -<p>The upstream/downstream mate orientations for a valid paired-end alignment against the forward reference strand. E.g., if <code>--fr</code> is specified and there is a candidate paired-end alignment where mate 1 appears upstream of the reverse complement of mate 2 and the fragment length constraints (<a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a>) are met, that alignment is valid. Also, if mate 2 appears upstream of the reverse complement of mate 1 and all other constraints are met, that too is valid. <code>--rf</code> likewise requires that an upstream mate1 be reverse-complemented and a downstream mate2 be forward-oriented. <code>--ff</code> requires both an upstream mate 1 and a downstream mate 2 to be forward-oriented. Default: <code>--fr</code> (appropriate for Illumina's Paired-end Sequencing Assay).</p> +<p>The upstream/downstream mate orientations for a valid paired-end +alignment against the forward reference strand. E.g., if +<code>--fr</code> is specified and there is a candidate paired-end +alignment where mate 1 appears upstream of the reverse complement of +mate 2 and the fragment length constraints (<a +href="#bowtie2-options-I"><code>-I</code></a> and <a +href="#bowtie2-options-X"><code>-X</code></a>) are met, that alignment +is valid. Also, if mate 2 appears upstream of the reverse complement of +mate 1 and all other constraints are met, that too is valid. +<code>--rf</code> likewise requires that an upstream mate1 be +reverse-complemented and a downstream mate2 be forward-oriented. +<code>--ff</code> requires both an upstream mate 1 and a downstream mate +2 to be forward-oriented. Default: <code>--fr</code> (appropriate for +Illumina's Paired-end Sequencing Assay).</p> </td> </tr> <tr> @@ -841,7 +1803,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--no-mixed</code></pre> </td> <td> -<p>By default, when <code>bowtie2</code> cannot find a concordant or discordant alignment for a pair, it then tries to find alignments for the individual mates. This option disables that behavior.</p> +<p>By default, when <code>bowtie2</code> cannot find a concordant or +discordant alignment for a pair, it then tries to find alignments for +the individual mates. This option disables that behavior.</p> </td> </tr> <tr> @@ -849,7 +1813,14 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--no-discordant</code></pre> </td> <td> -<p>By default, <code>bowtie2</code> looks for discordant alignments if it cannot find any concordant alignments. A discordant alignment is an alignment where both mates align uniquely, but that does not satisfy the paired-end constraints (<a href="#bowtie2-options-fr"><code>--fr</code>/<code>--rf</code>/<code>--ff</code></a>, <a href="#bowtie2-options-I"><code>-I</code></a>, <a href="#bowtie2-options-X"><code>-X</code></a>). This option disables that behavior.</p> +<p>By default, <code>bowtie2</code> looks for discordant alignments if +it cannot find any concordant alignments. A discordant alignment is an +alignment where both mates align uniquely, but that does not satisfy the +paired-end constraints (<a +href="#bowtie2-options-fr"><code>--fr</code>/<code>--rf</code>/<code>--ff</code></a>, +<a href="#bowtie2-options-I"><code>-I</code></a>, <a +href="#bowtie2-options-X"><code>-X</code></a>). This option disables +that behavior.</p> </td> </tr> <tr> @@ -857,7 +1828,12 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--dovetail</code></pre> </td> <td> -<p>If the mates "dovetail", that is if one mate alignment extends past the beginning of the other such that the wrong mate begins upstream, consider that to be concordant. See also: <a href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can overlap, contain or dovetail each other</a>. Default: mates cannot dovetail in a concordant alignment.</p> +<p>If the mates "dovetail", that is if one mate alignment extends past +the beginning of the other such that the wrong mate begins upstream, +consider that to be concordant. See also: <a +href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can +overlap, contain or dovetail each other</a>. Default: mates cannot +dovetail in a concordant alignment.</p> </td> </tr> <tr> @@ -865,7 +1841,11 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--no-contain</code></pre> </td> <td> -<p>If one mate alignment contains the other, consider that to be non-concordant. See also: <a href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can overlap, contain or dovetail each other</a>. Default: a mate can contain the other in a concordant alignment.</p> +<p>If one mate alignment contains the other, consider that to be +non-concordant. See also: <a +href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can +overlap, contain or dovetail each other</a>. Default: a mate can contain +the other in a concordant alignment.</p> </td> </tr> <tr> @@ -873,7 +1853,11 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--no-overlap</code></pre> </td> <td> -<p>If one mate alignment overlaps the other at all, consider that to be non-concordant. See also: <a href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can overlap, contain or dovetail each other</a>. Default: mates can overlap in a concordant alignment.</p> +<p>If one mate alignment overlaps the other at all, consider that to be +non-concordant. See also: <a +href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can +overlap, contain or dovetail each other</a>. Default: mates can overlap +in a concordant alignment.</p> </td> </tr> </table> @@ -884,7 +1868,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--align-paired-reads</code></pre> </td> <td> -<p>Bowtie 2 will, by default, attempt to align unpaired BAM reads. Use this option to align paired-end reads instead.</p> +<p>Bowtie 2 will, by default, attempt to align unpaired BAM reads. Use +this option to align paired-end reads instead.</p> </td> </tr> <tr> @@ -892,7 +1877,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--preserve-tags</code></pre> </td> <td> -<p>Preserve tags from the original BAM record by appending them to the end of the corresponding Bowtie 2 SAM output.</p> +<p>Preserve tags from the original BAM record by appending them to the +end of the corresponding Bowtie 2 SAM output.</p> </td> </tr> </table> @@ -903,7 +1889,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-t/--time</code></pre> </td> <td> -<p>Print the wall-clock time required to load the index files and align the reads. This is printed to the "standard error" ("stderr") filehandle. Default: off.</p> +<p>Print the wall-clock time required to load the index files and align +the reads. This is printed to the "standard error" ("stderr") +filehandle. Default: off.</p> </td> </tr> <tr> @@ -914,7 +1902,17 @@ Seed 4 rc: TTATGCATGA</code></pre> --un-lz4 <path></code></pre> </td> <td> -<p>Write unpaired reads that fail to align to file at <code><path></code>. These reads correspond to the SAM records with the FLAGS <code>0x4</code> bit set and neither the <code>0x40</code> nor <code>0x80</code> bits set. If <code>--un-gz</code> is specified, output will be gzip compressed. If <code>--un-bz2</code> or <code>--un-lz4</code> is specified, output will be bzip2 or lz4 compressed. Reads written in this way will appear exactly as they did in the input file, without any modification (same sequence, same name, same quality string, same quality encoding). Reads will not necessarily appear in the same order as they did in the input.</p> +<p>Write unpaired reads that fail to align to file at +<code><path></code>. These reads correspond to the SAM records +with the FLAGS <code>0x4</code> bit set and neither the +<code>0x40</code> nor <code>0x80</code> bits set. If +<code>--un-gz</code> is specified, output will be gzip compressed. If +<code>--un-bz2</code> or <code>--un-lz4</code> is specified, output will +be bzip2 or lz4 compressed. Reads written in this way will appear +exactly as they did in the input file, without any modification (same +sequence, same name, same quality string, same quality encoding). Reads +will not necessarily appear in the same order as they did in the +input.</p> </td> </tr> <tr> @@ -925,7 +1923,17 @@ Seed 4 rc: TTATGCATGA</code></pre> --al-lz4 <path></code></pre> </td> <td> -<p>Write unpaired reads that align at least once to file at <code><path></code>. These reads correspond to the SAM records with the FLAGS <code>0x4</code>, <code>0x40</code>, and <code>0x80</code> bits unset. If <code>--al-gz</code> is specified, output will be gzip compressed. If <code>--al-bz2</code> is specified, output will be bzip2 compressed. Similarly if <code>--al-lz4</code> is specified, output will be lz4 compressed. Reads written in this way will appear exactly as they did in the input file, without any modification (same sequence, same name, same quality string, same quality encoding). Reads will not necessarily appear in the same order as they did in the input.</p> +<p>Write unpaired reads that align at least once to file at +<code><path></code>. These reads correspond to the SAM records +with the FLAGS <code>0x4</code>, <code>0x40</code>, and +<code>0x80</code> bits unset. If <code>--al-gz</code> is specified, +output will be gzip compressed. If <code>--al-bz2</code> is specified, +output will be bzip2 compressed. Similarly if <code>--al-lz4</code> is +specified, output will be lz4 compressed. Reads written in this way will +appear exactly as they did in the input file, without any modification +(same sequence, same name, same quality string, same quality encoding). +Reads will not necessarily appear in the same order as they did in the +input.</p> </td> </tr> <tr> @@ -936,7 +1944,20 @@ Seed 4 rc: TTATGCATGA</code></pre> --un-conc-lz4 <path></code></pre> </td> <td> -<p>Write paired-end reads that fail to align concordantly to file(s) at <code><path></code>. These reads correspond to the SAM records with the FLAGS <code>0x4</code> bit set and either the <code>0x40</code> or <code>0x80</code> bit set (depending on whether it's mate #1 or #2). <code>.1</code> and <code>.2</code> strings are added to the filename to distinguish which file contains mate #1 and mate #2. If a percent symbol, <code>%</code>, is used in <code><path></code>, the percent symbol is replaced with <code>1</code> or <code>2</code> to make the per-mate filenames. Otherwise, <code>.1</code> or <code>.2</code> are added before the final dot in <code><path></code> to make the per-mate filenames. Reads written in this way will appear exactly as they did in the input files, without any modification (same sequence, same name, same quality string, same quality encoding). Reads will not necessarily appear in the same order as they did in the inputs.</p> +<p>Write paired-end reads that fail to align concordantly to file(s) at +<code><path></code>. These reads correspond to the SAM records +with the FLAGS <code>0x4</code> bit set and either the <code>0x40</code> +or <code>0x80</code> bit set (depending on whether it's mate #1 or #2). +<code>.1</code> and <code>.2</code> strings are added to the filename to +distinguish which file contains mate #1 and mate #2. If a percent +symbol, <code>%</code>, is used in <code><path></code>, the +percent symbol is replaced with <code>1</code> or <code>2</code> to make +the per-mate filenames. Otherwise, <code>.1</code> or <code>.2</code> +are added before the final dot in <code><path></code> to make the +per-mate filenames. Reads written in this way will appear exactly as +they did in the input files, without any modification (same sequence, +same name, same quality string, same quality encoding). Reads will not +necessarily appear in the same order as they did in the inputs.</p> </td> </tr> <tr> @@ -947,7 +1968,21 @@ Seed 4 rc: TTATGCATGA</code></pre> --al-conc-lz4 <path></code></pre> </td> <td> -<p>Write paired-end reads that align concordantly at least once to file(s) at <code><path></code>. These reads correspond to the SAM records with the FLAGS <code>0x4</code> bit unset and either the <code>0x40</code> or <code>0x80</code> bit set (depending on whether it's mate #1 or #2). <code>.1</code> and <code>.2</code> strings are added to the filename to distinguish which file contains mate #1 and mate #2. If a percent symbol, <code>%</code>, is used in <code><path></code>, the percent symbol is replaced with <code>1</code> or <code>2</code> to make the per-mate filenames. Otherwise, <code>.1</code> or <code>.2</code> are added before the final dot in <code><path></code> to make the per-mate filenames. Reads written in this way will appear exactly as they did in the input files, without any modification (same sequence, same name, same quality string, same quality encoding). Reads will not necessarily appear in the same order as they did in the inputs.</p> +<p>Write paired-end reads that align concordantly at least once to +file(s) at <code><path></code>. These reads correspond to the SAM +records with the FLAGS <code>0x4</code> bit unset and either the +<code>0x40</code> or <code>0x80</code> bit set (depending on whether +it's mate #1 or #2). <code>.1</code> and <code>.2</code> strings are +added to the filename to distinguish which file contains mate #1 and +mate #2. If a percent symbol, <code>%</code>, is used in +<code><path></code>, the percent symbol is replaced with +<code>1</code> or <code>2</code> to make the per-mate filenames. +Otherwise, <code>.1</code> or <code>.2</code> are added before the final +dot in <code><path></code> to make the per-mate filenames. Reads +written in this way will appear exactly as they did in the input files, +without any modification (same sequence, same name, same quality string, +same quality encoding). Reads will not necessarily appear in the same +order as they did in the inputs.</p> </td> </tr> <tr> @@ -963,7 +1998,11 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--met-file <path></code></pre> </td> <td> -<p>Write <code>bowtie2</code> metrics to file <code><path></code>. Having alignment metric can be useful for debugging certain problems, especially performance issues. See also: <a href="#bowtie2-options-met"><code>--met</code></a>. Default: metrics disabled.</p> +<p>Write <code>bowtie2</code> metrics to file <code><path></code>. +Having alignment metric can be useful for debugging certain problems, +especially performance issues. See also: <a +href="#bowtie2-options-met"><code>--met</code></a>. Default: metrics +disabled.</p> </td> </tr> <tr> @@ -971,7 +2010,13 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--met-stderr <path></code></pre> </td> <td> -<p>Write <code>bowtie2</code> metrics to the "standard error" ("stderr") filehandle. This is not mutually exclusive with <a href="#bowtie2-options-met-file"><code>--met-file</code></a>. Having alignment metric can be useful for debugging certain problems, especially performance issues. See also: <a href="#bowtie2-options-met"><code>--met</code></a>. Default: metrics disabled.</p> +<p>Write <code>bowtie2</code> metrics to the "standard error" ("stderr") +filehandle. This is not mutually exclusive with <a +href="#bowtie2-options-met-file"><code>--met-file</code></a>. Having +alignment metric can be useful for debugging certain problems, +especially performance issues. See also: <a +href="#bowtie2-options-met"><code>--met</code></a>. Default: metrics +disabled.</p> </td> </tr> <tr> @@ -979,7 +2024,11 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--met <int></code></pre> </td> <td> -<p>Write a new <code>bowtie2</code> metrics record every <code><int></code> seconds. Only matters if either <a href="#bowtie2-options-met-stderr"><code>--met-stderr</code></a> or <a href="#bowtie2-options-met-file"><code>--met-file</code></a> are specified. Default: 1.</p> +<p>Write a new <code>bowtie2</code> metrics record every +<code><int></code> seconds. Only matters if either <a +href="#bowtie2-options-met-stderr"><code>--met-stderr</code></a> or <a +href="#bowtie2-options-met-file"><code>--met-file</code></a> are +specified. Default: 1.</p> </td> </tr> </table> @@ -1014,7 +2063,12 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--rg-id <text></code></pre> </td> <td> -<p>Set the read group ID to <code><text></code>. This causes the SAM <code>@RG</code> header line to be printed, with <code><text></code> as the value associated with the <code>ID:</code> tag. It also causes the <code>RG:Z:</code> extra field to be attached to each SAM output record, with value set to <code><text></code>.</p> +<p>Set the read group ID to <code><text></code>. This causes the +SAM <code>@RG</code> header line to be printed, with +<code><text></code> as the value associated with the +<code>ID:</code> tag. It also causes the <code>RG:Z:</code> extra field +to be attached to each SAM output record, with value set to +<code><text></code>.</p> </td> </tr> <tr> @@ -1022,7 +2076,16 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--rg <text></code></pre> </td> <td> -<p>Add <code><text></code> (usually of the form <code>TAG:VAL</code>, e.g. <code>SM:Pool1</code>) as a field on the <code>@RG</code> header line. Note: in order for the <code>@RG</code> line to appear, <a href="#bowtie2-options-rg-id"><code>--rg-id</code></a> must also be specified. This is because the <code>ID</code> tag is required by the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM Spec</a>. Specify <code>--rg</code> multiple times to set multiple fields. See the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM Spec</a> for details about what fields are legal.</p> +<p>Add <code><text></code> (usually of the form +<code>TAG:VAL</code>, e.g. <code>SM:Pool1</code>) as a field on the +<code>@RG</code> header line. Note: in order for the <code>@RG</code> +line to appear, <a +href="#bowtie2-options-rg-id"><code>--rg-id</code></a> must also be +specified. This is because the <code>ID</code> tag is required by the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM Spec</a>. Specify +<code>--rg</code> multiple times to set multiple fields. See the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM Spec</a> for details +about what fields are legal.</p> </td> </tr> <tr> @@ -1030,7 +2093,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--omit-sec-seq</code></pre> </td> <td> -<p>When printing secondary alignments, Bowtie 2 by default will write out the <code>SEQ</code> and <code>QUAL</code> strings. Specifying this option causes Bowtie 2 to print an asterisk in those fields instead.</p> +<p>When printing secondary alignments, Bowtie 2 by default will write +out the <code>SEQ</code> and <code>QUAL</code> strings. Specifying this +option causes Bowtie 2 to print an asterisk in those fields instead.</p> </td> </tr> <tr> @@ -1038,7 +2103,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--soft-clipped-unmapped-tlen</code></pre> </td> <td> -<p>Consider soft-clipped bases unmapped when calculating <code>TLEN</code>. Only available in <a href="#bowtie2-options-local"><code>--local</code></a> mode.</p> +<p>Consider soft-clipped bases unmapped when calculating +<code>TLEN</code>. Only available in <a +href="#bowtie2-options-local"><code>--local</code></a> mode.</p> </td> </tr> <tr> @@ -1046,7 +2113,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--sam-no-qname-trunc</code></pre> </td> <td> -<p>Suppress standard behavior of truncating readname at first whitespace at the expense of generating non-standard SAM</p> +<p>Suppress standard behavior of truncating readname at first whitespace +at the expense of generating non-standard SAM</p> </td> </tr> <tr> @@ -1054,7 +2122,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--xeq</code></pre> </td> <td> -<p>Use <code>'='/'X'</code>, instead of <code>'M'</code>, to specify matches/mismatches in SAM record</p> +<p>Use <code>'='/'X'</code>, instead of <code>'M'</code>, to specify +matches/mismatches in SAM record</p> </td> </tr> <tr> @@ -1062,7 +2131,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--sam-append-comment</code></pre> </td> <td> -<p>Append FASTA/FASTQ comment to SAM record, where a comment is everything after the first space in the read name.</p> +<p>Append FASTA/FASTQ comment to SAM record, where a comment is +everything after the first space in the read name.</p> </td> </tr> </table> @@ -1073,7 +2143,12 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-o/--offrate <int></code></pre> </td> <td> -<p>Override the offrate of the index with <code><int></code>. If <code><int></code> is greater than the offrate used to build the index, then some row markings are discarded when the index is read into memory. This reduces the memory footprint of the aligner but requires more time to calculate text offsets. <code><int></code> must be greater than the value used to build the index.</p> +<p>Override the offrate of the index with <code><int></code>. If +<code><int></code> is greater than the offrate used to build the +index, then some row markings are discarded when the index is read into +memory. This reduces the memory footprint of the aligner but requires +more time to calculate text offsets. <code><int></code> must be +greater than the value used to build the index.</p> </td> </tr> <tr> @@ -1081,7 +2156,16 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-p/--threads NTHREADS</code></pre> </td> <td> -<p>Launch <code>NTHREADS</code> parallel search threads (default: 1). Threads will run on separate processors/cores and synchronize when parsing reads and outputting alignments. Searching for alignments is highly parallel, and speedup is close to linear. Increasing <code>-p</code> increases Bowtie 2's memory footprint. E.g. when aligning to a human genome index, increasing <code>-p</code> from 1 to 8 increases the memory footprint by a few hundred megabytes. This option is only available if <code>bowtie</code> is linked with the <code>pthreads</code> library (i.e. if <code>BOWTIE_PTHREADS=0</code> is not specified at build time).</p> +<p>Launch <code>NTHREADS</code> parallel search threads (default: 1). +Threads will run on separate processors/cores and synchronize when +parsing reads and outputting alignments. Searching for alignments is +highly parallel, and speedup is close to linear. Increasing +<code>-p</code> increases Bowtie 2's memory footprint. E.g. when +aligning to a human genome index, increasing <code>-p</code> from 1 to 8 +increases the memory footprint by a few hundred megabytes. This option +is only available if <code>bowtie</code> is linked with the +<code>pthreads</code> library (i.e. if <code>BOWTIE_PTHREADS=0</code> is +not specified at build time).</p> </td> </tr> <tr> @@ -1089,7 +2173,15 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--reorder</code></pre> </td> <td> -<p>Guarantees that output SAM records are printed in an order corresponding to the order of the reads in the original input file, even when <a href="#bowtie2-options-p"><code>-p</code></a> is set greater than 1. Specifying <code>--reorder</code> and setting <a href="#bowtie2-options-p"><code>-p</code></a> greater than 1 causes Bowtie 2 to run somewhat slower and use somewhat more memory than if <code>--reorder</code> were not specified. Has no effect if <a href="#bowtie2-options-p"><code>-p</code></a> is set to 1, since output order will naturally correspond to input order in that case.</p> +<p>Guarantees that output SAM records are printed in an order +corresponding to the order of the reads in the original input file, even +when <a href="#bowtie2-options-p"><code>-p</code></a> is set greater +than 1. Specifying <code>--reorder</code> and setting <a +href="#bowtie2-options-p"><code>-p</code></a> greater than 1 causes +Bowtie 2 to run somewhat slower and use somewhat more memory than if +<code>--reorder</code> were not specified. Has no effect if <a +href="#bowtie2-options-p"><code>-p</code></a> is set to 1, since output +order will naturally correspond to input order in that case.</p> </td> </tr> <tr> @@ -1097,7 +2189,13 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--mm</code></pre> </td> <td> -<p>Use memory-mapped I/O to load the index, rather than typical file I/O. Memory-mapping allows many concurrent <code>bowtie</code> processes on the same computer to share the same memory image of the index (i.e. you pay the memory overhead just once). This facilitates memory-efficient parallelization of <code>bowtie</code> in situations where using <a href="#bowtie2-options-p"><code>-p</code></a> is not possible or not preferable.</p> +<p>Use memory-mapped I/O to load the index, rather than typical file +I/O. Memory-mapping allows many concurrent <code>bowtie</code> processes +on the same computer to share the same memory image of the index (i.e. +you pay the memory overhead just once). This facilitates +memory-efficient parallelization of <code>bowtie</code> in situations +where using <a href="#bowtie2-options-p"><code>-p</code></a> is not +possible or not preferable.</p> </td> </tr> </table> @@ -1108,7 +2206,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--qc-filter</code></pre> </td> <td> -<p>Filter out reads for which the QSEQ filter field is non-zero. Only has an effect when read format is <a href="#bowtie2-options-qseq"><code>--qseq</code></a>. Default: off.</p> +<p>Filter out reads for which the QSEQ filter field is non-zero. Only +has an effect when read format is <a +href="#bowtie2-options-qseq"><code>--qseq</code></a>. Default: off.</p> </td> </tr> <tr> @@ -1116,7 +2216,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--seed <int></code></pre> </td> <td> -<p>Use <code><int></code> as the seed for pseudo-random number generator. Default: 0.</p> +<p>Use <code><int></code> as the seed for pseudo-random number +generator. Default: 0.</p> </td> </tr> <tr> @@ -1124,7 +2225,19 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--non-deterministic</code></pre> </td> <td> -<p>Normally, Bowtie 2 re-initializes its pseudo-random generator for each read. It seeds the generator with a number derived from (a) the read name, (b) the nucleotide sequence, (c) the quality sequence, (d) the value of the <a href="#bowtie2-options-seed"><code>--seed</code></a> option. This means that if two reads are identical (same name, same nucleotides, same qualities) Bowtie 2 will find and report the same alignment(s) for both, even if there was ambiguity. When <code>--non-deterministic</code> is specified, Bowtie 2 re-initializes its pseudo-random generator for each read using the current time. This means that Bowtie 2 will not necessarily report the same alignment for two identical reads. This is counter-intuitive for some users, but might be more appropriate in situations where the input consists of many identical reads.</p> +<p>Normally, Bowtie 2 re-initializes its pseudo-random generator for +each read. It seeds the generator with a number derived from (a) the +read name, (b) the nucleotide sequence, (c) the quality sequence, (d) +the value of the <a href="#bowtie2-options-seed"><code>--seed</code></a> +option. This means that if two reads are identical (same name, same +nucleotides, same qualities) Bowtie 2 will find and report the same +alignment(s) for both, even if there was ambiguity. When +<code>--non-deterministic</code> is specified, Bowtie 2 re-initializes +its pseudo-random generator for each read using the current time. This +means that Bowtie 2 will not necessarily report the same alignment for +two identical reads. This is counter-intuitive for some users, but might +be more appropriate in situations where the input consists of many +identical reads.</p> </td> </tr> <tr> @@ -1145,12 +2258,30 @@ Seed 4 rc: TTATGCATGA</code></pre> </tr> </table> <h2 id="sam-output">SAM output</h2> -<p>Following is a brief description of the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> format as output by <code>bowtie2</code>. For more details, see the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM format specification</a>.</p> -<p>By default, <code>bowtie2</code> prints a SAM header with <code>@HD</code>, <code>@SQ</code> and <code>@PG</code> lines. When one or more <a href="#bowtie2-options-rg"><code>--rg</code></a> arguments are specified, <code>bowtie2</code> will also print an <code>@RG</code> line that includes all user-specified <a href="#bowtie2-options-rg"><code>--rg</code></a> tokens separated by tabs.</p> -<p>Each subsequent line describes an alignment or, if the read failed to align, a read. Each line is a collection of at least 12 fields separated by tabs; from left to right, the fields are:</p> +<p>Following is a brief description of the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> format as output +by <code>bowtie2</code>. For more details, see the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM format +specification</a>.</p> +<p>By default, <code>bowtie2</code> prints a SAM header with +<code>@HD</code>, <code>@SQ</code> and <code>@PG</code> lines. When one +or more <a href="#bowtie2-options-rg"><code>--rg</code></a> arguments +are specified, <code>bowtie2</code> will also print an <code>@RG</code> +line that includes all user-specified <a +href="#bowtie2-options-rg"><code>--rg</code></a> tokens separated by +tabs.</p> +<p>Each subsequent line describes an alignment or, if the read failed to +align, a read. Each line is a collection of at least 12 fields separated +by tabs; from left to right, the fields are:</p> <ol type="1"> <li><p>Name of read that aligned.</p> -<p>Note that the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> disallows whitespace in the read name. If the read name contains any whitespace characters, Bowtie 2 will truncate the name at the first whitespace character. This is similar to the behavior of other tools. The standard behavior of truncating at the first whitespace can be suppressed with <code>--sam-no-qname-trunc</code> at the expense of generating non-standard SAM.</p></li> +<p>Note that the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM +specification</a> disallows whitespace in the read name. If the read +name contains any whitespace characters, Bowtie 2 will truncate the name +at the first whitespace character. This is similar to the behavior of +other tools. The standard behavior of truncating at the first whitespace +can be suppressed with <code>--sam-no-qname-trunc</code> at the expense +of generating non-standard SAM.</p></li> <li><p>Sum of all applicable flags. Flags relevant to Bowtie are:</p> <table> <tr> @@ -1198,7 +2329,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>32</code></pre> </td> <td> -<p>The other mate in the paired-end alignment is aligned to the reverse reference strand</p> +<p>The other mate in the paired-end alignment is aligned to the reverse +reference strand</p> </td> </tr> <tr> @@ -1218,17 +2350,36 @@ Seed 4 rc: TTATGCATGA</code></pre> </td> </tr> </table> -<p>Thus, an unpaired read that aligns to the reverse reference strand will have flag 16. A paired-end read that aligns and is the first mate in the pair will have flag 83 (= 64 + 16 + 2 + 1).</p></li> +<p>Thus, an unpaired read that aligns to the reverse reference strand +will have flag 16. A paired-end read that aligns and is the first mate +in the pair will have flag 83 (= 64 + 16 + 2 + 1).</p></li> <li><p>Name of reference sequence where alignment occurs</p></li> -<li><p>1-based offset into the forward reference strand where leftmost character of the alignment occurs</p></li> +<li><p>1-based offset into the forward reference strand where leftmost +character of the alignment occurs</p></li> <li><p>Mapping quality</p></li> <li><p>CIGAR string representation of alignment</p></li> -<li><p>Name of reference sequence where mate's alignment occurs. Set to <code>=</code> if the mate's reference sequence is the same as this alignment's, or <code>*</code> if there is no mate.</p></li> -<li><p>1-based offset into the forward reference strand where leftmost character of the mate's alignment occurs. Offset is 0 if there is no mate.</p></li> -<li><p>Inferred fragment length. Size is negative if the mate's alignment occurs upstream of this alignment. Size is 0 if the mates did not align concordantly. However, size is non-0 if the mates aligned discordantly to the same chromosome.</p></li> -<li><p>Read sequence (reverse-complemented if aligned to the reverse strand)</p></li> -<li><p>ASCII-encoded read qualities (reverse-complemented if the read aligned to the reverse strand). The encoded quality values are on the <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred quality</a> scale and the encoding is ASCII-offset by 33 (ASCII char <code>!</code>), similarly to a <a href="http://en.wikipedia.org/wiki/FASTQ_format">FASTQ</a> file.</p></li> -<li><p>Optional fields. Fields are tab-separated. <code>bowtie2</code> outputs zero or more of these optional fields for each alignment, depending on the type of the alignment:</p></li> +<li><p>Name of reference sequence where mate's alignment occurs. Set to +<code>=</code> if the mate's reference sequence is the same as this +alignment's, or <code>*</code> if there is no mate.</p></li> +<li><p>1-based offset into the forward reference strand where leftmost +character of the mate's alignment occurs. Offset is 0 if there is no +mate.</p></li> +<li><p>Inferred fragment length. Size is negative if the mate's +alignment occurs upstream of this alignment. Size is 0 if the mates did +not align concordantly. However, size is non-0 if the mates aligned +discordantly to the same chromosome.</p></li> +<li><p>Read sequence (reverse-complemented if aligned to the reverse +strand)</p></li> +<li><p>ASCII-encoded read qualities (reverse-complemented if the read +aligned to the reverse strand). The encoded quality values are on the <a +href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred +quality</a> scale and the encoding is ASCII-offset by 33 (ASCII char +<code>!</code>), similarly to a <a +href="http://en.wikipedia.org/wiki/FASTQ_format">FASTQ</a> +file.</p></li> +<li><p>Optional fields. Fields are tab-separated. <code>bowtie2</code> +outputs zero or more of these optional fields for each alignment, +depending on the type of the alignment:</p></li> </ol> <table> <tr> @@ -1236,7 +2387,10 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>AS:i:<N></code></pre> </td> <td> -<p>Alignment score. Can be negative. Can be greater than 0 in <a href="#bowtie2-options-local"><code>--local</code></a> mode (but not in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode). Only present if SAM record is for an aligned read.</p> +<p>Alignment score. Can be negative. Can be greater than 0 in <a +href="#bowtie2-options-local"><code>--local</code></a> mode (but not in +<a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> +mode). Only present if SAM record is for an aligned read.</p> </td> </tr> <tr> @@ -1244,7 +2398,14 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>XS:i:<N></code></pre> </td> <td> -<p>Alignment score for the best-scoring alignment found other than the alignment reported. Can be negative. Can be greater than 0 in <a href="#bowtie2-options-local"><code>--local</code></a> mode (but not in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode). Only present if the SAM record is for an aligned read and more than one alignment was found for the read. Note that, when the read is part of a concordantly-aligned pair, this score could be greater than <a href="#bowtie2-build-opt-fields-as"><code>AS:i</code></a>.</p> +<p>Alignment score for the best-scoring alignment found other than the +alignment reported. Can be negative. Can be greater than 0 in <a +href="#bowtie2-options-local"><code>--local</code></a> mode (but not in +<a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> +mode). Only present if the SAM record is for an aligned read and more +than one alignment was found for the read. Note that, when the read is +part of a concordantly-aligned pair, this score could be greater than <a +href="#bowtie2-build-opt-fields-as"><code>AS:i</code></a>.</p> </td> </tr> <tr> @@ -1252,7 +2413,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>YS:i:<N></code></pre> </td> <td> -<p>Alignment score for opposite mate in the paired-end alignment. Only present if the SAM record is for a read that aligned as part of a paired-end alignment.</p> +<p>Alignment score for opposite mate in the paired-end alignment. Only +present if the SAM record is for a read that aligned as part of a +paired-end alignment.</p> </td> </tr> <tr> @@ -1260,7 +2423,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>XN:i:<N></code></pre> </td> <td> -<p>The number of ambiguous bases in the reference covering this alignment. Only present if SAM record is for an aligned read.</p> +<p>The number of ambiguous bases in the reference covering this +alignment. Only present if SAM record is for an aligned read.</p> </td> </tr> <tr> @@ -1268,7 +2432,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>XM:i:<N></code></pre> </td> <td> -<p>The number of mismatches in the alignment. Only present if SAM record is for an aligned read.</p> +<p>The number of mismatches in the alignment. Only present if SAM record +is for an aligned read.</p> </td> </tr> <tr> @@ -1276,7 +2441,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>XO:i:<N></code></pre> </td> <td> -<p>The number of gap opens, for both read and reference gaps, in the alignment. Only present if SAM record is for an aligned read.</p> +<p>The number of gap opens, for both read and reference gaps, in the +alignment. Only present if SAM record is for an aligned read.</p> </td> </tr> <tr> @@ -1284,7 +2450,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>XG:i:<N></code></pre> </td> <td> -<p>The number of gap extensions, for both read and reference gaps, in the alignment. Only present if SAM record is for an aligned read.</p> +<p>The number of gap extensions, for both read and reference gaps, in +the alignment. Only present if SAM record is for an aligned read.</p> </td> </tr> <tr> @@ -1292,7 +2459,10 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>NM:i:<N></code></pre> </td> <td> -<p>The edit distance; that is, the minimal number of one-nucleotide edits (substitutions, insertions and deletions) needed to transform the read string into the reference string. Only present if SAM record is for an aligned read.</p> +<p>The edit distance; that is, the minimal number of one-nucleotide +edits (substitutions, insertions and deletions) needed to transform the +read string into the reference string. Only present if SAM record is for +an aligned read.</p> </td> </tr> <tr> @@ -1300,7 +2470,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>YF:Z:<S></code></pre> </td> <td> -<p>String indicating reason why the read was filtered out. See also: <a href="#filtering">Filtering</a>. Only appears for reads that were filtered out.</p> +<p>String indicating reason why the read was filtered out. See also: <a +href="#filtering">Filtering</a>. Only appears for reads that were +filtered out.</p> </td> </tr> <tr> @@ -1308,7 +2480,12 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>YT:Z:<S></code></pre> </td> <td> -<p>Value of <code>UU</code> indicates the read was not part of a pair. Value of <code>CP</code> indicates the read was part of a pair and the pair aligned concordantly. Value of <code>DP</code> indicates the read was part of a pair and the pair aligned discordantly. Value of <code>UP</code> indicates the read was part of a pair but the pair failed to aligned either concordantly or discordantly.</p> +<p>Value of <code>UU</code> indicates the read was not part of a pair. +Value of <code>CP</code> indicates the read was part of a pair and the +pair aligned concordantly. Value of <code>DP</code> indicates the read +was part of a pair and the pair aligned discordantly. Value of +<code>UP</code> indicates the read was part of a pair but the pair +failed to aligned either concordantly or discordantly.</p> </td> </tr> <tr> @@ -1316,17 +2493,68 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>MD:Z:<S></code></pre> </td> <td> -<p>A string representation of the mismatched reference bases in the alignment. See <a href="https://samtools.github.io/hts-specs/SAMtags.pdf">SAM Tags format specification</a> for details. Only present if SAM record is for an aligned read.</p> +<p>A string representation of the mismatched reference bases in the +alignment. See <a +href="https://samtools.github.io/hts-specs/SAMtags.pdf">SAM Tags format +specification</a> for details. Only present if SAM record is for an +aligned read.</p> </td> </tr> </table> -<h1 id="the-bowtie2-build-indexer">The <code>bowtie2-build</code> indexer</h1> -<p><code>bowtie2-build</code> builds a Bowtie index from a set of DNA sequences. <code>bowtie2-build</code> outputs a set of 6 files with suffixes <code>.1.bt2</code>, <code>.2.bt2</code>, <code>.3.bt2</code>, <code>.4.bt2</code>, <code>.rev.1.bt2</code>, and <code>.rev.2.bt2</code>. In the case of a large index these suffixes will have a <code>bt2l</code> termination. These files together constitute the index: they are all that is needed to align reads to that reference. The original sequence <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files are no longer used by Bowtie 2 once the index is built.</p> -<p>Bowtie 2's <code>.bt2</code> index format is different from Bowtie 1's <code>.ebwt</code> format, and they are not compatible with each other.</p> -<p>Use of Karkkainen's <a href="http://portal.acm.org/citation.cfm?id=1314852">blockwise algorithm</a> allows <code>bowtie2-build</code> to trade off between running time and memory usage. <code>bowtie2-build</code> has three options governing how it makes this trade: <a href="#bowtie2-build-options-p"><code>-p</code>/<code>--packed</code></a>, <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>/<a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>, and <a href="#bowtie2-build-options-dcv"><code>--dcv</code></a>. By default, <code>bowtie2-build</code> will automatically search for the settings that yield the best running time without exhausting memory. This behavior can be disabled using the <a href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> option.</p> -<p>The indexer provides options pertaining to the "shape" of the index, e.g. <a href="#bowtie2-build-options-o"><code>--offrate</code></a> governs the fraction of <a href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> rows that are "marked" (i.e., the density of the suffix-array sample; see the original <a href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> paper for details). All of these options are potentially profitable trade-offs depending on the application. They have been set to defaults that are reasonable for most cases according to our experiments. See <a href="#performance-tuning">Performance tuning</a> for details.</p> -<p><code>bowtie2-build</code> can generate either <a href="#small-and-large-indexes">small or large indexes</a>. The wrapper will decide which based on the length of the input genome. If the reference does not exceed 4 billion characters but a large index is preferred, the user can specify <a href="#bowtie2-build-options-large-index"><code>--large-index</code></a> to force <code>bowtie2-build</code> to build a large index instead.</p> -<p>The Bowtie 2 index is based on the <a href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> of Ferragina and Manzini, which in turn is based on the <a href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> transform. The algorithm used to build the index is based on the <a href="http://portal.acm.org/citation.cfm?id=1314852">blockwise algorithm</a> of Karkkainen.</p> +<h1 id="the-bowtie2-build-indexer">The <code>bowtie2-build</code> +indexer</h1> +<p><code>bowtie2-build</code> builds a Bowtie index from a set of DNA +sequences. <code>bowtie2-build</code> outputs a set of 6 files with +suffixes <code>.1.bt2</code>, <code>.2.bt2</code>, <code>.3.bt2</code>, +<code>.4.bt2</code>, <code>.rev.1.bt2</code>, and +<code>.rev.2.bt2</code>. In the case of a large index these suffixes +will have a <code>bt2l</code> termination. These files together +constitute the index: they are all that is needed to align reads to that +reference. The original sequence <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files +are no longer used by Bowtie 2 once the index is built.</p> +<p>Bowtie 2's <code>.bt2</code> index format is different from Bowtie +1's <code>.ebwt</code> format, and they are not compatible with each +other.</p> +<p>Use of Karkkainen's <a +href="http://portal.acm.org/citation.cfm?id=1314852">blockwise +algorithm</a> allows <code>bowtie2-build</code> to trade off between +running time and memory usage. <code>bowtie2-build</code> has three +options governing how it makes this trade: <a +href="#bowtie2-build-options-p"><code>-p</code>/<code>--packed</code></a>, +<a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>/<a +href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>, and +<a href="#bowtie2-build-options-dcv"><code>--dcv</code></a>. By default, +<code>bowtie2-build</code> will automatically search for the settings +that yield the best running time without exhausting memory. This +behavior can be disabled using the <a +href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> +option.</p> +<p>The indexer provides options pertaining to the "shape" of the index, +e.g. <a href="#bowtie2-build-options-o"><code>--offrate</code></a> +governs the fraction of <a +href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> +rows that are "marked" (i.e., the density of the suffix-array sample; +see the original <a +href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> paper +for details). All of these options are potentially profitable trade-offs +depending on the application. They have been set to defaults that are +reasonable for most cases according to our experiments. See <a +href="#performance-tuning">Performance tuning</a> for details.</p> +<p><code>bowtie2-build</code> can generate either <a +href="#small-and-large-indexes">small or large indexes</a>. The wrapper +will decide which based on the length of the input genome. If the +reference does not exceed 4 billion characters but a large index is +preferred, the user can specify <a +href="#bowtie2-build-options-large-index"><code>--large-index</code></a> +to force <code>bowtie2-build</code> to build a large index instead.</p> +<p>The Bowtie 2 index is based on the <a +href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> of +Ferragina and Manzini, which in turn is based on the <a +href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> +transform. The algorithm used to build the index is based on the <a +href="http://portal.acm.org/citation.cfm?id=1314852">blockwise +algorithm</a> of Karkkainen.</p> <h2 id="command-line-1">Command Line</h2> <p>Usage:</p> <pre><code>bowtie2-build [options]* <reference_in> <bt2_base></code></pre> @@ -1337,7 +2565,14 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code><reference_in></code></pre> </td> <td> -<p>A comma-separated list of <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files containing the reference sequences to be aligned to, or, if <a href="#bowtie2-build-options-c"><code>-c</code></a> is specified, the sequences themselves. E.g., <code><reference_in></code> might be <code>chr1.fa,chr2.fa,chrX.fa,chrY.fa</code>, or, if <a href="#bowtie2-build-options-c"><code>-c</code></a> is specified, this might be <code>GGTCATCCT,ACGGGTCGT,CCGTTCTATGCGGCTTA</code>.</p> +<p>A comma-separated list of <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files +containing the reference sequences to be aligned to, or, if <a +href="#bowtie2-build-options-c"><code>-c</code></a> is specified, the +sequences themselves. E.g., <code><reference_in></code> might be +<code>chr1.fa,chr2.fa,chrX.fa,chrY.fa</code>, or, if <a +href="#bowtie2-build-options-c"><code>-c</code></a> is specified, this +might be <code>GGTCATCCT,ACGGGTCGT,CCGTTCTATGCGGCTTA</code>.</p> </td> </tr> <tr> @@ -1345,7 +2580,12 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code><bt2_base></code></pre> </td> <td> -<p>The basename of the index files to write. By default, <code>bowtie2-build</code> writes files named <code>NAME.1.bt2</code>, <code>NAME.2.bt2</code>, <code>NAME.3.bt2</code>, <code>NAME.4.bt2</code>, <code>NAME.rev.1.bt2</code>, and <code>NAME.rev.2.bt2</code>, where <code>NAME</code> is <code><bt2_base></code>.</p> +<p>The basename of the index files to write. By default, +<code>bowtie2-build</code> writes files named <code>NAME.1.bt2</code>, +<code>NAME.2.bt2</code>, <code>NAME.3.bt2</code>, +<code>NAME.4.bt2</code>, <code>NAME.rev.1.bt2</code>, and +<code>NAME.rev.2.bt2</code>, where <code>NAME</code> is +<code><bt2_base></code>.</p> </td> </tr> </table> @@ -1356,7 +2596,11 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-f</code></pre> </td> <td> -<p>The reference input files (specified as <code><reference_in></code>) are <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files (usually having extension <code>.fa</code>, <code>.mfa</code>, <code>.fna</code> or similar).</p> +<p>The reference input files (specified as +<code><reference_in></code>) are <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files +(usually having extension <code>.fa</code>, <code>.mfa</code>, +<code>.fna</code> or similar).</p> </td> </tr> <tr> @@ -1364,7 +2608,11 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-c</code></pre> </td> <td> -<p>The reference sequences are given on the command line. I.e. <code><reference_in></code> is a comma-separated list of sequences rather than a list of <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files.</p> +<p>The reference sequences are given on the command line. I.e. +<code><reference_in></code> is a comma-separated list of sequences +rather than a list of <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> +files.</p> </td> </tr> <tr> @@ -1372,7 +2620,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--large-index</code></pre> </td> <td> -<p>Force <code>bowtie2-build</code> to build a <a href="#small-and-large-indexes">large index</a>, even if the reference is less than ~ 4 billion nucleotides inlong.</p> +<p>Force <code>bowtie2-build</code> to build a <a +href="#small-and-large-indexes">large index</a>, even if the reference +is less than ~ 4 billion nucleotides inlong.</p> </td> </tr> <tr> @@ -1380,7 +2630,14 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-a/--noauto</code></pre> </td> <td> -<p>Disable the default behavior whereby <code>bowtie2-build</code> automatically selects values for the <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, <a href="#bowtie2-build-options-dcv"><code>--dcv</code></a> and <a href="#bowtie2-build-options-p"><code>--packed</code></a> parameters according to available memory. Instead, user may specify values for those parameters. If memory is exhausted during indexing, an error message will be printed; it is up to the user to try new parameters.</p> +<p>Disable the default behavior whereby <code>bowtie2-build</code> +automatically selects values for the <a +href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, <a +href="#bowtie2-build-options-dcv"><code>--dcv</code></a> and <a +href="#bowtie2-build-options-p"><code>--packed</code></a> parameters +according to available memory. Instead, user may specify values for +those parameters. If memory is exhausted during indexing, an error +message will be printed; it is up to the user to try new parameters.</p> </td> </tr> <tr> @@ -1388,7 +2645,11 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-p/--packed</code></pre> </td> <td> -<p>Use a packed (2-bits-per-nucleotide) representation for DNA strings. This saves memory but makes indexing 2-3 times slower. Default: off. This is configured automatically by default; use <a href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> to configure manually.</p> +<p>Use a packed (2-bits-per-nucleotide) representation for DNA strings. +This saves memory but makes indexing 2-3 times slower. Default: off. +This is configured automatically by default; use <a +href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> +to configure manually.</p> </td> </tr> <tr> @@ -1396,7 +2657,18 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--bmax <int></code></pre> </td> <td> -<p>The maximum number of suffixes allowed in a block. Allowing more suffixes per block makes indexing faster, but increases peak memory usage. Setting this option overrides any previous setting for <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, or <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>. Default (in terms of the <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> parameter) is <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> 4 * number of threads. This is configured automatically by default; use <a href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> to configure manually.</p> +<p>The maximum number of suffixes allowed in a block. Allowing more +suffixes per block makes indexing faster, but increases peak memory +usage. Setting this option overrides any previous setting for <a +href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, or <a +href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>. +Default (in terms of the <a +href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> +parameter) is <a +href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> 4 * +number of threads. This is configured automatically by default; use <a +href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> +to configure manually.</p> </td> </tr> <tr> @@ -1404,7 +2676,16 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--bmaxdivn <int></code></pre> </td> <td> -<p>The maximum number of suffixes allowed in a block, expressed as a fraction of the length of the reference. Setting this option overrides any previous setting for <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, or <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>. Default: <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> 4 * number of threads. This is configured automatically by default; use <a href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> to configure manually.</p> +<p>The maximum number of suffixes allowed in a block, expressed as a +fraction of the length of the reference. Setting this option overrides +any previous setting for <a +href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, or <a +href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>. +Default: <a +href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> 4 * +number of threads. This is configured automatically by default; use <a +href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> +to configure manually.</p> </td> </tr> <tr> @@ -1412,7 +2693,13 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--dcv <int></code></pre> </td> <td> -<p>Use <code><int></code> as the period for the difference-cover sample. A larger period yields less memory overhead, but may make suffix sorting slower, especially if repeats are present. Must be a power of 2 no greater than 4096. Default: 1024. This is configured automatically by default; use <a href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> to configure manually.</p> +<p>Use <code><int></code> as the period for the difference-cover +sample. A larger period yields less memory overhead, but may make suffix +sorting slower, especially if repeats are present. Must be a power of 2 +no greater than 4096. Default: 1024. This is configured automatically by +default; use <a +href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> +to configure manually.</p> </td> </tr> <tr> @@ -1420,7 +2707,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--nodc</code></pre> </td> <td> -<p>Disable use of the difference-cover sample. Suffix sorting becomes quadratic-time in the worst case (where the worst case is an extremely repetitive reference). Default: off.</p> +<p>Disable use of the difference-cover sample. Suffix sorting becomes +quadratic-time in the worst case (where the worst case is an extremely +repetitive reference). Default: off.</p> </td> </tr> <tr> @@ -1428,7 +2717,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-r/--noref</code></pre> </td> <td> -<p>Do not build the <code>NAME.3.bt2</code> and <code>NAME.4.bt2</code> portions of the index, which contain a bitpacked version of the reference sequences and are used for paired-end alignment.</p> +<p>Do not build the <code>NAME.3.bt2</code> and <code>NAME.4.bt2</code> +portions of the index, which contain a bitpacked version of the +reference sequences and are used for paired-end alignment.</p> </td> </tr> <tr> @@ -1436,7 +2727,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-3/--justref</code></pre> </td> <td> -<p>Build only the <code>NAME.3.bt2</code> and <code>NAME.4.bt2</code> portions of the index, which contain a bitpacked version of the reference sequences and are used for paired-end alignment.</p> +<p>Build only the <code>NAME.3.bt2</code> and <code>NAME.4.bt2</code> +portions of the index, which contain a bitpacked version of the +reference sequences and are used for paired-end alignment.</p> </td> </tr> <tr> @@ -1444,7 +2737,16 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-o/--offrate <int></code></pre> </td> <td> -<p>To map alignments back to positions on the reference sequences, it's necessary to annotate ("mark") some or all of the <a href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> rows with their corresponding location on the genome. <a href="#bowtie2-build-options-o"><code>-o</code>/<code>--offrate</code></a> governs how many rows get marked: the indexer will mark every 2^<code><int></code> rows. Marking more rows makes reference-position lookups faster, but requires more memory to hold the annotations at runtime. The default is 5 (every 32nd row is marked; for human genome, annotations occupy about 340 megabytes).</p> +<p>To map alignments back to positions on the reference sequences, it's +necessary to annotate ("mark") some or all of the <a +href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> +rows with their corresponding location on the genome. <a +href="#bowtie2-build-options-o"><code>-o</code>/<code>--offrate</code></a> +governs how many rows get marked: the indexer will mark every +2^<code><int></code> rows. Marking more rows makes +reference-position lookups faster, but requires more memory to hold the +annotations at runtime. The default is 5 (every 32nd row is marked; for +human genome, annotations occupy about 340 megabytes).</p> </td> </tr> <tr> @@ -1452,7 +2754,13 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-t/--ftabchars <int></code></pre> </td> <td> -<p>The ftab is the lookup table used to calculate an initial <a href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> range with respect to the first <code><int></code> characters of the query. A larger <code><int></code> yields a larger lookup table but faster query times. The ftab has size 4^(<code><int></code>+1) bytes. The default setting is 10 (ftab is 4MB).</p> +<p>The ftab is the lookup table used to calculate an initial <a +href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> +range with respect to the first <code><int></code> characters of +the query. A larger <code><int></code> yields a larger lookup +table but faster query times. The ftab has size +4^(<code><int></code>+1) bytes. The default setting is 10 (ftab is +4MB).</p> </td> </tr> <tr> @@ -1460,7 +2768,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--seed <int></code></pre> </td> <td> -<p>Use <code><int></code> as the seed for pseudo-random number generator.</p> +<p>Use <code><int></code> as the seed for pseudo-random number +generator.</p> </td> </tr> <tr> @@ -1468,7 +2777,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--cutoff <int></code></pre> </td> <td> -<p>Index only the first <code><int></code> bases of the reference sequences (cumulative across sequences) and ignore the rest.</p> +<p>Index only the first <code><int></code> bases of the reference +sequences (cumulative across sequences) and ignore the rest.</p> </td> </tr> <tr> @@ -1476,7 +2786,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-q/--quiet</code></pre> </td> <td> -<p><code>bowtie2-build</code> is verbose by default. With this option <code>bowtie2-build</code> will print only error messages.</p> +<p><code>bowtie2-build</code> is verbose by default. With this option +<code>bowtie2-build</code> will print only error messages.</p> </td> </tr> <tr> @@ -1484,7 +2795,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--threads <int></code></pre> </td> <td> -<p>By default <code>bowtie2-build</code> is using only one thread. Increasing the number of threads will speed up the index building considerably in most cases.</p> +<p>By default <code>bowtie2-build</code> is using only one thread. +Increasing the number of threads will speed up the index building +considerably in most cases.</p> </td> </tr> <tr> @@ -1504,8 +2817,20 @@ Seed 4 rc: TTATGCATGA</code></pre> </td> </tr> </table> -<h1 id="the-bowtie2-inspect-index-inspector">The <code>bowtie2-inspect</code> index inspector</h1> -<p><code>bowtie2-inspect</code> extracts information from a Bowtie index about what kind of index it is and what reference sequences were used to build it. When run without any options, the tool will output a <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> file containing the sequences of the original references (with all non-<code>A</code>/<code>C</code>/<code>G</code>/<code>T</code> characters converted to <code>N</code>s). It can also be used to extract just the reference sequence names using the <a href="#bowtie2-inspect-options-n"><code>-n</code>/<code>--names</code></a> option or a more verbose summary using the <a href="#bowtie2-inspect-options-s"><code>-s</code>/<code>--summary</code></a> option.</p> +<h1 id="the-bowtie2-inspect-index-inspector">The +<code>bowtie2-inspect</code> index inspector</h1> +<p><code>bowtie2-inspect</code> extracts information from a Bowtie index +about what kind of index it is and what reference sequences were used to +build it. When run without any options, the tool will output a <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> file +containing the sequences of the original references (with all +non-<code>A</code>/<code>C</code>/<code>G</code>/<code>T</code> +characters converted to <code>N</code>s). It can also be used to extract +just the reference sequence names using the <a +href="#bowtie2-inspect-options-n"><code>-n</code>/<code>--names</code></a> +option or a more verbose summary using the <a +href="#bowtie2-inspect-options-s"><code>-s</code>/<code>--summary</code></a> +option.</p> <h2 id="command-line-2">Command Line</h2> <p>Usage:</p> <pre><code>bowtie2-inspect [options]* <bt2_base></code></pre> @@ -1516,7 +2841,12 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code><bt2_base></code></pre> </td> <td> -<p>The basename of the index to be inspected. The basename is name of any of the index files but with the <code>.X.bt2</code> or <code>.rev.X.bt2</code> suffix omitted. <code>bowtie2-inspect</code> first looks in the current directory for the index files, then in the directory specified in the <code>BOWTIE2_INDEXES</code> environment variable.</p> +<p>The basename of the index to be inspected. The basename is name of +any of the index files but with the <code>.X.bt2</code> or +<code>.rev.X.bt2</code> suffix omitted. <code>bowtie2-inspect</code> +first looks in the current directory for the index files, then in the +directory specified in the <code>BOWTIE2_INDEXES</code> environment +variable.</p> </td> </tr> </table> @@ -1527,7 +2857,10 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-a/--across <int></code></pre> </td> <td> -<p>When printing <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> output, output a newline character every <code><int></code> bases (default: 60).</p> +<p>When printing <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> +output, output a newline character every <code><int></code> bases +(default: 60).</p> </td> </tr> <tr> @@ -1543,7 +2876,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-s/--summary</code></pre> </td> <td> -<p>Print a summary that includes information about index settings, as well as the names and lengths of the input sequences. The summary has this format:</p> +<p>Print a summary that includes information about index settings, as +well as the names and lengths of the input sequences. The summary has +this format:</p> <pre><code>Colorspace <0 or 1> SA-Sample 1 in <sample> FTab-Chars <chars> @@ -1551,7 +2886,8 @@ Sequence-1 <name> <len> Sequence-2 <name> <len> ... Sequence-N <name> <len></code></pre> -<p>Fields are separated by tabs. Colorspace is always set to 0 for Bowtie 2.</p> +<p>Fields are separated by tabs. Colorspace is always set to 0 for +Bowtie 2.</p> </td> </tr> <tr> @@ -1587,18 +2923,59 @@ Sequence-N <name> <len></code></pre> </td> </tr> </table> -<h1 id="getting-started-with-bowtie-2-lambda-phage-example">Getting started with Bowtie 2: Lambda phage example</h1> -<p>Bowtie 2 comes with some example files to get you started. The example files are not scientifically significant; we use the <a href="http://en.wikipedia.org/wiki/Lambda_phage">Lambda phage</a> reference genome simply because it's short, and the reads were generated by a computer program, not a sequencer. However, these files will let you start running Bowtie 2 and downstream tools right away.</p> -<p>First follow the manual instructions to <a href="#obtaining-bowtie-2">obtain Bowtie 2</a>. Set the <code>BT2_HOME</code> environment variable to point to the new Bowtie 2 directory containing the <code>bowtie2</code>, <code>bowtie2-build</code> and <code>bowtie2-inspect</code> binaries. This is important, as the <code>BT2_HOME</code> variable is used in the commands below to refer to that directory.</p> +<h1 id="getting-started-with-bowtie-2-lambda-phage-example">Getting +started with Bowtie 2: Lambda phage example</h1> +<p>Bowtie 2 comes with some example files to get you started. The +example files are not scientifically significant; we use the <a +href="http://en.wikipedia.org/wiki/Lambda_phage">Lambda phage</a> +reference genome simply because it's short, and the reads were generated +by a computer program, not a sequencer. However, these files will let +you start running Bowtie 2 and downstream tools right away.</p> +<p>First follow the manual instructions to <a +href="#obtaining-bowtie-2">obtain Bowtie 2</a>. Set the +<code>BT2_HOME</code> environment variable to point to the new Bowtie 2 +directory containing the <code>bowtie2</code>, +<code>bowtie2-build</code> and <code>bowtie2-inspect</code> binaries. +This is important, as the <code>BT2_HOME</code> variable is used in the +commands below to refer to that directory.</p> <h2 id="indexing-a-reference-genome">Indexing a reference genome</h2> -<p>To create an index for the <a href="http://en.wikipedia.org/wiki/Lambda_phage">Lambda phage</a> reference genome included with Bowtie 2, create a new temporary directory (it doesn't matter where), change into that directory, and run:</p> +<p>To create an index for the <a +href="http://en.wikipedia.org/wiki/Lambda_phage">Lambda phage</a> +reference genome included with Bowtie 2, create a new temporary +directory (it doesn't matter where), change into that directory, and +run:</p> <pre><code>$BT2_HOME/bowtie2-build $BT2_HOME/example/reference/lambda_virus.fa lambda_virus</code></pre> -<p>The command should print many lines of output then quit. When the command completes, the current directory will contain four new files that all start with <code>lambda_virus</code> and end with <code>.1.bt2</code>, <code>.2.bt2</code>, <code>.3.bt2</code>, <code>.4.bt2</code>, <code>.rev.1.bt2</code>, and <code>.rev.2.bt2</code>. These files constitute the index - you're done!</p> -<p>You can use <code>bowtie2-build</code> to create an index for a set of <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files obtained from any source, including sites such as <a href="http://genome.ucsc.edu/cgi-bin/hgGateway">UCSC</a>, <a href="http://www.ncbi.nlm.nih.gov/sites/genome">NCBI</a>, and <a href="http://www.ensembl.org/">Ensembl</a>. When indexing multiple <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files, specify all the files using commas to separate file names. For more details on how to create an index with <code>bowtie2-build</code>, see the <a href="#the-bowtie2-build-indexer">manual section on index building</a>. You may also want to bypass this process by obtaining a pre-built index. See <a href="#using-a-pre-built-index">using a pre-built index</a> below for an example.</p> +<p>The command should print many lines of output then quit. When the +command completes, the current directory will contain four new files +that all start with <code>lambda_virus</code> and end with +<code>.1.bt2</code>, <code>.2.bt2</code>, <code>.3.bt2</code>, +<code>.4.bt2</code>, <code>.rev.1.bt2</code>, and +<code>.rev.2.bt2</code>. These files constitute the index - you're +done!</p> +<p>You can use <code>bowtie2-build</code> to create an index for a set +of <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> +files obtained from any source, including sites such as <a +href="http://genome.ucsc.edu/cgi-bin/hgGateway">UCSC</a>, <a +href="http://www.ncbi.nlm.nih.gov/sites/genome">NCBI</a>, and <a +href="http://www.ensembl.org/">Ensembl</a>. When indexing multiple <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files, +specify all the files using commas to separate file names. For more +details on how to create an index with <code>bowtie2-build</code>, see +the <a href="#the-bowtie2-build-indexer">manual section on index +building</a>. You may also want to bypass this process by obtaining a +pre-built index. See <a href="#using-a-pre-built-index">using a +pre-built index</a> below for an example.</p> <h2 id="aligning-example-reads">Aligning example reads</h2> -<p>Stay in the directory created in the previous step, which now contains the <code>lambda_virus</code> index files. Next, run:</p> +<p>Stay in the directory created in the previous step, which now +contains the <code>lambda_virus</code> index files. Next, run:</p> <pre><code>$BT2_HOME/bowtie2 -x lambda_virus -U $BT2_HOME/example/reads/reads_1.fq -S eg1.sam</code></pre> -<p>This runs the Bowtie 2 aligner, which aligns a set of unpaired reads to the <a href="http://en.wikipedia.org/wiki/Lambda_phage">Lambda phage</a> reference genome using the index generated in the previous step. The alignment results in SAM format are written to the file <code>eg1.sam</code>, and a short alignment summary is written to the console. (Actually, the summary is written to the "standard error" or "stderr" filehandle, which is typically printed to the console.)</p> +<p>This runs the Bowtie 2 aligner, which aligns a set of unpaired reads +to the <a href="http://en.wikipedia.org/wiki/Lambda_phage">Lambda +phage</a> reference genome using the index generated in the previous +step. The alignment results in SAM format are written to the file +<code>eg1.sam</code>, and a short alignment summary is written to the +console. (Actually, the summary is written to the "standard error" or +"stderr" filehandle, which is typically printed to the console.)</p> <p>To see the first few lines of the SAM output, run:</p> <pre><code>head eg1.sam</code></pre> <p>You will see something like this:</p> @@ -1612,28 +2989,61 @@ r4 0 gi|9626243|ref|NC_001416.1| 40075 42 184M * 0 0 GGGCCAATGCGC r5 0 gi|9626243|ref|NC_001416.1| 48010 42 138M * 0 0 GTCAGGAAAGTGGTAAAACTGCAACTCAATTACTGCAATGCCCTCGTAATTAAGTGAATTTACAATATCGTCCTGTTCGGAGGGAAGAACGCGGGATGTTCATTCTTCATCACTTTTAATTGATGTATATGCTCTCTT 9''%<D)A03E1-*7=),:F/0!6,D9:H,<9D%:0B(%'E,(8EFG$E89B$27G8F*2+4,-!,0D5()&=(FGG:5;3*@/.0F-G#5#3->('FDFEG?)5.!)"AGADB3?6(@H(:B<>6!>;>6>G,."?% AS:i:0 XN:i:0 XM:i:0 XO:i:0 XG:i:0 NM:i:0 MD:Z:138 YT:Z:UU r6 16 gi|9626243|ref|NC_001416.1| 41607 42 72M2D119M * 0 0 TCGATTTGCAAATACCGGAACATCTCGGTAACTGCATATTCTGCATTAAAAAATCAACGCAAAAAATCGGACGCCTGCAAAGATGAGGAGGGATTGCAGCGTGTTTTTAATGAGGTCATCACGGGATNCCATGTGCGTGACGGNCATCGGGAAACGCCAAAGGAGATTATGTACCGAGGAAGAATGTCGCT 1H#G;H"$E*E#&"*)2%66?=9/9'=;4)4/>@%+5#@#$4A*!<D=="8#1*A9BA=:(1+#C&.#(3#H=9E)AC*5,AC#E'536*2?)H14?>9'B=7(3H/B:+A:8%1-+#(E%&$$&14"76D?>7(&20H5%*&CF8!G5B+A4F$7(:"'?0$?G+$)B-?2<0<F=D!38BH,%=8&5@+ AS:i:-13 XN:i:0 XM:i:2 XO:i:1 XG:i:2 NM:i:4 MD:Z:72^TT55C15A47 YT:Z:UU r7 16 gi|9626243|ref|NC_001416.1| 4692 42 143M * 0 0 TCAGCCGGACGCGGGCGCTGCAGCCGTACTCGGGGATGACCGGTTACAACGGCATTATCGCCCGTCTGCAACAGGCTGCCAGCGATCCGATGGTGGACAGCATTCTGCTCGATATGGACANGCCCGGCGGGATGGTGGCGGGG -"/@*7A0)>2,AAH@&"%B)*5*23B/,)90.B@%=FE,E063C9?,:26$-0:,.,1849'4.;F>FA;76+5&$<C":$!A*,<B,<)@<'85D%C*:)30@85;?.B$05=@95DCDH<53!8G:F:B7/A.E':434> AS:i:-6 XN:i:0 XM:i:2 XO:i:0 XG:i:0 NM:i:2 MD:Z:98G21C22 YT:Z:UU</code></pre> -<p>The first few lines (beginning with <code>@</code>) are SAM header lines, and the rest of the lines are SAM alignments, one line per read or mate. See the <a href="#sam-output">Bowtie 2 manual section on SAM output</a> and the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> for details about how to interpret the SAM file format.</p> +<p>The first few lines (beginning with <code>@</code>) are SAM header +lines, and the rest of the lines are SAM alignments, one line per read +or mate. See the <a href="#sam-output">Bowtie 2 manual section on SAM +output</a> and the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> +for details about how to interpret the SAM file format.</p> <h2 id="paired-end-example">Paired-end example</h2> -<p>To align paired-end reads included with Bowtie 2, stay in the same directory and run:</p> +<p>To align paired-end reads included with Bowtie 2, stay in the same +directory and run:</p> <pre><code>$BT2_HOME/bowtie2 -x lambda_virus -1 $BT2_HOME/example/reads/reads_1.fq -2 $BT2_HOME/example/reads/reads_2.fq -S eg2.sam</code></pre> -<p>This aligns a set of paired-end reads to the reference genome, with results written to the file <code>eg2.sam</code>.</p> +<p>This aligns a set of paired-end reads to the reference genome, with +results written to the file <code>eg2.sam</code>.</p> <h2 id="local-alignment-example-1">Local alignment example</h2> -<p>To use <a href="#end-to-end-alignment-versus-local-alignment">local alignment</a> to align some longer reads included with Bowtie 2, stay in the same directory and run:</p> +<p>To use <a href="#end-to-end-alignment-versus-local-alignment">local +alignment</a> to align some longer reads included with Bowtie 2, stay in +the same directory and run:</p> <pre><code>$BT2_HOME/bowtie2 --local -x lambda_virus -U $BT2_HOME/example/reads/longreads.fq -S eg3.sam</code></pre> -<p>This aligns the long reads to the reference genome using local alignment, with results written to the file <code>eg3.sam</code>.</p> -<h2 id="using-samtoolsbcftools-downstream">Using SAMtools/BCFtools downstream</h2> -<p><a href="http://samtools.sourceforge.net">SAMtools</a> is a collection of tools for manipulating and analyzing SAM and BAM alignment files. <a href="http://samtools.sourceforge.net/mpileup.shtml">BCFtools</a> is a collection of tools for calling variants and manipulating VCF and BCF files, and it is typically distributed with <a href="http://samtools.sourceforge.net">SAMtools</a>. Using these tools together allows you to get from alignments in SAM format to variant calls in VCF format. This example assumes that <code>samtools</code> and <code>bcftools</code> are installed and that the directories containing these binaries are in your <a href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH environment variable</a>.</p> +<p>This aligns the long reads to the reference genome using local +alignment, with results written to the file <code>eg3.sam</code>.</p> +<h2 id="using-samtoolsbcftools-downstream">Using SAMtools/BCFtools +downstream</h2> +<p><a href="http://samtools.sourceforge.net">SAMtools</a> is a +collection of tools for manipulating and analyzing SAM and BAM alignment +files. <a +href="http://samtools.sourceforge.net/mpileup.shtml">BCFtools</a> is a +collection of tools for calling variants and manipulating VCF and BCF +files, and it is typically distributed with <a +href="http://samtools.sourceforge.net">SAMtools</a>. Using these tools +together allows you to get from alignments in SAM format to variant +calls in VCF format. This example assumes that <code>samtools</code> and +<code>bcftools</code> are installed and that the directories containing +these binaries are in your <a +href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH environment +variable</a>.</p> <p>Run the paired-end example:</p> <pre><code>$BT2_HOME/bowtie2 -x $BT2_HOME/example/index/lambda_virus -1 $BT2_HOME/example/reads/reads_1.fq -2 $BT2_HOME/example/reads/reads_2.fq -S eg2.sam</code></pre> -<p>Use <code>samtools view</code> to convert the SAM file into a BAM file. BAM is the binary format corresponding to the SAM text format. Run:</p> +<p>Use <code>samtools view</code> to convert the SAM file into a BAM +file. BAM is the binary format corresponding to the SAM text format. +Run:</p> <pre><code>samtools view -bS eg2.sam > eg2.bam</code></pre> -<p>Use <code>samtools sort</code> to convert the BAM file to a sorted BAM file.</p> +<p>Use <code>samtools sort</code> to convert the BAM file to a sorted +BAM file.</p> <pre><code>samtools sort eg2.bam -o eg2.sorted.bam</code></pre> -<p>We now have a sorted BAM file called <code>eg2.sorted.bam</code>. Sorted BAM is a useful format because the alignments are (a) compressed, which is convenient for long-term storage, and (b) sorted, which is conveneint for variant discovery. To generate variant calls in VCF format, run:</p> +<p>We now have a sorted BAM file called <code>eg2.sorted.bam</code>. +Sorted BAM is a useful format because the alignments are (a) compressed, +which is convenient for long-term storage, and (b) sorted, which is +conveneint for variant discovery. To generate variant calls in VCF +format, run:</p> <pre><code>bcftools mpileup -f $BT2_HOME/example/reference/lambda_virus.fa eg2.sorted.bam | bcftools view -Ov - > eg2.raw.bcf</code></pre> <p>Then to view the variants, run:</p> <pre><code>bcftools view eg2.raw.bcf</code></pre> -<p>See the official SAMtools guide to <a href="http://samtools.sourceforge.net/mpileup.shtml">Calling SNPs/INDELs with SAMtools/BCFtools</a> for more details and variations on this process.</p> +<p>See the official SAMtools guide to <a +href="http://samtools.sourceforge.net/mpileup.shtml">Calling SNPs/INDELs +with SAMtools/BCFtools</a> for more details and variations on this +process.</p> <!-- fasta-continuous option does not exist but was added to prevent warnings due to the case insensitive nature of markdown URLs --> </body> diff --git a/doc/website/manual.ssi b/doc/website/manual.ssi index 1a578a9..4bf1ed4 100644 --- a/doc/website/manual.ssi +++ b/doc/website/manual.ssi @@ -1,90 +1,162 @@ <nav id="TOC" role="doc-toc"> <ul> -<li><a href="#introduction">Introduction</a> +<li><a href="#introduction" id="toc-introduction">Introduction</a> <ul> -<li><a href="#how-is-bowtie-2-different-from-bowtie-1">How is Bowtie 2 different from Bowtie 1?</a></li> -<li><a href="#what-isnt-bowtie-2">What isn't Bowtie 2?</a></li> +<li><a href="#how-is-bowtie-2-different-from-bowtie-1" +id="toc-how-is-bowtie-2-different-from-bowtie-1">How is Bowtie 2 +different from Bowtie 1?</a></li> +<li><a href="#what-isnt-bowtie-2" id="toc-what-isnt-bowtie-2">What isn't +Bowtie 2?</a></li> </ul></li> -<li><a href="#obtaining-bowtie-2">Obtaining Bowtie 2</a> +<li><a href="#obtaining-bowtie-2" id="toc-obtaining-bowtie-2">Obtaining +Bowtie 2</a> <ul> -<li><a href="#building-from-source">Building from source</a></li> -<li><a href="#adding-to-path">Adding to PATH</a></li> +<li><a href="#building-from-source" +id="toc-building-from-source">Building from source</a></li> +<li><a href="#adding-to-path" id="toc-adding-to-path">Adding to +PATH</a></li> </ul></li> -<li><a href="#the-bowtie2-aligner">The <code>bowtie2</code> aligner</a> +<li><a href="#the-bowtie2-aligner" id="toc-the-bowtie2-aligner">The +<code>bowtie2</code> aligner</a> <ul> -<li><a href="#end-to-end-alignment-versus-local-alignment">End-to-end alignment versus local alignment</a> +<li><a href="#end-to-end-alignment-versus-local-alignment" +id="toc-end-to-end-alignment-versus-local-alignment">End-to-end +alignment versus local alignment</a> <ul> -<li><a href="#end-to-end-alignment-example">End-to-end alignment example</a></li> -<li><a href="#local-alignment-example">Local alignment example</a></li> +<li><a href="#end-to-end-alignment-example" +id="toc-end-to-end-alignment-example">End-to-end alignment +example</a></li> +<li><a href="#local-alignment-example" +id="toc-local-alignment-example">Local alignment example</a></li> </ul></li> -<li><a href="#scores-higher-more-similar">Scores: higher = more similar</a> +<li><a href="#scores-higher-more-similar" +id="toc-scores-higher-more-similar">Scores: higher = more similar</a> <ul> -<li><a href="#end-to-end-alignment-score-example">End-to-end alignment score example</a></li> -<li><a href="#local-alignment-score-example">Local alignment score example</a></li> -<li><a href="#valid-alignments-meet-or-exceed-the-minimum-score-threshold">Valid alignments meet or exceed the minimum score threshold</a></li> +<li><a href="#end-to-end-alignment-score-example" +id="toc-end-to-end-alignment-score-example">End-to-end alignment score +example</a></li> +<li><a href="#local-alignment-score-example" +id="toc-local-alignment-score-example">Local alignment score +example</a></li> +<li><a +href="#valid-alignments-meet-or-exceed-the-minimum-score-threshold" +id="toc-valid-alignments-meet-or-exceed-the-minimum-score-threshold">Valid +alignments meet or exceed the minimum score threshold</a></li> </ul></li> -<li><a href="#mapping-quality-higher-more-unique">Mapping quality: higher = more unique</a></li> -<li><a href="#aligning-pairs">Aligning pairs</a> +<li><a href="#mapping-quality-higher-more-unique" +id="toc-mapping-quality-higher-more-unique">Mapping quality: higher = +more unique</a></li> +<li><a href="#aligning-pairs" id="toc-aligning-pairs">Aligning pairs</a> <ul> -<li><a href="#paired-inputs">Paired inputs</a></li> -<li><a href="#paired-sam-output">Paired SAM output</a></li> -<li><a href="#concordant-pairs-match-pair-expectations-discordant-pairs-dont">Concordant pairs match pair expectations, discordant pairs don't</a></li> -<li><a href="#mixed-mode-paired-where-possible-unpaired-otherwise">Mixed mode: paired where possible, unpaired otherwise</a></li> -<li><a href="#some-sam-flags-describe-paired-end-properties">Some SAM FLAGS describe paired-end properties</a></li> -<li><a href="#some-sam-optional-fields-describe-more-paired-end-properties">Some SAM optional fields describe more paired-end properties</a></li> -<li><a href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can overlap, contain, or dovetail each other</a></li> +<li><a href="#paired-inputs" id="toc-paired-inputs">Paired +inputs</a></li> +<li><a href="#paired-sam-output" id="toc-paired-sam-output">Paired SAM +output</a></li> +<li><a +href="#concordant-pairs-match-pair-expectations-discordant-pairs-dont" +id="toc-concordant-pairs-match-pair-expectations-discordant-pairs-dont">Concordant +pairs match pair expectations, discordant pairs don't</a></li> +<li><a href="#mixed-mode-paired-where-possible-unpaired-otherwise" +id="toc-mixed-mode-paired-where-possible-unpaired-otherwise">Mixed mode: +paired where possible, unpaired otherwise</a></li> +<li><a href="#some-sam-flags-describe-paired-end-properties" +id="toc-some-sam-flags-describe-paired-end-properties">Some SAM FLAGS +describe paired-end properties</a></li> +<li><a +href="#some-sam-optional-fields-describe-more-paired-end-properties" +id="toc-some-sam-optional-fields-describe-more-paired-end-properties">Some +SAM optional fields describe more paired-end properties</a></li> +<li><a href="#mates-can-overlap-contain-or-dovetail-each-other" +id="toc-mates-can-overlap-contain-or-dovetail-each-other">Mates can +overlap, contain, or dovetail each other</a></li> </ul></li> -<li><a href="#reporting">Reporting</a> +<li><a href="#reporting" id="toc-reporting">Reporting</a> <ul> -<li><a href="#distinct-alignments-map-a-read-to-different-places">Distinct alignments map a read to different places</a></li> -<li><a href="#default-mode-search-for-multiple-alignments-report-the-best-one">Default mode: search for multiple alignments, report the best one</a></li> -<li><a href="#k-mode-search-for-one-or-more-alignments-report-each">-k mode: search for one or more alignments, report each</a></li> -<li><a href="#a-mode-search-for-and-report-all-alignments">-a mode: search for and report all alignments</a></li> -<li><a href="#randomness-in-bowtie-2">Randomness in Bowtie 2</a></li> +<li><a href="#distinct-alignments-map-a-read-to-different-places" +id="toc-distinct-alignments-map-a-read-to-different-places">Distinct +alignments map a read to different places</a></li> +<li><a +href="#default-mode-search-for-multiple-alignments-report-the-best-one" +id="toc-default-mode-search-for-multiple-alignments-report-the-best-one">Default +mode: search for multiple alignments, report the best one</a></li> +<li><a href="#k-mode-search-for-one-or-more-alignments-report-each" +id="toc-k-mode-search-for-one-or-more-alignments-report-each">-k mode: +search for one or more alignments, report each</a></li> +<li><a href="#a-mode-search-for-and-report-all-alignments" +id="toc-a-mode-search-for-and-report-all-alignments">-a mode: search for +and report all alignments</a></li> +<li><a href="#randomness-in-bowtie-2" +id="toc-randomness-in-bowtie-2">Randomness in Bowtie 2</a></li> </ul></li> -<li><a href="#multiseed-heuristic">Multiseed heuristic</a> +<li><a href="#multiseed-heuristic" +id="toc-multiseed-heuristic">Multiseed heuristic</a> <ul> -<li><a href="#fm-index-memory-footprint">FM Index memory footprint</a></li> +<li><a href="#fm-index-memory-footprint" +id="toc-fm-index-memory-footprint">FM Index memory footprint</a></li> </ul></li> -<li><a href="#ambiguous-characters">Ambiguous characters</a></li> -<li><a href="#presets-setting-many-settings-at-once">Presets: setting many settings at once</a></li> -<li><a href="#filtering">Filtering</a></li> -<li><a href="#alignment-summary">Alignment summary</a></li> -<li><a href="#wrapper-scripts">Wrapper scripts</a></li> -<li><a href="#small-and-large-indexes">Small and large indexes</a></li> -<li><a href="#performance-tuning">Performance tuning</a></li> -<li><a href="#command-line">Command Line</a> +<li><a href="#ambiguous-characters" +id="toc-ambiguous-characters">Ambiguous characters</a></li> +<li><a href="#presets-setting-many-settings-at-once" +id="toc-presets-setting-many-settings-at-once">Presets: setting many +settings at once</a></li> +<li><a href="#filtering" id="toc-filtering">Filtering</a></li> +<li><a href="#alignment-summary" id="toc-alignment-summary">Alignment +summary</a></li> +<li><a href="#wrapper-scripts" id="toc-wrapper-scripts">Wrapper +scripts</a></li> +<li><a href="#small-and-large-indexes" +id="toc-small-and-large-indexes">Small and large indexes</a></li> +<li><a href="#performance-tuning" +id="toc-performance-tuning">Performance tuning</a></li> +<li><a href="#command-line" id="toc-command-line">Command Line</a> <ul> -<li><a href="#setting-function-options">Setting function options</a></li> -<li><a href="#usage">Usage</a></li> -<li><a href="#main-arguments">Main arguments</a></li> -<li><a href="#options">Options</a></li> +<li><a href="#setting-function-options" +id="toc-setting-function-options">Setting function options</a></li> +<li><a href="#usage" id="toc-usage">Usage</a></li> +<li><a href="#main-arguments" id="toc-main-arguments">Main +arguments</a></li> +<li><a href="#options" id="toc-options">Options</a></li> </ul></li> -<li><a href="#sam-output">SAM output</a></li> +<li><a href="#sam-output" id="toc-sam-output">SAM output</a></li> </ul></li> -<li><a href="#the-bowtie2-build-indexer">The <code>bowtie2-build</code> indexer</a> +<li><a href="#the-bowtie2-build-indexer" +id="toc-the-bowtie2-build-indexer">The <code>bowtie2-build</code> +indexer</a> <ul> -<li><a href="#command-line-1">Command Line</a> +<li><a href="#command-line-1" id="toc-command-line-1">Command Line</a> <ul> -<li><a href="#main-arguments-1">Main arguments</a></li> -<li><a href="#options-1">Options</a></li> +<li><a href="#main-arguments-1" id="toc-main-arguments-1">Main +arguments</a></li> +<li><a href="#options-1" id="toc-options-1">Options</a></li> </ul></li> </ul></li> -<li><a href="#the-bowtie2-inspect-index-inspector">The <code>bowtie2-inspect</code> index inspector</a> +<li><a href="#the-bowtie2-inspect-index-inspector" +id="toc-the-bowtie2-inspect-index-inspector">The +<code>bowtie2-inspect</code> index inspector</a> <ul> -<li><a href="#command-line-2">Command Line</a> +<li><a href="#command-line-2" id="toc-command-line-2">Command Line</a> <ul> -<li><a href="#main-arguments-2">Main arguments</a></li> -<li><a href="#options-2">Options</a></li> +<li><a href="#main-arguments-2" id="toc-main-arguments-2">Main +arguments</a></li> +<li><a href="#options-2" id="toc-options-2">Options</a></li> </ul></li> </ul></li> -<li><a href="#getting-started-with-bowtie-2-lambda-phage-example">Getting started with Bowtie 2: Lambda phage example</a> +<li><a href="#getting-started-with-bowtie-2-lambda-phage-example" +id="toc-getting-started-with-bowtie-2-lambda-phage-example">Getting +started with Bowtie 2: Lambda phage example</a> <ul> -<li><a href="#indexing-a-reference-genome">Indexing a reference genome</a></li> -<li><a href="#aligning-example-reads">Aligning example reads</a></li> -<li><a href="#paired-end-example">Paired-end example</a></li> -<li><a href="#local-alignment-example-1">Local alignment example</a></li> -<li><a href="#using-samtoolsbcftools-downstream">Using SAMtools/BCFtools downstream</a></li> +<li><a href="#indexing-a-reference-genome" +id="toc-indexing-a-reference-genome">Indexing a reference +genome</a></li> +<li><a href="#aligning-example-reads" +id="toc-aligning-example-reads">Aligning example reads</a></li> +<li><a href="#paired-end-example" id="toc-paired-end-example">Paired-end +example</a></li> +<li><a href="#local-alignment-example-1" +id="toc-local-alignment-example-1">Local alignment example</a></li> +<li><a href="#using-samtoolsbcftools-downstream" +id="toc-using-samtoolsbcftools-downstream">Using SAMtools/BCFtools +downstream</a></li> </ul></li> </ul> </nav> @@ -94,58 +166,204 @@ ! of this text document, or see the HTML manual online. ! --> <h1 id="introduction">Introduction</h1> -<p><a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> is an ultrafast and memory-efficient tool for aligning sequencing reads to long reference sequences. It is particularly good at aligning reads of about 50 up to 100s of characters to relatively long (e.g. mammalian) genomes. Bowtie 2 indexes the genome with an <a href="http://en.wikipedia.org/wiki/FM-index">FM Index</a> (based on the <a href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler Transform</a> or <a href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">BWT</a>) to keep its memory footprint small: for the human genome, its memory footprint is typically around 3.2 gigabytes of RAM. Bowtie 2 supports gapped, local, and paired-end alignment modes. Multiple processors can be used simultaneously to achieve greater alignment speed.</p> -<p>Bowtie 2 outputs alignments in <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> format, enabling interoperation with a large number of other tools (e.g. <a href="http://samtools.sourceforge.net">SAMtools</a>, <a href="http://www.broadinstitute.org/gsa/wiki/index.php/The_Genome_Analysis_Toolkit">GATK</a>) that use SAM. Bowtie 2 is distributed under the <a href="http://www.gnu.org/licenses/gpl-3.0.html">GPLv3 license</a>, and it runs on the command line under Windows, Mac OS X and Linux and BSD.</p> -<p><a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> is often the first step in pipelines for comparative genomics, including for variation calling, ChIP-seq, RNA-seq, BS-seq. <a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> and <a href="http://bowtie-bio.sf.net">Bowtie</a> (also called "<a href="http://bowtie-bio.sf.net">Bowtie 1</a>" here) are also tightly integrated into many other tools, some of which <a href="http://bowtie-bio.sourceforge.net/bowtie2/other_tools.shtml">are listed here</a>.</p> -<p>If you use <a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> for your published research, please cite our work. Papers describing Bowtie 2 are:</p> +<p><a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> is an +ultrafast and memory-efficient tool for aligning sequencing reads to +long reference sequences. It is particularly good at aligning reads of +about 50 up to 100s of characters to relatively long (e.g. mammalian) +genomes. Bowtie 2 indexes the genome with an <a +href="http://en.wikipedia.org/wiki/FM-index">FM Index</a> (based on the +<a +href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler +Transform</a> or <a +href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">BWT</a>) +to keep its memory footprint small: for the human genome, its memory +footprint is typically around 3.2 gigabytes of RAM. Bowtie 2 supports +gapped, local, and paired-end alignment modes. Multiple processors can +be used simultaneously to achieve greater alignment speed.</p> +<p>Bowtie 2 outputs alignments in <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> format, enabling +interoperation with a large number of other tools (e.g. <a +href="http://samtools.sourceforge.net">SAMtools</a>, <a +href="http://www.broadinstitute.org/gsa/wiki/index.php/The_Genome_Analysis_Toolkit">GATK</a>) +that use SAM. Bowtie 2 is distributed under the <a +href="http://www.gnu.org/licenses/gpl-3.0.html">GPLv3 license</a>, and +it runs on the command line under Windows, Mac OS X and Linux and +BSD.</p> +<p><a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> is often the +first step in pipelines for comparative genomics, including for +variation calling, ChIP-seq, RNA-seq, BS-seq. <a +href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> and <a +href="http://bowtie-bio.sf.net">Bowtie</a> (also called "<a +href="http://bowtie-bio.sf.net">Bowtie 1</a>" here) are also tightly +integrated into many other tools, some of which <a +href="http://bowtie-bio.sourceforge.net/bowtie2/other_tools.shtml">are +listed here</a>.</p> +<p>If you use <a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> +for your published research, please cite our work. Papers describing +Bowtie 2 are:</p> <ul> -<li><p>Langmead B, Wilks C, Antonescu V, Charles R. <a href="https://doi.org/10.1093/bioinformatics/bty648">Scaling read aligners to hundreds of threads on general-purpose processors</a>. <em>Bioinformatics</em>. 2018 Jul 18. doi: 10.1093/bioinformatics/bty648.</p></li> -<li><p>Langmead B, Salzberg SL. <a href="https://www.nature.com/articles/nmeth.1923">Fast gapped-read alignment with Bowtie 2</a>. <em>Nature Methods</em>. 2012 Mar 4;9(4):357-9. doi: 10.1038/nmeth.1923.</p></li> +<li><p>Langmead B, Wilks C, Antonescu V, Charles R. <a +href="https://doi.org/10.1093/bioinformatics/bty648">Scaling read +aligners to hundreds of threads on general-purpose processors</a>. +<em>Bioinformatics</em>. 2018 Jul 18. doi: +10.1093/bioinformatics/bty648.</p></li> +<li><p>Langmead B, Salzberg SL. <a +href="https://www.nature.com/articles/nmeth.1923">Fast gapped-read +alignment with Bowtie 2</a>. <em>Nature Methods</em>. 2012 Mar +4;9(4):357-9. doi: 10.1038/nmeth.1923.</p></li> </ul> -<h2 id="how-is-bowtie-2-different-from-bowtie-1">How is Bowtie 2 different from Bowtie 1?</h2> -<p>Bowtie 1 was released in 2009 and was geared toward aligning the relatively short sequencing reads (up to 25-50 nucleotides) prevalent at the time. Since then, technology has improved both sequencing throughput (more nucleotides produced per sequencer per day) and read length (more nucleotides per read).</p> +<h2 id="how-is-bowtie-2-different-from-bowtie-1">How is Bowtie 2 +different from Bowtie 1?</h2> +<p>Bowtie 1 was released in 2009 and was geared toward aligning the +relatively short sequencing reads (up to 25-50 nucleotides) prevalent at +the time. Since then, technology has improved both sequencing throughput +(more nucleotides produced per sequencer per day) and read length (more +nucleotides per read).</p> <p>The chief differences between Bowtie 1 and Bowtie 2 are:</p> <ol type="1"> -<li><p>For reads longer than about 50 bp Bowtie 2 is generally faster, more sensitive, and uses less memory than Bowtie 1. For relatively short reads (e.g. less than 50 bp) Bowtie 1 is sometimes faster and/or more sensitive.</p></li> -<li><p>Bowtie 2 supports gapped alignment with affine gap penalties. Number of gaps and gap lengths are not restricted, except by way of the configurable scoring scheme. Bowtie 1 finds just ungapped alignments.</p></li> -<li><p>Bowtie 2 supports <a href="#end-to-end-alignment-versus-local-alignment">local alignment</a>, which doesn't require reads to align end-to-end. Local alignments might be "trimmed" ("soft clipped") at one or both extremes in a way that optimizes alignment score. Bowtie 2 also supports <a href="#end-to-end-alignment-versus-local-alignment">end-to-end alignment</a> which, like Bowtie 1, requires that the read align entirely.</p></li> -<li><p>There is no upper limit on read length in Bowtie 2. Bowtie 1 had an upper limit of around 1000 bp.</p></li> -<li><p>Bowtie 2 allows alignments to <a href="#ambiguous-characters">overlap ambiguous characters</a> (e.g. <code>N</code>s) in the reference. Bowtie 1 does not.</p></li> -<li><p>Bowtie 2 does away with Bowtie 1's notion of alignment "stratum", and its distinction between "Maq-like" and "end-to-end" modes. In Bowtie 2 all alignments lie along a continuous spectrum of alignment scores where the <a href="#scores-higher-more-similar">scoring scheme</a>, similar to <a href="http://en.wikipedia.org/wiki/Needleman-Wunsch_algorithm">Needleman-Wunsch</a> and <a href="http://en.wikipedia.org/wiki/Smith_waterman">Smith-Waterman</a>.</p></li> -<li><p>Bowtie 2's <a href="#aligning-pairs">paired-end alignment</a> is more flexible. E.g. for pairs that do not align in a paired fashion, Bowtie 2 attempts to find unpaired alignments for each mate.</p></li> -<li><p>Bowtie 2 reports a spectrum of mapping qualities, in contrast for Bowtie 1 which reports either 0 or high.</p></li> +<li><p>For reads longer than about 50 bp Bowtie 2 is generally faster, +more sensitive, and uses less memory than Bowtie 1. For relatively short +reads (e.g. less than 50 bp) Bowtie 1 is sometimes faster and/or more +sensitive.</p></li> +<li><p>Bowtie 2 supports gapped alignment with affine gap penalties. +Number of gaps and gap lengths are not restricted, except by way of the +configurable scoring scheme. Bowtie 1 finds just ungapped +alignments.</p></li> +<li><p>Bowtie 2 supports <a +href="#end-to-end-alignment-versus-local-alignment">local alignment</a>, +which doesn't require reads to align end-to-end. Local alignments might +be "trimmed" ("soft clipped") at one or both extremes in a way that +optimizes alignment score. Bowtie 2 also supports <a +href="#end-to-end-alignment-versus-local-alignment">end-to-end +alignment</a> which, like Bowtie 1, requires that the read align +entirely.</p></li> +<li><p>There is no upper limit on read length in Bowtie 2. Bowtie 1 had +an upper limit of around 1000 bp.</p></li> +<li><p>Bowtie 2 allows alignments to <a +href="#ambiguous-characters">overlap ambiguous characters</a> (e.g. +<code>N</code>s) in the reference. Bowtie 1 does not.</p></li> +<li><p>Bowtie 2 does away with Bowtie 1's notion of alignment "stratum", +and its distinction between "Maq-like" and "end-to-end" modes. In Bowtie +2 all alignments lie along a continuous spectrum of alignment scores +where the <a href="#scores-higher-more-similar">scoring scheme</a>, +similar to <a +href="http://en.wikipedia.org/wiki/Needleman-Wunsch_algorithm">Needleman-Wunsch</a> +and <a +href="http://en.wikipedia.org/wiki/Smith_waterman">Smith-Waterman</a>.</p></li> +<li><p>Bowtie 2's <a href="#aligning-pairs">paired-end alignment</a> is +more flexible. E.g. for pairs that do not align in a paired fashion, +Bowtie 2 attempts to find unpaired alignments for each mate.</p></li> +<li><p>Bowtie 2 reports a spectrum of mapping qualities, in contrast for +Bowtie 1 which reports either 0 or high.</p></li> <li><p>Bowtie 2 does not align colorspace reads.</p></li> </ol> -<p>Bowtie 2 is not a "drop-in" replacement for Bowtie 1. Bowtie 2's command-line arguments and genome index format are both different from Bowtie 1's.</p> +<p>Bowtie 2 is not a "drop-in" replacement for Bowtie 1. Bowtie 2's +command-line arguments and genome index format are both different from +Bowtie 1's.</p> <h2 id="what-isnt-bowtie-2">What isn't Bowtie 2?</h2> -<p>Bowtie 2 is geared toward aligning relatively short sequencing reads to long genomes. That said, it handles arbitrarily small reference sequences (e.g. amplicons) and very long reads (i.e. upwards of 10s or 100s of kilobases), though it is slower in those settings. It is optimized for the read lengths and error modes yielded by typical Illumina sequencers.</p> -<p>Bowtie 2 does not support alignment of colorspace reads. (Bowtie 1 does.)</p> +<p>Bowtie 2 is geared toward aligning relatively short sequencing reads +to long genomes. That said, it handles arbitrarily small reference +sequences (e.g. amplicons) and very long reads (i.e. upwards of 10s or +100s of kilobases), though it is slower in those settings. It is +optimized for the read lengths and error modes yielded by typical +Illumina sequencers.</p> +<p>Bowtie 2 does not support alignment of colorspace reads. (Bowtie 1 +does.)</p> <h1 id="obtaining-bowtie-2">Obtaining Bowtie 2</h1> -<p>Bowtie 2 is available from various package managers, notably <a href="https://anaconda.org/bioconda/bowtie2">Bioconda</a>. With Bioconda installed, you should be able to install Bowtie 2 with <code>conda install bowtie2</code>.</p> -<p>Containerized versions of Bowtie 2 are also available via the <a href="https://BioContainers.pro">Biocontainers</a> project (e.g. <a href="https://hub.docker.com/r/biocontainers/bowtie2/">via Docker Hub</a>).</p> -<p>You can also download Bowtie 2 sources and binaries from the <a href="https://sourceforge.net/projects/bowtie-bio/files/bowtie2/">Download</a> section of the Sourceforge site. Binaries are available for the <code>x86_64</code> architecture running Linux, Mac OS X, and Windows. FreeBSD users can obtain the latest version of Bowtie 2 from ports using <code>pkg install bowtie2</code>. If you plan to compile Bowtie 2 yourself, make sure to get the source package, i.e., the filename that ends in "-source.zip".</p> +<p>Bowtie 2 is available from various package managers, notably <a +href="https://anaconda.org/bioconda/bowtie2">Bioconda</a>. With Bioconda +installed, you should be able to install Bowtie 2 with +<code>conda install bowtie2</code>.</p> +<p>You can also download Bowtie 2 sources and binaries from the <a +href="https://sourceforge.net/projects/bowtie-bio/files/bowtie2/">Download</a> +section of the Sourceforge site. Binaries are available for the +<code>x86_64</code> architecture running Linux, Mac OS X, and Windows. +FreeBSD users can obtain the latest version of Bowtie 2 from ports using +<code>pkg install bowtie2</code>. If you plan to compile Bowtie 2 +yourself, make sure to get the source package, i.e., the filename that +ends in "-source.zip".</p> <h2 id="building-from-source">Building from source</h2> <p>Building from source</p> -<p>Building Bowtie 2 from source requires a GNU-like environment with Clang/GCC, GNU Make and other basics. It should be possible to build Bowtie 2 on most vanilla *NIX installations or on a Mac installation with Xcode installed. Bowtie 2 can also be built on Windows using a 64-bit MinGW distribution and MSYS. In order to simplify the MinGW setup it might be worth investigating popular MinGW personal builds since these are coming already prepared with most of the toolchains needed.</p> -<p>First, download the source package from the <a href="https://sourceforge.net/projects/bowtie-bio/files/bowtie2/">sourceforge site</a>. Make sure you're getting the source package; the file downloaded should end in <code>-source.zip</code>. Unzip the file, change to the unzipped directory, and build the Bowtie 2 tools by running GNU <code>make</code> (usually with the command <code>make</code>, but sometimes with <code>gmake</code>) with no arguments. If building with MinGW, run <code>make</code> from the MSYS environment.</p> -<p>The Bowtie 2 Makefile also includes recipes for basic automatic dependency management. Running <code>make static-libs && make STATIC_BUILD=1</code> will issue a series of commands that will: 1. download zstd and zlib 2. compile them as static libraries 3. link the resulting libraries to the compiled Bowtie 2 binaries</p> -<p>As of version 2.3.5 bowtie2 now supports aligning SRA reads. Prepackaged builds will include a package that supports SRA. If you're building bowtie2 from source please make sure that the Java runtime is available on your system. You can then proceed with the build by running <code>make sra-deps && make USE_SRA=1</code>.</p> +<p>Building Bowtie 2 from source requires a GNU-like environment with +Clang/GCC, GNU Make and other basics. It should be possible to build +Bowtie 2 on most vanilla *NIX installations or on a Mac installation +with Xcode installed. Bowtie 2 can also be built on Windows using a +64-bit MinGW distribution and MSYS. In order to simplify the MinGW setup +it might be worth investigating popular MinGW personal builds since +these are coming already prepared with most of the toolchains +needed.</p> +<p>First, download the source package from the <a +href="https://sourceforge.net/projects/bowtie-bio/files/bowtie2/">sourceforge +site</a>. Make sure you're getting the source package; the file +downloaded should end in <code>-source.zip</code>. Unzip the file, +change to the unzipped directory, and build the Bowtie 2 tools by +running GNU <code>make</code> (usually with the command +<code>make</code>, but sometimes with <code>gmake</code>) with no +arguments. If building with MinGW, run <code>make</code> from the MSYS +environment.</p> +<p>The Bowtie 2 Makefile also includes recipes for basic automatic +dependency management. Running +<code>make static-libs && make STATIC_BUILD=1</code> will issue +a series of commands that will: 1. download zstd and zlib 2. compile +them as static libraries 3. link the resulting libraries to the compiled +Bowtie 2 binaries</p> +<p>As of version 2.3.5 bowtie2 now supports aligning SRA reads. +Prepackaged builds will include a package that supports SRA. If you're +building bowtie2 from source please make sure that the Java runtime is +available on your system. You can then proceed with the build by running +<code>make sra-deps && make USE_SRA=1</code>.</p> <h2 id="adding-to-path">Adding to PATH</h2> -<p>By adding your new Bowtie 2 directory to your <a href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH environment variable</a>, you ensure that whenever you run <code>bowtie2</code>, <code>bowtie2-build</code> or <code>bowtie2-inspect</code> from the command line, you will get the version you just installed without having to specify the entire path. This is recommended for most users. To do this, follow your operating system's instructions for adding the directory to your <a href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH</a>.</p> -<p>If you would like to install Bowtie 2 by copying the Bowtie 2 executable files to an existing directory in your <a href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH</a>, make sure that you copy all the executables, including <code>bowtie2</code>, <code>bowtie2-align-s</code>, <code>bowtie2-align-l</code>, <code>bowtie2-build</code>, <code>bowtie2-build-s</code>, <code>bowtie2-build-l</code>, <code>bowtie2-inspect</code>, <code>bowtie2-inspect-s</code> and <code>bowtie2-inspect-l</code>.</p> +<p>By adding your new Bowtie 2 directory to your <a +href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH environment +variable</a>, you ensure that whenever you run <code>bowtie2</code>, +<code>bowtie2-build</code> or <code>bowtie2-inspect</code> from the +command line, you will get the version you just installed without having +to specify the entire path. This is recommended for most users. To do +this, follow your operating system's instructions for adding the +directory to your <a +href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH</a>.</p> +<p>If you would like to install Bowtie 2 by copying the Bowtie 2 +executable files to an existing directory in your <a +href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH</a>, make sure +that you copy all the executables, including <code>bowtie2</code>, +<code>bowtie2-align-s</code>, <code>bowtie2-align-l</code>, +<code>bowtie2-build</code>, <code>bowtie2-build-s</code>, +<code>bowtie2-build-l</code>, <code>bowtie2-inspect</code>, +<code>bowtie2-inspect-s</code> and <code>bowtie2-inspect-l</code>.</p> <h1 id="the-bowtie2-aligner">The <code>bowtie2</code> aligner</h1> -<p><code>bowtie2</code> takes a Bowtie 2 index and a set of sequencing read files and outputs a set of alignments in SAM format.</p> -<p>"Alignment" is the process by which we discover how and where the read sequences are similar to the reference sequence. An "alignment" is a result from this process, specifically: an alignment is a way of "lining up" some or all of the characters in the read with some characters from the reference in a way that reveals how they're similar. For example:</p> +<p><code>bowtie2</code> takes a Bowtie 2 index and a set of sequencing +read files and outputs a set of alignments in SAM format.</p> +<p>"Alignment" is the process by which we discover how and where the +read sequences are similar to the reference sequence. An "alignment" is +a result from this process, specifically: an alignment is a way of +"lining up" some or all of the characters in the read with some +characters from the reference in a way that reveals how they're similar. +For example:</p> <pre><code> Read: GACTGGGCGATCTCGACTTCG ||||| |||||||||| ||| Reference: GACTG--CGATCTCGACATCG</code></pre> -<p>Where dash symbols represent gaps and vertical bars show where aligned characters match.</p> -<p>We use alignment to make an educated guess as to where a read originated with respect to the reference genome. It's not always possible to determine this with certainty. For instance, if the reference genome contains several long stretches of As (<code>AAAAAAAAA</code> etc.) and the read sequence is a short stretch of As (<code>AAAAAAA</code>), we cannot know for certain exactly where in the sea of <code>A</code>s the read originated.</p> -<h2 id="end-to-end-alignment-versus-local-alignment">End-to-end alignment versus local alignment</h2> -<p>By default, Bowtie 2 performs end-to-end read alignment. That is, it searches for alignments involving all of the read characters. This is also called an "untrimmed" or "unclipped" alignment.</p> -<p>When the --local option is specified, Bowtie 2 performs local read alignment. In this mode, Bowtie 2 might "trim" or "clip" some read characters from one or both ends of the alignment if doing so maximizes the alignment score.</p> +<p>Where dash symbols represent gaps and vertical bars show where +aligned characters match.</p> +<p>We use alignment to make an educated guess as to where a read +originated with respect to the reference genome. It's not always +possible to determine this with certainty. For instance, if the +reference genome contains several long stretches of As +(<code>AAAAAAAAA</code> etc.) and the read sequence is a short stretch +of As (<code>AAAAAAA</code>), we cannot know for certain exactly where +in the sea of <code>A</code>s the read originated.</p> +<h2 id="end-to-end-alignment-versus-local-alignment">End-to-end +alignment versus local alignment</h2> +<p>By default, Bowtie 2 performs end-to-end read alignment. That is, it +searches for alignments involving all of the read characters. This is +also called an "untrimmed" or "unclipped" alignment.</p> +<p>When the --local option is specified, Bowtie 2 performs local read +alignment. In this mode, Bowtie 2 might "trim" or "clip" some read +characters from one or both ends of the alignment if doing so maximizes +the alignment score.</p> <h3 id="end-to-end-alignment-example">End-to-end alignment example</h3> -<p>The following is an "end-to-end" alignment because it involves all the characters in the read. Such an alignment can be produced by Bowtie 2 in either end-to-end mode or in local mode.</p> +<p>The following is an "end-to-end" alignment because it involves all +the characters in the read. Such an alignment can be produced by Bowtie +2 in either end-to-end mode or in local mode.</p> <pre><code>Read: GACTGGGCGATCTCGACTTCG Reference: GACTGCGATCTCGACATCG @@ -154,7 +372,11 @@ Alignment: ||||| |||||||||| ||| Reference: GACTG--CGATCTCGACATCG</code></pre> <h3 id="local-alignment-example">Local alignment example</h3> -<p>The following is a "local" alignment because some of the characters at the ends of the read do not participate. In this case, 4 characters are omitted (or "soft trimmed" or "soft clipped") from the beginning and 3 characters are omitted from the end. This sort of alignment can be produced by Bowtie 2 only in local mode.</p> +<p>The following is a "local" alignment because some of the characters +at the ends of the read do not participate. In this case, 4 characters +are omitted (or "soft trimmed" or "soft clipped") from the beginning and +3 characters are omitted from the end. This sort of alignment can be +produced by Bowtie 2 only in local mode.</p> <pre><code>Read: ACGGTTGCGTTAATCCGCCACG Reference: TAACTTGCGTTAAATCCGCCTGG @@ -163,47 +385,204 @@ Alignment: ||||||||| |||||| Reference: TAACTTGCGTTAAATCCGCCTGG</code></pre> <h2 id="scores-higher-more-similar">Scores: higher = more similar</h2> -<p>An alignment score quantifies how similar the read sequence is to the reference sequence aligned to. The higher the score, the more similar they are. A score is calculated by subtracting penalties for each difference (mismatch, gap, etc.) and, in local alignment mode, adding bonuses for each match.</p> -<p>The scores can be configured with the <a href="#bowtie2-options-ma"><code>--ma</code></a> (match bonus), <a href="#bowtie2-options-mp"><code>--mp</code></a> (mismatch penalty), <a href="#bowtie2-options-np"><code>--np</code></a> (penalty for having an N in either the read or the reference), <a href="#bowtie2-options-rdg"><code>--rdg</code></a> (affine read gap penalty) and <a href="#bowtie2-options-rfg"><code>--rfg</code></a> (affine reference gap penalty) options.</p> -<h3 id="end-to-end-alignment-score-example">End-to-end alignment score example</h3> -<p>A mismatched base at a high-quality position in the read receives a penalty of -6 by default. A length-2 read gap receives a penalty of -11 by default (-5 for the gap open, -3 for the first extension, -3 for the second extension). Thus, in end-to-end alignment mode, if the read is 50 bp long and it matches the reference exactly except for one mismatch at a high-quality position and one length-2 read gap, then the overall score is -(6 + 11) = -17.</p> -<p>The best possible alignment score in end-to-end mode is 0, which happens when there are no differences between the read and the reference.</p> -<h3 id="local-alignment-score-example">Local alignment score example</h3> -<p>A mismatched base at a high-quality position in the read receives a penalty of -6 by default. A length-2 read gap receives a penalty of -11 by default (-5 for the gap open, -3 for the first extension, -3 for the second extension). A base that matches receives a bonus of +2 be default. Thus, in local alignment mode, if the read is 50 bp long and it matches the reference exactly except for one mismatch at a high-quality position and one length-2 read gap, then the overall score equals the total bonus, 2 * 49, minus the total penalty, 6 + 11, = 81.</p> -<p>The best possible score in local mode equals the match bonus times the length of the read. This happens when there are no differences between the read and the reference.</p> -<h3 id="valid-alignments-meet-or-exceed-the-minimum-score-threshold">Valid alignments meet or exceed the minimum score threshold</h3> -<p>For an alignment to be considered "valid" (i.e. "good enough") by Bowtie 2, it must have an alignment score no less than the minimum score threshold. The threshold is configurable and is expressed as a function of the read length. In end-to-end alignment mode, the default minimum score threshold is <code>-0.6 + -0.6 * L</code>, where <code>L</code> is the read length. In local alignment mode, the default minimum score threshold is <code>20 + 8.0 * ln(L)</code>, where L is the read length. This can be configured with the <a href="#bowtie2-options-score-min"><code>--score-min</code></a> option. For details on how to set options like <code>--score-min</code> that correspond to functions, see the section on <a href="#setting-function-options">setting function options</a>.</p> -<h2 id="mapping-quality-higher-more-unique">Mapping quality: higher = more unique</h2> -<p>The aligner cannot always assign a read to its point of origin with high confidence. For instance, a read that originated inside a repeat element might align equally well to many occurrences of the element throughout the genome, leaving the aligner with no basis for preferring one over the others.</p> -<p>Aligners characterize their degree of confidence in the point of origin by reporting a mapping quality: a non-negative integer Q = -10 log10 p, where p is an estimate of the probability that the alignment does not correspond to the read's true point of origin. Mapping quality is sometimes abbreviated MAPQ, and is recorded in the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> <code>MAPQ</code> field.</p> -<p>Mapping quality is related to "uniqueness." We say an alignment is unique if it has a much higher alignment score than all the other possible alignments. The bigger the gap between the best alignment's score and the second-best alignment's score, the more unique the best alignment, and the higher its mapping quality should be.</p> -<p>Accurate mapping qualities are useful for downstream tools like variant callers. For instance, a variant caller might choose to ignore evidence from alignments with mapping quality less than, say, 10. A mapping quality of 10 or less indicates that there is at least a 1 in 10 chance that the read truly originated elsewhere.</p> +<p>An alignment score quantifies how similar the read sequence is to the +reference sequence aligned to. The higher the score, the more similar +they are. A score is calculated by subtracting penalties for each +difference (mismatch, gap, etc.) and, in local alignment mode, adding +bonuses for each match.</p> +<p>The scores can be configured with the <a +href="#bowtie2-options-ma"><code>--ma</code></a> (match bonus), <a +href="#bowtie2-options-mp"><code>--mp</code></a> (mismatch penalty), <a +href="#bowtie2-options-np"><code>--np</code></a> (penalty for having an +N in either the read or the reference), <a +href="#bowtie2-options-rdg"><code>--rdg</code></a> (affine read gap +penalty) and <a href="#bowtie2-options-rfg"><code>--rfg</code></a> +(affine reference gap penalty) options.</p> +<h3 id="end-to-end-alignment-score-example">End-to-end alignment score +example</h3> +<p>A mismatched base at a high-quality position in the read receives a +penalty of -6 by default. A length-2 read gap receives a penalty of -11 +by default (-5 for the gap open, -3 for the first extension, -3 for the +second extension). Thus, in end-to-end alignment mode, if the read is 50 +bp long and it matches the reference exactly except for one mismatch at +a high-quality position and one length-2 read gap, then the overall +score is -(6 + 11) = -17.</p> +<p>The best possible alignment score in end-to-end mode is 0, which +happens when there are no differences between the read and the +reference.</p> +<h3 id="local-alignment-score-example">Local alignment score +example</h3> +<p>A mismatched base at a high-quality position in the read receives a +penalty of -6 by default. A length-2 read gap receives a penalty of -11 +by default (-5 for the gap open, -3 for the first extension, -3 for the +second extension). A base that matches receives a bonus of +2 be +default. Thus, in local alignment mode, if the read is 50 bp long and it +matches the reference exactly except for one mismatch at a high-quality +position and one length-2 read gap, then the overall score equals the +total bonus, 2 * 49, minus the total penalty, 6 + 11, = 81.</p> +<p>The best possible score in local mode equals the match bonus times +the length of the read. This happens when there are no differences +between the read and the reference.</p> +<h3 +id="valid-alignments-meet-or-exceed-the-minimum-score-threshold">Valid +alignments meet or exceed the minimum score threshold</h3> +<p>For an alignment to be considered "valid" (i.e. "good enough") by +Bowtie 2, it must have an alignment score no less than the minimum score +threshold. The threshold is configurable and is expressed as a function +of the read length. In end-to-end alignment mode, the default minimum +score threshold is <code>-0.6 + -0.6 * L</code>, where <code>L</code> is +the read length. In local alignment mode, the default minimum score +threshold is <code>20 + 8.0 * ln(L)</code>, where L is the read length. +This can be configured with the <a +href="#bowtie2-options-score-min"><code>--score-min</code></a> option. +For details on how to set options like <code>--score-min</code> that +correspond to functions, see the section on <a +href="#setting-function-options">setting function options</a>.</p> +<h2 id="mapping-quality-higher-more-unique">Mapping quality: higher = +more unique</h2> +<p>The aligner cannot always assign a read to its point of origin with +high confidence. For instance, a read that originated inside a repeat +element might align equally well to many occurrences of the element +throughout the genome, leaving the aligner with no basis for preferring +one over the others.</p> +<p>Aligners characterize their degree of confidence in the point of +origin by reporting a mapping quality: a non-negative integer Q = -10 +log10 p, where p is an estimate of the probability that the alignment +does not correspond to the read's true point of origin. Mapping quality +is sometimes abbreviated MAPQ, and is recorded in the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> +<code>MAPQ</code> field.</p> +<p>Mapping quality is related to "uniqueness." We say an alignment is +unique if it has a much higher alignment score than all the other +possible alignments. The bigger the gap between the best alignment's +score and the second-best alignment's score, the more unique the best +alignment, and the higher its mapping quality should be.</p> +<p>Accurate mapping qualities are useful for downstream tools like +variant callers. For instance, a variant caller might choose to ignore +evidence from alignments with mapping quality less than, say, 10. A +mapping quality of 10 or less indicates that there is at least a 1 in 10 +chance that the read truly originated elsewhere.</p> <h2 id="aligning-pairs">Aligning pairs</h2> -<p>A "paired-end" or "mate-pair" read consists of pair of mates, called mate 1 and mate 2. Pairs come with a prior expectation about (a) the relative orientation of the mates, and (b) the distance separating them on the original DNA molecule. Exactly what expectations hold for a given dataset depends on the lab procedures used to generate the data. For example, a common lab procedure for producing pairs is Illumina's Paired-end Sequencing Assay, which yields pairs with a relative orientation of FR ("forward, reverse") meaning that if mate 1 came from the Watson strand, mate 2 very likely came from the Crick strand and vice versa. Also, this protocol yields pairs where the expected genomic distance from end to end is about 200-500 base pairs.</p> -<p>For simplicity, this manual uses the term "paired-end" to refer to any pair of reads with some expected relative orientation and distance. Depending on the protocol, these might actually be referred to as "paired-end" or "mate-paired." Also, we always refer to the individual sequences making up the pair as "mates."</p> +<p>A "paired-end" or "mate-pair" read consists of pair of mates, called +mate 1 and mate 2. Pairs come with a prior expectation about (a) the +relative orientation of the mates, and (b) the distance separating them +on the original DNA molecule. Exactly what expectations hold for a given +dataset depends on the lab procedures used to generate the data. For +example, a common lab procedure for producing pairs is Illumina's +Paired-end Sequencing Assay, which yields pairs with a relative +orientation of FR ("forward, reverse") meaning that if mate 1 came from +the Watson strand, mate 2 very likely came from the Crick strand and +vice versa. Also, this protocol yields pairs where the expected genomic +distance from end to end is about 200-500 base pairs.</p> +<p>For simplicity, this manual uses the term "paired-end" to refer to +any pair of reads with some expected relative orientation and distance. +Depending on the protocol, these might actually be referred to as +"paired-end" or "mate-paired." Also, we always refer to the individual +sequences making up the pair as "mates."</p> <h3 id="paired-inputs">Paired inputs</h3> -<p>Pairs are often stored in a pair of files, one file containing the mate 1s and the other containing the mates 2s. The first mate in the file for mate 1 forms a pair with the first mate in the file for mate 2, the second with the second, and so on. When aligning pairs with Bowtie 2, specify the file with the mate 1s mates using the <a href="#bowtie2-options-1"><code>-1</code></a> argument and the file with the mate 2s using the <a href="#bowtie2-options-2"><code>-2</code></a> argument. This causes Bowtie 2 to take the paired nature of the reads into account when aligning them.</p> +<p>Pairs are often stored in a pair of files, one file containing the +mate 1s and the other containing the mates 2s. The first mate in the +file for mate 1 forms a pair with the first mate in the file for mate 2, +the second with the second, and so on. When aligning pairs with Bowtie +2, specify the file with the mate 1s mates using the <a +href="#bowtie2-options-1"><code>-1</code></a> argument and the file with +the mate 2s using the <a href="#bowtie2-options-2"><code>-2</code></a> +argument. This causes Bowtie 2 to take the paired nature of the reads +into account when aligning them.</p> <h3 id="paired-sam-output">Paired SAM output</h3> -<p>When Bowtie 2 prints a SAM alignment for a pair, it prints two records (i.e. two lines of output), one for each mate. The first record describes the alignment for mate 1 and the second record describes the alignment for mate 2. In both records, some of the fields of the SAM record describe various properties of the alignment; for instance, the 7th and 8th fields (<code>RNEXT</code> and <code>PNEXT</code> respectively) indicate the reference name and position where the other mate aligned, and the 9th field indicates the inferred length of the DNA fragment from which the two mates were sequenced. See the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> for more details regarding these fields.</p> -<h3 id="concordant-pairs-match-pair-expectations-discordant-pairs-dont">Concordant pairs match pair expectations, discordant pairs don't</h3> -<p>A pair that aligns with the expected relative mate orientation and with the expected range of distances between mates is said to align "concordantly". If both mates have unique alignments, but the alignments do not match paired-end expectations (i.e. the mates aren't in the expected relative orientation, or aren't within the expected distance range, or both), the pair is said to align "discordantly". Discordant alignments may be of particular interest, for instance, when seeking <a href="http://www.ncbi.nlm.nih.gov/dbvar/content/overview/">structural variants</a>.</p> -<p>The expected relative orientation of the mates is set using the <a href="#bowtie2-options-fr"><code>--ff</code></a>, <a href="#bowtie2-options-fr"><code>--fr</code></a>, or <a href="#bowtie2-options-fr"><code>--rf</code></a> options. The expected range of inter-mates distances (as measured from the furthest extremes of the mates; also called "outer distance") is set with the <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a> options. Note that setting <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a> far apart makes Bowtie 2 slower. See documentation for <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a>.</p> -<p>To declare that a pair aligns discordantly, Bowtie 2 requires that both mates align uniquely. This is a conservative threshold, but this is often desirable when seeking structural variants.</p> -<p>By default, Bowtie 2 searches for both concordant and discordant alignments, though searching for discordant alignments can be disabled with the <a href="#bowtie2-options-no-discordant"><code>--no-discordant</code></a> option.</p> -<h3 id="mixed-mode-paired-where-possible-unpaired-otherwise">Mixed mode: paired where possible, unpaired otherwise</h3> -<p>If Bowtie 2 cannot find a paired-end alignment for a pair, by default it will go on to look for unpaired alignments for the constituent mates. This is called "mixed mode." To disable mixed mode, set the <a href="#bowtie2-options-no-mixed"><code>--no-mixed</code></a> option.</p> -<p>Bowtie 2 runs a little faster in <code>--no-mixed</code> mode, but will only consider alignment status of pairs per se, not individual mates.</p> -<h3 id="some-sam-flags-describe-paired-end-properties">Some SAM FLAGS describe paired-end properties</h3> -<p>The SAM <code>FLAGS</code> field, the second field in a SAM record, has multiple bits that describe the paired-end nature of the read and alignment. The first (least significant) bit (1 in decimal, 0x1 in hexadecimal) is set if the read is part of a pair. The second bit (2 in decimal, 0x2 in hexadecimal) is set if the read is part of a pair that aligned in a paired-end fashion. The fourth bit (8 in decimal, 0x8 in hexadecimal) is set if the read is part of a pair and the other mate in the pair had at least one valid alignment. The sixth bit (32 in decimal, 0x20 in hexadecimal) is set if the read is part of a pair and the other mate in the pair aligned to the Crick strand (or, equivalently, if the reverse complement of the other mate aligned to the Watson strand). The seventh bit (64 in decimal, 0x40 in hexadecimal) is set if the read is mate 1 in a pair. The eighth bit (128 in decimal, 0x80 in hexadecimal) is set if the read is mate 2 in a pair. See the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> for a more detailed description of the <code>FLAGS</code> field.</p> -<h3 id="some-sam-optional-fields-describe-more-paired-end-properties">Some SAM optional fields describe more paired-end properties</h3> -<p>The last several fields of each SAM record usually contain SAM optional fields, which are simply tab-separated strings conveying additional information about the reads and alignments. A SAM optional field is formatted like this: "XP:i:1" where "XP" is the <code>TAG</code>, "i" is the <code>TYPE</code> ("integer" in this case), and "1" is the <code>VALUE</code>. See the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> for details regarding SAM optional fields.</p> -<h3 id="mates-can-overlap-contain-or-dovetail-each-other">Mates can overlap, contain, or dovetail each other</h3> -<p>The fragment and read lengths might be such that alignments for the two mates from a pair overlap each other. Consider this example:</p> -<p>(For these examples, assume we expect mate 1 to align to the left of mate 2.)</p> +<p>When Bowtie 2 prints a SAM alignment for a pair, it prints two +records (i.e. two lines of output), one for each mate. The first record +describes the alignment for mate 1 and the second record describes the +alignment for mate 2. In both records, some of the fields of the SAM +record describe various properties of the alignment; for instance, the +7th and 8th fields (<code>RNEXT</code> and <code>PNEXT</code> +respectively) indicate the reference name and position where the other +mate aligned, and the 9th field indicates the inferred length of the DNA +fragment from which the two mates were sequenced. See the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> +for more details regarding these fields.</p> +<h3 +id="concordant-pairs-match-pair-expectations-discordant-pairs-dont">Concordant +pairs match pair expectations, discordant pairs don't</h3> +<p>A pair that aligns with the expected relative mate orientation and +with the expected range of distances between mates is said to align +"concordantly". If both mates have unique alignments, but the alignments +do not match paired-end expectations (i.e. the mates aren't in the +expected relative orientation, or aren't within the expected distance +range, or both), the pair is said to align "discordantly". Discordant +alignments may be of particular interest, for instance, when seeking <a +href="http://www.ncbi.nlm.nih.gov/dbvar/content/overview/">structural +variants</a>.</p> +<p>The expected relative orientation of the mates is set using the <a +href="#bowtie2-options-fr"><code>--ff</code></a>, <a +href="#bowtie2-options-fr"><code>--fr</code></a>, or <a +href="#bowtie2-options-fr"><code>--rf</code></a> options. The expected +range of inter-mates distances (as measured from the furthest extremes +of the mates; also called "outer distance") is set with the <a +href="#bowtie2-options-I"><code>-I</code></a> and <a +href="#bowtie2-options-X"><code>-X</code></a> options. Note that setting +<a href="#bowtie2-options-I"><code>-I</code></a> and <a +href="#bowtie2-options-X"><code>-X</code></a> far apart makes Bowtie 2 +slower. See documentation for <a +href="#bowtie2-options-I"><code>-I</code></a> and <a +href="#bowtie2-options-X"><code>-X</code></a>.</p> +<p>To declare that a pair aligns discordantly, Bowtie 2 requires that +both mates align uniquely. This is a conservative threshold, but this is +often desirable when seeking structural variants.</p> +<p>By default, Bowtie 2 searches for both concordant and discordant +alignments, though searching for discordant alignments can be disabled +with the <a +href="#bowtie2-options-no-discordant"><code>--no-discordant</code></a> +option.</p> +<h3 id="mixed-mode-paired-where-possible-unpaired-otherwise">Mixed mode: +paired where possible, unpaired otherwise</h3> +<p>If Bowtie 2 cannot find a paired-end alignment for a pair, by default +it will go on to look for unpaired alignments for the constituent mates. +This is called "mixed mode." To disable mixed mode, set the <a +href="#bowtie2-options-no-mixed"><code>--no-mixed</code></a> option.</p> +<p>Bowtie 2 runs a little faster in <code>--no-mixed</code> mode, but +will only consider alignment status of pairs per se, not individual +mates.</p> +<h3 id="some-sam-flags-describe-paired-end-properties">Some SAM FLAGS +describe paired-end properties</h3> +<p>The SAM <code>FLAGS</code> field, the second field in a SAM record, +has multiple bits that describe the paired-end nature of the read and +alignment. The first (least significant) bit (1 in decimal, 0x1 in +hexadecimal) is set if the read is part of a pair. The second bit (2 in +decimal, 0x2 in hexadecimal) is set if the read is part of a pair that +aligned in a paired-end fashion. The fourth bit (8 in decimal, 0x8 in +hexadecimal) is set if the read is part of a pair and the other mate in +the pair had at least one valid alignment. The sixth bit (32 in decimal, +0x20 in hexadecimal) is set if the read is part of a pair and the other +mate in the pair aligned to the Crick strand (or, equivalently, if the +reverse complement of the other mate aligned to the Watson strand). The +seventh bit (64 in decimal, 0x40 in hexadecimal) is set if the read is +mate 1 in a pair. The eighth bit (128 in decimal, 0x80 in hexadecimal) +is set if the read is mate 2 in a pair. See the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> +for a more detailed description of the <code>FLAGS</code> field.</p> +<h3 +id="some-sam-optional-fields-describe-more-paired-end-properties">Some +SAM optional fields describe more paired-end properties</h3> +<p>The last several fields of each SAM record usually contain SAM +optional fields, which are simply tab-separated strings conveying +additional information about the reads and alignments. A SAM optional +field is formatted like this: "XP:i:1" where "XP" is the +<code>TAG</code>, "i" is the <code>TYPE</code> ("integer" in this case), +and "1" is the <code>VALUE</code>. See the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> +for details regarding SAM optional fields.</p> +<h3 id="mates-can-overlap-contain-or-dovetail-each-other">Mates can +overlap, contain, or dovetail each other</h3> +<p>The fragment and read lengths might be such that alignments for the +two mates from a pair overlap each other. Consider this example:</p> +<p>(For these examples, assume we expect mate 1 to align to the left of +mate 2.)</p> <pre><code>Mate 1: GCAGATTATATGAGTCAGCTACGATATTGTT Mate 2: TGTTTGGGGTGACACATTACGCGTCTTTGAC Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pre> -<p>It's also possible, though unusual, for one mate alignment to contain the other, as in these examples:</p> +<p>It's also possible, though unusual, for one mate alignment to contain +the other, as in these examples:</p> <pre><code>Mate 1: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGC Mate 2: TGTTTGGGGTGACACATTACGC Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC @@ -211,64 +590,269 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC Mate 1: CAGCTACGATATTGTTTGGGGTGACACATTACGC Mate 2: CTACGATATTGTTTGGGGTGAC Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pre> -<p>And it's also possible, though unusual, for the mates to "dovetail", with the mates seemingly extending "past" each other as in this example:</p> +<p>And it's also possible, though unusual, for the mates to "dovetail", +with the mates seemingly extending "past" each other as in this +example:</p> <pre><code>Mate 1: GTCAGCTACGATATTGTTTGGGGTGACACATTACGC Mate 2: TATGAGTCAGCTACGATATTGTTTGGGGTGACACAT Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pre> -<p>In some situations, it's desirable for the aligner to consider all these cases as "concordant" as long as other paired-end constraints are not violated. Bowtie 2's default behavior is to consider overlapping and containing as being consistent with concordant alignment. By default, dovetailing is considered inconsistent with concordant alignment.</p> -<p>These defaults can be overridden. Setting <a href="#bowtie2-options-no-overlap"><code>--no-overlap</code></a> causes Bowtie 2 to consider overlapping mates as non-concordant. Setting <a href="#bowtie2-options-no-contain"><code>--no-contain</code></a> causes Bowtie 2 to consider cases where one mate alignment contains the other as non-concordant. Setting <a href="#bowtie2-options-dovetail"><code>--dovetail</code></a> causes Bowtie 2 to consider cases where the mate alignments dovetail as concordant.</p> +<p>In some situations, it's desirable for the aligner to consider all +these cases as "concordant" as long as other paired-end constraints are +not violated. Bowtie 2's default behavior is to consider overlapping and +containing as being consistent with concordant alignment. By default, +dovetailing is considered inconsistent with concordant alignment.</p> +<p>These defaults can be overridden. Setting <a +href="#bowtie2-options-no-overlap"><code>--no-overlap</code></a> causes +Bowtie 2 to consider overlapping mates as non-concordant. Setting <a +href="#bowtie2-options-no-contain"><code>--no-contain</code></a> causes +Bowtie 2 to consider cases where one mate alignment contains the other +as non-concordant. Setting <a +href="#bowtie2-options-dovetail"><code>--dovetail</code></a> causes +Bowtie 2 to consider cases where the mate alignments dovetail as +concordant.</p> <h2 id="reporting">Reporting</h2> -<p>The reporting mode governs how many alignments Bowtie 2 looks for, and how to report them. Bowtie 2 has three distinct reporting modes. The default reporting mode is similar to the default reporting mode of many other read alignment tools, including <a href="http://bio-bwa.sourceforge.net/">BWA</a>. It is also similar to Bowtie 1's <code>-M</code> alignment mode.</p> -<p>In general, when we say that a read has an alignment, we mean that it has a <a href="#valid-alignments-meet-or-exceed-the-minimum-score-threshold">valid alignment</a>. When we say that a read has multiple alignments, we mean that it has multiple alignments that are valid and distinct from one another.</p> -<h3 id="distinct-alignments-map-a-read-to-different-places">Distinct alignments map a read to different places</h3> -<p>Two alignments for the same individual read are "distinct" if they map the same read to different places. Specifically, we say that two alignments are distinct if there are no alignment positions where a particular read offset is aligned opposite a particular reference offset in both alignments with the same orientation. E.g. if the first alignment is in the forward orientation and aligns the read character at read offset 10 to the reference character at chromosome 3, offset 3,445,245, and the second alignment is also in the forward orientation and also aligns the read character at read offset 10 to the reference character at chromosome 3, offset 3,445,245, they are not distinct alignments.</p> -<p>Two alignments for the same pair are distinct if either the mate 1s in the two paired-end alignments are distinct or the mate 2s in the two alignments are distinct or both.</p> -<h3 id="default-mode-search-for-multiple-alignments-report-the-best-one">Default mode: search for multiple alignments, report the best one</h3> -<p>By default, Bowtie 2 searches for distinct, valid alignments for each read. When it finds a valid alignment, it generally will continue to look for alignments that are nearly as good or better. It will eventually stop looking, either because it exceeded a limit placed on search effort (see <a href="#bowtie2-options-D"><code>-D</code></a> and <a href="#bowtie2-options-R"><code>-R</code></a>) or because it already knows all it needs to know to report an alignment. Information from the best alignments are used to estimate mapping quality (the <code>MAPQ</code> <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> field) and to set SAM optional fields, such as <a href="#bowtie2-build-opt-fields-as"><code>AS:i</code></a> and <a href="#bowtie2-build-opt-fields-xs"><code>XS:i</code></a>. Bowtie 2 does not guarantee that the alignment reported is the best possible in terms of alignment score.</p> -<p>See also: <a href="#bowtie2-options-D"><code>-D</code></a>, which puts an upper limit on the number of dynamic programming problems (i.e. seed extensions) that can "fail" in a row before Bowtie 2 stops searching. Increasing <a href="#bowtie2-options-D"><code>-D</code></a> makes Bowtie 2 slower, but increases the likelihood that it will report the correct alignment for a read that aligns many places.</p> -<p>See also: <a href="#bowtie2-options-R"><code>-R</code></a>, which sets the maximum number of times Bowtie 2 will "re-seed" when attempting to align a read with repetitive seeds. Increasing <a href="#bowtie2-options-R"><code>-R</code></a> makes Bowtie 2 slower, but increases the likelihood that it will report the correct alignment for a read that aligns many places.</p> -<h3 id="k-mode-search-for-one-or-more-alignments-report-each">-k mode: search for one or more alignments, report each</h3> -<p>In <a href="#bowtie2-options-k"><code>-k</code></a> mode, Bowtie 2 searches for up to N distinct, valid alignments for each read, where N equals the integer specified with the <code>-k</code> parameter. That is, if <code>-k 2</code> is specified, Bowtie 2 will search for at most 2 distinct alignments. It reports all alignments found, in descending order by alignment score. The alignment score for a paired-end alignment equals the sum of the alignment scores of the individual mates. Each reported read or pair alignment beyond the first has the SAM 'secondary' bit (which equals 256) set in its FLAGS field. Supplementary alignments will also be assigned a MAPQ of 255. See the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> for details.</p> -<p>Bowtie 2 does not "find" alignments in any specific order, so for reads that have more than N distinct, valid alignments, Bowtie 2 does not guarantee that the N alignments reported are the best possible in terms of alignment score. Still, this mode can be effective and fast in situations where the user cares more about whether a read aligns (or aligns a certain number of times) than where exactly it originated.</p> -<h3 id="a-mode-search-for-and-report-all-alignments">-a mode: search for and report all alignments</h3> -<p><a href="#bowtie2-options-a"><code>-a</code></a> mode is similar to <a href="#bowtie2-options-k"><code>-k</code></a> mode except that there is no upper limit on the number of alignments Bowtie 2 should report. Alignments are reported in descending order by alignment score. The alignment score for a paired-end alignment equals the sum of the alignment scores of the individual mates. Each reported read or pair alignment beyond the first has the SAM 'secondary' bit (which equals 256) set in its FLAGS field. Supplementary alignments will be assigned a MAPQ of 255. See the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> for details.</p> -<p>Some tools are designed with this reporting mode in mind. Bowtie 2 is not! For very large genomes, this mode is very slow.</p> +<p>The reporting mode governs how many alignments Bowtie 2 looks for, +and how to report them. Bowtie 2 has three distinct reporting modes. The +default reporting mode is similar to the default reporting mode of many +other read alignment tools, including <a +href="http://bio-bwa.sourceforge.net/">BWA</a>. It is also similar to +Bowtie 1's <code>-M</code> alignment mode.</p> +<p>In general, when we say that a read has an alignment, we mean that it +has a <a +href="#valid-alignments-meet-or-exceed-the-minimum-score-threshold">valid +alignment</a>. When we say that a read has multiple alignments, we mean +that it has multiple alignments that are valid and distinct from one +another.</p> +<h3 id="distinct-alignments-map-a-read-to-different-places">Distinct +alignments map a read to different places</h3> +<p>Two alignments for the same individual read are "distinct" if they +map the same read to different places. Specifically, we say that two +alignments are distinct if there are no alignment positions where a +particular read offset is aligned opposite a particular reference offset +in both alignments with the same orientation. E.g. if the first +alignment is in the forward orientation and aligns the read character at +read offset 10 to the reference character at chromosome 3, offset +3,445,245, and the second alignment is also in the forward orientation +and also aligns the read character at read offset 10 to the reference +character at chromosome 3, offset 3,445,245, they are not distinct +alignments.</p> +<p>Two alignments for the same pair are distinct if either the mate 1s +in the two paired-end alignments are distinct or the mate 2s in the two +alignments are distinct or both.</p> +<h3 +id="default-mode-search-for-multiple-alignments-report-the-best-one">Default +mode: search for multiple alignments, report the best one</h3> +<p>By default, Bowtie 2 searches for distinct, valid alignments for each +read. When it finds a valid alignment, it generally will continue to +look for alignments that are nearly as good or better. It will +eventually stop looking, either because it exceeded a limit placed on +search effort (see <a href="#bowtie2-options-D"><code>-D</code></a> and +<a href="#bowtie2-options-R"><code>-R</code></a>) or because it already +knows all it needs to know to report an alignment. Information from the +best alignments are used to estimate mapping quality (the +<code>MAPQ</code> <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> field) and to +set SAM optional fields, such as <a +href="#bowtie2-build-opt-fields-as"><code>AS:i</code></a> and <a +href="#bowtie2-build-opt-fields-xs"><code>XS:i</code></a>. Bowtie 2 does +not guarantee that the alignment reported is the best possible in terms +of alignment score.</p> +<p>See also: <a href="#bowtie2-options-D"><code>-D</code></a>, which +puts an upper limit on the number of dynamic programming problems (i.e. +seed extensions) that can "fail" in a row before Bowtie 2 stops +searching. Increasing <a href="#bowtie2-options-D"><code>-D</code></a> +makes Bowtie 2 slower, but increases the likelihood that it will report +the correct alignment for a read that aligns many places.</p> +<p>See also: <a href="#bowtie2-options-R"><code>-R</code></a>, which +sets the maximum number of times Bowtie 2 will "re-seed" when attempting +to align a read with repetitive seeds. Increasing <a +href="#bowtie2-options-R"><code>-R</code></a> makes Bowtie 2 slower, but +increases the likelihood that it will report the correct alignment for a +read that aligns many places.</p> +<h3 id="k-mode-search-for-one-or-more-alignments-report-each">-k mode: +search for one or more alignments, report each</h3> +<p>In <a href="#bowtie2-options-k"><code>-k</code></a> mode, Bowtie 2 +searches for up to N distinct, valid alignments for each read, where N +equals the integer specified with the <code>-k</code> parameter. That +is, if <code>-k 2</code> is specified, Bowtie 2 will search for at most +2 distinct alignments. It reports all alignments found, in descending +order by alignment score. The alignment score for a paired-end alignment +equals the sum of the alignment scores of the individual mates. Each +reported read or pair alignment beyond the first has the SAM 'secondary' +bit (which equals 256) set in its FLAGS field. Supplementary alignments +will also be assigned a MAPQ of 255. See the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> +for details.</p> +<p>Bowtie 2 does not "find" alignments in any specific order, so for +reads that have more than N distinct, valid alignments, Bowtie 2 does +not guarantee that the N alignments reported are the best possible in +terms of alignment score. Still, this mode can be effective and fast in +situations where the user cares more about whether a read aligns (or +aligns a certain number of times) than where exactly it originated.</p> +<h3 id="a-mode-search-for-and-report-all-alignments">-a mode: search for +and report all alignments</h3> +<p><a href="#bowtie2-options-a"><code>-a</code></a> mode is similar to +<a href="#bowtie2-options-k"><code>-k</code></a> mode except that there +is no upper limit on the number of alignments Bowtie 2 should report. +Alignments are reported in descending order by alignment score. The +alignment score for a paired-end alignment equals the sum of the +alignment scores of the individual mates. Each reported read or pair +alignment beyond the first has the SAM 'secondary' bit (which equals +256) set in its FLAGS field. Supplementary alignments will be assigned a +MAPQ of 255. See the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> +for details.</p> +<p>Some tools are designed with this reporting mode in mind. Bowtie 2 is +not! For very large genomes, this mode is very slow.</p> <h3 id="randomness-in-bowtie-2">Randomness in Bowtie 2</h3> -<p>Bowtie 2's search for alignments for a given read is "randomized." That is, when Bowtie 2 encounters a set of equally-good choices, it uses a pseudo-random number to choose. For example, if Bowtie 2 discovers a set of 3 equally-good alignments and wants to decide which to report, it picks a pseudo-random integer 0, 1 or 2 and reports the corresponding alignment. Arbitrary choices can crop up at various points during alignment.</p> -<p>The pseudo-random number generator is re-initialized for every read, and the seed used to initialize it is a function of the read name, nucleotide string, quality string, and the value specified with <a href="#bowtie2-options-seed"><code>--seed</code></a>. If you run the same version of Bowtie 2 on two reads with identical names, nucleotide strings, and quality strings, and if <a href="#bowtie2-options-seed"><code>--seed</code></a> is set the same for both runs, Bowtie 2 will produce the same output; i.e., it will align the read to the same place, even if there are multiple equally good alignments. This is intuitive and desirable in most cases. Most users expect Bowtie to produce the same output when run twice on the same input.</p> -<p>However, when the user specifies the <a href="#bowtie2-options-non-deterministic"><code>--non-deterministic</code></a> option, Bowtie 2 will use the current time to re-initialize the pseudo-random number generator. When this is specified, Bowtie 2 might report different alignments for identical reads. This is counter-intuitive for some users, but might be more appropriate in situations where the input consists of many identical reads.</p> +<p>Bowtie 2's search for alignments for a given read is "randomized." +That is, when Bowtie 2 encounters a set of equally-good choices, it uses +a pseudo-random number to choose. For example, if Bowtie 2 discovers a +set of 3 equally-good alignments and wants to decide which to report, it +picks a pseudo-random integer 0, 1 or 2 and reports the corresponding +alignment. Arbitrary choices can crop up at various points during +alignment.</p> +<p>The pseudo-random number generator is re-initialized for every read, +and the seed used to initialize it is a function of the read name, +nucleotide string, quality string, and the value specified with <a +href="#bowtie2-options-seed"><code>--seed</code></a>. If you run the +same version of Bowtie 2 on two reads with identical names, nucleotide +strings, and quality strings, and if <a +href="#bowtie2-options-seed"><code>--seed</code></a> is set the same for +both runs, Bowtie 2 will produce the same output; i.e., it will align +the read to the same place, even if there are multiple equally good +alignments. This is intuitive and desirable in most cases. Most users +expect Bowtie to produce the same output when run twice on the same +input.</p> +<p>However, when the user specifies the <a +href="#bowtie2-options-non-deterministic"><code>--non-deterministic</code></a> +option, Bowtie 2 will use the current time to re-initialize the +pseudo-random number generator. When this is specified, Bowtie 2 might +report different alignments for identical reads. This is +counter-intuitive for some users, but might be more appropriate in +situations where the input consists of many identical reads.</p> <h2 id="multiseed-heuristic">Multiseed heuristic</h2> -<p>To rapidly narrow the number of possible alignments that must be considered, Bowtie 2 begins by extracting substrings ("seeds") from the read and its reverse complement and aligning them in an ungapped fashion with the help of the <a href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a>. This is "multiseed alignment" and it is similar to what <a href="http://genomebiology.com/2009/10/3/R25">Bowtie 1 does</a>, except Bowtie 1 attempts to align the entire read this way.</p> -<p>This initial step makes Bowtie 2 much faster than it would be without such a filter, but at the expense of missing some valid alignments. For instance, it is possible for a read to have a valid overall alignment but to have no valid seed alignments because each potential seed alignment is interrupted by too many mismatches or gaps.</p> -<p>The trade-off between speed and sensitivity/accuracy can be adjusted by setting the seed length (<a href="#bowtie2-options-L"><code>-L</code></a>), the interval between extracted seeds (<a href="#bowtie2-options-i"><code>-i</code></a>), and the number of mismatches permitted per seed (<a href="#bowtie2-options-N"><code>-N</code></a>). For more sensitive alignment, set these parameters to (a) make the seeds closer together, (b) make the seeds shorter, and/or (c) allow more mismatches. You can adjust these options one-by-one, though Bowtie 2 comes with some useful combinations of options prepackaged as "<a href="#presets-setting-many-settings-at-once">preset options</a>."</p> -<p><a href="#bowtie2-options-D"><code>-D</code></a> and <a href="#bowtie2-options-R"><code>-R</code></a> are also options that adjust the trade-off between speed and sensitivity/accuracy.</p> +<p>To rapidly narrow the number of possible alignments that must be +considered, Bowtie 2 begins by extracting substrings ("seeds") from the +read and its reverse complement and aligning them in an ungapped fashion +with the help of the <a +href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a>. This +is "multiseed alignment" and it is similar to what <a +href="http://genomebiology.com/2009/10/3/R25">Bowtie 1 does</a>, except +Bowtie 1 attempts to align the entire read this way.</p> +<p>This initial step makes Bowtie 2 much faster than it would be without +such a filter, but at the expense of missing some valid alignments. For +instance, it is possible for a read to have a valid overall alignment +but to have no valid seed alignments because each potential seed +alignment is interrupted by too many mismatches or gaps.</p> +<p>The trade-off between speed and sensitivity/accuracy can be adjusted +by setting the seed length (<a +href="#bowtie2-options-L"><code>-L</code></a>), the interval between +extracted seeds (<a href="#bowtie2-options-i"><code>-i</code></a>), and +the number of mismatches permitted per seed (<a +href="#bowtie2-options-N"><code>-N</code></a>). For more sensitive +alignment, set these parameters to (a) make the seeds closer together, +(b) make the seeds shorter, and/or (c) allow more mismatches. You can +adjust these options one-by-one, though Bowtie 2 comes with some useful +combinations of options prepackaged as "<a +href="#presets-setting-many-settings-at-once">preset options</a>."</p> +<p><a href="#bowtie2-options-D"><code>-D</code></a> and <a +href="#bowtie2-options-R"><code>-R</code></a> are also options that +adjust the trade-off between speed and sensitivity/accuracy.</p> <h3 id="fm-index-memory-footprint">FM Index memory footprint</h3> -<p>Bowtie 2 uses the <a href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> to find ungapped alignments for seeds. This step accounts for the bulk of Bowtie 2's memory footprint, as the <a href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> itself is typically the largest data structure used. For instance, the memory footprint of the <a href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> for the human genome is about 3.2 gigabytes of RAM.</p> +<p>Bowtie 2 uses the <a +href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> to find +ungapped alignments for seeds. This step accounts for the bulk of Bowtie +2's memory footprint, as the <a +href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> itself +is typically the largest data structure used. For instance, the memory +footprint of the <a +href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> for the +human genome is about 3.2 gigabytes of RAM.</p> <h2 id="ambiguous-characters">Ambiguous characters</h2> -<p>Non-whitespace characters besides A, C, G or T are considered "ambiguous." N is a common ambiguous character that appears in reference sequences. Bowtie 2 considers all ambiguous characters in the reference (including <a href="http://www.bioinformatics.org/sms/iupac.html">IUPAC nucleotide codes</a>) to be Ns.</p> -<p>Bowtie 2 allows alignments to overlap ambiguous characters in the reference. An alignment position that contains an ambiguous character in the read, reference, or both, is penalized according to <a href="#bowtie2-options-np"><code>--np</code></a>. <a href="#bowtie2-options-n-ceil"><code>--n-ceil</code></a> sets an upper limit on the number of positions that may contain ambiguous reference characters in a valid alignment. The optional field <a href="#bowtie2-build-opt-fields-xn"><code>XN:i</code></a> reports the number of ambiguous reference characters overlapped by an alignment.</p> -<p>Note that the <a href="#multiseed-heuristic">multiseed heuristic</a> cannot find <em>seed</em> alignments that overlap ambiguous reference characters. For an alignment overlapping an ambiguous reference character to be found, it must have one or more seed alignments that do not overlap ambiguous reference characters.</p> -<h2 id="presets-setting-many-settings-at-once">Presets: setting many settings at once</h2> -<p>Bowtie 2 comes with some useful combinations of parameters packaged into shorter "preset" parameters. For example, running Bowtie 2 with the <a href="#bowtie2-options-very-sensitive"><code>--very-sensitive</code></a> option is the same as running with options: <code>-D 20 -R 3 -N 0 -L 20 -i S,1,0.50</code>. The preset options that come with Bowtie 2 are designed to cover a wide area of the speed/sensitivity/accuracy trade-off space, with the presets ending in <code>fast</code> generally being faster but less sensitive and less accurate, and the presets ending in <code>sensitive</code> generally being slower but more sensitive and more accurate. See the <a href="#preset-options-in---end-to-end-mode">documentation for the preset options</a> for details.</p> -<p>As of Bowtie2 v2.4.0, individual preset values can be overridden by providing the specific options e.g. the configured seed length of 20 in the [<code>--very-senitive</code>] preset above can be changed to 25 by also specifying the <code>-L 25</code> parameter anywhere on the command line.</p> +<p>Non-whitespace characters besides A, C, G or T are considered +"ambiguous." N is a common ambiguous character that appears in reference +sequences. Bowtie 2 considers all ambiguous characters in the reference +(including <a href="http://www.bioinformatics.org/sms/iupac.html">IUPAC +nucleotide codes</a>) to be Ns.</p> +<p>Bowtie 2 allows alignments to overlap ambiguous characters in the +reference. An alignment position that contains an ambiguous character in +the read, reference, or both, is penalized according to <a +href="#bowtie2-options-np"><code>--np</code></a>. <a +href="#bowtie2-options-n-ceil"><code>--n-ceil</code></a> sets an upper +limit on the number of positions that may contain ambiguous reference +characters in a valid alignment. The optional field <a +href="#bowtie2-build-opt-fields-xn"><code>XN:i</code></a> reports the +number of ambiguous reference characters overlapped by an alignment.</p> +<p>Note that the <a href="#multiseed-heuristic">multiseed heuristic</a> +cannot find <em>seed</em> alignments that overlap ambiguous reference +characters. For an alignment overlapping an ambiguous reference +character to be found, it must have one or more seed alignments that do +not overlap ambiguous reference characters.</p> +<h2 id="presets-setting-many-settings-at-once">Presets: setting many +settings at once</h2> +<p>Bowtie 2 comes with some useful combinations of parameters packaged +into shorter "preset" parameters. For example, running Bowtie 2 with the +<a +href="#bowtie2-options-very-sensitive"><code>--very-sensitive</code></a> +option is the same as running with options: +<code>-D 20 -R 3 -N 0 -L 20 -i S,1,0.50</code>. The preset options that +come with Bowtie 2 are designed to cover a wide area of the +speed/sensitivity/accuracy trade-off space, with the presets ending in +<code>fast</code> generally being faster but less sensitive and less +accurate, and the presets ending in <code>sensitive</code> generally +being slower but more sensitive and more accurate. See the <a +href="#preset-options-in---end-to-end-mode">documentation for the preset +options</a> for details.</p> +<p>As of Bowtie2 v2.4.0, individual preset values can be overridden by +providing the specific options e.g. the configured seed length of 20 in +the [<code>--very-senitive</code>] preset above can be changed to 25 by +also specifying the <code>-L 25</code> parameter anywhere on the command +line.</p> <h2 id="filtering">Filtering</h2> -<p>Some reads are skipped or "filtered out" by Bowtie 2. For example, reads may be filtered out because they are extremely short or have a high proportion of ambiguous nucleotides. Bowtie 2 will still print a SAM record for such a read, but no alignment will be reported and the <code>YF:i</code> SAM optional field will be set to indicate the reason the read was filtered.</p> +<p>Some reads are skipped or "filtered out" by Bowtie 2. For example, +reads may be filtered out because they are extremely short or have a +high proportion of ambiguous nucleotides. Bowtie 2 will still print a +SAM record for such a read, but no alignment will be reported and the +<code>YF:i</code> SAM optional field will be set to indicate the reason +the read was filtered.</p> <ul> -<li><code>YF:Z:LN</code>: the read was filtered because it had length less than or equal to the number of seed mismatches set with the <a href="#bowtie2-options-N"><code>-N</code></a> option.</li> -<li><code>YF:Z:NS</code>: the read was filtered because it contains a number of ambiguous characters (usually <code>N</code> or <code>.</code>) greater than the ceiling specified with <a href="#bowtie2-options-n-ceil"><code>--n-ceil</code></a>.</li> -<li><code>YF:Z:SC</code>: the read was filtered because the read length and the match bonus (set with <a href="#bowtie2-options-ma"><code>--ma</code></a>) are such that the read can't possibly earn an alignment score greater than or equal to the threshold set with <a href="#bowtie2-options-score-min"><code>--score-min</code></a></li> -<li><code>YF:Z:QC</code>: the read was filtered because it was marked as failing quality control and the user specified the <a href="#bowtie2-options-qc-filter"><code>--qc-filter</code></a> option. This only happens when the input is in Illumina's QSEQ format (i.e. when <a href="#bowtie2-options-qseq"><code>--qseq</code></a> is specified) and the last (11th) field of the read's QSEQ record contains <code>1</code>.</li> +<li><code>YF:Z:LN</code>: the read was filtered because it had length +less than or equal to the number of seed mismatches set with the <a +href="#bowtie2-options-N"><code>-N</code></a> option.</li> +<li><code>YF:Z:NS</code>: the read was filtered because it contains a +number of ambiguous characters (usually <code>N</code> or +<code>.</code>) greater than the ceiling specified with <a +href="#bowtie2-options-n-ceil"><code>--n-ceil</code></a>.</li> +<li><code>YF:Z:SC</code>: the read was filtered because the read length +and the match bonus (set with <a +href="#bowtie2-options-ma"><code>--ma</code></a>) are such that the read +can't possibly earn an alignment score greater than or equal to the +threshold set with <a +href="#bowtie2-options-score-min"><code>--score-min</code></a></li> +<li><code>YF:Z:QC</code>: the read was filtered because it was marked as +failing quality control and the user specified the <a +href="#bowtie2-options-qc-filter"><code>--qc-filter</code></a> option. +This only happens when the input is in Illumina's QSEQ format (i.e. when +<a href="#bowtie2-options-qseq"><code>--qseq</code></a> is specified) +and the last (11th) field of the read's QSEQ record contains +<code>1</code>.</li> </ul> -<p>If a read could be filtered for more than one reason, the value <code>YF:Z</code> flag will reflect only one of those reasons.</p> +<p>If a read could be filtered for more than one reason, the value +<code>YF:Z</code> flag will reflect only one of those reasons.</p> <h2 id="alignment-summary">Alignment summary</h2> -<p>When Bowtie 2 finishes running, it prints messages summarizing what happened. These messages are printed to the "standard error" ("stderr") filehandle. For datasets consisting of unpaired reads, the summary might look like this:</p> +<p>When Bowtie 2 finishes running, it prints messages summarizing what +happened. These messages are printed to the "standard error" ("stderr") +filehandle. For datasets consisting of unpaired reads, the summary might +look like this:</p> <pre><code>20000 reads; of these: 20000 (100.00%) were unpaired; of these: 1247 (6.24%) aligned 0 times 18739 (93.69%) aligned exactly 1 time 14 (0.07%) aligned >1 times 93.77% overall alignment rate</code></pre> -<p>For datasets consisting of pairs, the summary might look like this:</p> +<p>For datasets consisting of pairs, the summary might look like +this:</p> <pre><code>10000 reads; of these: 10000 (100.00%) were paired; of these: 650 (6.50%) aligned concordantly 0 times @@ -286,27 +870,82 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr 96.70% overall alignment rate</code></pre> <p>The indentation indicates how subtotals relate to totals.</p> <h2 id="wrapper-scripts">Wrapper scripts</h2> -<p>The <code>bowtie2</code>, <code>bowtie2-build</code> and <code>bowtie2-inspect</code> executables are actually wrapper scripts that call binary programs as appropriate. The wrappers shield users from having to distinguish between "small" and "large" index formats, discussed briefly in the following section. Also, the <code>bowtie2</code> wrapper provides some key functionality, like the ability to handle compressed inputs, and the functionality for <a href="#bowtie2-options-un"><code>--un</code></a>, <a href="#bowtie2-options-al"><code>--al</code></a> and related options.</p> -<p>It is recommended that you always run the bowtie2 wrappers and not run the binaries directly.</p> +<p>The <code>bowtie2</code>, <code>bowtie2-build</code> and +<code>bowtie2-inspect</code> executables are actually wrapper scripts +that call binary programs as appropriate. The wrappers shield users from +having to distinguish between "small" and "large" index formats, +discussed briefly in the following section. Also, the +<code>bowtie2</code> wrapper provides some key functionality, like the +ability to handle compressed inputs, and the functionality for <a +href="#bowtie2-options-un"><code>--un</code></a>, <a +href="#bowtie2-options-al"><code>--al</code></a> and related +options.</p> +<p>It is recommended that you always run the bowtie2 wrappers and not +run the binaries directly.</p> <h2 id="small-and-large-indexes">Small and large indexes</h2> -<p><code>bowtie2-build</code> can index reference genomes of any size. For genomes less than about 4 billion nucleotides in length, <code>bowtie2-build</code> builds a "small" index using 32-bit numbers in various parts of the index. When the genome is longer, <code>bowtie2-build</code> builds a "large" index using 64-bit numbers. Small indexes are stored in files with the <code>.bt2</code> extension, and large indexes are stored in files with the <code>.bt2l</code> extension. The user need not worry about whether a particular index is small or large; the wrapper scripts will automatically build and use the appropriate index.</p> +<p><code>bowtie2-build</code> can index reference genomes of any size. +For genomes less than about 4 billion nucleotides in length, +<code>bowtie2-build</code> builds a "small" index using 32-bit numbers +in various parts of the index. When the genome is longer, +<code>bowtie2-build</code> builds a "large" index using 64-bit numbers. +Small indexes are stored in files with the <code>.bt2</code> extension, +and large indexes are stored in files with the <code>.bt2l</code> +extension. The user need not worry about whether a particular index is +small or large; the wrapper scripts will automatically build and use the +appropriate index.</p> <h2 id="performance-tuning">Performance tuning</h2> <ol type="1"> -<li><p>If your computer has multiple processors/cores, use <code>-p</code></p> -<p>The <a href="#bowtie2-options-p"><code>-p</code></a> option causes Bowtie 2 to launch a specified number of parallel search threads. Each thread runs on a different processor/core and all threads find alignments in parallel, increasing alignment throughput by approximately a multiple of the number of threads (though in practice, speedup is somewhat worse than linear).</p></li> -<li><p>If reporting many alignments per read, try reducing <code>bowtie2-build --offrate</code></p> -<p>If you are using <a href="#bowtie2-options-k"><code>-k</code></a> or <a href="#bowtie2-options-a"><code>-a</code></a> options and Bowtie 2 is reporting many alignments per read, using an index with a denser SA sample can speed things up considerably. To do this, specify a smaller-than-default <a href="#bowtie2-options-o"><code>-o</code>/<code>--offrate</code></a> value when running <code>bowtie2-build</code>. A denser SA sample yields a larger index, but is also particularly effective at speeding up alignment when many alignments are reported per read.</p></li> -<li><p>If <code>bowtie2</code> "thrashes", try increasing <code>bowtie2-build --offrate</code></p> -<p>If <code>bowtie2</code> runs very slowly on a relatively low-memory computer, try setting <a href="#bowtie2-options-o"><code>-o</code>/<code>--offrate</code></a> to a <em>larger</em> value when building the index. This decreases the memory footprint of the index.</p></li> +<li><p>If your computer has multiple processors/cores, use +<code>-p</code></p> +<p>The <a href="#bowtie2-options-p"><code>-p</code></a> option causes +Bowtie 2 to launch a specified number of parallel search threads. Each +thread runs on a different processor/core and all threads find +alignments in parallel, increasing alignment throughput by approximately +a multiple of the number of threads (though in practice, speedup is +somewhat worse than linear).</p></li> +<li><p>If reporting many alignments per read, try reducing +<code>bowtie2-build --offrate</code></p> +<p>If you are using <a href="#bowtie2-options-k"><code>-k</code></a> or +<a href="#bowtie2-options-a"><code>-a</code></a> options and Bowtie 2 is +reporting many alignments per read, using an index with a denser SA +sample can speed things up considerably. To do this, specify a +smaller-than-default <a +href="#bowtie2-options-o"><code>-o</code>/<code>--offrate</code></a> +value when running <code>bowtie2-build</code>. A denser SA sample yields +a larger index, but is also particularly effective at speeding up +alignment when many alignments are reported per read.</p></li> +<li><p>If <code>bowtie2</code> "thrashes", try increasing +<code>bowtie2-build --offrate</code></p> +<p>If <code>bowtie2</code> runs very slowly on a relatively low-memory +computer, try setting <a +href="#bowtie2-options-o"><code>-o</code>/<code>--offrate</code></a> to +a <em>larger</em> value when building the index. This decreases the +memory footprint of the index.</p></li> </ol> <h2 id="command-line">Command Line</h2> <h3 id="setting-function-options">Setting function options</h3> -<p>Some Bowtie 2 options specify a function rather than an individual number or setting. In these cases the user specifies three parameters: (a) a function type <code>F</code>, (b) a constant term <code>B</code>, and (c) a coefficient <code>A</code>. The available function types are constant (<code>C</code>), linear (<code>L</code>), square-root (<code>S</code>), and natural log (<code>G</code>). The parameters are specified as <code>F,B,A</code> - that is, the function type, the constant term, and the coefficient are separated by commas with no whitespace. The constant term and coefficient may be negative and/or floating-point numbers.</p> -<p>For example, if the function specification is <code>L,-0.4,-0.6</code>, then the function defined is:</p> +<p>Some Bowtie 2 options specify a function rather than an individual +number or setting. In these cases the user specifies three parameters: +(a) a function type <code>F</code>, (b) a constant term <code>B</code>, +and (c) a coefficient <code>A</code>. The available function types are +constant (<code>C</code>), linear (<code>L</code>), square-root +(<code>S</code>), and natural log (<code>G</code>). The parameters are +specified as <code>F,B,A</code> - that is, the function type, the +constant term, and the coefficient are separated by commas with no +whitespace. The constant term and coefficient may be negative and/or +floating-point numbers.</p> +<p>For example, if the function specification is +<code>L,-0.4,-0.6</code>, then the function defined is:</p> <pre><code>f(x) = -0.4 + -0.6 * x</code></pre> -<p>If the function specification is <code>G,1,5.4</code>, then the function defined is:</p> +<p>If the function specification is <code>G,1,5.4</code>, then the +function defined is:</p> <pre><code>f(x) = 1.0 + 5.4 * ln(x)</code></pre> -<p>See the documentation for the option in question to learn what the parameter <code>x</code> is for. For example, in the case if the <a href="#bowtie2-options-score-min"><code>--score-min</code></a> option, the function <code>f(x)</code> sets the minimum alignment score necessary for an alignment to be considered valid, and <code>x</code> is the read length.</p> +<p>See the documentation for the option in question to learn what the +parameter <code>x</code> is for. For example, in the case if the <a +href="#bowtie2-options-score-min"><code>--score-min</code></a> option, +the function <code>f(x)</code> sets the minimum alignment score +necessary for an alignment to be considered valid, and <code>x</code> is +the read length.</p> <h3 id="usage">Usage</h3> <pre><code>bowtie2 [options]* -x <bt2-idx> {-1 <m1> -2 <m2> | -U <r> | --interleaved <i> | --sra-acc <acc> | b <bam>} -S [<sam>]</code></pre> <h3 id="main-arguments">Main arguments</h3> @@ -316,7 +955,12 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-x <bt2-idx></code></pre> </td> <td> -<p>The basename of the index for the reference genome. The basename is the name of any of the index files up to but not including the final <code>.1.bt2</code> / <code>.rev.1.bt2</code> / etc. <code>bowtie2</code> looks for the specified index first in the current directory, then in the directory specified in the <code>BOWTIE2_INDEXES</code> environment variable.</p> +<p>The basename of the index for the reference genome. The basename is +the name of any of the index files up to but not including the final +<code>.1.bt2</code> / <code>.rev.1.bt2</code> / etc. +<code>bowtie2</code> looks for the specified index first in the current +directory, then in the directory specified in the +<code>BOWTIE2_INDEXES</code> environment variable.</p> </td> </tr> <tr> @@ -324,7 +968,13 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-1 <m1></code></pre> </td> <td> -<p>Comma-separated list of files containing mate 1s (filename usually includes <code>_1</code>), e.g. <code>-1 flyA_1.fq,flyB_1.fq</code>. Sequences specified with this option must correspond file-for-file and read-for-read with those specified in <code><m2></code>. Reads may be a mix of different lengths. If <code>-</code> is specified, <code>bowtie2</code> will read the mate 1s from the "standard in" or "stdin" filehandle.</p> +<p>Comma-separated list of files containing mate 1s (filename usually +includes <code>_1</code>), e.g. <code>-1 flyA_1.fq,flyB_1.fq</code>. +Sequences specified with this option must correspond file-for-file and +read-for-read with those specified in <code><m2></code>. Reads may +be a mix of different lengths. If <code>-</code> is specified, +<code>bowtie2</code> will read the mate 1s from the "standard in" or +"stdin" filehandle.</p> </td> </tr> <tr> @@ -332,7 +982,13 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-2 <m2></code></pre> </td> <td> -<p>Comma-separated list of files containing mate 2s (filename usually includes <code>_2</code>), e.g. <code>-2 flyA_2.fq,flyB_2.fq</code>. Sequences specified with this option must correspond file-for-file and read-for-read with those specified in <code><m1></code>. Reads may be a mix of different lengths. If <code>-</code> is specified, <code>bowtie2</code> will read the mate 2s from the "standard in" or "stdin" filehandle.</p> +<p>Comma-separated list of files containing mate 2s (filename usually +includes <code>_2</code>), e.g. <code>-2 flyA_2.fq,flyB_2.fq</code>. +Sequences specified with this option must correspond file-for-file and +read-for-read with those specified in <code><m1></code>. Reads may +be a mix of different lengths. If <code>-</code> is specified, +<code>bowtie2</code> will read the mate 2s from the "standard in" or +"stdin" filehandle.</p> </td> </tr> <tr> @@ -340,7 +996,11 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-U <r></code></pre> </td> <td> -<p>Comma-separated list of files containing unpaired reads to be aligned, e.g. <code>lane1.fq,lane2.fq,lane3.fq,lane4.fq</code>. Reads may be a mix of different lengths. If <code>-</code> is specified, <code>bowtie2</code> gets the reads from the "standard in" or "stdin" filehandle.</p> +<p>Comma-separated list of files containing unpaired reads to be +aligned, e.g. <code>lane1.fq,lane2.fq,lane3.fq,lane4.fq</code>. Reads +may be a mix of different lengths. If <code>-</code> is specified, +<code>bowtie2</code> gets the reads from the "standard in" or "stdin" +filehandle.</p> </td> </tr> <tr> @@ -348,7 +1008,8 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>--interleaved</code></pre> </td> <td> -<p>Reads interleaved FASTQ files where the first two records (8 lines) represent a mate pair.</p> +<p>Reads interleaved FASTQ files where the first two records (8 lines) +represent a mate pair.</p> </td> </tr> <tr> @@ -356,8 +1017,14 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>--sra-acc</code></pre> </td> <td> -<p>Reads are SRA accessions. If the accession provided cannot be found in local storage it will be fetched from the NCBI database. If you find that SRA alignments are long running please rerun your command with the <a href="#bowtie2-options-p"><code>-p</code>/<code>--threads</code></a> parameter set to desired number of threads.</p> -<p>NB: this option is only available if bowtie 2 is compiled with the necessary SRA libraries. See <a href="#obtaining-bowtie-2">Obtaining Bowtie 2</a> for details.</p> +<p>Reads are SRA accessions. If the accession provided cannot be found +in local storage it will be fetched from the NCBI database. If you find +that SRA alignments are long running please rerun your command with the +<a href="#bowtie2-options-p"><code>-p</code>/<code>--threads</code></a> +parameter set to desired number of threads.</p> +<p>NB: this option is only available if bowtie 2 is compiled with the +necessary SRA libraries. See <a href="#obtaining-bowtie-2">Obtaining +Bowtie 2</a> for details.</p> </td> </tr> <tr> @@ -365,7 +1032,11 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-b <bam></code></pre> </td> <td> -<p>Reads are unaligned BAM records sorted by read name. The <a href="#bowtie2-options-align-paired-reads"><code>--align-paired-reads</code></a> and <a href="#bowtie2-options-preserve-tags"><code>--preserve-tags</code></a> options affect the way Bowtie 2 processes records.</p> +<p>Reads are unaligned BAM records sorted by read name. The <a +href="#bowtie2-options-align-paired-reads"><code>--align-paired-reads</code></a> +and <a +href="#bowtie2-options-preserve-tags"><code>--preserve-tags</code></a> +options affect the way Bowtie 2 processes records.</p> </td> </tr> <tr> @@ -373,7 +1044,8 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-S <sam></code></pre> </td> <td> -<p>File to write SAM alignments to. By default, alignments are written to the "standard out" or "stdout" filehandle (i.e. the console).</p> +<p>File to write SAM alignments to. By default, alignments are written +to the "standard out" or "stdout" filehandle (i.e. the console).</p> </td> </tr> </table> @@ -385,7 +1057,12 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-q</code></pre> </td> <td> -<p>Reads (specified with <code><m1></code>, <code><m2></code>, <code><s></code>) are FASTQ files. FASTQ files usually have extension <code>.fq</code> or <code>.fastq</code>. FASTQ is the default format. See also: <a href="#bowtie2-options-solexa-quals"><code>--solexa-quals</code></a> and <a href="#bowtie2-options-int-quals"><code>--int-quals</code></a>.</p> +<p>Reads (specified with <code><m1></code>, +<code><m2></code>, <code><s></code>) are FASTQ files. FASTQ +files usually have extension <code>.fq</code> or <code>.fastq</code>. +FASTQ is the default format. See also: <a +href="#bowtie2-options-solexa-quals"><code>--solexa-quals</code></a> and +<a href="#bowtie2-options-int-quals"><code>--int-quals</code></a>.</p> </td> </tr> <tr> @@ -393,7 +1070,11 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>--tab5</code></pre> </td> <td> -<p>Each read or pair is on a single line. An unpaired read line is <code>[name]\t[seq]\t[qual]\n</code>. A paired-end read line is <code>[name]\t[seq1]\t[qual1]\t[seq2]\t[qual2]\n</code>. An input file can be a mix of unpaired and paired-end reads and Bowtie 2 recognizes each according to the number of fields, handling each as it should.</p> +<p>Each read or pair is on a single line. An unpaired read line is +<code>[name]\t[seq]\t[qual]\n</code>. A paired-end read line is +<code>[name]\t[seq1]\t[qual1]\t[seq2]\t[qual2]\n</code>. An input file +can be a mix of unpaired and paired-end reads and Bowtie 2 recognizes +each according to the number of fields, handling each as it should.</p> </td> </tr> <tr> @@ -401,7 +1082,10 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>--tab6</code></pre> </td> <td> -<p>Similar to <a href="#bowtie2-options-tab5"><code>--tab5</code></a> except, for paired-end reads, the second end can have a different name from the first: <code>[name1]\t[seq1]\t[qual1]\t[name2]\t[seq2]\t[qual2]\n</code></p> +<p>Similar to <a href="#bowtie2-options-tab5"><code>--tab5</code></a> +except, for paired-end reads, the second end can have a different name +from the first: +<code>[name1]\t[seq1]\t[qual1]\t[name2]\t[seq2]\t[qual2]\n</code></p> </td> </tr> <tr> @@ -409,7 +1093,11 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>--qseq</code></pre> </td> <td> -<p>Reads (specified with <code><m1></code>, <code><m2></code>, <code><s></code>) are QSEQ files. QSEQ files usually end in <code>_qseq.txt</code>. See also: <a href="#bowtie2-options-solexa-quals"><code>--solexa-quals</code></a> and <a href="#bowtie2-options-int-quals"><code>--int-quals</code></a>.</p> +<p>Reads (specified with <code><m1></code>, +<code><m2></code>, <code><s></code>) are QSEQ files. QSEQ +files usually end in <code>_qseq.txt</code>. See also: <a +href="#bowtie2-options-solexa-quals"><code>--solexa-quals</code></a> and +<a href="#bowtie2-options-int-quals"><code>--int-quals</code></a>.</p> </td> </tr> <tr> @@ -417,7 +1105,17 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-f</code></pre> </td> <td> -<p>Reads (specified with <code><m1></code>, <code><m2></code>, <code><s></code>) are <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files. <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files usually have extension <code>.fa</code>, <code>.fasta</code>, <code>.mfa</code>, <code>.fna</code> or similar. <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files do not have a way of specifying quality values, so when <code>-f</code> is set, the result is as if <a href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a> is also set.</p> +<p>Reads (specified with <code><m1></code>, +<code><m2></code>, <code><s></code>) are <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files. +<a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> +files usually have extension <code>.fa</code>, <code>.fasta</code>, +<code>.mfa</code>, <code>.fna</code> or similar. <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files +do not have a way of specifying quality values, so when <code>-f</code> +is set, the result is as if <a +href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a> is +also set.</p> </td> </tr> <tr> @@ -425,7 +1123,12 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-r</code></pre> </td> <td> -<p>Reads (specified with <code><m1></code>, <code><m2></code>, <code><s></code>) are files with one input sequence per line, without any other information (no read names, no qualities). When <code>-r</code> is set, the result is as if <a href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a> is also set.</p> +<p>Reads (specified with <code><m1></code>, +<code><m2></code>, <code><s></code>) are files with one +input sequence per line, without any other information (no read names, +no qualities). When <code>-r</code> is set, the result is as if <a +href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a> is +also set.</p> </td> </tr> <tr> @@ -433,7 +1136,17 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr <pre><code>-F k:<int>,i:<int></code></pre> </td> <td> -Reads are substrings (k-mers) extracted from a FASTA file <code><s></code>. Specifically, for every reference sequence in FASTA file <code><s></code>, Bowtie 2 aligns the k-mers at offsets 1, 1+i, 1+2i, ... until reaching the end of the reference. Each k-mer is aligned as a separate read. Quality values are set to all Is (40 on Phred scale). Each k-mer (read) is given a name like <code><sequence>_<offset></code>, where <code><sequence></code> is the name of the FASTA sequence it was drawn from and <code><offset></code> is its 0-based offset of origin with respect to the sequence. Only single k-mers, i.e. unpaired reads, can be aligned in this way. +Reads are substrings (k-mers) extracted from a FASTA file +<code><s></code>. Specifically, for every reference sequence in +FASTA file <code><s></code>, Bowtie 2 aligns the k-mers at offsets +1, 1+i, 1+2i, ... until reaching the end of the reference. Each k-mer is +aligned as a separate read. Quality values are set to all Is (40 on +Phred scale). Each k-mer (read) is given a name like +<code><sequence>_<offset></code>, where +<code><sequence></code> is the name of the FASTA sequence it was +drawn from and <code><offset></code> is its 0-based offset of +origin with respect to the sequence. Only single k-mers, i.e. unpaired +reads, can be aligned in this way. </td> </tr> <tr> @@ -441,7 +1154,12 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>-c</code></pre> </td> <td> -<p>The read sequences are given on command line. I.e. <code><m1></code>, <code><m2></code> and <code><singles></code> are comma-separated lists of reads rather than lists of read files. There is no way to specify read names or qualities, so <code>-c</code> also implies <a href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a>.</p> +<p>The read sequences are given on command line. I.e. +<code><m1></code>, <code><m2></code> and +<code><singles></code> are comma-separated lists of reads rather +than lists of read files. There is no way to specify read names or +qualities, so <code>-c</code> also implies <a +href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a>.</p> </td> </tr> <tr> @@ -449,7 +1167,8 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>-s/--skip <int></code></pre> </td> <td> -<p>Skip (i.e. do not align) the first <code><int></code> reads or pairs in the input.</p> +<p>Skip (i.e. do not align) the first <code><int></code> reads or +pairs in the input.</p> </td> </tr> <tr> @@ -457,7 +1176,10 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>-u/--qupto <int></code></pre> </td> <td> -<p>Align the first <code><int></code> reads or read pairs from the input (after the <a href="#bowtie2-options-s"><code>-s</code>/<code>--skip</code></a> reads or pairs have been skipped), then stop. Default: no limit.</p> +<p>Align the first <code><int></code> reads or read pairs from the +input (after the <a +href="#bowtie2-options-s"><code>-s</code>/<code>--skip</code></a> reads +or pairs have been skipped), then stop. Default: no limit.</p> </td> </tr> <tr> @@ -465,7 +1187,8 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>-5/--trim5 <int></code></pre> </td> <td> -<p>Trim <code><int></code> bases from 5' (left) end of each read before alignment (default: 0).</p> +<p>Trim <code><int></code> bases from 5' (left) end of each read +before alignment (default: 0).</p> </td> </tr> <tr> @@ -473,7 +1196,8 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>-3/--trim3 <int></code></pre> </td> <td> -<p>Trim <code><int></code> bases from 3' (right) end of each read before alignment (default: 0).</p> +<p>Trim <code><int></code> bases from 3' (right) end of each read +before alignment (default: 0).</p> </td> </tr> <tr> @@ -481,7 +1205,14 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>--trim-to [3:|5:]<int></code></pre> </td> <td> -<p>Trim reads exceeding <code><int></code> bases. Bases will be trimmed from either the 3' (right) or 5' (left) end of the read. If the read end if not specified, bowtie 2 will default to trimming from the 3' (right) end of the read. <a href="#bowtie2-options-trim-to"><code>--trim-to</code></a> and <a href="#bowtie2-options-3"><code>-3</code></a>/<a href="#bowtie2-options-5"><code>-5</code></a> are mutually exclusive.</p> +<p>Trim reads exceeding <code><int></code> bases. Bases will be +trimmed from either the 3' (right) or 5' (left) end of the read. If the +read end if not specified, bowtie 2 will default to trimming from the 3' +(right) end of the read. <a +href="#bowtie2-options-trim-to"><code>--trim-to</code></a> and <a +href="#bowtie2-options-3"><code>-3</code></a>/<a +href="#bowtie2-options-5"><code>-5</code></a> are mutually +exclusive.</p> </td> </tr> <tr> @@ -489,7 +1220,10 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>--phred33</code></pre> </td> <td> -<p>Input qualities are ASCII chars equal to the <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred quality</a> plus 33. This is also called the "Phred+33" encoding, which is used by the very latest Illumina pipelines.</p> +<p>Input qualities are ASCII chars equal to the <a +href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred +quality</a> plus 33. This is also called the "Phred+33" encoding, which +is used by the very latest Illumina pipelines.</p> </td> </tr> <tr> @@ -497,7 +1231,9 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>--phred64</code></pre> </td> <td> -<p>Input qualities are ASCII chars equal to the <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred quality</a> plus 64. This is also called the "Phred+64" encoding.</p> +<p>Input qualities are ASCII chars equal to the <a +href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred +quality</a> plus 64. This is also called the "Phred+64" encoding.</p> </td> </tr> <tr> @@ -505,7 +1241,12 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>--solexa-quals</code></pre> </td> <td> -<p>Convert input qualities from <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Solexa</a> (which can be negative) to <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred</a> (which can't). This scheme was used in older Illumina GA Pipeline versions (prior to 1.3). Default: off.</p> +<p>Convert input qualities from <a +href="http://en.wikipedia.org/wiki/Phred_quality_score">Solexa</a> +(which can be negative) to <a +href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred</a> (which +can't). This scheme was used in older Illumina GA Pipeline versions +(prior to 1.3). Default: off.</p> </td> </tr> <tr> @@ -513,11 +1254,19 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>--int-quals</code></pre> </td> <td> -<p>Quality values are represented in the read input file as space-separated ASCII integers, e.g., <code>40 40 30 40</code>..., rather than ASCII characters, e.g., <code>II?I</code>.... Integers are treated as being on the <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred quality</a> scale unless <a href="#bowtie2-options-solexa-quals"><code>--solexa-quals</code></a> is also specified. Default: off.</p> +<p>Quality values are represented in the read input file as +space-separated ASCII integers, e.g., <code>40 40 30 40</code>..., +rather than ASCII characters, e.g., <code>II?I</code>.... Integers are +treated as being on the <a +href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred +quality</a> scale unless <a +href="#bowtie2-options-solexa-quals"><code>--solexa-quals</code></a> is +also specified. Default: off.</p> </td> </tr> </table> -<h4 id="preset-options-in---end-to-end-mode">Preset options in <code>--end-to-end</code> mode</h4> +<h4 id="preset-options-in---end-to-end-mode">Preset options in +<code>--end-to-end</code> mode</h4> <table> <tr> <td id="bowtie2-options-very-fast"> @@ -540,7 +1289,9 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>--sensitive</code></pre> </td> <td> -<p>Same as: <code>-D 15 -R 2 -N 0 -L 22 -i S,1,1.15</code> (default in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode)</p> +<p>Same as: <code>-D 15 -R 2 -N 0 -L 22 -i S,1,1.15</code> (default in +<a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> +mode)</p> </td> </tr> <tr> @@ -552,7 +1303,8 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> </td> </tr> </table> -<h4 id="preset-options-in---local-mode">Preset options in <code>--local</code> mode</h4> +<h4 id="preset-options-in---local-mode">Preset options in +<code>--local</code> mode</h4> <table> <tr> <td id="bowtie2-options-very-fast-local"> @@ -575,7 +1327,8 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>--sensitive-local</code></pre> </td> <td> -<p>Same as: <code>-D 15 -R 2 -N 0 -L 20 -i S,1,0.75</code> (default in <a href="#bowtie2-options-local"><code>--local</code></a> mode)</p> +<p>Same as: <code>-D 15 -R 2 -N 0 -L 20 -i S,1,0.75</code> (default in +<a href="#bowtie2-options-local"><code>--local</code></a> mode)</p> </td> </tr> <tr> @@ -594,7 +1347,10 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>-N <int></code></pre> </td> <td> -<p>Sets the number of mismatches to allowed in a seed alignment during <a href="#multiseed-heuristic">multiseed alignment</a>. Can be set to 0 or 1. Setting this higher makes alignment slower (often much slower) but increases sensitivity. Default: 0.</p> +<p>Sets the number of mismatches to allowed in a seed alignment during +<a href="#multiseed-heuristic">multiseed alignment</a>. Can be set to 0 +or 1. Setting this higher makes alignment slower (often much slower) but +increases sensitivity. Default: 0.</p> </td> </tr> <tr> @@ -602,7 +1358,14 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>-L <int></code></pre> </td> <td> -<p>Sets the length of the seed substrings to align during <a href="#multiseed-heuristic">multiseed alignment</a>. Smaller values make alignment slower but more sensitive. Default: the <a href="#bowtie2-options-sensitive"><code>--sensitive</code></a> preset is used by default, which sets <code>-L</code> to 22 and 20 in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode and in <a href="#bowtie2-options-local"><code>--local</code></a> mode.</p> +<p>Sets the length of the seed substrings to align during <a +href="#multiseed-heuristic">multiseed alignment</a>. Smaller values make +alignment slower but more sensitive. Default: the <a +href="#bowtie2-options-sensitive"><code>--sensitive</code></a> preset is +used by default, which sets <code>-L</code> to 22 and 20 in <a +href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode +and in <a href="#bowtie2-options-local"><code>--local</code></a> +mode.</p> </td> </tr> <tr> @@ -610,7 +1373,10 @@ Reads are substrings (k-mers) extracted from a FASTA file <code><s></code> <pre><code>-i <func></code></pre> </td> <td> -<p>Sets a function governing the interval between seed substrings to use during <a href="#multiseed-heuristic">multiseed alignment</a>. For instance, if the read has 30 characters, and seed length is 10, and the seed interval is 6, the seeds extracted will be:</p> +<p>Sets a function governing the interval between seed substrings to use +during <a href="#multiseed-heuristic">multiseed alignment</a>. For +instance, if the read has 30 characters, and seed length is 10, and the +seed interval is 6, the seeds extracted will be:</p> <pre><code>Read: TAGCTACGCTCTACGCTATCATGCATAAAC Seed 1 fw: TAGCTACGCT Seed 1 rc: AGCGTAGCTA @@ -620,7 +1386,19 @@ Seed 3 fw: ACGCTATCAT Seed 3 rc: ATGATAGCGT Seed 4 fw: TCATGCATAA Seed 4 rc: TTATGCATGA</code></pre> -<p>Since it's best to use longer intervals for longer reads, this parameter sets the interval as a function of the read length, rather than a single one-size-fits-all number. For instance, specifying <code>-i S,1,2.5</code> sets the interval function <code>f</code> to <code>f(x) = 1 + 2.5 * sqrt(x)</code>, where x is the read length. See also: <a href="#setting-function-options">setting function options</a>. If the function returns a result less than 1, it is rounded up to 1. Default: the <a href="#bowtie2-options-sensitive"><code>--sensitive</code></a> preset is used by default, which sets <code>-i</code> to <code>S,1,1.15</code> in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode to <code>-i S,1,0.75</code> in <a href="#bowtie2-options-local"><code>--local</code></a> mode.</p> +<p>Since it's best to use longer intervals for longer reads, this +parameter sets the interval as a function of the read length, rather +than a single one-size-fits-all number. For instance, specifying +<code>-i S,1,2.5</code> sets the interval function <code>f</code> to +<code>f(x) = 1 + 2.5 * sqrt(x)</code>, where x is the read length. See +also: <a href="#setting-function-options">setting function options</a>. +If the function returns a result less than 1, it is rounded up to 1. +Default: the <a +href="#bowtie2-options-sensitive"><code>--sensitive</code></a> preset is +used by default, which sets <code>-i</code> to <code>S,1,1.15</code> in +<a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode +to <code>-i S,1,0.75</code> in <a +href="#bowtie2-options-local"><code>--local</code></a> mode.</p> </td> </tr> <tr> @@ -628,7 +1406,14 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--n-ceil <func></code></pre> </td> <td> -<p>Sets a function governing the maximum number of ambiguous characters (usually <code>N</code>s and/or <code>.</code>s) allowed in a read as a function of read length. For instance, specifying <code>-L,0,0.15</code> sets the N-ceiling function <code>f</code> to <code>f(x) = 0 + 0.15 * x</code>, where x is the read length. See also: <a href="#setting-function-options">setting function options</a>. Reads exceeding this ceiling are <a href="#filtering">filtered out</a>. Default: <code>L,0,0.15</code>.</p> +<p>Sets a function governing the maximum number of ambiguous characters +(usually <code>N</code>s and/or <code>.</code>s) allowed in a read as a +function of read length. For instance, specifying <code>-L,0,0.15</code> +sets the N-ceiling function <code>f</code> to +<code>f(x) = 0 + 0.15 * x</code>, where x is the read length. See also: +<a href="#setting-function-options">setting function options</a>. Reads +exceeding this ceiling are <a href="#filtering">filtered out</a>. +Default: <code>L,0,0.15</code>.</p> </td> </tr> <tr> @@ -636,7 +1421,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--dpad <int></code></pre> </td> <td> -<p>"Pads" dynamic programming problems by <code><int></code> columns on either side to allow gaps. Default: 15.</p> +<p>"Pads" dynamic programming problems by <code><int></code> +columns on either side to allow gaps. Default: 15.</p> </td> </tr> <tr> @@ -644,7 +1430,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--gbar <int></code></pre> </td> <td> -<p>Disallow gaps within <code><int></code> positions of the beginning or end of the read. Default: 4.</p> +<p>Disallow gaps within <code><int></code> positions of the +beginning or end of the read. Default: 4.</p> </td> </tr> <tr> @@ -652,7 +1439,14 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--ignore-quals</code></pre> </td> <td> -<p>When calculating a mismatch penalty, always consider the quality value at the mismatched position to be the highest possible, regardless of the actual value. I.e. input is treated as though all quality values are high. This is also the default behavior when the input doesn't specify quality values (e.g. in <a href="#bowtie2-options-f"><code>-f</code></a>, <a href="#bowtie2-options-r"><code>-r</code></a>, or <a href="#bowtie2-options-c"><code>-c</code></a> modes).</p> +<p>When calculating a mismatch penalty, always consider the quality +value at the mismatched position to be the highest possible, regardless +of the actual value. I.e. input is treated as though all quality values +are high. This is also the default behavior when the input doesn't +specify quality values (e.g. in <a +href="#bowtie2-options-f"><code>-f</code></a>, <a +href="#bowtie2-options-r"><code>-r</code></a>, or <a +href="#bowtie2-options-c"><code>-c</code></a> modes).</p> </td> </tr> <tr> @@ -660,7 +1454,15 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--nofw/--norc</code></pre> </td> <td> -<p>If <code>--nofw</code> is specified, <code>bowtie2</code> will not attempt to align unpaired reads to the forward (Watson) reference strand. If <code>--norc</code> is specified, <code>bowtie2</code> will not attempt to align unpaired reads against the reverse-complement (Crick) reference strand. In paired-end mode, <code>--nofw</code> and <code>--norc</code> pertain to the fragments; i.e. specifying <code>--nofw</code> causes <code>bowtie2</code> to explore only those paired-end configurations corresponding to fragments from the reverse-complement (Crick) strand. Default: both strands enabled.</p> +<p>If <code>--nofw</code> is specified, <code>bowtie2</code> will not +attempt to align unpaired reads to the forward (Watson) reference +strand. If <code>--norc</code> is specified, <code>bowtie2</code> will +not attempt to align unpaired reads against the reverse-complement +(Crick) reference strand. In paired-end mode, <code>--nofw</code> and +<code>--norc</code> pertain to the fragments; i.e. specifying +<code>--nofw</code> causes <code>bowtie2</code> to explore only those +paired-end configurations corresponding to fragments from the +reverse-complement (Crick) strand. Default: both strands enabled.</p> </td> </tr> <tr> @@ -668,7 +1470,24 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--no-1mm-upfront</code></pre> </td> <td> -<p>By default, Bowtie 2 will attempt to find either an exact or a 1-mismatch end-to-end alignment for the read <em>before</em> trying the <a href="#multiseed-heuristic">multiseed heuristic</a>. Such alignments can be found very quickly, and many short read alignments have exact or near-exact end-to-end alignments. However, this can lead to unexpected alignments when the user also sets options governing the <a href="#multiseed-heuristic">multiseed heuristic</a>, like <a href="#bowtie2-options-L"><code>-L</code></a> and <a href="#bowtie2-options-N"><code>-N</code></a>. For instance, if the user specifies <code>-N 0</code> and <code>-L</code> equal to the length of the read, the user will be surprised to find 1-mismatch alignments reported. This option prevents Bowtie 2 from searching for 1-mismatch end-to-end alignments before using the <a href="#multiseed-heuristic">multiseed heuristic</a>, which leads to the expected behavior when combined with options such as <a href="#bowtie2-options-L"><code>-L</code></a> and <a href="#bowtie2-options-N"><code>-N</code></a>. This comes at the expense of speed.</p> +<p>By default, Bowtie 2 will attempt to find either an exact or a +1-mismatch end-to-end alignment for the read <em>before</em> trying the +<a href="#multiseed-heuristic">multiseed heuristic</a>. Such alignments +can be found very quickly, and many short read alignments have exact or +near-exact end-to-end alignments. However, this can lead to unexpected +alignments when the user also sets options governing the <a +href="#multiseed-heuristic">multiseed heuristic</a>, like <a +href="#bowtie2-options-L"><code>-L</code></a> and <a +href="#bowtie2-options-N"><code>-N</code></a>. For instance, if the user +specifies <code>-N 0</code> and <code>-L</code> equal to the length of +the read, the user will be surprised to find 1-mismatch alignments +reported. This option prevents Bowtie 2 from searching for 1-mismatch +end-to-end alignments before using the <a +href="#multiseed-heuristic">multiseed heuristic</a>, which leads to the +expected behavior when combined with options such as <a +href="#bowtie2-options-L"><code>-L</code></a> and <a +href="#bowtie2-options-N"><code>-N</code></a>. This comes at the expense +of speed.</p> </td> </tr> <tr> @@ -676,7 +1495,14 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--end-to-end</code></pre> </td> <td> -<p>In this mode, Bowtie 2 requires that the entire read align from one end to the other, without any trimming (or "soft clipping") of characters from either end. The match bonus <a href="#bowtie2-options-ma"><code>--ma</code></a> always equals 0 in this mode, so all alignment scores are less than or equal to 0, and the greatest possible alignment score is 0. This is mutually exclusive with <a href="#bowtie2-options-local"><code>--local</code></a>. <code>--end-to-end</code> is the default mode.</p> +<p>In this mode, Bowtie 2 requires that the entire read align from one +end to the other, without any trimming (or "soft clipping") of +characters from either end. The match bonus <a +href="#bowtie2-options-ma"><code>--ma</code></a> always equals 0 in this +mode, so all alignment scores are less than or equal to 0, and the +greatest possible alignment score is 0. This is mutually exclusive with +<a href="#bowtie2-options-local"><code>--local</code></a>. +<code>--end-to-end</code> is the default mode.</p> </td> </tr> <tr> @@ -684,7 +1510,19 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--local</code></pre> </td> <td> -<p>In this mode, Bowtie 2 does not require that the entire read align from one end to the other. Rather, some characters may be omitted ("soft clipped") from the ends in order to achieve the greatest possible alignment score. The match bonus <a href="#bowtie2-options-ma"><code>--ma</code></a> is used in this mode, and the best possible alignment score is equal to the match bonus (<a href="#bowtie2-options-ma"><code>--ma</code></a>) times the length of the read. Specifying <code>--local</code> and one of the presets (e.g. <code>--local --very-fast</code>) is equivalent to specifying the local version of the preset (<code>--very-fast-local</code>). This is mutually exclusive with <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a>. <code>--end-to-end</code> is the default mode.</p> +<p>In this mode, Bowtie 2 does not require that the entire read align +from one end to the other. Rather, some characters may be omitted ("soft +clipped") from the ends in order to achieve the greatest possible +alignment score. The match bonus <a +href="#bowtie2-options-ma"><code>--ma</code></a> is used in this mode, +and the best possible alignment score is equal to the match bonus (<a +href="#bowtie2-options-ma"><code>--ma</code></a>) times the length of +the read. Specifying <code>--local</code> and one of the presets (e.g. +<code>--local --very-fast</code>) is equivalent to specifying the local +version of the preset (<code>--very-fast-local</code>). This is mutually +exclusive with <a +href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a>. +<code>--end-to-end</code> is the default mode.</p> </td> </tr> </table> @@ -695,7 +1533,13 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--ma <int></code></pre> </td> <td> -<p>Sets the match bonus. In <a href="#bowtie2-options-local"><code>--local</code></a> mode <code><int></code> is added to the alignment score for each position where a read character aligns to a reference character and the characters match. Not used in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode. Default: 2.</p> +<p>Sets the match bonus. In <a +href="#bowtie2-options-local"><code>--local</code></a> mode +<code><int></code> is added to the alignment score for each +position where a read character aligns to a reference character and the +characters match. Not used in <a +href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode. +Default: 2.</p> </td> </tr> <tr> @@ -703,7 +1547,18 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--mp MX,MN</code></pre> </td> <td> -<p>Sets the maximum (<code>MX</code>) and minimum (<code>MN</code>) mismatch penalties, both integers. A number less than or equal to <code>MX</code> and greater than or equal to <code>MN</code> is subtracted from the alignment score for each position where a read character aligns to a reference character, the characters do not match, and neither is an <code>N</code>. If <a href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a> is specified, the number subtracted quals <code>MX</code>. Otherwise, the number subtracted is <code>MN + floor( (MX-MN)(MIN(Q, 40.0)/40.0) )</code> where Q is the Phred quality value. Default: <code>MX</code> = 6, <code>MN</code> = 2.</p> +<p>Sets the maximum (<code>MX</code>) and minimum (<code>MN</code>) +mismatch penalties, both integers. A number less than or equal to +<code>MX</code> and greater than or equal to <code>MN</code> is +subtracted from the alignment score for each position where a read +character aligns to a reference character, the characters do not match, +and neither is an <code>N</code>. If <a +href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a> is +specified, the number subtracted quals <code>MX</code>. Otherwise, the +number subtracted is +<code>MN + floor( (MX-MN)(MIN(Q, 40.0)/40.0) )</code> where Q is the +Phred quality value. Default: <code>MX</code> = 6, <code>MN</code> = +2.</p> </td> </tr> <tr> @@ -711,7 +1566,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--np <int></code></pre> </td> <td> -<p>Sets penalty for positions where the read, reference, or both, contain an ambiguous character such as <code>N</code>. Default: 1.</p> +<p>Sets penalty for positions where the read, reference, or both, +contain an ambiguous character such as <code>N</code>. Default: 1.</p> </td> </tr> <tr> @@ -719,7 +1575,10 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--rdg <int1>,<int2></code></pre> </td> <td> -<p>Sets the read gap open (<code><int1></code>) and extend (<code><int2></code>) penalties. A read gap of length N gets a penalty of <code><int1></code> + N * <code><int2></code>. Default: 5, 3.</p> +<p>Sets the read gap open (<code><int1></code>) and extend +(<code><int2></code>) penalties. A read gap of length N gets a +penalty of <code><int1></code> + N * <code><int2></code>. +Default: 5, 3.</p> </td> </tr> <tr> @@ -727,7 +1586,10 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--rfg <int1>,<int2></code></pre> </td> <td> -<p>Sets the reference gap open (<code><int1></code>) and extend (<code><int2></code>) penalties. A reference gap of length N gets a penalty of <code><int1></code> + N * <code><int2></code>. Default: 5, 3.</p> +<p>Sets the reference gap open (<code><int1></code>) and extend +(<code><int2></code>) penalties. A reference gap of length N gets +a penalty of <code><int1></code> + N * <code><int2></code>. +Default: 5, 3.</p> </td> </tr> <tr> @@ -735,7 +1597,17 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--score-min <func></code></pre> </td> <td> -<p>Sets a function governing the minimum alignment score needed for an alignment to be considered "valid" (i.e. good enough to report). This is a function of read length. For instance, specifying <code>L,0,-0.6</code> sets the minimum-score function <code>f</code> to <code>f(x) = 0 + -0.6 * x</code>, where <code>x</code> is the read length. See also: <a href="#setting-function-options">setting function options</a>. The default in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode is <code>L,-0.6,-0.6</code> and the default in <a href="#bowtie2-options-local"><code>--local</code></a> mode is <code>G,20,8</code>.</p> +<p>Sets a function governing the minimum alignment score needed for an +alignment to be considered "valid" (i.e. good enough to report). This is +a function of read length. For instance, specifying +<code>L,0,-0.6</code> sets the minimum-score function <code>f</code> to +<code>f(x) = 0 + -0.6 * x</code>, where <code>x</code> is the read +length. See also: <a href="#setting-function-options">setting function +options</a>. The default in <a +href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode is +<code>L,-0.6,-0.6</code> and the default in <a +href="#bowtie2-options-local"><code>--local</code></a> mode is +<code>G,20,8</code>.</p> </td> </tr> </table> @@ -746,9 +1618,32 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-k <int></code></pre> </td> <td> -<p>By default, <code>bowtie2</code> searches for distinct, valid alignments for each read. When it finds a valid alignment, it continues looking for alignments that are nearly as good or better. The best alignment found is reported (randomly selected from among best if tied). Information about the best alignments is used to estimate mapping quality and to set SAM optional fields, such as <a href="#bowtie2-build-opt-fields-as"><code>AS:i</code></a> and <a href="#bowtie2-build-opt-fields-xs"><code>XS:i</code></a>.</p> -<p>When <code>-k</code> is specified, however, <code>bowtie2</code> behaves differently. Instead, it searches for at most <code><int></code> distinct, valid alignments for each read. The search terminates when it can't find more distinct valid alignments, or when it finds <code><int></code>, whichever happens first. All alignments found are reported in descending order by alignment score. The alignment score for a paired-end alignment equals the sum of the alignment scores of the individual mates. Each reported read or pair alignment beyond the first has the SAM 'secondary' bit (which equals 256) set in its FLAGS field. For reads that have more than <code><int></code> distinct, valid alignments, <code>bowtie2</code> does not guarantee that the <code><int></code> alignments reported are the best possible in terms of alignment score. <code>-k</code> is mutually exclusive with <a href="#bowtie2-options-a"><code>-a</code></a>.</p> -<p>Note: Bowtie 2 is not designed with large values for <code>-k</code> in mind, and when aligning reads to long, repetitive genomes large <code>-k</code> can be very, very slow.</p> +<p>By default, <code>bowtie2</code> searches for distinct, valid +alignments for each read. When it finds a valid alignment, it continues +looking for alignments that are nearly as good or better. The best +alignment found is reported (randomly selected from among best if tied). +Information about the best alignments is used to estimate mapping +quality and to set SAM optional fields, such as <a +href="#bowtie2-build-opt-fields-as"><code>AS:i</code></a> and <a +href="#bowtie2-build-opt-fields-xs"><code>XS:i</code></a>.</p> +<p>When <code>-k</code> is specified, however, <code>bowtie2</code> +behaves differently. Instead, it searches for at most +<code><int></code> distinct, valid alignments for each read. The +search terminates when it can't find more distinct valid alignments, or +when it finds <code><int></code>, whichever happens first. All +alignments found are reported in descending order by alignment score. +The alignment score for a paired-end alignment equals the sum of the +alignment scores of the individual mates. Each reported read or pair +alignment beyond the first has the SAM 'secondary' bit (which equals +256) set in its FLAGS field. For reads that have more than +<code><int></code> distinct, valid alignments, +<code>bowtie2</code> does not guarantee that the +<code><int></code> alignments reported are the best possible in +terms of alignment score. <code>-k</code> is mutually exclusive with <a +href="#bowtie2-options-a"><code>-a</code></a>.</p> +<p>Note: Bowtie 2 is not designed with large values for <code>-k</code> +in mind, and when aligning reads to long, repetitive genomes large +<code>-k</code> can be very, very slow.</p> </td> </tr> <tr> @@ -756,8 +1651,13 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-a</code></pre> </td> <td> -<p>Like <a href="#bowtie2-options-k"><code>-k</code></a> but with no upper limit on number of alignments to search for. <code>-a</code> is mutually exclusive with <a href="#bowtie2-options-k"><code>-k</code></a>.</p> -<p>Note: Bowtie 2 is not designed with <code>-a</code> mode in mind, and when aligning reads to long, repetitive genomes this mode can be very, very slow.</p> +<p>Like <a href="#bowtie2-options-k"><code>-k</code></a> but with no +upper limit on number of alignments to search for. <code>-a</code> is +mutually exclusive with <a +href="#bowtie2-options-k"><code>-k</code></a>.</p> +<p>Note: Bowtie 2 is not designed with <code>-a</code> mode in mind, and +when aligning reads to long, repetitive genomes this mode can be very, +very slow.</p> </td> </tr> </table> @@ -768,7 +1668,11 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-D <int></code></pre> </td> <td> -<p>Up to <code><int></code> consecutive seed extension attempts can "fail" before Bowtie 2 moves on, using the alignments found so far. A seed extension "fails" if it does not yield a new best or a new second-best alignment. This limit is automatically adjusted up when -k or -a are specified. Default: 15.</p> +<p>Up to <code><int></code> consecutive seed extension attempts +can "fail" before Bowtie 2 moves on, using the alignments found so far. +A seed extension "fails" if it does not yield a new best or a new +second-best alignment. This limit is automatically adjusted up when -k +or -a are specified. Default: 15.</p> </td> </tr> <tr> @@ -776,7 +1680,13 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-R <int></code></pre> </td> <td> -<p><code><int></code> is the maximum number of times Bowtie 2 will "re-seed" reads with repetitive seeds. When "re-seeding," Bowtie 2 simply chooses a new set of reads (same length, same number of mismatches allowed) at different offsets and searches for more alignments. A read is considered to have repetitive seeds if the total number of seed hits divided by the number of seeds that aligned at least once is greater than 300. Default: 2.</p> +<p><code><int></code> is the maximum number of times Bowtie 2 will +"re-seed" reads with repetitive seeds. When "re-seeding," Bowtie 2 +simply chooses a new set of reads (same length, same number of +mismatches allowed) at different offsets and searches for more +alignments. A read is considered to have repetitive seeds if the total +number of seed hits divided by the number of seeds that aligned at least +once is greater than 300. Default: 2.</p> </td> </tr> </table> @@ -787,8 +1697,25 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-I/--minins <int></code></pre> </td> <td> -<p>The minimum fragment length for valid paired-end alignments. E.g. if <code>-I 60</code> is specified and a paired-end alignment consists of two 20-bp alignments in the appropriate orientation with a 20-bp gap between them, that alignment is considered valid (as long as <a href="#bowtie2-options-X"><code>-X</code></a> is also satisfied). A 19-bp gap would not be valid in that case. If trimming options <a href="#bowtie2-options-3"><code>-3</code></a> or <a href="#bowtie2-options-5"><code>-5</code></a> are also used, the <a href="#bowtie2-options-I"><code>-I</code></a> constraint is applied with respect to the untrimmed mates.</p> -<p>The larger the difference between <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a>, the slower Bowtie 2 will run. This is because larger differences between <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a> require that Bowtie 2 scan a larger window to determine if a concordant alignment exists. For typical fragment length ranges (200 to 400 nucleotides), Bowtie 2 is very efficient.</p> +<p>The minimum fragment length for valid paired-end alignments. E.g. if +<code>-I 60</code> is specified and a paired-end alignment consists of +two 20-bp alignments in the appropriate orientation with a 20-bp gap +between them, that alignment is considered valid (as long as <a +href="#bowtie2-options-X"><code>-X</code></a> is also satisfied). A +19-bp gap would not be valid in that case. If trimming options <a +href="#bowtie2-options-3"><code>-3</code></a> or <a +href="#bowtie2-options-5"><code>-5</code></a> are also used, the <a +href="#bowtie2-options-I"><code>-I</code></a> constraint is applied with +respect to the untrimmed mates.</p> +<p>The larger the difference between <a +href="#bowtie2-options-I"><code>-I</code></a> and <a +href="#bowtie2-options-X"><code>-X</code></a>, the slower Bowtie 2 will +run. This is because larger differences between <a +href="#bowtie2-options-I"><code>-I</code></a> and <a +href="#bowtie2-options-X"><code>-X</code></a> require that Bowtie 2 scan +a larger window to determine if a concordant alignment exists. For +typical fragment length ranges (200 to 400 nucleotides), Bowtie 2 is +very efficient.</p> <p>Default: 0 (essentially imposing no minimum)</p> </td> </tr> @@ -797,8 +1724,25 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-X/--maxins <int></code></pre> </td> <td> -<p>The maximum fragment length for valid paired-end alignments. E.g. if <code>-X 100</code> is specified and a paired-end alignment consists of two 20-bp alignments in the proper orientation with a 60-bp gap between them, that alignment is considered valid (as long as <a href="#bowtie2-options-I"><code>-I</code></a> is also satisfied). A 61-bp gap would not be valid in that case. If trimming options <a href="#bowtie2-options-3"><code>-3</code></a> or <a href="#bowtie2-options-5"><code>-5</code></a> are also used, the <code>-X</code> constraint is applied with respect to the untrimmed mates, not the trimmed mates.</p> -<p>The larger the difference between <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a>, the slower Bowtie 2 will run. This is because larger differences between <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a> require that Bowtie 2 scan a larger window to determine if a concordant alignment exists. For typical fragment length ranges (200 to 400 nucleotides), Bowtie 2 is very efficient.</p> +<p>The maximum fragment length for valid paired-end alignments. E.g. if +<code>-X 100</code> is specified and a paired-end alignment consists of +two 20-bp alignments in the proper orientation with a 60-bp gap between +them, that alignment is considered valid (as long as <a +href="#bowtie2-options-I"><code>-I</code></a> is also satisfied). A +61-bp gap would not be valid in that case. If trimming options <a +href="#bowtie2-options-3"><code>-3</code></a> or <a +href="#bowtie2-options-5"><code>-5</code></a> are also used, the +<code>-X</code> constraint is applied with respect to the untrimmed +mates, not the trimmed mates.</p> +<p>The larger the difference between <a +href="#bowtie2-options-I"><code>-I</code></a> and <a +href="#bowtie2-options-X"><code>-X</code></a>, the slower Bowtie 2 will +run. This is because larger differences between <a +href="#bowtie2-options-I"><code>-I</code></a> and <a +href="#bowtie2-options-X"><code>-X</code></a> require that Bowtie 2 scan +a larger window to determine if a concordant alignment exists. For +typical fragment length ranges (200 to 400 nucleotides), Bowtie 2 is +very efficient.</p> <p>Default: 500.</p> </td> </tr> @@ -807,7 +1751,20 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--fr/--rf/--ff</code></pre> </td> <td> -<p>The upstream/downstream mate orientations for a valid paired-end alignment against the forward reference strand. E.g., if <code>--fr</code> is specified and there is a candidate paired-end alignment where mate 1 appears upstream of the reverse complement of mate 2 and the fragment length constraints (<a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a>) are met, that alignment is valid. Also, if mate 2 appears upstream of the reverse complement of mate 1 and all other constraints are met, that too is valid. <code>--rf</code> likewise requires that an upstream mate1 be reverse-complemented and a downstream mate2 be forward-oriented. <code>--ff</code> requires both an upstream mate 1 and a downstream mate 2 to be forward-oriented. Default: <code>--fr</code> (appropriate for Illumina's Paired-end Sequencing Assay).</p> +<p>The upstream/downstream mate orientations for a valid paired-end +alignment against the forward reference strand. E.g., if +<code>--fr</code> is specified and there is a candidate paired-end +alignment where mate 1 appears upstream of the reverse complement of +mate 2 and the fragment length constraints (<a +href="#bowtie2-options-I"><code>-I</code></a> and <a +href="#bowtie2-options-X"><code>-X</code></a>) are met, that alignment +is valid. Also, if mate 2 appears upstream of the reverse complement of +mate 1 and all other constraints are met, that too is valid. +<code>--rf</code> likewise requires that an upstream mate1 be +reverse-complemented and a downstream mate2 be forward-oriented. +<code>--ff</code> requires both an upstream mate 1 and a downstream mate +2 to be forward-oriented. Default: <code>--fr</code> (appropriate for +Illumina's Paired-end Sequencing Assay).</p> </td> </tr> <tr> @@ -815,7 +1772,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--no-mixed</code></pre> </td> <td> -<p>By default, when <code>bowtie2</code> cannot find a concordant or discordant alignment for a pair, it then tries to find alignments for the individual mates. This option disables that behavior.</p> +<p>By default, when <code>bowtie2</code> cannot find a concordant or +discordant alignment for a pair, it then tries to find alignments for +the individual mates. This option disables that behavior.</p> </td> </tr> <tr> @@ -823,7 +1782,14 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--no-discordant</code></pre> </td> <td> -<p>By default, <code>bowtie2</code> looks for discordant alignments if it cannot find any concordant alignments. A discordant alignment is an alignment where both mates align uniquely, but that does not satisfy the paired-end constraints (<a href="#bowtie2-options-fr"><code>--fr</code>/<code>--rf</code>/<code>--ff</code></a>, <a href="#bowtie2-options-I"><code>-I</code></a>, <a href="#bowtie2-options-X"><code>-X</code></a>). This option disables that behavior.</p> +<p>By default, <code>bowtie2</code> looks for discordant alignments if +it cannot find any concordant alignments. A discordant alignment is an +alignment where both mates align uniquely, but that does not satisfy the +paired-end constraints (<a +href="#bowtie2-options-fr"><code>--fr</code>/<code>--rf</code>/<code>--ff</code></a>, +<a href="#bowtie2-options-I"><code>-I</code></a>, <a +href="#bowtie2-options-X"><code>-X</code></a>). This option disables +that behavior.</p> </td> </tr> <tr> @@ -831,7 +1797,12 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--dovetail</code></pre> </td> <td> -<p>If the mates "dovetail", that is if one mate alignment extends past the beginning of the other such that the wrong mate begins upstream, consider that to be concordant. See also: <a href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can overlap, contain or dovetail each other</a>. Default: mates cannot dovetail in a concordant alignment.</p> +<p>If the mates "dovetail", that is if one mate alignment extends past +the beginning of the other such that the wrong mate begins upstream, +consider that to be concordant. See also: <a +href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can +overlap, contain or dovetail each other</a>. Default: mates cannot +dovetail in a concordant alignment.</p> </td> </tr> <tr> @@ -839,7 +1810,11 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--no-contain</code></pre> </td> <td> -<p>If one mate alignment contains the other, consider that to be non-concordant. See also: <a href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can overlap, contain or dovetail each other</a>. Default: a mate can contain the other in a concordant alignment.</p> +<p>If one mate alignment contains the other, consider that to be +non-concordant. See also: <a +href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can +overlap, contain or dovetail each other</a>. Default: a mate can contain +the other in a concordant alignment.</p> </td> </tr> <tr> @@ -847,7 +1822,11 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--no-overlap</code></pre> </td> <td> -<p>If one mate alignment overlaps the other at all, consider that to be non-concordant. See also: <a href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can overlap, contain or dovetail each other</a>. Default: mates can overlap in a concordant alignment.</p> +<p>If one mate alignment overlaps the other at all, consider that to be +non-concordant. See also: <a +href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can +overlap, contain or dovetail each other</a>. Default: mates can overlap +in a concordant alignment.</p> </td> </tr> </table> @@ -858,7 +1837,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--align-paired-reads</code></pre> </td> <td> -<p>Bowtie 2 will, by default, attempt to align unpaired BAM reads. Use this option to align paired-end reads instead.</p> +<p>Bowtie 2 will, by default, attempt to align unpaired BAM reads. Use +this option to align paired-end reads instead.</p> </td> </tr> <tr> @@ -866,7 +1846,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--preserve-tags</code></pre> </td> <td> -<p>Preserve tags from the original BAM record by appending them to the end of the corresponding Bowtie 2 SAM output.</p> +<p>Preserve tags from the original BAM record by appending them to the +end of the corresponding Bowtie 2 SAM output.</p> </td> </tr> </table> @@ -877,7 +1858,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-t/--time</code></pre> </td> <td> -<p>Print the wall-clock time required to load the index files and align the reads. This is printed to the "standard error" ("stderr") filehandle. Default: off.</p> +<p>Print the wall-clock time required to load the index files and align +the reads. This is printed to the "standard error" ("stderr") +filehandle. Default: off.</p> </td> </tr> <tr> @@ -888,7 +1871,17 @@ Seed 4 rc: TTATGCATGA</code></pre> --un-lz4 <path></code></pre> </td> <td> -<p>Write unpaired reads that fail to align to file at <code><path></code>. These reads correspond to the SAM records with the FLAGS <code>0x4</code> bit set and neither the <code>0x40</code> nor <code>0x80</code> bits set. If <code>--un-gz</code> is specified, output will be gzip compressed. If <code>--un-bz2</code> or <code>--un-lz4</code> is specified, output will be bzip2 or lz4 compressed. Reads written in this way will appear exactly as they did in the input file, without any modification (same sequence, same name, same quality string, same quality encoding). Reads will not necessarily appear in the same order as they did in the input.</p> +<p>Write unpaired reads that fail to align to file at +<code><path></code>. These reads correspond to the SAM records +with the FLAGS <code>0x4</code> bit set and neither the +<code>0x40</code> nor <code>0x80</code> bits set. If +<code>--un-gz</code> is specified, output will be gzip compressed. If +<code>--un-bz2</code> or <code>--un-lz4</code> is specified, output will +be bzip2 or lz4 compressed. Reads written in this way will appear +exactly as they did in the input file, without any modification (same +sequence, same name, same quality string, same quality encoding). Reads +will not necessarily appear in the same order as they did in the +input.</p> </td> </tr> <tr> @@ -899,7 +1892,17 @@ Seed 4 rc: TTATGCATGA</code></pre> --al-lz4 <path></code></pre> </td> <td> -<p>Write unpaired reads that align at least once to file at <code><path></code>. These reads correspond to the SAM records with the FLAGS <code>0x4</code>, <code>0x40</code>, and <code>0x80</code> bits unset. If <code>--al-gz</code> is specified, output will be gzip compressed. If <code>--al-bz2</code> is specified, output will be bzip2 compressed. Similarly if <code>--al-lz4</code> is specified, output will be lz4 compressed. Reads written in this way will appear exactly as they did in the input file, without any modification (same sequence, same name, same quality string, same quality encoding). Reads will not necessarily appear in the same order as they did in the input.</p> +<p>Write unpaired reads that align at least once to file at +<code><path></code>. These reads correspond to the SAM records +with the FLAGS <code>0x4</code>, <code>0x40</code>, and +<code>0x80</code> bits unset. If <code>--al-gz</code> is specified, +output will be gzip compressed. If <code>--al-bz2</code> is specified, +output will be bzip2 compressed. Similarly if <code>--al-lz4</code> is +specified, output will be lz4 compressed. Reads written in this way will +appear exactly as they did in the input file, without any modification +(same sequence, same name, same quality string, same quality encoding). +Reads will not necessarily appear in the same order as they did in the +input.</p> </td> </tr> <tr> @@ -910,7 +1913,20 @@ Seed 4 rc: TTATGCATGA</code></pre> --un-conc-lz4 <path></code></pre> </td> <td> -<p>Write paired-end reads that fail to align concordantly to file(s) at <code><path></code>. These reads correspond to the SAM records with the FLAGS <code>0x4</code> bit set and either the <code>0x40</code> or <code>0x80</code> bit set (depending on whether it's mate #1 or #2). <code>.1</code> and <code>.2</code> strings are added to the filename to distinguish which file contains mate #1 and mate #2. If a percent symbol, <code>%</code>, is used in <code><path></code>, the percent symbol is replaced with <code>1</code> or <code>2</code> to make the per-mate filenames. Otherwise, <code>.1</code> or <code>.2</code> are added before the final dot in <code><path></code> to make the per-mate filenames. Reads written in this way will appear exactly as they did in the input files, without any modification (same sequence, same name, same quality string, same quality encoding). Reads will not necessarily appear in the same order as they did in the inputs.</p> +<p>Write paired-end reads that fail to align concordantly to file(s) at +<code><path></code>. These reads correspond to the SAM records +with the FLAGS <code>0x4</code> bit set and either the <code>0x40</code> +or <code>0x80</code> bit set (depending on whether it's mate #1 or #2). +<code>.1</code> and <code>.2</code> strings are added to the filename to +distinguish which file contains mate #1 and mate #2. If a percent +symbol, <code>%</code>, is used in <code><path></code>, the +percent symbol is replaced with <code>1</code> or <code>2</code> to make +the per-mate filenames. Otherwise, <code>.1</code> or <code>.2</code> +are added before the final dot in <code><path></code> to make the +per-mate filenames. Reads written in this way will appear exactly as +they did in the input files, without any modification (same sequence, +same name, same quality string, same quality encoding). Reads will not +necessarily appear in the same order as they did in the inputs.</p> </td> </tr> <tr> @@ -921,7 +1937,21 @@ Seed 4 rc: TTATGCATGA</code></pre> --al-conc-lz4 <path></code></pre> </td> <td> -<p>Write paired-end reads that align concordantly at least once to file(s) at <code><path></code>. These reads correspond to the SAM records with the FLAGS <code>0x4</code> bit unset and either the <code>0x40</code> or <code>0x80</code> bit set (depending on whether it's mate #1 or #2). <code>.1</code> and <code>.2</code> strings are added to the filename to distinguish which file contains mate #1 and mate #2. If a percent symbol, <code>%</code>, is used in <code><path></code>, the percent symbol is replaced with <code>1</code> or <code>2</code> to make the per-mate filenames. Otherwise, <code>.1</code> or <code>.2</code> are added before the final dot in <code><path></code> to make the per-mate filenames. Reads written in this way will appear exactly as they did in the input files, without any modification (same sequence, same name, same quality string, same quality encoding). Reads will not necessarily appear in the same order as they did in the inputs.</p> +<p>Write paired-end reads that align concordantly at least once to +file(s) at <code><path></code>. These reads correspond to the SAM +records with the FLAGS <code>0x4</code> bit unset and either the +<code>0x40</code> or <code>0x80</code> bit set (depending on whether +it's mate #1 or #2). <code>.1</code> and <code>.2</code> strings are +added to the filename to distinguish which file contains mate #1 and +mate #2. If a percent symbol, <code>%</code>, is used in +<code><path></code>, the percent symbol is replaced with +<code>1</code> or <code>2</code> to make the per-mate filenames. +Otherwise, <code>.1</code> or <code>.2</code> are added before the final +dot in <code><path></code> to make the per-mate filenames. Reads +written in this way will appear exactly as they did in the input files, +without any modification (same sequence, same name, same quality string, +same quality encoding). Reads will not necessarily appear in the same +order as they did in the inputs.</p> </td> </tr> <tr> @@ -937,7 +1967,11 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--met-file <path></code></pre> </td> <td> -<p>Write <code>bowtie2</code> metrics to file <code><path></code>. Having alignment metric can be useful for debugging certain problems, especially performance issues. See also: <a href="#bowtie2-options-met"><code>--met</code></a>. Default: metrics disabled.</p> +<p>Write <code>bowtie2</code> metrics to file <code><path></code>. +Having alignment metric can be useful for debugging certain problems, +especially performance issues. See also: <a +href="#bowtie2-options-met"><code>--met</code></a>. Default: metrics +disabled.</p> </td> </tr> <tr> @@ -945,7 +1979,13 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--met-stderr <path></code></pre> </td> <td> -<p>Write <code>bowtie2</code> metrics to the "standard error" ("stderr") filehandle. This is not mutually exclusive with <a href="#bowtie2-options-met-file"><code>--met-file</code></a>. Having alignment metric can be useful for debugging certain problems, especially performance issues. See also: <a href="#bowtie2-options-met"><code>--met</code></a>. Default: metrics disabled.</p> +<p>Write <code>bowtie2</code> metrics to the "standard error" ("stderr") +filehandle. This is not mutually exclusive with <a +href="#bowtie2-options-met-file"><code>--met-file</code></a>. Having +alignment metric can be useful for debugging certain problems, +especially performance issues. See also: <a +href="#bowtie2-options-met"><code>--met</code></a>. Default: metrics +disabled.</p> </td> </tr> <tr> @@ -953,7 +1993,11 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--met <int></code></pre> </td> <td> -<p>Write a new <code>bowtie2</code> metrics record every <code><int></code> seconds. Only matters if either <a href="#bowtie2-options-met-stderr"><code>--met-stderr</code></a> or <a href="#bowtie2-options-met-file"><code>--met-file</code></a> are specified. Default: 1.</p> +<p>Write a new <code>bowtie2</code> metrics record every +<code><int></code> seconds. Only matters if either <a +href="#bowtie2-options-met-stderr"><code>--met-stderr</code></a> or <a +href="#bowtie2-options-met-file"><code>--met-file</code></a> are +specified. Default: 1.</p> </td> </tr> </table> @@ -988,7 +2032,12 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--rg-id <text></code></pre> </td> <td> -<p>Set the read group ID to <code><text></code>. This causes the SAM <code>@RG</code> header line to be printed, with <code><text></code> as the value associated with the <code>ID:</code> tag. It also causes the <code>RG:Z:</code> extra field to be attached to each SAM output record, with value set to <code><text></code>.</p> +<p>Set the read group ID to <code><text></code>. This causes the +SAM <code>@RG</code> header line to be printed, with +<code><text></code> as the value associated with the +<code>ID:</code> tag. It also causes the <code>RG:Z:</code> extra field +to be attached to each SAM output record, with value set to +<code><text></code>.</p> </td> </tr> <tr> @@ -996,7 +2045,16 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--rg <text></code></pre> </td> <td> -<p>Add <code><text></code> (usually of the form <code>TAG:VAL</code>, e.g. <code>SM:Pool1</code>) as a field on the <code>@RG</code> header line. Note: in order for the <code>@RG</code> line to appear, <a href="#bowtie2-options-rg-id"><code>--rg-id</code></a> must also be specified. This is because the <code>ID</code> tag is required by the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM Spec</a>. Specify <code>--rg</code> multiple times to set multiple fields. See the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM Spec</a> for details about what fields are legal.</p> +<p>Add <code><text></code> (usually of the form +<code>TAG:VAL</code>, e.g. <code>SM:Pool1</code>) as a field on the +<code>@RG</code> header line. Note: in order for the <code>@RG</code> +line to appear, <a +href="#bowtie2-options-rg-id"><code>--rg-id</code></a> must also be +specified. This is because the <code>ID</code> tag is required by the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM Spec</a>. Specify +<code>--rg</code> multiple times to set multiple fields. See the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM Spec</a> for details +about what fields are legal.</p> </td> </tr> <tr> @@ -1004,7 +2062,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--omit-sec-seq</code></pre> </td> <td> -<p>When printing secondary alignments, Bowtie 2 by default will write out the <code>SEQ</code> and <code>QUAL</code> strings. Specifying this option causes Bowtie 2 to print an asterisk in those fields instead.</p> +<p>When printing secondary alignments, Bowtie 2 by default will write +out the <code>SEQ</code> and <code>QUAL</code> strings. Specifying this +option causes Bowtie 2 to print an asterisk in those fields instead.</p> </td> </tr> <tr> @@ -1012,7 +2072,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--soft-clipped-unmapped-tlen</code></pre> </td> <td> -<p>Consider soft-clipped bases unmapped when calculating <code>TLEN</code>. Only available in <a href="#bowtie2-options-local"><code>--local</code></a> mode.</p> +<p>Consider soft-clipped bases unmapped when calculating +<code>TLEN</code>. Only available in <a +href="#bowtie2-options-local"><code>--local</code></a> mode.</p> </td> </tr> <tr> @@ -1020,7 +2082,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--sam-no-qname-trunc</code></pre> </td> <td> -<p>Suppress standard behavior of truncating readname at first whitespace at the expense of generating non-standard SAM</p> +<p>Suppress standard behavior of truncating readname at first whitespace +at the expense of generating non-standard SAM</p> </td> </tr> <tr> @@ -1028,7 +2091,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--xeq</code></pre> </td> <td> -<p>Use <code>'='/'X'</code>, instead of <code>'M'</code>, to specify matches/mismatches in SAM record</p> +<p>Use <code>'='/'X'</code>, instead of <code>'M'</code>, to specify +matches/mismatches in SAM record</p> </td> </tr> <tr> @@ -1036,7 +2100,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--sam-append-comment</code></pre> </td> <td> -<p>Append FASTA/FASTQ comment to SAM record, where a comment is everything after the first space in the read name.</p> +<p>Append FASTA/FASTQ comment to SAM record, where a comment is +everything after the first space in the read name.</p> </td> </tr> </table> @@ -1047,7 +2112,12 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-o/--offrate <int></code></pre> </td> <td> -<p>Override the offrate of the index with <code><int></code>. If <code><int></code> is greater than the offrate used to build the index, then some row markings are discarded when the index is read into memory. This reduces the memory footprint of the aligner but requires more time to calculate text offsets. <code><int></code> must be greater than the value used to build the index.</p> +<p>Override the offrate of the index with <code><int></code>. If +<code><int></code> is greater than the offrate used to build the +index, then some row markings are discarded when the index is read into +memory. This reduces the memory footprint of the aligner but requires +more time to calculate text offsets. <code><int></code> must be +greater than the value used to build the index.</p> </td> </tr> <tr> @@ -1055,7 +2125,16 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-p/--threads NTHREADS</code></pre> </td> <td> -<p>Launch <code>NTHREADS</code> parallel search threads (default: 1). Threads will run on separate processors/cores and synchronize when parsing reads and outputting alignments. Searching for alignments is highly parallel, and speedup is close to linear. Increasing <code>-p</code> increases Bowtie 2's memory footprint. E.g. when aligning to a human genome index, increasing <code>-p</code> from 1 to 8 increases the memory footprint by a few hundred megabytes. This option is only available if <code>bowtie</code> is linked with the <code>pthreads</code> library (i.e. if <code>BOWTIE_PTHREADS=0</code> is not specified at build time).</p> +<p>Launch <code>NTHREADS</code> parallel search threads (default: 1). +Threads will run on separate processors/cores and synchronize when +parsing reads and outputting alignments. Searching for alignments is +highly parallel, and speedup is close to linear. Increasing +<code>-p</code> increases Bowtie 2's memory footprint. E.g. when +aligning to a human genome index, increasing <code>-p</code> from 1 to 8 +increases the memory footprint by a few hundred megabytes. This option +is only available if <code>bowtie</code> is linked with the +<code>pthreads</code> library (i.e. if <code>BOWTIE_PTHREADS=0</code> is +not specified at build time).</p> </td> </tr> <tr> @@ -1063,7 +2142,15 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--reorder</code></pre> </td> <td> -<p>Guarantees that output SAM records are printed in an order corresponding to the order of the reads in the original input file, even when <a href="#bowtie2-options-p"><code>-p</code></a> is set greater than 1. Specifying <code>--reorder</code> and setting <a href="#bowtie2-options-p"><code>-p</code></a> greater than 1 causes Bowtie 2 to run somewhat slower and use somewhat more memory than if <code>--reorder</code> were not specified. Has no effect if <a href="#bowtie2-options-p"><code>-p</code></a> is set to 1, since output order will naturally correspond to input order in that case.</p> +<p>Guarantees that output SAM records are printed in an order +corresponding to the order of the reads in the original input file, even +when <a href="#bowtie2-options-p"><code>-p</code></a> is set greater +than 1. Specifying <code>--reorder</code> and setting <a +href="#bowtie2-options-p"><code>-p</code></a> greater than 1 causes +Bowtie 2 to run somewhat slower and use somewhat more memory than if +<code>--reorder</code> were not specified. Has no effect if <a +href="#bowtie2-options-p"><code>-p</code></a> is set to 1, since output +order will naturally correspond to input order in that case.</p> </td> </tr> <tr> @@ -1071,7 +2158,13 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--mm</code></pre> </td> <td> -<p>Use memory-mapped I/O to load the index, rather than typical file I/O. Memory-mapping allows many concurrent <code>bowtie</code> processes on the same computer to share the same memory image of the index (i.e. you pay the memory overhead just once). This facilitates memory-efficient parallelization of <code>bowtie</code> in situations where using <a href="#bowtie2-options-p"><code>-p</code></a> is not possible or not preferable.</p> +<p>Use memory-mapped I/O to load the index, rather than typical file +I/O. Memory-mapping allows many concurrent <code>bowtie</code> processes +on the same computer to share the same memory image of the index (i.e. +you pay the memory overhead just once). This facilitates +memory-efficient parallelization of <code>bowtie</code> in situations +where using <a href="#bowtie2-options-p"><code>-p</code></a> is not +possible or not preferable.</p> </td> </tr> </table> @@ -1082,7 +2175,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--qc-filter</code></pre> </td> <td> -<p>Filter out reads for which the QSEQ filter field is non-zero. Only has an effect when read format is <a href="#bowtie2-options-qseq"><code>--qseq</code></a>. Default: off.</p> +<p>Filter out reads for which the QSEQ filter field is non-zero. Only +has an effect when read format is <a +href="#bowtie2-options-qseq"><code>--qseq</code></a>. Default: off.</p> </td> </tr> <tr> @@ -1090,7 +2185,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--seed <int></code></pre> </td> <td> -<p>Use <code><int></code> as the seed for pseudo-random number generator. Default: 0.</p> +<p>Use <code><int></code> as the seed for pseudo-random number +generator. Default: 0.</p> </td> </tr> <tr> @@ -1098,7 +2194,19 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--non-deterministic</code></pre> </td> <td> -<p>Normally, Bowtie 2 re-initializes its pseudo-random generator for each read. It seeds the generator with a number derived from (a) the read name, (b) the nucleotide sequence, (c) the quality sequence, (d) the value of the <a href="#bowtie2-options-seed"><code>--seed</code></a> option. This means that if two reads are identical (same name, same nucleotides, same qualities) Bowtie 2 will find and report the same alignment(s) for both, even if there was ambiguity. When <code>--non-deterministic</code> is specified, Bowtie 2 re-initializes its pseudo-random generator for each read using the current time. This means that Bowtie 2 will not necessarily report the same alignment for two identical reads. This is counter-intuitive for some users, but might be more appropriate in situations where the input consists of many identical reads.</p> +<p>Normally, Bowtie 2 re-initializes its pseudo-random generator for +each read. It seeds the generator with a number derived from (a) the +read name, (b) the nucleotide sequence, (c) the quality sequence, (d) +the value of the <a href="#bowtie2-options-seed"><code>--seed</code></a> +option. This means that if two reads are identical (same name, same +nucleotides, same qualities) Bowtie 2 will find and report the same +alignment(s) for both, even if there was ambiguity. When +<code>--non-deterministic</code> is specified, Bowtie 2 re-initializes +its pseudo-random generator for each read using the current time. This +means that Bowtie 2 will not necessarily report the same alignment for +two identical reads. This is counter-intuitive for some users, but might +be more appropriate in situations where the input consists of many +identical reads.</p> </td> </tr> <tr> @@ -1119,12 +2227,30 @@ Seed 4 rc: TTATGCATGA</code></pre> </tr> </table> <h2 id="sam-output">SAM output</h2> -<p>Following is a brief description of the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> format as output by <code>bowtie2</code>. For more details, see the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM format specification</a>.</p> -<p>By default, <code>bowtie2</code> prints a SAM header with <code>@HD</code>, <code>@SQ</code> and <code>@PG</code> lines. When one or more <a href="#bowtie2-options-rg"><code>--rg</code></a> arguments are specified, <code>bowtie2</code> will also print an <code>@RG</code> line that includes all user-specified <a href="#bowtie2-options-rg"><code>--rg</code></a> tokens separated by tabs.</p> -<p>Each subsequent line describes an alignment or, if the read failed to align, a read. Each line is a collection of at least 12 fields separated by tabs; from left to right, the fields are:</p> +<p>Following is a brief description of the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> format as output +by <code>bowtie2</code>. For more details, see the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM format +specification</a>.</p> +<p>By default, <code>bowtie2</code> prints a SAM header with +<code>@HD</code>, <code>@SQ</code> and <code>@PG</code> lines. When one +or more <a href="#bowtie2-options-rg"><code>--rg</code></a> arguments +are specified, <code>bowtie2</code> will also print an <code>@RG</code> +line that includes all user-specified <a +href="#bowtie2-options-rg"><code>--rg</code></a> tokens separated by +tabs.</p> +<p>Each subsequent line describes an alignment or, if the read failed to +align, a read. Each line is a collection of at least 12 fields separated +by tabs; from left to right, the fields are:</p> <ol type="1"> <li><p>Name of read that aligned.</p> -<p>Note that the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> disallows whitespace in the read name. If the read name contains any whitespace characters, Bowtie 2 will truncate the name at the first whitespace character. This is similar to the behavior of other tools. The standard behavior of truncating at the first whitespace can be suppressed with <code>--sam-no-qname-trunc</code> at the expense of generating non-standard SAM.</p></li> +<p>Note that the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM +specification</a> disallows whitespace in the read name. If the read +name contains any whitespace characters, Bowtie 2 will truncate the name +at the first whitespace character. This is similar to the behavior of +other tools. The standard behavior of truncating at the first whitespace +can be suppressed with <code>--sam-no-qname-trunc</code> at the expense +of generating non-standard SAM.</p></li> <li><p>Sum of all applicable flags. Flags relevant to Bowtie are:</p> <table> <tr> @@ -1172,7 +2298,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>32</code></pre> </td> <td> -<p>The other mate in the paired-end alignment is aligned to the reverse reference strand</p> +<p>The other mate in the paired-end alignment is aligned to the reverse +reference strand</p> </td> </tr> <tr> @@ -1192,17 +2319,36 @@ Seed 4 rc: TTATGCATGA</code></pre> </td> </tr> </table> -<p>Thus, an unpaired read that aligns to the reverse reference strand will have flag 16. A paired-end read that aligns and is the first mate in the pair will have flag 83 (= 64 + 16 + 2 + 1).</p></li> +<p>Thus, an unpaired read that aligns to the reverse reference strand +will have flag 16. A paired-end read that aligns and is the first mate +in the pair will have flag 83 (= 64 + 16 + 2 + 1).</p></li> <li><p>Name of reference sequence where alignment occurs</p></li> -<li><p>1-based offset into the forward reference strand where leftmost character of the alignment occurs</p></li> +<li><p>1-based offset into the forward reference strand where leftmost +character of the alignment occurs</p></li> <li><p>Mapping quality</p></li> <li><p>CIGAR string representation of alignment</p></li> -<li><p>Name of reference sequence where mate's alignment occurs. Set to <code>=</code> if the mate's reference sequence is the same as this alignment's, or <code>*</code> if there is no mate.</p></li> -<li><p>1-based offset into the forward reference strand where leftmost character of the mate's alignment occurs. Offset is 0 if there is no mate.</p></li> -<li><p>Inferred fragment length. Size is negative if the mate's alignment occurs upstream of this alignment. Size is 0 if the mates did not align concordantly. However, size is non-0 if the mates aligned discordantly to the same chromosome.</p></li> -<li><p>Read sequence (reverse-complemented if aligned to the reverse strand)</p></li> -<li><p>ASCII-encoded read qualities (reverse-complemented if the read aligned to the reverse strand). The encoded quality values are on the <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred quality</a> scale and the encoding is ASCII-offset by 33 (ASCII char <code>!</code>), similarly to a <a href="http://en.wikipedia.org/wiki/FASTQ_format">FASTQ</a> file.</p></li> -<li><p>Optional fields. Fields are tab-separated. <code>bowtie2</code> outputs zero or more of these optional fields for each alignment, depending on the type of the alignment:</p></li> +<li><p>Name of reference sequence where mate's alignment occurs. Set to +<code>=</code> if the mate's reference sequence is the same as this +alignment's, or <code>*</code> if there is no mate.</p></li> +<li><p>1-based offset into the forward reference strand where leftmost +character of the mate's alignment occurs. Offset is 0 if there is no +mate.</p></li> +<li><p>Inferred fragment length. Size is negative if the mate's +alignment occurs upstream of this alignment. Size is 0 if the mates did +not align concordantly. However, size is non-0 if the mates aligned +discordantly to the same chromosome.</p></li> +<li><p>Read sequence (reverse-complemented if aligned to the reverse +strand)</p></li> +<li><p>ASCII-encoded read qualities (reverse-complemented if the read +aligned to the reverse strand). The encoded quality values are on the <a +href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred +quality</a> scale and the encoding is ASCII-offset by 33 (ASCII char +<code>!</code>), similarly to a <a +href="http://en.wikipedia.org/wiki/FASTQ_format">FASTQ</a> +file.</p></li> +<li><p>Optional fields. Fields are tab-separated. <code>bowtie2</code> +outputs zero or more of these optional fields for each alignment, +depending on the type of the alignment:</p></li> </ol> <table> <tr> @@ -1210,7 +2356,10 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>AS:i:<N></code></pre> </td> <td> -<p>Alignment score. Can be negative. Can be greater than 0 in <a href="#bowtie2-options-local"><code>--local</code></a> mode (but not in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode). Only present if SAM record is for an aligned read.</p> +<p>Alignment score. Can be negative. Can be greater than 0 in <a +href="#bowtie2-options-local"><code>--local</code></a> mode (but not in +<a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> +mode). Only present if SAM record is for an aligned read.</p> </td> </tr> <tr> @@ -1218,7 +2367,14 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>XS:i:<N></code></pre> </td> <td> -<p>Alignment score for the best-scoring alignment found other than the alignment reported. Can be negative. Can be greater than 0 in <a href="#bowtie2-options-local"><code>--local</code></a> mode (but not in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode). Only present if the SAM record is for an aligned read and more than one alignment was found for the read. Note that, when the read is part of a concordantly-aligned pair, this score could be greater than <a href="#bowtie2-build-opt-fields-as"><code>AS:i</code></a>.</p> +<p>Alignment score for the best-scoring alignment found other than the +alignment reported. Can be negative. Can be greater than 0 in <a +href="#bowtie2-options-local"><code>--local</code></a> mode (but not in +<a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> +mode). Only present if the SAM record is for an aligned read and more +than one alignment was found for the read. Note that, when the read is +part of a concordantly-aligned pair, this score could be greater than <a +href="#bowtie2-build-opt-fields-as"><code>AS:i</code></a>.</p> </td> </tr> <tr> @@ -1226,7 +2382,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>YS:i:<N></code></pre> </td> <td> -<p>Alignment score for opposite mate in the paired-end alignment. Only present if the SAM record is for a read that aligned as part of a paired-end alignment.</p> +<p>Alignment score for opposite mate in the paired-end alignment. Only +present if the SAM record is for a read that aligned as part of a +paired-end alignment.</p> </td> </tr> <tr> @@ -1234,7 +2392,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>XN:i:<N></code></pre> </td> <td> -<p>The number of ambiguous bases in the reference covering this alignment. Only present if SAM record is for an aligned read.</p> +<p>The number of ambiguous bases in the reference covering this +alignment. Only present if SAM record is for an aligned read.</p> </td> </tr> <tr> @@ -1242,7 +2401,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>XM:i:<N></code></pre> </td> <td> -<p>The number of mismatches in the alignment. Only present if SAM record is for an aligned read.</p> +<p>The number of mismatches in the alignment. Only present if SAM record +is for an aligned read.</p> </td> </tr> <tr> @@ -1250,7 +2410,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>XO:i:<N></code></pre> </td> <td> -<p>The number of gap opens, for both read and reference gaps, in the alignment. Only present if SAM record is for an aligned read.</p> +<p>The number of gap opens, for both read and reference gaps, in the +alignment. Only present if SAM record is for an aligned read.</p> </td> </tr> <tr> @@ -1258,7 +2419,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>XG:i:<N></code></pre> </td> <td> -<p>The number of gap extensions, for both read and reference gaps, in the alignment. Only present if SAM record is for an aligned read.</p> +<p>The number of gap extensions, for both read and reference gaps, in +the alignment. Only present if SAM record is for an aligned read.</p> </td> </tr> <tr> @@ -1266,7 +2428,10 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>NM:i:<N></code></pre> </td> <td> -<p>The edit distance; that is, the minimal number of one-nucleotide edits (substitutions, insertions and deletions) needed to transform the read string into the reference string. Only present if SAM record is for an aligned read.</p> +<p>The edit distance; that is, the minimal number of one-nucleotide +edits (substitutions, insertions and deletions) needed to transform the +read string into the reference string. Only present if SAM record is for +an aligned read.</p> </td> </tr> <tr> @@ -1274,7 +2439,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>YF:Z:<S></code></pre> </td> <td> -<p>String indicating reason why the read was filtered out. See also: <a href="#filtering">Filtering</a>. Only appears for reads that were filtered out.</p> +<p>String indicating reason why the read was filtered out. See also: <a +href="#filtering">Filtering</a>. Only appears for reads that were +filtered out.</p> </td> </tr> <tr> @@ -1282,7 +2449,12 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>YT:Z:<S></code></pre> </td> <td> -<p>Value of <code>UU</code> indicates the read was not part of a pair. Value of <code>CP</code> indicates the read was part of a pair and the pair aligned concordantly. Value of <code>DP</code> indicates the read was part of a pair and the pair aligned discordantly. Value of <code>UP</code> indicates the read was part of a pair but the pair failed to aligned either concordantly or discordantly.</p> +<p>Value of <code>UU</code> indicates the read was not part of a pair. +Value of <code>CP</code> indicates the read was part of a pair and the +pair aligned concordantly. Value of <code>DP</code> indicates the read +was part of a pair and the pair aligned discordantly. Value of +<code>UP</code> indicates the read was part of a pair but the pair +failed to aligned either concordantly or discordantly.</p> </td> </tr> <tr> @@ -1290,17 +2462,68 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>MD:Z:<S></code></pre> </td> <td> -<p>A string representation of the mismatched reference bases in the alignment. See <a href="https://samtools.github.io/hts-specs/SAMtags.pdf">SAM Tags format specification</a> for details. Only present if SAM record is for an aligned read.</p> +<p>A string representation of the mismatched reference bases in the +alignment. See <a +href="https://samtools.github.io/hts-specs/SAMtags.pdf">SAM Tags format +specification</a> for details. Only present if SAM record is for an +aligned read.</p> </td> </tr> </table> -<h1 id="the-bowtie2-build-indexer">The <code>bowtie2-build</code> indexer</h1> -<p><code>bowtie2-build</code> builds a Bowtie index from a set of DNA sequences. <code>bowtie2-build</code> outputs a set of 6 files with suffixes <code>.1.bt2</code>, <code>.2.bt2</code>, <code>.3.bt2</code>, <code>.4.bt2</code>, <code>.rev.1.bt2</code>, and <code>.rev.2.bt2</code>. In the case of a large index these suffixes will have a <code>bt2l</code> termination. These files together constitute the index: they are all that is needed to align reads to that reference. The original sequence <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files are no longer used by Bowtie 2 once the index is built.</p> -<p>Bowtie 2's <code>.bt2</code> index format is different from Bowtie 1's <code>.ebwt</code> format, and they are not compatible with each other.</p> -<p>Use of Karkkainen's <a href="http://portal.acm.org/citation.cfm?id=1314852">blockwise algorithm</a> allows <code>bowtie2-build</code> to trade off between running time and memory usage. <code>bowtie2-build</code> has three options governing how it makes this trade: <a href="#bowtie2-build-options-p"><code>-p</code>/<code>--packed</code></a>, <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>/<a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>, and <a href="#bowtie2-build-options-dcv"><code>--dcv</code></a>. By default, <code>bowtie2-build</code> will automatically search for the settings that yield the best running time without exhausting memory. This behavior can be disabled using the <a href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> option.</p> -<p>The indexer provides options pertaining to the "shape" of the index, e.g. <a href="#bowtie2-build-options-o"><code>--offrate</code></a> governs the fraction of <a href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> rows that are "marked" (i.e., the density of the suffix-array sample; see the original <a href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> paper for details). All of these options are potentially profitable trade-offs depending on the application. They have been set to defaults that are reasonable for most cases according to our experiments. See <a href="#performance-tuning">Performance tuning</a> for details.</p> -<p><code>bowtie2-build</code> can generate either <a href="#small-and-large-indexes">small or large indexes</a>. The wrapper will decide which based on the length of the input genome. If the reference does not exceed 4 billion characters but a large index is preferred, the user can specify <a href="#bowtie2-build-options-large-index"><code>--large-index</code></a> to force <code>bowtie2-build</code> to build a large index instead.</p> -<p>The Bowtie 2 index is based on the <a href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> of Ferragina and Manzini, which in turn is based on the <a href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> transform. The algorithm used to build the index is based on the <a href="http://portal.acm.org/citation.cfm?id=1314852">blockwise algorithm</a> of Karkkainen.</p> +<h1 id="the-bowtie2-build-indexer">The <code>bowtie2-build</code> +indexer</h1> +<p><code>bowtie2-build</code> builds a Bowtie index from a set of DNA +sequences. <code>bowtie2-build</code> outputs a set of 6 files with +suffixes <code>.1.bt2</code>, <code>.2.bt2</code>, <code>.3.bt2</code>, +<code>.4.bt2</code>, <code>.rev.1.bt2</code>, and +<code>.rev.2.bt2</code>. In the case of a large index these suffixes +will have a <code>bt2l</code> termination. These files together +constitute the index: they are all that is needed to align reads to that +reference. The original sequence <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files +are no longer used by Bowtie 2 once the index is built.</p> +<p>Bowtie 2's <code>.bt2</code> index format is different from Bowtie +1's <code>.ebwt</code> format, and they are not compatible with each +other.</p> +<p>Use of Karkkainen's <a +href="http://portal.acm.org/citation.cfm?id=1314852">blockwise +algorithm</a> allows <code>bowtie2-build</code> to trade off between +running time and memory usage. <code>bowtie2-build</code> has three +options governing how it makes this trade: <a +href="#bowtie2-build-options-p"><code>-p</code>/<code>--packed</code></a>, +<a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>/<a +href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>, and +<a href="#bowtie2-build-options-dcv"><code>--dcv</code></a>. By default, +<code>bowtie2-build</code> will automatically search for the settings +that yield the best running time without exhausting memory. This +behavior can be disabled using the <a +href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> +option.</p> +<p>The indexer provides options pertaining to the "shape" of the index, +e.g. <a href="#bowtie2-build-options-o"><code>--offrate</code></a> +governs the fraction of <a +href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> +rows that are "marked" (i.e., the density of the suffix-array sample; +see the original <a +href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> paper +for details). All of these options are potentially profitable trade-offs +depending on the application. They have been set to defaults that are +reasonable for most cases according to our experiments. See <a +href="#performance-tuning">Performance tuning</a> for details.</p> +<p><code>bowtie2-build</code> can generate either <a +href="#small-and-large-indexes">small or large indexes</a>. The wrapper +will decide which based on the length of the input genome. If the +reference does not exceed 4 billion characters but a large index is +preferred, the user can specify <a +href="#bowtie2-build-options-large-index"><code>--large-index</code></a> +to force <code>bowtie2-build</code> to build a large index instead.</p> +<p>The Bowtie 2 index is based on the <a +href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> of +Ferragina and Manzini, which in turn is based on the <a +href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> +transform. The algorithm used to build the index is based on the <a +href="http://portal.acm.org/citation.cfm?id=1314852">blockwise +algorithm</a> of Karkkainen.</p> <h2 id="command-line-1">Command Line</h2> <p>Usage:</p> <pre><code>bowtie2-build [options]* <reference_in> <bt2_base></code></pre> @@ -1311,7 +2534,14 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code><reference_in></code></pre> </td> <td> -<p>A comma-separated list of <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files containing the reference sequences to be aligned to, or, if <a href="#bowtie2-build-options-c"><code>-c</code></a> is specified, the sequences themselves. E.g., <code><reference_in></code> might be <code>chr1.fa,chr2.fa,chrX.fa,chrY.fa</code>, or, if <a href="#bowtie2-build-options-c"><code>-c</code></a> is specified, this might be <code>GGTCATCCT,ACGGGTCGT,CCGTTCTATGCGGCTTA</code>.</p> +<p>A comma-separated list of <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files +containing the reference sequences to be aligned to, or, if <a +href="#bowtie2-build-options-c"><code>-c</code></a> is specified, the +sequences themselves. E.g., <code><reference_in></code> might be +<code>chr1.fa,chr2.fa,chrX.fa,chrY.fa</code>, or, if <a +href="#bowtie2-build-options-c"><code>-c</code></a> is specified, this +might be <code>GGTCATCCT,ACGGGTCGT,CCGTTCTATGCGGCTTA</code>.</p> </td> </tr> <tr> @@ -1319,7 +2549,12 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code><bt2_base></code></pre> </td> <td> -<p>The basename of the index files to write. By default, <code>bowtie2-build</code> writes files named <code>NAME.1.bt2</code>, <code>NAME.2.bt2</code>, <code>NAME.3.bt2</code>, <code>NAME.4.bt2</code>, <code>NAME.rev.1.bt2</code>, and <code>NAME.rev.2.bt2</code>, where <code>NAME</code> is <code><bt2_base></code>.</p> +<p>The basename of the index files to write. By default, +<code>bowtie2-build</code> writes files named <code>NAME.1.bt2</code>, +<code>NAME.2.bt2</code>, <code>NAME.3.bt2</code>, +<code>NAME.4.bt2</code>, <code>NAME.rev.1.bt2</code>, and +<code>NAME.rev.2.bt2</code>, where <code>NAME</code> is +<code><bt2_base></code>.</p> </td> </tr> </table> @@ -1330,7 +2565,11 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-f</code></pre> </td> <td> -<p>The reference input files (specified as <code><reference_in></code>) are <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files (usually having extension <code>.fa</code>, <code>.mfa</code>, <code>.fna</code> or similar).</p> +<p>The reference input files (specified as +<code><reference_in></code>) are <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files +(usually having extension <code>.fa</code>, <code>.mfa</code>, +<code>.fna</code> or similar).</p> </td> </tr> <tr> @@ -1338,7 +2577,11 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-c</code></pre> </td> <td> -<p>The reference sequences are given on the command line. I.e. <code><reference_in></code> is a comma-separated list of sequences rather than a list of <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files.</p> +<p>The reference sequences are given on the command line. I.e. +<code><reference_in></code> is a comma-separated list of sequences +rather than a list of <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> +files.</p> </td> </tr> <tr> @@ -1346,7 +2589,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--large-index</code></pre> </td> <td> -<p>Force <code>bowtie2-build</code> to build a <a href="#small-and-large-indexes">large index</a>, even if the reference is less than ~ 4 billion nucleotides inlong.</p> +<p>Force <code>bowtie2-build</code> to build a <a +href="#small-and-large-indexes">large index</a>, even if the reference +is less than ~ 4 billion nucleotides inlong.</p> </td> </tr> <tr> @@ -1354,7 +2599,14 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-a/--noauto</code></pre> </td> <td> -<p>Disable the default behavior whereby <code>bowtie2-build</code> automatically selects values for the <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, <a href="#bowtie2-build-options-dcv"><code>--dcv</code></a> and <a href="#bowtie2-build-options-p"><code>--packed</code></a> parameters according to available memory. Instead, user may specify values for those parameters. If memory is exhausted during indexing, an error message will be printed; it is up to the user to try new parameters.</p> +<p>Disable the default behavior whereby <code>bowtie2-build</code> +automatically selects values for the <a +href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, <a +href="#bowtie2-build-options-dcv"><code>--dcv</code></a> and <a +href="#bowtie2-build-options-p"><code>--packed</code></a> parameters +according to available memory. Instead, user may specify values for +those parameters. If memory is exhausted during indexing, an error +message will be printed; it is up to the user to try new parameters.</p> </td> </tr> <tr> @@ -1362,7 +2614,11 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-p/--packed</code></pre> </td> <td> -<p>Use a packed (2-bits-per-nucleotide) representation for DNA strings. This saves memory but makes indexing 2-3 times slower. Default: off. This is configured automatically by default; use <a href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> to configure manually.</p> +<p>Use a packed (2-bits-per-nucleotide) representation for DNA strings. +This saves memory but makes indexing 2-3 times slower. Default: off. +This is configured automatically by default; use <a +href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> +to configure manually.</p> </td> </tr> <tr> @@ -1370,7 +2626,18 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--bmax <int></code></pre> </td> <td> -<p>The maximum number of suffixes allowed in a block. Allowing more suffixes per block makes indexing faster, but increases peak memory usage. Setting this option overrides any previous setting for <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, or <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>. Default (in terms of the <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> parameter) is <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> 4 * number of threads. This is configured automatically by default; use <a href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> to configure manually.</p> +<p>The maximum number of suffixes allowed in a block. Allowing more +suffixes per block makes indexing faster, but increases peak memory +usage. Setting this option overrides any previous setting for <a +href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, or <a +href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>. +Default (in terms of the <a +href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> +parameter) is <a +href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> 4 * +number of threads. This is configured automatically by default; use <a +href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> +to configure manually.</p> </td> </tr> <tr> @@ -1378,7 +2645,16 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--bmaxdivn <int></code></pre> </td> <td> -<p>The maximum number of suffixes allowed in a block, expressed as a fraction of the length of the reference. Setting this option overrides any previous setting for <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, or <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>. Default: <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> 4 * number of threads. This is configured automatically by default; use <a href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> to configure manually.</p> +<p>The maximum number of suffixes allowed in a block, expressed as a +fraction of the length of the reference. Setting this option overrides +any previous setting for <a +href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, or <a +href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>. +Default: <a +href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> 4 * +number of threads. This is configured automatically by default; use <a +href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> +to configure manually.</p> </td> </tr> <tr> @@ -1386,7 +2662,13 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--dcv <int></code></pre> </td> <td> -<p>Use <code><int></code> as the period for the difference-cover sample. A larger period yields less memory overhead, but may make suffix sorting slower, especially if repeats are present. Must be a power of 2 no greater than 4096. Default: 1024. This is configured automatically by default; use <a href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> to configure manually.</p> +<p>Use <code><int></code> as the period for the difference-cover +sample. A larger period yields less memory overhead, but may make suffix +sorting slower, especially if repeats are present. Must be a power of 2 +no greater than 4096. Default: 1024. This is configured automatically by +default; use <a +href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> +to configure manually.</p> </td> </tr> <tr> @@ -1394,7 +2676,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--nodc</code></pre> </td> <td> -<p>Disable use of the difference-cover sample. Suffix sorting becomes quadratic-time in the worst case (where the worst case is an extremely repetitive reference). Default: off.</p> +<p>Disable use of the difference-cover sample. Suffix sorting becomes +quadratic-time in the worst case (where the worst case is an extremely +repetitive reference). Default: off.</p> </td> </tr> <tr> @@ -1402,7 +2686,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-r/--noref</code></pre> </td> <td> -<p>Do not build the <code>NAME.3.bt2</code> and <code>NAME.4.bt2</code> portions of the index, which contain a bitpacked version of the reference sequences and are used for paired-end alignment.</p> +<p>Do not build the <code>NAME.3.bt2</code> and <code>NAME.4.bt2</code> +portions of the index, which contain a bitpacked version of the +reference sequences and are used for paired-end alignment.</p> </td> </tr> <tr> @@ -1410,7 +2696,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-3/--justref</code></pre> </td> <td> -<p>Build only the <code>NAME.3.bt2</code> and <code>NAME.4.bt2</code> portions of the index, which contain a bitpacked version of the reference sequences and are used for paired-end alignment.</p> +<p>Build only the <code>NAME.3.bt2</code> and <code>NAME.4.bt2</code> +portions of the index, which contain a bitpacked version of the +reference sequences and are used for paired-end alignment.</p> </td> </tr> <tr> @@ -1418,7 +2706,16 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-o/--offrate <int></code></pre> </td> <td> -<p>To map alignments back to positions on the reference sequences, it's necessary to annotate ("mark") some or all of the <a href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> rows with their corresponding location on the genome. <a href="#bowtie2-build-options-o"><code>-o</code>/<code>--offrate</code></a> governs how many rows get marked: the indexer will mark every 2^<code><int></code> rows. Marking more rows makes reference-position lookups faster, but requires more memory to hold the annotations at runtime. The default is 5 (every 32nd row is marked; for human genome, annotations occupy about 340 megabytes).</p> +<p>To map alignments back to positions on the reference sequences, it's +necessary to annotate ("mark") some or all of the <a +href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> +rows with their corresponding location on the genome. <a +href="#bowtie2-build-options-o"><code>-o</code>/<code>--offrate</code></a> +governs how many rows get marked: the indexer will mark every +2^<code><int></code> rows. Marking more rows makes +reference-position lookups faster, but requires more memory to hold the +annotations at runtime. The default is 5 (every 32nd row is marked; for +human genome, annotations occupy about 340 megabytes).</p> </td> </tr> <tr> @@ -1426,7 +2723,13 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-t/--ftabchars <int></code></pre> </td> <td> -<p>The ftab is the lookup table used to calculate an initial <a href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> range with respect to the first <code><int></code> characters of the query. A larger <code><int></code> yields a larger lookup table but faster query times. The ftab has size 4^(<code><int></code>+1) bytes. The default setting is 10 (ftab is 4MB).</p> +<p>The ftab is the lookup table used to calculate an initial <a +href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> +range with respect to the first <code><int></code> characters of +the query. A larger <code><int></code> yields a larger lookup +table but faster query times. The ftab has size +4^(<code><int></code>+1) bytes. The default setting is 10 (ftab is +4MB).</p> </td> </tr> <tr> @@ -1434,7 +2737,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--seed <int></code></pre> </td> <td> -<p>Use <code><int></code> as the seed for pseudo-random number generator.</p> +<p>Use <code><int></code> as the seed for pseudo-random number +generator.</p> </td> </tr> <tr> @@ -1442,7 +2746,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--cutoff <int></code></pre> </td> <td> -<p>Index only the first <code><int></code> bases of the reference sequences (cumulative across sequences) and ignore the rest.</p> +<p>Index only the first <code><int></code> bases of the reference +sequences (cumulative across sequences) and ignore the rest.</p> </td> </tr> <tr> @@ -1450,7 +2755,8 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-q/--quiet</code></pre> </td> <td> -<p><code>bowtie2-build</code> is verbose by default. With this option <code>bowtie2-build</code> will print only error messages.</p> +<p><code>bowtie2-build</code> is verbose by default. With this option +<code>bowtie2-build</code> will print only error messages.</p> </td> </tr> <tr> @@ -1458,7 +2764,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>--threads <int></code></pre> </td> <td> -<p>By default <code>bowtie2-build</code> is using only one thread. Increasing the number of threads will speed up the index building considerably in most cases.</p> +<p>By default <code>bowtie2-build</code> is using only one thread. +Increasing the number of threads will speed up the index building +considerably in most cases.</p> </td> </tr> <tr> @@ -1478,8 +2786,20 @@ Seed 4 rc: TTATGCATGA</code></pre> </td> </tr> </table> -<h1 id="the-bowtie2-inspect-index-inspector">The <code>bowtie2-inspect</code> index inspector</h1> -<p><code>bowtie2-inspect</code> extracts information from a Bowtie index about what kind of index it is and what reference sequences were used to build it. When run without any options, the tool will output a <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> file containing the sequences of the original references (with all non-<code>A</code>/<code>C</code>/<code>G</code>/<code>T</code> characters converted to <code>N</code>s). It can also be used to extract just the reference sequence names using the <a href="#bowtie2-inspect-options-n"><code>-n</code>/<code>--names</code></a> option or a more verbose summary using the <a href="#bowtie2-inspect-options-s"><code>-s</code>/<code>--summary</code></a> option.</p> +<h1 id="the-bowtie2-inspect-index-inspector">The +<code>bowtie2-inspect</code> index inspector</h1> +<p><code>bowtie2-inspect</code> extracts information from a Bowtie index +about what kind of index it is and what reference sequences were used to +build it. When run without any options, the tool will output a <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> file +containing the sequences of the original references (with all +non-<code>A</code>/<code>C</code>/<code>G</code>/<code>T</code> +characters converted to <code>N</code>s). It can also be used to extract +just the reference sequence names using the <a +href="#bowtie2-inspect-options-n"><code>-n</code>/<code>--names</code></a> +option or a more verbose summary using the <a +href="#bowtie2-inspect-options-s"><code>-s</code>/<code>--summary</code></a> +option.</p> <h2 id="command-line-2">Command Line</h2> <p>Usage:</p> <pre><code>bowtie2-inspect [options]* <bt2_base></code></pre> @@ -1490,7 +2810,12 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code><bt2_base></code></pre> </td> <td> -<p>The basename of the index to be inspected. The basename is name of any of the index files but with the <code>.X.bt2</code> or <code>.rev.X.bt2</code> suffix omitted. <code>bowtie2-inspect</code> first looks in the current directory for the index files, then in the directory specified in the <code>BOWTIE2_INDEXES</code> environment variable.</p> +<p>The basename of the index to be inspected. The basename is name of +any of the index files but with the <code>.X.bt2</code> or +<code>.rev.X.bt2</code> suffix omitted. <code>bowtie2-inspect</code> +first looks in the current directory for the index files, then in the +directory specified in the <code>BOWTIE2_INDEXES</code> environment +variable.</p> </td> </tr> </table> @@ -1501,7 +2826,10 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-a/--across <int></code></pre> </td> <td> -<p>When printing <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> output, output a newline character every <code><int></code> bases (default: 60).</p> +<p>When printing <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> +output, output a newline character every <code><int></code> bases +(default: 60).</p> </td> </tr> <tr> @@ -1517,7 +2845,9 @@ Seed 4 rc: TTATGCATGA</code></pre> <pre><code>-s/--summary</code></pre> </td> <td> -<p>Print a summary that includes information about index settings, as well as the names and lengths of the input sequences. The summary has this format:</p> +<p>Print a summary that includes information about index settings, as +well as the names and lengths of the input sequences. The summary has +this format:</p> <pre><code>Colorspace <0 or 1> SA-Sample 1 in <sample> FTab-Chars <chars> @@ -1525,7 +2855,8 @@ Sequence-1 <name> <len> Sequence-2 <name> <len> ... Sequence-N <name> <len></code></pre> -<p>Fields are separated by tabs. Colorspace is always set to 0 for Bowtie 2.</p> +<p>Fields are separated by tabs. Colorspace is always set to 0 for +Bowtie 2.</p> </td> </tr> <tr> @@ -1561,18 +2892,59 @@ Sequence-N <name> <len></code></pre> </td> </tr> </table> -<h1 id="getting-started-with-bowtie-2-lambda-phage-example">Getting started with Bowtie 2: Lambda phage example</h1> -<p>Bowtie 2 comes with some example files to get you started. The example files are not scientifically significant; we use the <a href="http://en.wikipedia.org/wiki/Lambda_phage">Lambda phage</a> reference genome simply because it's short, and the reads were generated by a computer program, not a sequencer. However, these files will let you start running Bowtie 2 and downstream tools right away.</p> -<p>First follow the manual instructions to <a href="#obtaining-bowtie-2">obtain Bowtie 2</a>. Set the <code>BT2_HOME</code> environment variable to point to the new Bowtie 2 directory containing the <code>bowtie2</code>, <code>bowtie2-build</code> and <code>bowtie2-inspect</code> binaries. This is important, as the <code>BT2_HOME</code> variable is used in the commands below to refer to that directory.</p> +<h1 id="getting-started-with-bowtie-2-lambda-phage-example">Getting +started with Bowtie 2: Lambda phage example</h1> +<p>Bowtie 2 comes with some example files to get you started. The +example files are not scientifically significant; we use the <a +href="http://en.wikipedia.org/wiki/Lambda_phage">Lambda phage</a> +reference genome simply because it's short, and the reads were generated +by a computer program, not a sequencer. However, these files will let +you start running Bowtie 2 and downstream tools right away.</p> +<p>First follow the manual instructions to <a +href="#obtaining-bowtie-2">obtain Bowtie 2</a>. Set the +<code>BT2_HOME</code> environment variable to point to the new Bowtie 2 +directory containing the <code>bowtie2</code>, +<code>bowtie2-build</code> and <code>bowtie2-inspect</code> binaries. +This is important, as the <code>BT2_HOME</code> variable is used in the +commands below to refer to that directory.</p> <h2 id="indexing-a-reference-genome">Indexing a reference genome</h2> -<p>To create an index for the <a href="http://en.wikipedia.org/wiki/Lambda_phage">Lambda phage</a> reference genome included with Bowtie 2, create a new temporary directory (it doesn't matter where), change into that directory, and run:</p> +<p>To create an index for the <a +href="http://en.wikipedia.org/wiki/Lambda_phage">Lambda phage</a> +reference genome included with Bowtie 2, create a new temporary +directory (it doesn't matter where), change into that directory, and +run:</p> <pre><code>$BT2_HOME/bowtie2-build $BT2_HOME/example/reference/lambda_virus.fa lambda_virus</code></pre> -<p>The command should print many lines of output then quit. When the command completes, the current directory will contain four new files that all start with <code>lambda_virus</code> and end with <code>.1.bt2</code>, <code>.2.bt2</code>, <code>.3.bt2</code>, <code>.4.bt2</code>, <code>.rev.1.bt2</code>, and <code>.rev.2.bt2</code>. These files constitute the index - you're done!</p> -<p>You can use <code>bowtie2-build</code> to create an index for a set of <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files obtained from any source, including sites such as <a href="http://genome.ucsc.edu/cgi-bin/hgGateway">UCSC</a>, <a href="http://www.ncbi.nlm.nih.gov/sites/genome">NCBI</a>, and <a href="http://www.ensembl.org/">Ensembl</a>. When indexing multiple <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files, specify all the files using commas to separate file names. For more details on how to create an index with <code>bowtie2-build</code>, see the <a href="#the-bowtie2-build-indexer">manual section on index building</a>. You may also want to bypass this process by obtaining a pre-built index. See <a href="#using-a-pre-built-index">using a pre-built index</a> below for an example.</p> +<p>The command should print many lines of output then quit. When the +command completes, the current directory will contain four new files +that all start with <code>lambda_virus</code> and end with +<code>.1.bt2</code>, <code>.2.bt2</code>, <code>.3.bt2</code>, +<code>.4.bt2</code>, <code>.rev.1.bt2</code>, and +<code>.rev.2.bt2</code>. These files constitute the index - you're +done!</p> +<p>You can use <code>bowtie2-build</code> to create an index for a set +of <a href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> +files obtained from any source, including sites such as <a +href="http://genome.ucsc.edu/cgi-bin/hgGateway">UCSC</a>, <a +href="http://www.ncbi.nlm.nih.gov/sites/genome">NCBI</a>, and <a +href="http://www.ensembl.org/">Ensembl</a>. When indexing multiple <a +href="https://en.wikipedia.org/wiki/FASTA"><code>FASTA</code></a> files, +specify all the files using commas to separate file names. For more +details on how to create an index with <code>bowtie2-build</code>, see +the <a href="#the-bowtie2-build-indexer">manual section on index +building</a>. You may also want to bypass this process by obtaining a +pre-built index. See <a href="#using-a-pre-built-index">using a +pre-built index</a> below for an example.</p> <h2 id="aligning-example-reads">Aligning example reads</h2> -<p>Stay in the directory created in the previous step, which now contains the <code>lambda_virus</code> index files. Next, run:</p> +<p>Stay in the directory created in the previous step, which now +contains the <code>lambda_virus</code> index files. Next, run:</p> <pre><code>$BT2_HOME/bowtie2 -x lambda_virus -U $BT2_HOME/example/reads/reads_1.fq -S eg1.sam</code></pre> -<p>This runs the Bowtie 2 aligner, which aligns a set of unpaired reads to the <a href="http://en.wikipedia.org/wiki/Lambda_phage">Lambda phage</a> reference genome using the index generated in the previous step. The alignment results in SAM format are written to the file <code>eg1.sam</code>, and a short alignment summary is written to the console. (Actually, the summary is written to the "standard error" or "stderr" filehandle, which is typically printed to the console.)</p> +<p>This runs the Bowtie 2 aligner, which aligns a set of unpaired reads +to the <a href="http://en.wikipedia.org/wiki/Lambda_phage">Lambda +phage</a> reference genome using the index generated in the previous +step. The alignment results in SAM format are written to the file +<code>eg1.sam</code>, and a short alignment summary is written to the +console. (Actually, the summary is written to the "standard error" or +"stderr" filehandle, which is typically printed to the console.)</p> <p>To see the first few lines of the SAM output, run:</p> <pre><code>head eg1.sam</code></pre> <p>You will see something like this:</p> @@ -1586,25 +2958,60 @@ r4 0 gi|9626243|ref|NC_001416.1| 40075 42 184M * 0 0 GGGCCAATGCGC r5 0 gi|9626243|ref|NC_001416.1| 48010 42 138M * 0 0 GTCAGGAAAGTGGTAAAACTGCAACTCAATTACTGCAATGCCCTCGTAATTAAGTGAATTTACAATATCGTCCTGTTCGGAGGGAAGAACGCGGGATGTTCATTCTTCATCACTTTTAATTGATGTATATGCTCTCTT 9''%<D)A03E1-*7=),:F/0!6,D9:H,<9D%:0B(%'E,(8EFG$E89B$27G8F*2+4,-!,0D5()&=(FGG:5;3*@/.0F-G#5#3->('FDFEG?)5.!)"AGADB3?6(@H(:B<>6!>;>6>G,."?% AS:i:0 XN:i:0 XM:i:0 XO:i:0 XG:i:0 NM:i:0 MD:Z:138 YT:Z:UU r6 16 gi|9626243|ref|NC_001416.1| 41607 42 72M2D119M * 0 0 TCGATTTGCAAATACCGGAACATCTCGGTAACTGCATATTCTGCATTAAAAAATCAACGCAAAAAATCGGACGCCTGCAAAGATGAGGAGGGATTGCAGCGTGTTTTTAATGAGGTCATCACGGGATNCCATGTGCGTGACGGNCATCGGGAAACGCCAAAGGAGATTATGTACCGAGGAAGAATGTCGCT 1H#G;H"$E*E#&"*)2%66?=9/9'=;4)4/>@%+5#@#$4A*!<D=="8#1*A9BA=:(1+#C&.#(3#H=9E)AC*5,AC#E'536*2?)H14?>9'B=7(3H/B:+A:8%1-+#(E%&$$&14"76D?>7(&20H5%*&CF8!G5B+A4F$7(:"'?0$?G+$)B-?2<0<F=D!38BH,%=8&5@+ AS:i:-13 XN:i:0 XM:i:2 XO:i:1 XG:i:2 NM:i:4 MD:Z:72^TT55C15A47 YT:Z:UU r7 16 gi|9626243|ref|NC_001416.1| 4692 42 143M * 0 0 TCAGCCGGACGCGGGCGCTGCAGCCGTACTCGGGGATGACCGGTTACAACGGCATTATCGCCCGTCTGCAACAGGCTGCCAGCGATCCGATGGTGGACAGCATTCTGCTCGATATGGACANGCCCGGCGGGATGGTGGCGGGG -"/@*7A0)>2,AAH@&"%B)*5*23B/,)90.B@%=FE,E063C9?,:26$-0:,.,1849'4.;F>FA;76+5&$<C":$!A*,<B,<)@<'85D%C*:)30@85;?.B$05=@95DCDH<53!8G:F:B7/A.E':434> AS:i:-6 XN:i:0 XM:i:2 XO:i:0 XG:i:0 NM:i:2 MD:Z:98G21C22 YT:Z:UU</code></pre> -<p>The first few lines (beginning with <code>@</code>) are SAM header lines, and the rest of the lines are SAM alignments, one line per read or mate. See the <a href="#sam-output">Bowtie 2 manual section on SAM output</a> and the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> for details about how to interpret the SAM file format.</p> +<p>The first few lines (beginning with <code>@</code>) are SAM header +lines, and the rest of the lines are SAM alignments, one line per read +or mate. See the <a href="#sam-output">Bowtie 2 manual section on SAM +output</a> and the <a +href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> +for details about how to interpret the SAM file format.</p> <h2 id="paired-end-example">Paired-end example</h2> -<p>To align paired-end reads included with Bowtie 2, stay in the same directory and run:</p> +<p>To align paired-end reads included with Bowtie 2, stay in the same +directory and run:</p> <pre><code>$BT2_HOME/bowtie2 -x lambda_virus -1 $BT2_HOME/example/reads/reads_1.fq -2 $BT2_HOME/example/reads/reads_2.fq -S eg2.sam</code></pre> -<p>This aligns a set of paired-end reads to the reference genome, with results written to the file <code>eg2.sam</code>.</p> +<p>This aligns a set of paired-end reads to the reference genome, with +results written to the file <code>eg2.sam</code>.</p> <h2 id="local-alignment-example-1">Local alignment example</h2> -<p>To use <a href="#end-to-end-alignment-versus-local-alignment">local alignment</a> to align some longer reads included with Bowtie 2, stay in the same directory and run:</p> +<p>To use <a href="#end-to-end-alignment-versus-local-alignment">local +alignment</a> to align some longer reads included with Bowtie 2, stay in +the same directory and run:</p> <pre><code>$BT2_HOME/bowtie2 --local -x lambda_virus -U $BT2_HOME/example/reads/longreads.fq -S eg3.sam</code></pre> -<p>This aligns the long reads to the reference genome using local alignment, with results written to the file <code>eg3.sam</code>.</p> -<h2 id="using-samtoolsbcftools-downstream">Using SAMtools/BCFtools downstream</h2> -<p><a href="http://samtools.sourceforge.net">SAMtools</a> is a collection of tools for manipulating and analyzing SAM and BAM alignment files. <a href="http://samtools.sourceforge.net/mpileup.shtml">BCFtools</a> is a collection of tools for calling variants and manipulating VCF and BCF files, and it is typically distributed with <a href="http://samtools.sourceforge.net">SAMtools</a>. Using these tools together allows you to get from alignments in SAM format to variant calls in VCF format. This example assumes that <code>samtools</code> and <code>bcftools</code> are installed and that the directories containing these binaries are in your <a href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH environment variable</a>.</p> +<p>This aligns the long reads to the reference genome using local +alignment, with results written to the file <code>eg3.sam</code>.</p> +<h2 id="using-samtoolsbcftools-downstream">Using SAMtools/BCFtools +downstream</h2> +<p><a href="http://samtools.sourceforge.net">SAMtools</a> is a +collection of tools for manipulating and analyzing SAM and BAM alignment +files. <a +href="http://samtools.sourceforge.net/mpileup.shtml">BCFtools</a> is a +collection of tools for calling variants and manipulating VCF and BCF +files, and it is typically distributed with <a +href="http://samtools.sourceforge.net">SAMtools</a>. Using these tools +together allows you to get from alignments in SAM format to variant +calls in VCF format. This example assumes that <code>samtools</code> and +<code>bcftools</code> are installed and that the directories containing +these binaries are in your <a +href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH environment +variable</a>.</p> <p>Run the paired-end example:</p> <pre><code>$BT2_HOME/bowtie2 -x $BT2_HOME/example/index/lambda_virus -1 $BT2_HOME/example/reads/reads_1.fq -2 $BT2_HOME/example/reads/reads_2.fq -S eg2.sam</code></pre> -<p>Use <code>samtools view</code> to convert the SAM file into a BAM file. BAM is the binary format corresponding to the SAM text format. Run:</p> +<p>Use <code>samtools view</code> to convert the SAM file into a BAM +file. BAM is the binary format corresponding to the SAM text format. +Run:</p> <pre><code>samtools view -bS eg2.sam > eg2.bam</code></pre> -<p>Use <code>samtools sort</code> to convert the BAM file to a sorted BAM file.</p> +<p>Use <code>samtools sort</code> to convert the BAM file to a sorted +BAM file.</p> <pre><code>samtools sort eg2.bam -o eg2.sorted.bam</code></pre> -<p>We now have a sorted BAM file called <code>eg2.sorted.bam</code>. Sorted BAM is a useful format because the alignments are (a) compressed, which is convenient for long-term storage, and (b) sorted, which is conveneint for variant discovery. To generate variant calls in VCF format, run:</p> +<p>We now have a sorted BAM file called <code>eg2.sorted.bam</code>. +Sorted BAM is a useful format because the alignments are (a) compressed, +which is convenient for long-term storage, and (b) sorted, which is +conveneint for variant discovery. To generate variant calls in VCF +format, run:</p> <pre><code>bcftools mpileup -f $BT2_HOME/example/reference/lambda_virus.fa eg2.sorted.bam | bcftools view -Ov - > eg2.raw.bcf</code></pre> <p>Then to view the variants, run:</p> <pre><code>bcftools view eg2.raw.bcf</code></pre> -<p>See the official SAMtools guide to <a href="http://samtools.sourceforge.net/mpileup.shtml">Calling SNPs/INDELs with SAMtools/BCFtools</a> for more details and variations on this process.</p> +<p>See the official SAMtools guide to <a +href="http://samtools.sourceforge.net/mpileup.shtml">Calling SNPs/INDELs +with SAMtools/BCFtools</a> for more details and variations on this +process.</p> +<!-- fasta-continuous option does not exist but was added to prevent +warnings due to the case insensitive nature of markdown URLs --> diff --git a/doc/website/recent_news.ssi b/doc/website/recent_news.ssi index ae3f1ce..e4f7f01 100644 --- a/doc/website/recent_news.ssi +++ b/doc/website/recent_news.ssi @@ -1,3 +1,13 @@ +<h2>Version 2.5.0 - Oct 31, 2022</h2> +<h3 id="bowtie2">bowtie2</h3> +<ul> + <li>Overall improvements in the use of prefetch instructions. (contribution by Igor Sfiligoi)</li> + <li>Made input/output fully asynchronous by using a dedicated thread. (contribution by Igor Sfiligoi)</li> + <li>Added support for AVX2 256-bit instructions with can be enabled by setting the <code>SSE_AXV2<code> environment variable at compile time. (contribution by Igor Sfiligoi)</li> + <li>Fixed an issue causing <code>bowtie2</code> to crash when processing ZSTD files with high compression ratios.</li> + <li>Changed the way that unique alignments are counted in summary message to better match up with filters on SAM output</li> +</ul> + <h2>Version 2.4.5 - Jan 16, 2022</h2> <h3 id="bowtie2">bowtie2</h3> <ul> diff --git a/doc/website/rhsidebar.ssi b/doc/website/rhsidebar.ssi index d3525ec..4e7fd59 100644 --- a/doc/website/rhsidebar.ssi +++ b/doc/website/rhsidebar.ssi @@ -18,10 +18,10 @@ </tr> <tr> <td> - <a href="https://sourceforge.net/projects/bowtie-bio/files/bowtie2/2.4.5">Bowtie2 v2.4.5</a> + <a href="https://sourceforge.net/projects/bowtie-bio/files/bowtie2/2.5.0">Bowtie2 v2.4.5</a> </td> <td align="right"> - 16/01/22 + 10/31/22 </td> </tr> <tr> @@ -104,7 +104,16 @@ <tr> <td> - <a href="https://genome-idx.s3.amazonaws.com/bt/chm13.draft_v1.0_plusY.zip"><i>H. sapiens</i>, CHM13plusY</a> + <a href="https://genome-idx.s3.amazonaws.com/bt/chm13v2.0.zip"><i>H. sapiens</i>, CHM13 v2</a> + </td> + <td align="right"> + <a href="https://github.com/nanopore-wgs-consortium/CHM13">T2T</a> + </td> + </tr> + + <tr> + <td> + <a href="https://genome-idx.s3.amazonaws.com/bt/chm13.draft_v1.0_plusY.zip"><i>H. sapiens</i>, CHM13 v1, plus Y</a> </td> <td align="right"> <a href="https://github.com/nanopore-wgs-consortium/CHM13">T2T</a> @@ -23,6 +23,8 @@ #include <iostream> #include <fstream> #include <string> +#include <condition_variable> +#include <thread> #include <stdio.h> #include <string.h> #include <stdint.h> @@ -134,7 +136,7 @@ public: int get() { assert(_in != NULL || _zIn != NULL || _inf != NULL || _ins != NULL); #ifdef WITH_ZSTD - assert(zstdIn != NULL) + assert(_zstdIn != NULL); #endif int c = peek(); if(c != -1) { @@ -263,7 +265,7 @@ public: int peek() { assert(_in != NULL || _zIn != NULL || _inf != NULL || _ins != NULL); #ifdef WITH_ZSTD - assert(zstdIn != NULL); + assert(_zstdIn != NULL); #endif assert_leq(_cur, _buf_sz); if(_cur == _buf_sz) { @@ -629,7 +631,10 @@ public: * Open a new output stream to a file with given name. */ OutFileBuf(const std::string& out, bool binary = false) : - name_(out.c_str()), cur_(0), closed_(false) + name_(out.c_str()), out_(NULL), cur_(0), closed_(false), + asyncData_(out_), asynct_(writeAsync, &asyncData_), + buf1_(new char[BUF_SZ]), buf2_(new char[BUF_SZ]), cap1_(BUF_SZ), cap2_(BUF_SZ), + buf_(buf1_), cap_(cap1_) { out_ = fopen(out.c_str(), binary ? "wb" : "w"); if(out_ == NULL) { @@ -644,7 +649,10 @@ public: * Open a new output stream to a file with given name. */ OutFileBuf(const char *out, bool binary = false) : - name_(out), cur_(0), closed_(false) + name_(out), out_(NULL), cur_(0), closed_(false), + asyncData_(out_), asynct_(writeAsync, &asyncData_), + buf1_(new char[BUF_SZ]), buf2_(new char[BUF_SZ]), cap1_(BUF_SZ), cap2_(BUF_SZ), + buf_(buf1_), cap_(cap1_) { assert(out != NULL); out_ = fopen(out, binary ? "wb" : "w"); @@ -657,14 +665,22 @@ public: /** * Open a new output stream to standard out. */ - OutFileBuf() : name_("cout"), cur_(0), closed_(false) { - out_ = stdout; - } + OutFileBuf() : + name_("cout"), out_(stdout), cur_(0), closed_(false), + asyncData_(out_), asynct_(writeAsync, &asyncData_), + buf1_(new char[BUF_SZ]), buf2_(new char[BUF_SZ]), cap1_(BUF_SZ), cap2_(BUF_SZ), + buf_(buf1_), cap_(cap1_) {} /** * Close buffer when object is destroyed. */ - ~OutFileBuf() { close(); } + ~OutFileBuf() { + close(); + asyncData_.notifyAbort(); + asynct_.join(); + delete[] buf2_; + delete[] buf1_; + } /** * Open a new output stream to a file with given name. @@ -685,7 +701,14 @@ public: */ void write(char c) { assert(!closed_); - if(cur_ == BUF_SZ) flush(); + if(cur_ == cap_) { + if(flushBlocking() && (cap_<MAX_BUF_SZ)) { + // minimize blocking, increase buffer instead + increaseBuffer(BUF_SZ); + } else { + flush(); + } + } buf_[cur_++] = c; } @@ -695,23 +718,8 @@ public: void writeString(const std::string& s) { assert(!closed_); size_t slen = s.length(); - if(cur_ + slen > BUF_SZ) { - if(cur_ > 0) flush(); - if(slen >= BUF_SZ) { - if (slen != fwrite(s.c_str(), 1, slen, out_)) { - std::cerr << "Error: outputting data" << std::endl; - throw 1; - } - } else { - memcpy(&buf_[cur_], s.data(), slen); - assert_eq(0, cur_); - cur_ = slen; - } - } else { - memcpy(&buf_[cur_], s.data(), slen); - cur_ += slen; - } - assert_leq(cur_, BUF_SZ); + const char *cstr = s.c_str(); + writeChars(cstr, slen); } /** @@ -721,23 +729,8 @@ public: void writeString(const T& s) { assert(!closed_); size_t slen = s.length(); - if(cur_ + slen > BUF_SZ) { - if(cur_ > 0) flush(); - if(slen >= BUF_SZ) { - if (slen != fwrite(s.toZBuf(), 1, slen, out_)) { - std::cerr << "Error outputting data" << std::endl; - throw 1; - } - } else { - memcpy(&buf_[cur_], s.toZBuf(), slen); - assert_eq(0, cur_); - cur_ = slen; - } - } else { - memcpy(&buf_[cur_], s.toZBuf(), slen); - cur_ += slen; - } - assert_leq(cur_, BUF_SZ); + const char *zbuf = s.toZBuf(); + writeChars(zbuf, slen); } /** @@ -745,23 +738,20 @@ public: */ void writeChars(const char * s, size_t len) { assert(!closed_); - if(cur_ + len > BUF_SZ) { - if(cur_ > 0) flush(); - if(len >= BUF_SZ) { - if (fwrite(s, len, 1, out_) != 1) { - std::cerr << "Error outputting data" << std::endl; - throw 1; - } + if(cur_ + len > cap_) { + if(flushBlocking() && (cap_<MAX_BUF_SZ)) { + // minimize blocking, increase buffer instead + resizeBuffer(cur_ + len + BUF_SZ); // add a little spare } else { - memcpy(&buf_[cur_], s, len); - assert_eq(0, cur_); - cur_ = len; + if(cur_ > 0) flush(); + // expand the buffer, if needed + // so we can keep using async writes + if(len >= cap_) resizeBufferNoCopy(len+BUF_SZ); // add a little spare } - } else { - memcpy(&buf_[cur_], s, len); - cur_ += len; } - assert_leq(cur_, BUF_SZ); + memcpy(&buf_[cur_], s, len); + cur_ += len; + assert_leq(cur_, cap_); } /** @@ -777,6 +767,7 @@ public: void close() { if(closed_) return; if(cur_ > 0) flush(); + asyncData_.waitIdle(); closed_ = true; if(out_ != stdout) { fclose(out_); @@ -792,12 +783,17 @@ public: } void flush() { - if(cur_ != fwrite((const void *)buf_, 1, cur_, out_)) { - if (errno == EPIPE) { - exit(EXIT_SUCCESS); - } - std::cerr << "Error while flushing and closing output" << std::endl; - throw 1; + // there still could have been an outstanding async write + asyncData_.waitIdle(); + // start the async write + asyncData_.setBuf(buf_, cur_); + // switch to the other buffer + if(buf_==buf1_) { + buf_ = buf2_; + cap_ = cap2_; + } else { + buf_ = buf1_; + cap_ = cap1_; } cur_ = 0; } @@ -816,15 +812,175 @@ public: return name_; } + bool flushBlocking() const { + return asyncData_.buf!=NULL; + } + private: + size_t roundBufferSize(size_t len) const { + // do exponential increases to reduce number of resizes during the process lifetime + // but only up to MAX_BUF_SZ + // Note: Cannot use std::min or max, as it will require MAX_BUF_SZ to have storage (before c++17) + size_t dsize = cap_*2; + if (dsize>MAX_BUF_SZ) dsize = MAX_BUF_SZ; + if (len<dsize) len=dsize; + // round up to multiple of BUF_SZ + return ((len+BUF_SZ-1)/BUF_SZ)*BUF_SZ; + } + + static char* _resizeSpecificBufferNoCopy( + const size_t newCap, + char* &buf, size_t &cap) { // buffer to resize + delete[] buf; + buf = new char[newCap]; + cap = newCap; + return buf; + } + + + + // increase the current buffer to at least len + void resizeBufferNoCopy(size_t len) { + assert_eq(cur_, 0); + // round up to multiple of BUF_SZ + const size_t newCap = roundBufferSize(len); + bool is1 = (buf_==buf1_); + buf_ = _resizeSpecificBufferNoCopy(newCap, + is1 ? buf1_ : buf2_, + is1 ? cap1_ : cap2_); + cap_ = newCap; + } + + + static char* _resizeSpecificBufferCopy( + const size_t newCap, + const size_t oldDataSize, // number of bytes to preserve + char* &buf, size_t &cap) { // buffer to resize + const char* oldBuf = buf; + buf = new char[newCap]; + memcpy(buf, oldBuf, oldDataSize); + delete[] oldBuf; + cap = newCap; + return buf; + } + + // increase the current buffer to at least len + // copy over the content + void resizeBuffer(size_t len) { + if(cur_==0) { + // nothing to copy, use the more efficient version + resizeBufferNoCopy(len); + return; + } + + // round up to multiple of BUF_SZ + size_t newCap = roundBufferSize(len); + bool is1 = (buf_==buf1_); + buf_ = _resizeSpecificBufferCopy(newCap, cur_, + is1 ? buf1_ : buf2_, + is1 ? cap1_ : cap2_); + cap_ = newCap; + } + + // increase the current buffer by delta + // copy over the content + void increaseBuffer(size_t delta) { + size_t newCap = cap_+delta; + resizeBuffer(newCap); + } + + class AsyncData { + public: + bool abort; // the async thread will abort when this is set to true + FILE* &out; + + const char* buf; + size_t cur; + + std::mutex m; + std::condition_variable cv; + + // m and cv default constructors are OK as-is + AsyncData(FILE* &_out) : abort(false), out(_out), buf(NULL) {} + + void notifyAbort() { + { + std::lock_guard<std::mutex> lk(m); + abort = true; + } + cv.notify_all(); + } + + void waitIdle() { + std::unique_lock<std::mutex> lk(m); + while(buf!=NULL) cv.wait(lk); + } + + void setBuf(const char* _buf, size_t _cur) { + { + std::lock_guard<std::mutex> lk(m); + buf = _buf; + cur = _cur; + } + cv.notify_all(); + } + + // returns abort + bool waitForBuf() { + std::unique_lock<std::mutex> lk(m); + while((buf==NULL)&&(!abort)) cv.wait(lk); + return abort; + } + + // returns abort + bool writeComplete() { + bool ret; + { + std::lock_guard<std::mutex> lk(m); + buf = NULL; + ret = abort; + } + cv.notify_all(); + return ret; + } + + + + }; + + static void writeAsync(AsyncData *asyncDataPtr) { + AsyncData &asyncData = *asyncDataPtr; + bool abort = false; + while(!abort) { + abort = asyncData.waitForBuf(); + if(abort) break; + if(asyncData.cur != fwrite((const void *)asyncData.buf, 1, asyncData.cur, asyncData.out)) { + if (errno == EPIPE) { + exit(EXIT_SUCCESS); + } + std::cerr << "Error while flushing and closing output" << std::endl; + throw 1; + } + abort = asyncData.writeComplete(); + } + + } - static const size_t BUF_SZ = 16 * 1024; + static constexpr size_t BUF_SZ = 16ul * 1024ul; + static constexpr size_t MAX_BUF_SZ = 16ul * 1024ul * 1024ul * 1024ul; const char *name_; FILE *out_; - size_t cur_; - char buf_[BUF_SZ]; // (large) input buffer + size_t cur_; // how much of the buffer is currently filled bool closed_; + AsyncData asyncData_; + std::thread asynct_; + char *buf1_; // (large) output buffer + char *buf2_; // (large) output buffer + size_t cap1_; //capacity of buf1_ + size_t cap2_; //capacity of buf2_ + char* buf_; // points to one of the two buffers below + size_t cap_; // capacity of the pointed buffer }; #endif /*ndef FILEBUF_H_*/ @@ -27,7 +27,8 @@ */ enum file_format { - FASTA = 1, + UNKNOWN = 0, + FASTA, FASTA_CONT, FASTQ, BAM, @@ -51,8 +52,8 @@ static const std::string file_format_names[] = { "Raw", "Command line", "Qseq", - "SRA Fasta", - "SRA Fastq" + "SRA Fasta", + "SRA Fastq" }; #endif /*FORMATS_H_*/ @@ -23,12 +23,14 @@ #include <stdio.h> #include <stdint.h> #include <unistd.h> +#include <condition_variable> #include <sys/stat.h> #include <zlib.h> #include <cassert> #include <string> #include <ctype.h> #include <vector> +#include <queue> #include "alphabet.h" #include "assert_helpers.h" #include "random_source.h" @@ -131,12 +133,11 @@ struct PatternParams { */ struct PerThreadReadBuf { - PerThreadReadBuf(size_t max_buf, int tid) : + PerThreadReadBuf(size_t max_buf) : max_buf_(max_buf), bufa_(max_buf), bufb_(max_buf), - rdid_(), - tid_(tid) + rdid_() { bufa_.resize(max_buf); bufb_.resize(max_buf); @@ -180,7 +181,7 @@ struct PerThreadReadBuf { /** * Return true when there's nothing left for next(). */ - bool exhausted() { + bool exhausted() const { assert_leq(cur_buf_, bufa_.size()); return cur_buf_ >= bufa_.size()-1 || bufa_[cur_buf_+1].readOrigBuf.empty(); } @@ -205,7 +206,6 @@ struct PerThreadReadBuf { EList<Read> bufb_; // Read buffer for mate bs size_t cur_buf_; // Read buffer currently active TReadId rdid_; // index of read at offset 0 of bufa_/bufb_ - int tid_; }; extern void wrongQualityFormat(const BTString& read_name); @@ -510,9 +510,9 @@ protected: if (fd == -1) return false; - int magic; + unsigned magic; - if (read(fd, &magic, sizeof(int)) != sizeof(int)) { + if (read(fd, &magic, sizeof(unsigned)) != sizeof(unsigned)) { std::cerr << "is_zstd_file: unable to read magic number" << std::endl; return false; } @@ -1153,13 +1153,16 @@ public: PatternSourcePerThread( PatternComposer& composer, - const PatternParams& pp, int tid) : + const PatternParams& pp) : composer_(composer), - buf_(pp.max_buf, tid), + buf_(pp.max_buf), pp_(pp), last_batch_(false), last_batch_size_(0) { } + // If it returns true, nextReadPair is non-blocking and should return success + bool nextReadPairReady() const {return !buf_.exhausted();} + /** * Use objects in the PatternSource and/or PatternComposer * hierarchies to populate the per-thread buffers. @@ -1243,16 +1246,15 @@ class PatternSourcePerThreadFactory { public: PatternSourcePerThreadFactory( PatternComposer& composer, - const PatternParams& pp, int tid) : + const PatternParams& pp) : composer_(composer), - pp_(pp), - tid_(tid) { } + pp_(pp) {} /** * Create a new heap-allocated PatternSourcePerThreads. */ virtual PatternSourcePerThread* create() const { - return new PatternSourcePerThread(composer_, pp_, tid_); + return new PatternSourcePerThread(composer_, pp_); } /** @@ -1262,7 +1264,7 @@ public: virtual EList<PatternSourcePerThread*>* create(uint32_t n) const { EList<PatternSourcePerThread*>* v = new EList<PatternSourcePerThread*>; for(size_t i = 0; i < n; i++) { - v->push_back(new PatternSourcePerThread(composer_, pp_, tid_)); + v->push_back(new PatternSourcePerThread(composer_, pp_)); assert(v->back() != NULL); } return v; @@ -1274,7 +1276,152 @@ private: /// Container for obtaining paired reads from PatternSources PatternComposer& composer_; const PatternParams& pp_; - int tid_; +}; + +class PatternSourceReadAheadFactory { +public: + class ReadElement { + public: + PatternSourcePerThread *ps; // not owned, just a pointer + std::pair<bool, bool> readResult; + }; + + PatternSourceReadAheadFactory( + PatternComposer& composer, + const PatternParams& pp, size_t n) : + psfact_(composer,pp), + psq_ready_(), + psq_idle_(psfact_,n), + asynct_(readAsync, this) {} + + ~PatternSourceReadAheadFactory() { + returnUnready(NULL); // this will signal asynct_ it is time to quit + asynct_.join(); + } + + // wait for data, if none in the queue + ReadElement nextReadPair() { + return psq_ready_.pop(); + } + + void returnUnready(PatternSourcePerThread* ps) { + psq_idle_.push(ps); + } + + void returnUnready(ReadElement& re) { + returnUnready(re.ps); + } + +private: + + template <typename T> + class LockedQueue { + public: + virtual ~LockedQueue() {} + + bool empty() { + bool ret = false; + { + std::unique_lock<std::mutex> lk(m_); + ret = q_.empty(); + } + return ret; + } + + void push(T& ps) { + { + std::unique_lock<std::mutex> lk(m_); + q_.push(ps); + } + cv_.notify_all(); + } + + // wait for data, if none in the queue + T pop() { + T ret; + { + std::unique_lock<std::mutex> lk(m_); + while (q_.empty()) cv_.wait(lk); + ret = q_.front(); + q_.pop(); + } + return ret; + } + + protected: + std::mutex m_; + std::condition_variable cv_; + std::queue<T> q_; + }; + + class LockedPSQueue : public LockedQueue<PatternSourcePerThread*> { + public: + LockedPSQueue(PatternSourcePerThreadFactory& psfact, size_t n) : + LockedQueue<PatternSourcePerThread*>() { + for (size_t i=0; i<n; i++) { + q_.push(psfact.create()); + } + } + + virtual ~LockedPSQueue() { + while (!q_.empty()) { + delete q_.front(); + q_.pop(); + } + } + }; + + class LockedREQueue : public LockedQueue<ReadElement> { + public: + virtual ~LockedREQueue() { + // we actually own ps while in the queue + while (!q_.empty()) { + delete q_.front().ps; + q_.pop(); + } + } + }; + + static void readAsync(PatternSourceReadAheadFactory *obj) { + LockedREQueue &psq_ready = obj->psq_ready_; + LockedPSQueue &psq_idle = obj->psq_idle_; + while(true) { + ReadElement re; + re.ps = psq_idle.pop(); + if (re.ps==NULL) break; // the destructor added this in the queue + + if (re.ps->nextReadPairReady()) { + // Should never get in here, but just in case + re.readResult = make_pair(true, false); + } else { + re.readResult = re.ps->nextReadPair(); + } + psq_ready.push(re); + } + } + + PatternSourcePerThreadFactory psfact_; + LockedREQueue psq_ready_; + LockedPSQueue psq_idle_; + std::thread asynct_; +}; + +// Simple wrapper for safely holding the result of PatternSourceReadAheadFactory +class PatternSourceReadAhead { +public: + PatternSourceReadAhead(PatternSourceReadAheadFactory& fact) : + fact_(fact), + re_(fact.nextReadPair()) {} + + ~PatternSourceReadAhead() { + fact_.returnUnready(re_); + } + + const std::pair<bool, bool>& readResult() const { return re_.readResult;} + PatternSourcePerThread* ptr() {return re_.ps;} +private: + PatternSourceReadAheadFactory& fact_; + PatternSourceReadAheadFactory::ReadElement re_; }; #ifdef USE_SRA @@ -325,8 +325,49 @@ public: for (i = 0; i < name.length() && !isspace(name[i]); i++) ; o.append('\t'); - o.append(name.toZBuf() + i + 1); + if (i < name.length()) { + const char *comment = name.toZBuf() + i + 1; + if (isIllumina(comment)) + o.append("BC:Z:"); + o.append(comment); + } + } + } + + bool isIllumina(const char *str) const { + const char *start, *end; + int field = 0; + + for (start = str, end = str; *end != '\0' && *end != ' '; end++) { + if (*end == ':') { + switch (field) { + case 0: { + char *endptr; + long read = strtol(start, &endptr, 10); + if (endptr != end || (read != 1 && read != 2)) + return false; + break; + } + case 1: + if (*start != 'N' && *start != 'Y') + return false; + break; + case 2: { + char *endptr; + long controlNumber = strtol(start, &endptr, 10); + if (endptr != end || controlNumber % 2 != 0) + return false; + break; + + } + default: + return false; + } + start = end + 1, field++; + } + } + return true; } template<typename T> diff --git a/scripts/bowtie2-hbb.sh b/scripts/bowtie2-hbb.sh index 0546a2b..70fbef3 100755 --- a/scripts/bowtie2-hbb.sh +++ b/scripts/bowtie2-hbb.sh @@ -44,10 +44,6 @@ fi # this variant creates static binaries with PIC source /hbb_exe_gc_hardened/activate -mkdir /mybin -echo 'res=`echo $@ | sed "s/-L.*$//"`; /opt/rh/devtoolset-8/root/usr/bin/ar $res;' > /mybin/ar -chmod +x /mybin/ar && export PATH=/mybin:$PATH - # this version of pandoc does not support 'smart' filters sed -i 's/-smart//' Makefile @@ -57,7 +53,7 @@ if [ $? -ne 0 ] ; then exit 1 fi -make -j4 bowtie2-bin-pkg STATIC_BUILD=1 WITH_ZSTD=1 USE_SRA=$use_sra +make bowtie2-bin-pkg STATIC_BUILD=1 WITH_ZSTD=1 USE_SRA=$use_sra if [ $? -ne 0 ] ; then echo "Unable to create bowtie2 package" exit 1 diff --git a/sse_util.cpp b/sse_util.cpp index d6310cf..2736078 100644 --- a/sse_util.cpp +++ b/sse_util.cpp @@ -25,9 +25,9 @@ * Given a column of filled-in cells, save the checkpointed cells in cs_. */ void Checkpointer::commitCol( - __m128i *pvH, - __m128i *pvE, - __m128i *pvF, + SSERegI *pvH, + SSERegI *pvE, + SSERegI *pvF, size_t coli) { } @@ -26,13 +26,13 @@ #include <iostream> #include "sse_wrap.h" -class EList_m128i { +class EList_sse { public: /** * Allocate initial default of S elements. */ - explicit EList_m128i(int cat = 0) : + explicit EList_sse (int cat = 0) : cat_(cat), last_alloc_(NULL), list_(NULL), sz_(0), cur_(0) { assert_geq(cat, 0); @@ -41,7 +41,7 @@ public: /** * Destructor. */ - ~EList_m128i() { free(); } + ~EList_sse () { free(); } /** * Return number of elements. @@ -103,7 +103,7 @@ public: */ void zero() { if(cur_ > 0) { - memset(list_, 0, cur_ * sizeof(__m128i)); + memset(list_, 0, cur_ * sizeof(SSERegI)); } } @@ -148,7 +148,7 @@ public: /** * Return a reference to the ith element. */ - inline __m128i& operator[](size_t i) { + inline SSERegI& operator[](size_t i) { assert_lt(i, cur_); return list_[i]; } @@ -156,7 +156,7 @@ public: /** * Return a reference to the ith element. */ - inline __m128i operator[](size_t i) const { + inline SSERegI operator[](size_t i) const { assert_lt(i, cur_); return list_[i]; } @@ -164,26 +164,26 @@ public: /** * Return a reference to the ith element. */ - inline __m128i& get(size_t i) { + inline SSERegI& get(size_t i) { return operator[](i); } /** * Return a reference to the ith element. */ - inline __m128i get(size_t i) const { + inline SSERegI get(size_t i) const { return operator[](i); } /** * Return a pointer to the beginning of the buffer. */ - __m128i *ptr() { return list_; } + SSERegI *ptr() { return list_; } /** * Return a const pointer to the beginning of the buffer. */ - const __m128i *ptr() const { return list_; } + const SSERegI *ptr() const { return list_; } /** * Return memory category. @@ -214,24 +214,25 @@ private: * Allocate a T array of length sz_ and store in list_. Also, * tally into the global memory tally. */ - __m128i *alloc(size_t sz) { - __m128i* last_alloc_; + SSERegI *alloc(size_t sz) { + SSERegI* last_alloc_; try { - last_alloc_ = new __m128i[sz + 2]; + last_alloc_ = new SSERegI[sz + 2]; } catch(std::bad_alloc& e) { - std::cerr << "Error: Out of memory allocating " << sz << " __m128i's for DP matrix: '" << e.what() << "'" << std::endl; + std::cerr << "Error: Out of memory allocating " << sz << " SSERegI's for DP matrix: '" << e.what() << "'" << std::endl; throw e; } this->last_alloc_ = last_alloc_; - __m128i* tmp = last_alloc_; + SSERegI* tmp = last_alloc_; size_t tmpint = (size_t)tmp; // Align it! - if((tmpint & 0xf) != 0) { - tmpint += 15; - tmpint &= (~0xf); - tmp = reinterpret_cast<__m128i*>(tmpint); + const size_t alignmask = NBYTES_PER_REG-1; + if((tmpint & alignmask) != 0) { + tmpint += alignmask; + tmpint &= (~alignmask); + tmp = reinterpret_cast<SSERegI*>(tmpint); } - assert_eq(0, (tmpint & 0xf)); // should be 16-byte aligned + assert_eq(0, (tmpint & alignmask)); // should be NBYTES_PER_REG-byte aligned assert(tmp != NULL); #ifdef USE_MEM_TALLY gMemTally.add(cat_, sz); @@ -272,8 +273,8 @@ private: */ void expandCopyExact(size_t newsz) { if(newsz <= sz_) return; - __m128i* prev_last_alloc = last_alloc_; - __m128i* tmp = alloc(newsz); + SSERegI* prev_last_alloc = last_alloc_; + SSERegI* tmp = alloc(newsz); assert(tmp != NULL); size_t cur = cur_; if(list_ != NULL) { @@ -281,7 +282,7 @@ private: // Note: operator= is used tmp[i] = list_[i]; } - __m128i* current_last_alloc = last_alloc_; + SSERegI* current_last_alloc = last_alloc_; last_alloc_ = prev_last_alloc; free(); last_alloc_ = current_last_alloc; @@ -312,7 +313,7 @@ private: assert(list_ != NULL); assert_gt(newsz, 0); free(); - __m128i* tmp = alloc(newsz); + SSERegI* tmp = alloc(newsz); assert(tmp != NULL); list_ = tmp; sz_ = newsz; @@ -320,8 +321,8 @@ private: } int cat_; // memory category, for accounting purposes - __m128i* last_alloc_; // what new[] originally returns - __m128i *list_; // list ptr, aligned version of what new[] returns + SSERegI* last_alloc_; // what new[] originally returns + SSERegI *list_; // list ptr, aligned version of what new[] returns size_t sz_; // capacity size_t cur_; // occupancy (AKA size) }; @@ -379,7 +380,7 @@ public: debug_ = true; commitMap_.clear(); firstCommit_ = true; - size_t perword = (is8 ? 16 : 8); + size_t perword = NBYTES_PER_REG / (is8 ? 1 : 2); is8_ = is8; niter_ = ((nrow_ + perword - 1) / perword); if(doTri) { @@ -408,11 +409,11 @@ public: */ int64_t debugCell(size_t row, size_t col, int hef) const { assert(debug_); - const __m128i* ptr = qcolsD_.ptr() + hef; + const SSERegI* ptr = qcolsD_.ptr() + hef; // Fast forward to appropriate column ptr += ((col * niter_) << 2); - size_t mod = row % niter_; // which m128i - size_t div = row / niter_; // offset into m128i + size_t mod = row % niter_; // which SSERegI + size_t div = row / niter_; // offset into SSERegI // Fast forward to appropriate word ptr += (mod << 2); // Extract score @@ -486,10 +487,10 @@ public: // It must be in a checkpointed column assert_eq(lomask_, (col & lomask_)); // Fast forward to appropriate column - const __m128i* ptr = qcols_.ptr() + hef; + const SSERegI* ptr = qcols_.ptr() + hef; ptr += (((col >> perpow2_) * niter_) << 2); - size_t mod = row % niter_; // which m128i - size_t div = row / niter_; // offset into m128i + size_t mod = row % niter_; // which SSERegI + size_t div = row / niter_; // offset into SSERegI // Fast forward to appropriate word ptr += (mod << 2); // Extract score @@ -516,7 +517,7 @@ public: /** * Given a column of filled-in cells, save the checkpointed cells in cs_. */ - void commitCol(__m128i *pvH, __m128i *pvE, __m128i *pvF, size_t coli); + void commitCol(SSERegI *pvH, SSERegI *pvE, SSERegI *pvF, size_t coli); /** * Reset the state of the Checkpointer. @@ -573,11 +574,11 @@ public: // We store columns in this way to reduce overhead of populating them bool is8_; // true -> fill used 8-bit cells - size_t niter_; // # __m128i words per column - EList_m128i qcols_; // checkpoint E/F/H values for select columns + size_t niter_; // # SSERegI words per column + EList_sse qcols_; // checkpoint E/F/H values for select columns bool debug_; // get debug checkpoints? (i.e. fill qcolsD_?) - EList_m128i qcolsD_; // checkpoint E/F/H values for all columns (debug) + EList_sse qcolsD_; // checkpoint E/F/H values for all columns (debug) }; #endif @@ -27,41 +27,220 @@ #ifndef SSE_WRAP_H_ #define SSE_WRAP_H_ +#ifdef SSE_AVX2 +#include <immintrin.h> +#define NBYTES_PER_REG 32 +#define BYTES_LOG2_PER_REG 5 +#define SSE_MASK_ALL ((int) 0xffffffff) + +typedef __m256i SSERegI; +#define sse_adds_epi16(x, y) _mm256_adds_epi16(x, y) +#define sse_adds_epu8(x, y) _mm256_adds_epu8(x, y) +#define sse_cmpeq_epi16(x, y) _mm256_cmpeq_epi16(x, y) +#define sse_cmpeq_epi8(x, y) _mm256_cmpeq_epi8(x, y) +#define sse_cmpgt_epi16(x, y) _mm256_cmpgt_epi16(x, y) +#define sse_cmpgt_epi8(x, y) _mm256_cmpgt_epi8(x, y) +#define sse_cmplt_epi16(x, y) _mm256_cmpgt_epi16(y,x) +#define sse_extract_epi16(x, y) _mm256_extract_epi16(x, y) +#define sse_insert_epi16(x, y, z) _mm256_insert_epi16(x, y, z) +#define sse_load_siall(x) _mm256_load_si256(x) +#define sse_max_epi16(x, y) _mm256_max_epi16(x, y) +#define sse_max_epu8(x, y) _mm256_max_epu8(x, y) +#define sse_movemask_epi8(x) _mm256_movemask_epi8(x) +#define sse_or_siall(x, y) _mm256_or_si256(x, y) +#define sse_setzero_siall() _mm256_setzero_si256() +#define sse_slli_epi16(x, y) _mm256_slli_epi16(x, y) +#define sse_srli_epi16(x, y) _mm256_srli_epi16(x, y) +#define sse_srli_epu8(x, y) _mm256_srli_epu8(x, y) +#define sse_store_siall(x, y) _mm256_store_si256(x, y) +#define sse_subs_epi16(x, y) _mm256_subs_epi16(x, y) +#define sse_subs_epu8(x, y) _mm256_subs_epu8(x, y) +#define sse_xor_siall(x, y) _mm256_xor_si256(x, y) +#define sse_set1_epi16(x) _mm256_set1_epi16(x) + +/* AVX2 does not have a native 256-bit shift instruction */ +/* Note only works for y<=16, which is OK for this code */ +#define sse_slli_siall(x, y) \ + _mm256_alignr_epi8(x, _mm256_permute2x128_si256(x, x, _MM_SHUFFLE(0, 0, 2, 0)), 16-y) + +#define sse_srli_siall(x, y) \ + _mm256_alignr_epi8(_mm256_permute2x128_si256(x, x, _MM_SHUFFLE(2, 0, 0, 1)), x, y) + +/* we can avoid one instruction, when y==16 */ +#define sse_slli_siall_16(x) \ + _mm256_permute2x128_si256(x, x, _MM_SHUFFLE(0, 0, 2, 0)) + +#define sse_srli_siall_16(x) \ + _mm256_permute2x128_si256(x, x, _MM_SHUFFLE(2, 0, 0, 1)) + +/* this operates on the 2x 128-bit lanes independenty */ +#define sse_slli_si128(x, y) _mm256_slli_si256(x, y) +#define sse_srli_si128(x, y) _mm256_srli_si256(x, y) + +/* compute the max val of a vector */ +#define sse_max_score_i16(inval, outval) { \ + SSERegI vlmax = inval; \ + SSERegI vltmp = sse_srli_siall(vlmax, 16); \ + vlmax = sse_max_epi16(vlmax, vltmp); \ + /* we use only 128-bit hers, use the fast version */ \ + vltmp = sse_srli_si128(vlmax, 8); \ + vlmax = sse_max_epi16(vlmax, vltmp); \ + vltmp = sse_srli_si128(vlmax, 4); \ + vlmax = sse_max_epi16(vlmax, vltmp); \ + vltmp = sse_srli_si128(vlmax, 2); \ + vlmax = sse_max_epi16(vlmax, vltmp); \ + outval = sse_extract_epi16(vlmax, 0); \ +} + +#define sse_max_score_u8(inval, outval) { \ + SSERegI vlmax = inval; \ + SSERegI vltmp = sse_srli_siall(vlmax, 16); \ + vlmax = sse_max_epu8(vlmax, vltmp); \ + /* we use only 128-bit hers, use the fast version */ \ + vltmp = sse_srli_si128(vlmax, 8); \ + vlmax = sse_max_epu8(vlmax, vltmp); \ + vltmp = sse_srli_si128(vlmax, 4); \ + vlmax = sse_max_epu8(vlmax, vltmp); \ + vltmp = sse_srli_si128(vlmax, 2); \ + vlmax = sse_max_epu8(vlmax, vltmp); \ + vltmp = sse_srli_si128(vlmax, 1); \ + vlmax = sse_max_epu8(vlmax, vltmp); \ + outval = sse_extract_epi16(vlmax, 0); \ + outval = outval & 0x00ff; \ +} + + +#else /* no SSE_AVX2 */ + #if defined(__aarch64__) || defined(__s390x__) || defined(__powerpc__) #include "simde/x86/sse2.h" #else #include <emmintrin.h> #endif +#define NBYTES_PER_REG 16 +#define BYTES_LOG2_PER_REG 4 +#define SSE_MASK_ALL 0xffff + #if defined(__aarch64__) || defined(__s390x__) || defined(__powerpc__) -typedef simde__m128i __m128i; -#define _mm_adds_epi16(x, y) simde_mm_adds_epi16(x, y) -#define _mm_adds_epu8(x, y) simde_mm_adds_epu8(x, y) -#define _mm_cmpeq_epi16(x, y) simde_mm_cmpeq_epi16(x, y) -#define _mm_cmpeq_epi8(x, y) simde_mm_cmpeq_epi8(x, y) -#define _mm_cmpgt_epi16(x, y) simde_mm_cmpgt_epi16(x, y) -#define _mm_cmpgt_epi8(x, y) simde_mm_cmpgt_epi8(x, y) -#define _mm_cmplt_epi16(x, y) simde_mm_cmplt_epi16(x, y) -#define _mm_cmplt_epu8(x, y) simde_mm_cmplt_epu8(x, y) -#define _mm_extract_epi16(x, y) simde_mm_extract_epi16(x, y) -#define _mm_insert_epi16(x, y, z) simde_mm_insert_epi16(x, y, z) -#define _mm_load_si128(x) simde_mm_load_si128(x) -#define _mm_max_epi16(x, y) simde_mm_max_epi16(x, y) -#define _mm_max_epu8(x, y) simde_mm_max_epu8(x, y) -#define _mm_movemask_epi8(x) simde_mm_movemask_epi8(x) -#define _mm_or_si128(x, y) simde_mm_or_si128(x, y) -#define _mm_setzero_si128() simde_mm_setzero_si128() -#define _mm_shuffle_epi32(x, y) simde_mm_shuffle_epi32(x, y) -#define _mm_shufflelo_epi16(x, y) simde_mm_shufflelo_epi16(x, y) -#define _mm_slli_epi16(x, y) simde_mm_slli_epi16(x, y) -#define _mm_slli_si128(x, y) simde_mm_slli_si128(x, y) -#define _mm_srli_epi16(x, y) simde_mm_srli_epi16(x, y) -#define _mm_srli_epu8(x, y) simde_mm_srli_epu8(x, y) -#define _mm_srli_si128(x, y) simde_mm_srli_si128(x, y) -#define _mm_store_si128(x, y) simde_mm_store_si128(x, y) -#define _mm_subs_epi16(x, y) simde_mm_subs_epi16(x, y) -#define _mm_subs_epu8(x, y) simde_mm_subs_epu8(x, y) -#define _mm_xor_si128(x, y) simde_mm_xor_si128(x, y) +typedef simde__m128i SSERegI; +#define sse_adds_epi16(x, y) simde_mm_adds_epi16(x, y) +#define sse_adds_epu8(x, y) simde_mm_adds_epu8(x, y) +#define sse_cmpeq_epi16(x, y) simde_mm_cmpeq_epi16(x, y) +#define sse_cmpeq_epi8(x, y) simde_mm_cmpeq_epi8(x, y) +#define sse_cmpgt_epi16(x, y) simde_mm_cmpgt_epi16(x, y) +#define sse_cmpgt_epi8(x, y) simde_mm_cmpgt_epi8(x, y) +#define sse_cmplt_epi16(x, y) simde_mm_cmplt_epi16(x, y) +#define sse_cmplt_epu8(x, y) simde_mm_cmplt_epu8(x, y) +#define sse_extract_epi16(x, y) simde_mm_extract_epi16(x, y) +#define sse_insert_epi16(x, y, z) simde_mm_insert_epi16(x, y, z) +#define sse_load_siall(x) simde_mm_load_si128(x) +#define sse_max_epi16(x, y) simde_mm_max_epi16(x, y) +#define sse_max_epu8(x, y) simde_mm_max_epu8(x, y) +#define sse_movemask_epi8(x) simde_mm_movemask_epi8(x) +#define sse_or_siall(x, y) simde_mm_or_si128(x, y) +#define sse_setzero_siall() simde_mm_setzero_si128() +#define sse_slli_epi16(x, y) simde_mm_slli_epi16(x, y) +#define sse_slli_siall(x, y) simde_mm_slli_si128(x, y) +#define sse_srli_epi16(x, y) simde_mm_srli_epi16(x, y) +#define sse_srli_epu8(x, y) simde_mm_srli_epu8(x, y) +#define sse_srli_siall(x, y) simde_mm_srli_si128(x, y) +#define sse_store_siall(x, y) simde_mm_store_si128(x, y) +#define sse_subs_epi16(x, y) simde_mm_subs_epi16(x, y) +#define sse_subs_epu8(x, y) simde_mm_subs_epu8(x, y) +#define sse_xor_siall(x, y) simde_mm_xor_si128(x, y) +#define sse_set1_epi16(x) simde_mm_set1_epi16(x) + +#else +typedef __m128i SSERegI; +#define sse_adds_epi16(x, y) _mm_adds_epi16(x, y) +#define sse_adds_epu8(x, y) _mm_adds_epu8(x, y) +#define sse_cmpeq_epi16(x, y) _mm_cmpeq_epi16(x, y) +#define sse_cmpeq_epi8(x, y) _mm_cmpeq_epi8(x, y) +#define sse_cmpgt_epi16(x, y) _mm_cmpgt_epi16(x, y) +#define sse_cmpgt_epi8(x, y) _mm_cmpgt_epi8(x, y) +#define sse_cmplt_epi16(x, y) _mm_cmplt_epi16(x, y) +#define sse_cmplt_epu8(x, y) _mm_cmplt_epu8(x, y) +#define sse_extract_epi16(x, y) _mm_extract_epi16(x, y) +#define sse_insert_epi16(x, y, z) _mm_insert_epi16(x, y, z) +#define sse_load_siall(x) _mm_load_si128(x) +#define sse_max_epi16(x, y) _mm_max_epi16(x, y) +#define sse_max_epu8(x, y) _mm_max_epu8(x, y) +#define sse_movemask_epi8(x) _mm_movemask_epi8(x) +#define sse_or_siall(x, y) _mm_or_si128(x, y) +#define sse_setzero_siall() _mm_setzero_si128() +#define sse_slli_epi16(x, y) _mm_slli_epi16(x, y) +#define sse_slli_siall(x, y) _mm_slli_si128(x, y) +#define sse_srli_epi16(x, y) _mm_srli_epi16(x, y) +#define sse_srli_epu8(x, y) _mm_srli_epu8(x, y) +#define sse_srli_siall(x, y) _mm_srli_si128(x, y) +#define sse_store_siall(x, y) _mm_store_si128(x, y) +#define sse_subs_epi16(x, y) _mm_subs_epi16(x, y) +#define sse_subs_epu8(x, y) _mm_subs_epu8(x, y) +#define sse_xor_siall(x, y) _mm_xor_si128(x, y) +#define sse_set1_epi16(x) _mm_set1_epi16(x) + #endif +/* compute the max val of a vector */ +#define sse_max_score_i16(inval, outval) { \ + SSERegI vlmax = inval; \ + SSERegI vltmp = sse_srli_siall(vlmax, 8); \ + vlmax = sse_max_epi16(vlmax, vltmp); \ + vltmp = sse_srli_siall(vlmax, 4); \ + vlmax = sse_max_epi16(vlmax, vltmp); \ + vltmp = sse_srli_siall(vlmax, 2); \ + vlmax = sse_max_epi16(vlmax, vltmp); \ + outval = sse_extract_epi16(vlmax, 0); \ +} + +#define sse_max_score_u8(inval, outval) { \ + SSERegI vlmax = inval; \ + SSERegI vltmp = sse_srli_siall(vlmax, 8); \ + vlmax = sse_max_epu8(vlmax, vltmp); \ + vltmp = sse_srli_siall(vlmax, 4); \ + vlmax = sse_max_epu8(vlmax, vltmp); \ + vltmp = sse_srli_siall(vlmax, 2); \ + vlmax = sse_max_epu8(vlmax, vltmp); \ + vltmp = sse_srli_siall(vlmax, 1); \ + vlmax = sse_max_epu8(vlmax, vltmp); \ + outval = sse_extract_epi16(vlmax, 0); \ + outval = outval & 0x00ff; \ +} + +#endif /* SSE_AVX2 */ + +/* Fill all elements in outval with inval */ +/* opt version will check for special ivals that can use shortcuts */ +#define sse_fill_i16(inval, outval) outval=sse_set1_epi16(inval) + +#define sse_fill_i16_opt(inval, outval) { \ + if (inval==0xffff) outval = sse_cmpeq_epi16(outval, outval); \ + else if (inval==0) outval = sse_xor_siall(outval, outval); \ + else sse_fill_i16(inval, outval); \ +} + +#define sse_fill_u8(inval, outval) {\ + int invalloc = inval; \ + int dup = (invalloc << 8) | (invalloc & 0x00ff); \ + sse_fill_i16(dup, outval); \ +} + +#define sse_fill_u8_opt(inval, outval) {\ + if (inval==0xff) outval = sse_cmpeq_epi16(outval, outval); \ + else if (inval==0) outval = sse_xor_siall(outval, outval); \ + else sse_fill_u8(inval, outval); \ +} + +/* Set the low element with invl, all others to 0 */ +#define sse_set_low_i16(inval, outval) { \ + outval = sse_setzero_siall(); \ + outval = sse_insert_epi16(outval, inval, 0); \ +} + +#define sse_set_low_u8(inval, outval) { \ + outval = sse_setzero_siall(); \ + outval = sse_insert_epi16(outval, inval, 0); \ +} + #endif /* SSE_WRAP_H_ */ diff --git a/zstd_decompress.cpp b/zstd_decompress.cpp index eb883a6..ff29549 100644 --- a/zstd_decompress.cpp +++ b/zstd_decompress.cpp @@ -1,3 +1,4 @@ +#include <cstdio> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -22,7 +23,12 @@ zstdStrm *zstdStrmInit() zstdClose(s); return NULL; } - if ((s->strm = ZSTD_createDStream()) == NULL) { + if ((s->o_buf = malloc(ZSTD_DStreamOutSize())) == NULL) { + zstdClose(s); + return NULL; + } + + if ((s->strm = ZSTD_createDCtx()) == NULL) { zstdClose(s); return NULL; } @@ -69,19 +75,21 @@ zstdStrm *zstdFdOpen(int fd) { int zstdDecompress(zstdStrm *s) { int ret; - ZSTD_inBuffer in = { s->i_buf, s->i_len, s->i_pos }; - ZSTD_outBuffer out = { s->o_buf, s->o_len, s->o_pos }; + ZSTD_outBuffer out = { s->o_buf, s->o_len, s->o_pos }; - ret = ZSTD_decompressStream(s->strm, &out, &in); - s->i_pos = in.pos; - s->o_pos = out.pos; + ret = ZSTD_decompressStream(s->strm, &out, &in); + s->i_pos = in.pos; + s->o_pos = out.pos; return ret; } int zstdRead(zstdStrm *s, void *buf, size_t len) { + if (s->i_pos == s->i_len && feof(s->fp)) + return -1; + s->o_pos = 0; s->o_buf = buf; s->o_len = len; @@ -94,8 +102,17 @@ int zstdRead(zstdStrm *s, void *buf, size_t len) int ret; if ((s->i_pos == 0 || s->i_pos == s->i_len) && !feof(s->fp)) { - s->i_pos = 0; - s->i_len = fread(s->i_buf, 1, ZSTD_DStreamInSize(), s->fp); + int nread; + size_t nleft = ZSTD_DStreamInSize(); + + s->i_len = s->i_pos = 0; + do { + nread = fread((char *)s->i_buf + s->i_len, 1, nleft, s->fp); + s->i_len += nread; + if (feof(s->fp)) + break; + nleft -= nread; + } while (nleft > 0); } ret = zstdDecompress(s); if (ret == 0) @@ -122,9 +139,18 @@ int zstdUngetc(int c, zstdStrm *s) int zstdGetc(zstdStrm *s) { - unsigned char c; + int ret; + - return zstdRead(s, &c, 1) == 1 ? c : EOF; + if (s->o_len == 0) { + ret = zstdRead(s, s->o_buf, ZSTD_DStreamOutSize()); + if (ret < 0) + return ret; + s->o_len = s->o_pos; + s->o_pos = 0; + } + s->o_len--; + return ((char *)s->o_buf)[s->o_pos++]; } int zstdClose(zstdStrm *s) @@ -135,6 +161,8 @@ int zstdClose(zstdStrm *s) fclose(s->fp); if (s->i_buf != NULL) free(s->i_buf); + if (s->o_buf != NULL) + free(s->o_buf); if (s->strm != NULL) ZSTD_freeDStream(s->strm); bzero(s, sizeof(zstdStrm)); diff --git a/zstd_decompress.h b/zstd_decompress.h index 273b389..77b4017 100644 --- a/zstd_decompress.h +++ b/zstd_decompress.h @@ -14,7 +14,7 @@ struct zstd_stream { size_t i_pos; void *i_buf; void *o_buf; - ZSTD_DStream *strm; + ZSTD_DCtx *strm; FILE *fp; }; |