From 1d0ff54794b5edea7cdf1d2d66710a0fa885bcc5 Mon Sep 17 00:00:00 2001 From: Teus Benschop Date: Sun, 28 Oct 2018 11:51:26 +0100 Subject: New upstream version 1.8.1 --- src/modules/common/Makefile.am | 18 +- src/modules/common/bz2comprs.cpp | 57 ++--- src/modules/common/entriesblk.cpp | 6 +- src/modules/common/rawstr.cpp | 32 +-- src/modules/common/rawstr4.cpp | 36 +-- src/modules/common/rawverse.cpp | 4 +- src/modules/common/rawverse4.cpp | 6 +- src/modules/common/swcomprs.cpp | 3 +- src/modules/common/xzcomprs.cpp | 130 ++++++---- src/modules/common/zipcomprs.cpp | 34 +-- src/modules/common/zstr.cpp | 42 ++-- src/modules/common/zverse.cpp | 19 +- src/modules/common/zverse4.cpp | 516 ++++++++++++++++++++++++++++++++++++++ 13 files changed, 718 insertions(+), 185 deletions(-) create mode 100644 src/modules/common/zverse4.cpp (limited to 'src/modules/common') diff --git a/src/modules/common/Makefile.am b/src/modules/common/Makefile.am index 90a3f98..0bca03e 100644 --- a/src/modules/common/Makefile.am +++ b/src/modules/common/Makefile.am @@ -6,18 +6,22 @@ libsword_la_SOURCES += $(commondir)/swcomprs.cpp libsword_la_SOURCES += $(commondir)/lzsscomprs.cpp if HAVE_LIBZ -SWZLIB = $(commondir)/zipcomprs.cpp -SWZLIB += $(commondir)/bz2comprs.cpp -SWZLIB += $(commondir)/xzcomprs.cpp -else -SWZLIB = +libsword_la_SOURCES += $(commondir)/zipcomprs.cpp endif -libsword_la_SOURCES += $(SWZLIB) + +if HAVE_BZIP2 +libsword_la_SOURCES += $(commondir)/bz2comprs.cpp +endif + +if HAVE_XZ +libsword_la_SOURCES += $(commondir)/xzcomprs.cpp +endif + libsword_la_SOURCES += $(commondir)/rawverse.cpp libsword_la_SOURCES += $(commondir)/rawverse4.cpp libsword_la_SOURCES += $(commondir)/swcipher.cpp libsword_la_SOURCES += $(commondir)/zverse.cpp +libsword_la_SOURCES += $(commondir)/zverse4.cpp libsword_la_SOURCES += $(commondir)/zstr.cpp libsword_la_SOURCES += $(commondir)/entriesblk.cpp libsword_la_SOURCES += $(commondir)/sapphire.cpp - diff --git a/src/modules/common/bz2comprs.cpp b/src/modules/common/bz2comprs.cpp index 7863e98..6b9374e 100644 --- a/src/modules/common/bz2comprs.cpp +++ b/src/modules/common/bz2comprs.cpp @@ -3,7 +3,7 @@ * bz2comprs.cpp - Bzip2Compress, a driver class that provides bzip2 * compression (Burrows–Wheeler with Huffman coding) * - * $Id: bz2comprs.cpp 3156 2014-04-17 03:50:37Z greg.hellings $ + * $Id: bz2comprs.cpp 3515 2017-11-01 11:38:09Z scribe $ * * Copyright 2000-2014 CrossWire Bible Society (http://www.crosswire.org) * CrossWire Bible Society @@ -26,7 +26,7 @@ #include #include #include -#include +#include SWORD_NAMESPACE_START @@ -36,6 +36,7 @@ SWORD_NAMESPACE_START */ Bzip2Compress::Bzip2Compress() : SWCompress() { + level = 9; } @@ -59,20 +60,6 @@ Bzip2Compress::~Bzip2Compress() { void Bzip2Compress::Encode(void) { -/* -ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, - const Bytef *source, uLong sourceLen)); - Compresses the source buffer into the destination buffer. sourceLen is - the byte length of the source buffer. Upon entry, destLen is the total - size of the destination buffer, which must be at least 0.1% larger than - sourceLen plus 12 bytes. Upon exit, destLen is the actual size of the - compressed buffer. - This function can be used to compress a whole file at once if the - input file is mmap'ed. - compress returns Z_OK if success, Z_MEM_ERROR if there was not - enough memory, Z_BUF_ERROR if there was not enough room in the output - buffer. -*/ direct = 0; // set direction needed by parent [Get|Send]Chars() // get buffer @@ -91,12 +78,12 @@ ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, } - zlen = (long) (len*1.001)+15; + zlen = (long) (len*1.01)+600; char *zbuf = new char[zlen+1]; if (len) { //printf("Doing compress\n"); - if (compress((Bytef*)zbuf, &zlen, (const Bytef*)buf, len) != Z_OK) + if (BZ2_bzBuffToBuffCompress(zbuf, (unsigned int*)&zlen, buf, (unsigned int)len, level, 0, 0) != BZ_OK) { printf("ERROR in compression\n"); } @@ -123,23 +110,7 @@ ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, void Bzip2Compress::Decode(void) { -/* -ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, - const Bytef *source, uLong sourceLen)); - Decompresses the source buffer into the destination buffer. sourceLen is - the byte length of the source buffer. Upon entry, destLen is the total - size of the destination buffer, which must be large enough to hold the - entire uncompressed data. (The size of the uncompressed data must have - been saved previously by the compressor and transmitted to the decompressor - by some mechanism outside the scope of this compression library.) - Upon exit, destLen is the actual size of the compressed buffer. - This function can be used to decompress a whole file at once if the - input file is mmap'ed. - - uncompress returns Z_OK if success, Z_MEM_ERROR if there was not - enough memory, Z_BUF_ERROR if there was not enough room in the output - buffer, or Z_DATA_ERROR if the input data was corrupted. -*/ + direct = 1; // set direction needed by parent [Get|Send]Chars() // get buffer char chunk[1024]; @@ -147,7 +118,7 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, char *chunkbuf = zbuf; int chunklen; unsigned long zlen = 0; - while((chunklen = GetChars(chunk, 1023))) { + while((chunklen = (int)GetChars(chunk, 1023))) { memcpy(chunkbuf, chunk, chunklen); zlen += chunklen; if (chunklen < 1023) @@ -158,16 +129,16 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, //printf("Decoding complength{%ld} uncomp{%ld}\n", zlen, blen); if (zlen) { - unsigned long blen = zlen*20; // trust compression is less than 1000% + unsigned int blen = (unsigned int)(zlen*20); // trust compression is less than 1000% char *buf = new char[blen]; //printf("Doing decompress {%s}\n", zbuf); slen = 0; - switch (uncompress((Bytef*)buf, &blen, (Bytef*)zbuf, zlen)){ - case Z_OK: SendChars(buf, blen); slen = blen; break; - case Z_MEM_ERROR: fprintf(stderr, "ERROR: not enough memory during decompression.\n"); break; - case Z_BUF_ERROR: fprintf(stderr, "ERROR: not enough room in the out buffer during decompression.\n"); break; - case Z_DATA_ERROR: fprintf(stderr, "ERROR: corrupt data during decompression.\n"); break; - default: fprintf(stderr, "ERROR: an unknown error occured during decompression.\n"); break; + switch (BZ2_bzBuffToBuffDecompress(buf, &blen, zbuf, (unsigned int)zlen, 0, 0)){ + case BZ_OK: SendChars(buf, blen); slen = blen; break; + case BZ_MEM_ERROR: fprintf(stderr, "ERROR: not enough memory during decompression.\n"); break; + case BZ_OUTBUFF_FULL: fprintf(stderr, "ERROR: not enough room in the out buffer during decompression.\n"); break; + case BZ_DATA_ERROR: fprintf(stderr, "ERROR: corrupt data during decompression.\n"); break; + default: fprintf(stderr, "ERROR: an unknown error occurred during decompression.\n"); break; } delete [] buf; } diff --git a/src/modules/common/entriesblk.cpp b/src/modules/common/entriesblk.cpp index 4872d28..85b7e0d 100644 --- a/src/modules/common/entriesblk.cpp +++ b/src/modules/common/entriesblk.cpp @@ -2,7 +2,7 @@ * * entriesblk.cpp - EntriesBlock facilitates compressed lex/dict modules * - * $Id: entriesblk.cpp 2833 2013-06-29 06:40:28Z chrislit $ + * $Id: entriesblk.cpp 3439 2016-10-23 08:32:02Z scribe $ * * Copyright 2001-2013 CrossWire Bible Society (http://www.crosswire.org) * CrossWire Bible Society @@ -84,8 +84,8 @@ void EntriesBlock::getMetaEntry(int index, unsigned long *offset, unsigned long void EntriesBlock::setMetaEntry(int index, unsigned long offset, unsigned long size) { - __u32 rawOffset = archtosword32(offset); - __u32 rawSize = archtosword32(size); + __u32 rawOffset = (__u32)archtosword32(offset); + __u32 rawSize = (__u32)archtosword32(size); if (index >= getCount()) // assert index < count return; diff --git a/src/modules/common/rawstr.cpp b/src/modules/common/rawstr.cpp index 12371ab..7f5015f 100644 --- a/src/modules/common/rawstr.cpp +++ b/src/modules/common/rawstr.cpp @@ -5,7 +5,7 @@ * and provides lookup and parsing functions based on * class StrKey * - * $Id: rawstr.cpp 3181 2014-04-17 04:27:57Z greg.hellings $ + * $Id: rawstr.cpp 3524 2017-11-07 03:08:49Z scribe $ * * Copyright 1998-2013 CrossWire Bible Society (http://www.crosswire.org) * CrossWire Bible Society @@ -74,8 +74,10 @@ RawStr::RawStr(const char *ipath, int fileMode, bool caseSensitive) : caseSensit buf.setFormatted("%s.dat", path); datfd = FileMgr::getSystemFileMgr()->open(buf, fileMode, true); - if (datfd < 0) { - SWLog::getSystemLog()->logError("%d", errno); + if (!datfd || datfd->getFd() < 0) { + // couldn't find datafile but this might be fine if we're + // merely instantiating a remote InstallMgr SWMgr + SWLog::getSystemLog()->logDebug("Couldn't open file: %s. errno: %d", buf.c_str(), errno); } instance++; @@ -111,7 +113,7 @@ void RawStr::getIDXBufDat(long ioffset, char **buf) const { int size; char ch; - if (datfd > 0) { + if (datfd && datfd->getFd() >= 0) { datfd->seek(ioffset, SEEK_SET); for (size = 0; datfd->read(&ch, 1) == 1; size++) { if ((ch == '\\') || (ch == 10) || (ch == 13)) @@ -145,7 +147,7 @@ void RawStr::getIDXBuf(long ioffset, char **buf) const { __u32 offset; - if (idxfd > 0) { + if (idxfd && idxfd->getFd() >= 0) { idxfd->seek(ioffset, SEEK_SET); idxfd->read(&offset, 4); @@ -183,9 +185,9 @@ signed char RawStr::findOffset(const char *ikey, __u32 *start, __u16 *size, long headoff = 0; stdstr(&key, ikey, 3); - if (!caseSensitive) toupperstr_utf8(key, strlen(key)*3); + if (!caseSensitive) toupperstr_utf8(key, (unsigned int)(strlen(key)*3)); - int keylen = strlen(key); + int keylen = (int)strlen(key); bool substr = false; trybuf = maxbuf = 0; @@ -243,7 +245,7 @@ signed char RawStr::findOffset(const char *ikey, __u32 *start, __u16 *size, long idxfd->read(&tmpStart, 4); idxfd->read(&tmpSize, 2); if (idxoff) - *idxoff = tryoff; + *idxoff = (__u32)tryoff; *start = swordtoarch32(tmpStart); *size = swordtoarch16(tmpSize); @@ -262,17 +264,17 @@ signed char RawStr::findOffset(const char *ikey, __u32 *start, __u16 *size, long if (bad) { if(!awayFromSubstrCheck) retval = -1; - *start = laststart; + *start = (__u32)laststart; *size = lastsize; tryoff = lasttry; if (idxoff) - *idxoff = tryoff; + *idxoff = (__u32)tryoff; break; } idxfd->read(&tmpStart, 4); idxfd->read(&tmpSize, 2); if (idxoff) - *idxoff = tryoff; + *idxoff = (__u32)tryoff; *start = swordtoarch32(tmpStart); *size = swordtoarch16(tmpSize); @@ -346,7 +348,7 @@ void RawStr::readText(__u32 istart, __u16 *isize, char **idxbuf, SWBuf &buf) con while (true); // while we're resolving links if (idxbuflocal) { - int localsize = strlen(idxbuflocal); + int localsize = (int)strlen(idxbuflocal); localsize = (localsize < (*isize - 1)) ? localsize : (*isize - 1); strncpy(*idxbuf, idxbuflocal, localsize); (*idxbuf)[localsize] = 0; @@ -381,7 +383,7 @@ void RawStr::doSetText(const char *ikey, const char *buf, long len) char errorStatus = findOffset(ikey, &start, &size, 0, &idxoff); stdstr(&key, ikey, 2); - if (!caseSensitive) toupperstr_utf8(key, strlen(key)*2); + if (!caseSensitive) toupperstr_utf8(key, (unsigned int)(strlen(key)*2)); len = (len < 0) ? strlen(buf) : len; @@ -424,7 +426,7 @@ void RawStr::doSetText(const char *ikey, const char *buf, long len) while (true); // while we're resolving links } - endoff = idxfd->seek(0, SEEK_END); + endoff = (__u32)idxfd->seek(0, SEEK_END); shiftSize = endoff - idxoff; @@ -440,7 +442,7 @@ void RawStr::doSetText(const char *ikey, const char *buf, long len) memcpy(outbuf + size, buf, len); size = outsize = size + (len); - start = outstart = datfd->seek(0, SEEK_END); + start = outstart = (__u32)datfd->seek(0, SEEK_END); outstart = archtosword32(start); outsize = archtosword16(size); diff --git a/src/modules/common/rawstr4.cpp b/src/modules/common/rawstr4.cpp index c88e70d..0b54c00 100644 --- a/src/modules/common/rawstr4.cpp +++ b/src/modules/common/rawstr4.cpp @@ -5,7 +5,7 @@ * and provides lookup and parsing functions based on * class StrKey * - * $Id: rawstr4.cpp 3181 2014-04-17 04:27:57Z greg.hellings $ + * $Id: rawstr4.cpp 3524 2017-11-07 03:08:49Z scribe $ * * Copyright 2001-2013 CrossWire Bible Society (http://www.crosswire.org) * CrossWire Bible Society @@ -74,8 +74,10 @@ RawStr4::RawStr4(const char *ipath, int fileMode, bool caseSensitive) : caseSens buf.setFormatted("%s.dat", path); datfd = FileMgr::getSystemFileMgr()->open(buf, fileMode, true); - if (datfd < 0) { - SWLog::getSystemLog()->logError("%d", errno); + if (!datfd || datfd->getFd() < 0) { + // couldn't find datafile but this might be fine if we're + // merely instantiating a remote InstallMgr SWMgr + SWLog::getSystemLog()->logDebug("Couldn't open file: %s. errno: %d", buf.c_str(), errno); } instance++; @@ -111,7 +113,7 @@ void RawStr4::getIDXBufDat(long ioffset, char **buf) const { int size; char ch; - if (datfd > 0) { + if ((unsigned long)datfd > 0) { datfd->seek(ioffset, SEEK_SET); for (size = 0; datfd->read(&ch, 1) == 1; size++) { if ((ch == '\\') || (ch == 10) || (ch == 13)) @@ -145,7 +147,7 @@ void RawStr4::getIDXBuf(long ioffset, char **buf) const { __u32 offset; - if (idxfd > 0) { + if ((unsigned long)idxfd > 0) { idxfd->seek(ioffset, SEEK_SET); idxfd->read(&offset, 4); @@ -193,9 +195,9 @@ signed char RawStr4::findOffset(const char *ikey, __u32 *start, __u32 *size, lon headoff = 0; stdstr(&key, ikey, 3); - if (!caseSensitive) toupperstr_utf8(key, strlen(key)*3); + if (!caseSensitive) toupperstr_utf8(key, (unsigned int)(strlen(key)*3)); - int keylen = strlen(key); + int keylen = (int)strlen(key); bool substr = false; trybuf = maxbuf = 0; @@ -252,7 +254,7 @@ signed char RawStr4::findOffset(const char *ikey, __u32 *start, __u32 *size, lon idxfd->read(&tmpStart, 4); idxfd->read(&tmpSize, 4); if (idxoff) - *idxoff = tryoff; + *idxoff = (__u32)tryoff; *start = swordtoarch32(tmpStart); *size = swordtoarch32(tmpSize); @@ -271,17 +273,17 @@ signed char RawStr4::findOffset(const char *ikey, __u32 *start, __u32 *size, lon if (bad) { if(!awayFromSubstrCheck) retval = -1; - *start = laststart; - *size = lastsize; + *start = (__u32)laststart; + *size = (__u32)lastsize; tryoff = lasttry; if (idxoff) - *idxoff = tryoff; + *idxoff = (__u32)tryoff; break; } idxfd->read(&tmpStart, 4); idxfd->read(&tmpSize, 4); if (idxoff) - *idxoff = tryoff; + *idxoff = (__u32)tryoff; *start = swordtoarch32(tmpStart); *size = swordtoarch32(tmpSize); @@ -355,7 +357,7 @@ void RawStr4::readText(__u32 istart, __u32 *isize, char **idxbuf, SWBuf &buf) co while (true); // while we're resolving links if (idxbuflocal) { - unsigned int localsize = strlen(idxbuflocal); + unsigned int localsize = (unsigned int)strlen(idxbuflocal); localsize = (localsize < (*isize - 1)) ? localsize : (*isize - 1); strncpy(*idxbuf, idxbuflocal, localsize); (*idxbuf)[localsize] = 0; @@ -389,7 +391,7 @@ void RawStr4::doSetText(const char *ikey, const char *buf, long len) { char errorStatus = findOffset(ikey, &start, &size, 0, &idxoff); stdstr(&key, ikey, 3); - if (!caseSensitive) toupperstr_utf8(key, strlen(key)*3); + if (!caseSensitive) toupperstr_utf8(key, (unsigned int)(strlen(key)*3)); len = (len < 0) ? strlen(buf) : len; getIDXBufDat(start, &dbKey); @@ -432,7 +434,7 @@ void RawStr4::doSetText(const char *ikey, const char *buf, long len) { while (true); // while we're resolving links } - endoff = idxfd->seek(0, SEEK_END); + endoff = (__u32)idxfd->seek(0, SEEK_END); shiftSize = endoff - idxoff; @@ -446,9 +448,9 @@ void RawStr4::doSetText(const char *ikey, const char *buf, long len) { sprintf(outbuf, "%s%c%c", key, 13, 10); size = strlen(outbuf); memcpy(outbuf + size, buf, len); - size = outsize = size + len; + size = outsize = size + (__u32)len; - start = outstart = datfd->seek(0, SEEK_END); + start = outstart = (__u32)datfd->seek(0, SEEK_END); outstart = archtosword32(start); outsize = archtosword32(size); diff --git a/src/modules/common/rawverse.cpp b/src/modules/common/rawverse.cpp index 34fba10..a4e8008 100644 --- a/src/modules/common/rawverse.cpp +++ b/src/modules/common/rawverse.cpp @@ -5,7 +5,7 @@ * and provides lookup and parsing functions based on * class VerseKey * - * $Id: rawverse.cpp 3181 2014-04-17 04:27:57Z greg.hellings $ + * $Id: rawverse.cpp 3439 2016-10-23 08:32:02Z scribe $ * * Copyright 1997-2013 CrossWire Bible Society (http://www.crosswire.org) * CrossWire Bible Society @@ -186,7 +186,7 @@ void RawVerse::doSetText(char testmt, long idxoff, const char *buf, long len) size = (len < 0) ? strlen(buf) : len; - start = textfp[testmt-1]->seek(0, SEEK_END); + start = (__s32)textfp[testmt-1]->seek(0, SEEK_END); idxfp[testmt-1]->seek(idxoff, SEEK_SET); if (size) { diff --git a/src/modules/common/rawverse4.cpp b/src/modules/common/rawverse4.cpp index 100ef6b..92775b7 100644 --- a/src/modules/common/rawverse4.cpp +++ b/src/modules/common/rawverse4.cpp @@ -6,7 +6,7 @@ * and provides lookup and parsing functions based on * class VerseKey * - * $Id: rawverse4.cpp 3182 2014-04-17 04:28:36Z greg.hellings $ + * $Id: rawverse4.cpp 3439 2016-10-23 08:32:02Z scribe $ * * Copyright 2007-2013 CrossWire Bible Society (http://www.crosswire.org) * CrossWire Bible Society @@ -185,9 +185,9 @@ void RawVerse4::doSetText(char testmt, long idxoff, const char *buf, long len) if (!testmt) testmt = ((idxfp[1]) ? 1:2); - size = (len < 0) ? strlen(buf) : len; + size = (__u32)((len < 0) ? strlen(buf) : len); - start = textfp[testmt-1]->seek(0, SEEK_END); + start = (__u32)textfp[testmt-1]->seek(0, SEEK_END); idxfp[testmt-1]->seek(idxoff, SEEK_SET); if (size) { diff --git a/src/modules/common/swcomprs.cpp b/src/modules/common/swcomprs.cpp index a9ac72e..ef7e27f 100644 --- a/src/modules/common/swcomprs.cpp +++ b/src/modules/common/swcomprs.cpp @@ -2,7 +2,7 @@ * * swcomprs.cpp - a driver class that provides compression utilities * - * $Id: swcomprs.cpp 3175 2014-04-17 04:21:31Z greg.hellings $ + * $Id: swcomprs.cpp 3121 2014-03-13 09:44:43Z chrislit $ * * Copyright 1996-2014 CrossWire Bible Society (http://www.crosswire.org) * CrossWire Bible Society @@ -35,6 +35,7 @@ SWORD_NAMESPACE_START SWCompress::SWCompress() { buf = zbuf = 0; + level = 6; Init(); } diff --git a/src/modules/common/xzcomprs.cpp b/src/modules/common/xzcomprs.cpp index 513e170..d29057d 100644 --- a/src/modules/common/xzcomprs.cpp +++ b/src/modules/common/xzcomprs.cpp @@ -3,7 +3,7 @@ * xzcomprs.cpp - XzCompress, a driver class that provides xz (LZMA2) * compression * - * $Id: xzcomprs.cpp 3156 2014-04-17 03:50:37Z greg.hellings $ + * $Id: xzcomprs.cpp 3515 2017-11-01 11:38:09Z scribe $ * * Copyright 2000-2014 CrossWire Bible Society (http://www.crosswire.org) * CrossWire Bible Society @@ -21,12 +21,13 @@ * */ - #include #include #include #include -#include + +#define LZMA_API_STATIC +#include SWORD_NAMESPACE_START @@ -36,6 +37,24 @@ SWORD_NAMESPACE_START */ XzCompress::XzCompress() : SWCompress() { + level = 3; + + // start with the estimated memory usage for our preset + memlimit = lzma_easy_decoder_memusage(level | LZMA_PRESET_EXTREME); + + // and round up to a power of 2-- + // bit twiddle hack to determine next greatest power of 2 from: + // http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 + memlimit--; + memlimit |= memlimit >> 1; + memlimit |= memlimit >> 2; + memlimit |= memlimit >> 4; + memlimit |= memlimit >> 8; + memlimit |= memlimit >> 16; + memlimit++; + + // double that for safety's sake + memlimit <<= 1; } @@ -59,20 +78,6 @@ XzCompress::~XzCompress() { void XzCompress::Encode(void) { -/* -ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, - const Bytef *source, uLong sourceLen)); - Compresses the source buffer into the destination buffer. sourceLen is - the byte length of the source buffer. Upon entry, destLen is the total - size of the destination buffer, which must be at least 0.1% larger than - sourceLen plus 12 bytes. Upon exit, destLen is the actual size of the - compressed buffer. - This function can be used to compress a whole file at once if the - input file is mmap'ed. - compress returns Z_OK if success, Z_MEM_ERROR if there was not - enough memory, Z_BUF_ERROR if there was not enough room in the output - buffer. -*/ direct = 0; // set direction needed by parent [Get|Send]Chars() // get buffer @@ -90,18 +95,22 @@ ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, chunkbuf = buf+len; } - - zlen = (long) (len*1.001)+15; + zlen = (long)lzma_stream_buffer_bound(len); char *zbuf = new char[zlen+1]; + size_t zpos = 0; + if (len) { //printf("Doing compress\n"); - if (compress((Bytef*)zbuf, &zlen, (const Bytef*)buf, len) != Z_OK) - { - printf("ERROR in compression\n"); - } - else { - SendChars(zbuf, zlen); + switch (lzma_easy_buffer_encode(level | LZMA_PRESET_EXTREME, LZMA_CHECK_CRC64, NULL, (const uint8_t*)buf, (size_t)len, (uint8_t*)zbuf, &zpos, (size_t)zlen)) { + case LZMA_OK: SendChars(zbuf, zpos); break; + case LZMA_BUF_ERROR: fprintf(stderr, "ERROR: not enough room in the out buffer during compression.\n"); break; + case LZMA_UNSUPPORTED_CHECK: fprintf(stderr, "ERROR: unsupported_check error encountered during decompression.\n"); break; + case LZMA_OPTIONS_ERROR: fprintf(stderr, "ERROR: options error encountered during decompression.\n"); break; + case LZMA_MEM_ERROR: fprintf(stderr, "ERROR: not enough memory during compression.\n"); break; + case LZMA_DATA_ERROR: fprintf(stderr, "ERROR: corrupt data during compression.\n"); break; + case LZMA_PROG_ERROR: fprintf(stderr, "ERROR: program error encountered during decompression.\n"); break; + default: fprintf(stderr, "ERROR: an unknown error occurred during compression.\n"); break; } } else @@ -123,23 +132,7 @@ ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, void XzCompress::Decode(void) { -/* -ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, - const Bytef *source, uLong sourceLen)); - Decompresses the source buffer into the destination buffer. sourceLen is - the byte length of the source buffer. Upon entry, destLen is the total - size of the destination buffer, which must be large enough to hold the - entire uncompressed data. (The size of the uncompressed data must have - been saved previously by the compressor and transmitted to the decompressor - by some mechanism outside the scope of this compression library.) - Upon exit, destLen is the actual size of the compressed buffer. - This function can be used to decompress a whole file at once if the - input file is mmap'ed. - - uncompress returns Z_OK if success, Z_MEM_ERROR if there was not - enough memory, Z_BUF_ERROR if there was not enough room in the output - buffer, or Z_DATA_ERROR if the input data was corrupted. -*/ + direct = 1; // set direction needed by parent [Get|Send]Chars() // get buffer char chunk[1024]; @@ -158,16 +151,25 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, //printf("Decoding complength{%ld} uncomp{%ld}\n", zlen, blen); if (zlen) { - unsigned long blen = zlen*20; // trust compression is less than 1000% + unsigned long blen = zlen*20; // trust compression is less than 2000% char *buf = new char[blen]; //printf("Doing decompress {%s}\n", zbuf); slen = 0; - switch (uncompress((Bytef*)buf, &blen, (Bytef*)zbuf, zlen)){ - case Z_OK: SendChars(buf, blen); slen = blen; break; - case Z_MEM_ERROR: fprintf(stderr, "ERROR: not enough memory during decompression.\n"); break; - case Z_BUF_ERROR: fprintf(stderr, "ERROR: not enough room in the out buffer during decompression.\n"); break; - case Z_DATA_ERROR: fprintf(stderr, "ERROR: corrupt data during decompression.\n"); break; - default: fprintf(stderr, "ERROR: an unknown error occured during decompression.\n"); break; + size_t zpos = 0; + size_t bpos = 0; + + switch (lzma_stream_buffer_decode((uint64_t *)&memlimit, 0, NULL, (const uint8_t*)zbuf, &zpos, (size_t)zlen, (uint8_t*)buf, &bpos, (size_t)&blen)){ + case LZMA_OK: SendChars(buf, bpos); slen = bpos; break; + case LZMA_FORMAT_ERROR: fprintf(stderr, "ERROR: format error encountered during decompression.\n"); break; + case LZMA_OPTIONS_ERROR: fprintf(stderr, "ERROR: options error encountered during decompression.\n"); break; + case LZMA_DATA_ERROR: fprintf(stderr, "ERROR: corrupt data during decompression.\n"); break; + case LZMA_NO_CHECK: fprintf(stderr, "ERROR: no_check error encountered during decompression.\n"); break; + case LZMA_UNSUPPORTED_CHECK: fprintf(stderr, "ERROR: unsupported_check error encountered during decompression.\n"); break; + case LZMA_MEMLIMIT_ERROR: fprintf(stderr, "ERROR: memlimit error encountered during decompression.\n"); break; + case LZMA_MEM_ERROR: fprintf(stderr, "ERROR: not enough memory during decompression.\n"); break; + case LZMA_BUF_ERROR: fprintf(stderr, "ERROR: not enough room in the out buffer during decompression.\n"); break; + case LZMA_PROG_ERROR: fprintf(stderr, "ERROR: program error encountered during decompression.\n"); break; + default: fprintf(stderr, "ERROR: an unknown error occurred during decompression.\n"); break; } delete [] buf; } @@ -178,4 +180,34 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, free (zbuf); } + +/****************************************************************************** + * XzCompress::SetLevel - This function sets the compression level of the + * compressor. + */ + +void XzCompress::setLevel(int l) { + level = l; + + // having changed the compression level, we need to adjust our memlimit accordingly, + // as in the constructor: + + // start with the estimated memory usage for our preset + memlimit = lzma_easy_decoder_memusage(level | LZMA_PRESET_EXTREME); + + // and round up to a power of 2-- + // bit twiddle hack to determine next greatest power of 2 from: + // http://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2 + memlimit--; + memlimit |= memlimit >> 1; + memlimit |= memlimit >> 2; + memlimit |= memlimit >> 4; + memlimit |= memlimit >> 8; + memlimit |= memlimit >> 16; + memlimit++; + + // double that for safety's sake + memlimit <<= 1; +}; + SWORD_NAMESPACE_END diff --git a/src/modules/common/zipcomprs.cpp b/src/modules/common/zipcomprs.cpp index 28471d8..557239e 100644 --- a/src/modules/common/zipcomprs.cpp +++ b/src/modules/common/zipcomprs.cpp @@ -3,7 +3,7 @@ * zipcomprs.cpp - ZipCompress, a driver class that provides zlib * compression * - * $Id: zipcomprs.cpp 3175 2014-04-17 04:21:31Z greg.hellings $ + * $Id: zipcomprs.cpp 3515 2017-11-01 11:38:09Z scribe $ * * Copyright 2000-2014 CrossWire Bible Society (http://www.crosswire.org) * CrossWire Bible Society @@ -38,6 +38,7 @@ SWORD_NAMESPACE_START ZipCompress::ZipCompress() : SWCompress() { // fprintf(stderr, "init compress\n"); + level = Z_DEFAULT_COMPRESSION; } @@ -62,18 +63,20 @@ ZipCompress::~ZipCompress() { void ZipCompress::Encode(void) { /* -ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, - const Bytef *source, uLong sourceLen)); - Compresses the source buffer into the destination buffer. sourceLen is - the byte length of the source buffer. Upon entry, destLen is the total - size of the destination buffer, which must be at least 0.1% larger than - sourceLen plus 12 bytes. Upon exit, destLen is the actual size of the +ZEXTERN int ZEXPORT compress2 OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen, + int level)); + + Compresses the source buffer into the destination buffer. The level + parameter has the same meaning as in deflateInit. sourceLen is the byte + length of the source buffer. Upon entry, destLen is the total size of the + destination buffer, which must be at least the value returned by + compressBound(sourceLen). Upon exit, destLen is the actual size of the compressed buffer. - This function can be used to compress a whole file at once if the - input file is mmap'ed. - compress returns Z_OK if success, Z_MEM_ERROR if there was not - enough memory, Z_BUF_ERROR if there was not enough room in the output - buffer. + + compress2 returns Z_OK if success, Z_MEM_ERROR if there was not enough + memory, Z_BUF_ERROR if there was not enough room in the output buffer, + Z_STREAM_ERROR if the level parameter is invalid. */ direct = 0; // set direction needed by parent [Get|Send]Chars() @@ -98,7 +101,7 @@ ZEXTERN int ZEXPORT compress OF((Bytef *dest, uLongf *destLen, if (len) { //printf("Doing compress\n"); - if (compress((Bytef*)zbuf, &zlen, (const Bytef*)buf, len) != Z_OK) + if (compress2((Bytef*)zbuf, &zlen, (const Bytef*)buf, len, level) != Z_OK) { printf("ERROR in compression\n"); } @@ -142,6 +145,7 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, enough memory, Z_BUF_ERROR if there was not enough room in the output buffer, or Z_DATA_ERROR if the input data was corrupted. */ + direct = 1; // set direction needed by parent [Get|Send]Chars() // get buffer char chunk[1024]; @@ -149,7 +153,7 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, char *chunkbuf = zbuf; int chunklen; unsigned long zlen = 0; - while((chunklen = GetChars(chunk, 1023))) { + while((chunklen = (int)GetChars(chunk, 1023))) { memcpy(chunkbuf, chunk, chunklen); zlen += chunklen; if (chunklen < 1023) @@ -169,7 +173,7 @@ ZEXTERN int ZEXPORT uncompress OF((Bytef *dest, uLongf *destLen, case Z_MEM_ERROR: fprintf(stderr, "ERROR: not enough memory during decompression.\n"); break; case Z_BUF_ERROR: fprintf(stderr, "ERROR: not enough room in the out buffer during decompression.\n"); break; case Z_DATA_ERROR: fprintf(stderr, "ERROR: corrupt data during decompression.\n"); break; - default: fprintf(stderr, "ERROR: an unknown error occured during decompression.\n"); break; + default: fprintf(stderr, "ERROR: an unknown error occurred during decompression.\n"); break; } delete [] buf; } diff --git a/src/modules/common/zstr.cpp b/src/modules/common/zstr.cpp index a745502..0b4f0f6 100644 --- a/src/modules/common/zstr.cpp +++ b/src/modules/common/zstr.cpp @@ -4,7 +4,7 @@ * files and provides lookup and parsing functions based on * class StrKey * - * $Id: zstr.cpp 2980 2013-09-14 21:51:47Z scribe $ + * $Id: zstr.cpp 3524 2017-11-07 03:08:49Z scribe $ * * Copyright 2001-2013 CrossWire Bible Society (http://www.crosswire.org) * CrossWire Bible Society @@ -83,8 +83,10 @@ zStr::zStr(const char *ipath, int fileMode, long blockCount, SWCompress *icomp, buf.setFormatted("%s.zdt", path); zdtfd = FileMgr::getSystemFileMgr()->open(buf, fileMode, true); - if (datfd <= 0) { - SWLog::getSystemLog()->logError("%d", errno); + if (!zdtfd || zdtfd->getFd() < 0) { + // couldn't find datafile but this might be fine if we're + // merely instantiating a remote InstallMgr SWMgr + SWLog::getSystemLog()->logDebug("Couldn't open file: %s. errno: %d", buf.c_str(), errno); } cacheBlock = 0; @@ -133,7 +135,7 @@ void zStr::getKeyFromDatOffset(long ioffset, char **buf) const { int size; char ch; - if (datfd > 0) { + if (datfd && datfd->getFd() >= 0) { datfd->seek(ioffset, SEEK_SET); for (size = 0; datfd->read(&ch, 1) == 1; size++) { if ((ch == '\\') || (ch == 10) || (ch == 13)) @@ -167,7 +169,7 @@ void zStr::getKeyFromIdxOffset(long ioffset, char **buf) const { __u32 offset; - if (idxfd > 0) { + if (idxfd && idxfd->getFd() >= 0) { idxfd->seek(ioffset, SEEK_SET); idxfd->read(&offset, 4); offset = swordtoarch32(offset); @@ -198,19 +200,19 @@ signed char zStr::findKeyIndex(const char *ikey, long *idxoff, long away) const bool awayFromSubstrCheck = false; if (idxfd->getFd() >= 0) { - tailoff = maxoff = idxfd->seek(0, SEEK_END) - IDXENTRYSIZE; + tailoff = maxoff = (__s32)idxfd->seek(0, SEEK_END) - IDXENTRYSIZE; if (*ikey) { headoff = 0; stdstr(&key, ikey, 3); - if (!caseSensitive) toupperstr_utf8(key, strlen(key)*3); + if (!caseSensitive) toupperstr_utf8(key, (unsigned int)(strlen(key)*3)); - int keylen = strlen(key); + int keylen = (int)strlen(key); bool substr = false; getKeyFromIdxOffset(maxoff, &maxbuf); while (headoff < tailoff) { - tryoff = (lastoff == -1) ? headoff + (((((tailoff / IDXENTRYSIZE) - (headoff / IDXENTRYSIZE))) / 2) * IDXENTRYSIZE) : lastoff; + tryoff = ((__s32)lastoff == -1) ? headoff + (((((tailoff / IDXENTRYSIZE) - (headoff / IDXENTRYSIZE))) / 2) * IDXENTRYSIZE) : (__s32)lastoff; lastoff = -1; getKeyFromIdxOffset(tryoff, &trybuf); @@ -295,7 +297,7 @@ signed char zStr::findKeyIndex(const char *ikey, long *idxoff, long away) const *idxoff = tryoff; - if (((laststart != start) || (lastsize != size)) && (start >= 0) && (size)) + if (((laststart != start) || (lastsize != size)) && size) away += (away < 0) ? 1 : -1; } @@ -366,7 +368,7 @@ void zStr::getText(long offset, char **idxbuf, char **buf) const { while (true); // while we're resolving links if (idxbuflocal) { - __u32 localsize = strlen(idxbuflocal); + __u32 localsize = (__u32)strlen(idxbuflocal); localsize = (localsize < (size - 1)) ? localsize : (size - 1); strncpy(*idxbuf, idxbuflocal, localsize); (*idxbuf)[localsize] = 0; @@ -416,7 +418,7 @@ void zStr::getCompressedText(long block, long entry, char **buf) const { cacheBlock = new EntriesBlock(rawBuf, len); cacheBlockIndex = block; } - size = cacheBlock->getEntrySize(entry); + size = (__u32)cacheBlock->getEntrySize(entry); *buf = (*buf) ? (char *)realloc(*buf, size*2 + 1) : (char *)malloc(size*2 + 1); strcpy(*buf, cacheBlock->getEntry(entry)); } @@ -448,7 +450,7 @@ void zStr::setText(const char *ikey, const char *buf, long len) { len = (len < 0) ? strlen(buf) : len; stdstr(&key, ikey, 3); - if (!caseSensitive) toupperstr_utf8(key, strlen(key)*3); + if (!caseSensitive) toupperstr_utf8(key, (unsigned int)(strlen(key)*3)); char notFound = findKeyIndex(ikey, &idxoff, 0); if (!notFound) { @@ -497,9 +499,9 @@ void zStr::setText(const char *ikey, const char *buf, long len) { } } - endoff = idxfd->seek(0, SEEK_END); + endoff = (__s32)idxfd->seek(0, SEEK_END); - shiftSize = endoff - idxoff; + shiftSize = endoff - (__s32)idxoff; if (shiftSize > 0) { idxBytes = new char [ shiftSize ]; @@ -509,7 +511,7 @@ void zStr::setText(const char *ikey, const char *buf, long len) { outbuf = new char [ len + strlen(key) + 5 ]; sprintf(outbuf, "%s%c%c", key, 13, 10); - size = strlen(outbuf); + size = (__u32)strlen(outbuf); if (len > 0) { // NOT a link if (!cacheBlock) { flushCache(); @@ -523,7 +525,7 @@ void zStr::setText(const char *ikey, const char *buf, long len) { } __u32 entry = cacheBlock->addEntry(buf); cacheDirty = true; - outstart = archtosword32(cacheBlockIndex); + outstart = (__u32)archtosword32(cacheBlockIndex); outsize = archtosword32(entry); memcpy (outbuf + size, &outstart, sizeof(__u32)); memcpy (outbuf + size + sizeof(__u32), &outsize, sizeof(__u32)); @@ -534,7 +536,7 @@ void zStr::setText(const char *ikey, const char *buf, long len) { size += len; } - start = datfd->seek(0, SEEK_END); + start = (__u32)datfd->seek(0, SEEK_END); outstart = archtosword32(start); outsize = archtosword32(size); @@ -609,7 +611,7 @@ void zStr::flushCache() const { unsigned long zdtSize = zdtfd->seek(0, SEEK_END); if ((cacheBlockIndex * ZDXENTRYSIZE) > (zdxSize - ZDXENTRYSIZE)) { // New Block - start = zdtSize; + start = (__u32)zdtSize; } else { zdxfd->seek(cacheBlockIndex * ZDXENTRYSIZE, SEEK_SET); @@ -624,7 +626,7 @@ void zStr::flushCache() const { size = outsize; } else { // middle and bigger-- we have serious problems, for now let's put it at the end = lots of wasted space - start = zdtSize; + start = (__u32)zdtSize; } } diff --git a/src/modules/common/zverse.cpp b/src/modules/common/zverse.cpp index cd63eca..e975a81 100644 --- a/src/modules/common/zverse.cpp +++ b/src/modules/common/zverse.cpp @@ -5,7 +5,7 @@ * and provides lookup and parsing functions based on * class VerseKey for compressed modules * - * $Id: zverse.cpp 3166 2014-04-17 04:08:45Z greg.hellings $ + * $Id: zverse.cpp 3439 2016-10-23 08:32:02Z scribe $ * * Copyright 1996-2013 CrossWire Bible Society (http://www.crosswire.org) * CrossWire Bible Society @@ -67,7 +67,6 @@ zVerse::zVerse(const char *ipath, int fileMode, int blockType, SWCompress *icomp SWBuf buf; - nl = '\n'; path = 0; cacheBufIdx = -1; cacheTestament = 0; @@ -273,7 +272,7 @@ void zVerse::zReadText(char testmt, long start, unsigned short size, unsigned lo compressor->Buf(0, &len); cacheBuf = (char *)calloc(len + 1, 1); memcpy(cacheBuf, compressor->Buf(), len); - cacheBufSize = strlen(cacheBuf); // TODO: can we just use len? + cacheBufSize = (int)strlen(cacheBuf); // TODO: can we just use len? cacheTestament = testmt; cacheBufIdx = ulBuffNum; } @@ -315,12 +314,12 @@ void zVerse::doSetText(char testmt, long idxoff, const char *buf, long len) { __u32 start; __u16 size; - __u32 outBufIdx = cacheBufIdx; + __u32 outBufIdx = (__u32)cacheBufIdx; idxoff *= 10; size = len; - start = strlen(cacheBuf); + start = (__u32)strlen(cacheBuf); if (!size) start = outBufIdx = 0; @@ -344,9 +343,9 @@ void zVerse::flushCache() const { __u32 size, outsize; __u32 zsize, outzsize; - idxoff = cacheBufIdx * 12; + idxoff = (__u32)cacheBufIdx * 12; if (cacheBuf) { - size = outsize = zsize = outzsize = strlen(cacheBuf); + size = outsize = zsize = outzsize = (__u32)strlen(cacheBuf); if (size) { // if (compressor) { // delete compressor; @@ -355,16 +354,16 @@ void zVerse::flushCache() const { compressor->Buf(cacheBuf); unsigned long tmpSize; compressor->zBuf(&tmpSize); - outzsize = zsize = tmpSize; + outzsize = zsize = (__u32)tmpSize; SWBuf buf; buf.setSize(zsize + 5); memcpy(buf.getRawData(), compressor->zBuf(&tmpSize), tmpSize); - outzsize = zsize = tmpSize; + outzsize = zsize = (__u32)tmpSize; buf.setSize(zsize); rawZFilter(buf, 1); // 1 = encipher - start = outstart = textfp[cacheTestament-1]->seek(0, SEEK_END); + start = outstart = (__u32)textfp[cacheTestament-1]->seek(0, SEEK_END); outstart = archtosword32(start); outsize = archtosword32(size); diff --git a/src/modules/common/zverse4.cpp b/src/modules/common/zverse4.cpp new file mode 100644 index 0000000..ff10b2d --- /dev/null +++ b/src/modules/common/zverse4.cpp @@ -0,0 +1,516 @@ +/****************************************************************************** + * + * zverse4.cpp - code for class 'zVerse4'- a module that reads raw text + * files: ot and nt using indexs ??.bks ??.cps ??.vss + * and provides lookup and parsing functions based on + * class VerseKey for compressed modules + * + * $Id: zverse4.cpp 3439 2016-10-23 08:32:02Z scribe $ + * + * Copyright 1996-2014 CrossWire Bible Society (http://www.crosswire.org) + * CrossWire Bible Society + * P. O. Box 2528 + * Tempe, AZ 85280-2528 + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License as published by the + * Free Software Foundation version 2. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + * + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + + +SWORD_NAMESPACE_START + +/****************************************************************************** + * zVerse4 Statics + */ + +int zVerse4::instance = 0; + +const char zVerse4::uniqueIndexID[] = {'X', 'r', 'v', 'c', 'b'}; + +/****************************************************************************** + * zVerse4 Constructor - Initializes data for instance of zVerse4 + * + * ENT: ipath - path of the directory where data and index files are located. + * be sure to include the trailing separator (e.g. '/' or '\') + * (e.g. 'modules/texts/rawtext/webster/') + * fileMode - open mode for the files (FileMgr::RDONLY, etc.) + * blockType - verse, chapter, book, etc. + */ + +zVerse4::zVerse4(const char *ipath, int fileMode, int blockType, SWCompress *icomp) +{ + // this line, instead of just defaulting, to keep FileMgr out of header + if (fileMode == -1) fileMode = FileMgr::RDONLY; + + SWBuf buf; + + path = 0; + cacheBufIdx = -1; + cacheTestament = 0; + cacheBuf = 0; + dirtyCache = false; + stdstr(&path, ipath); + + if ((path[strlen(path)-1] == '/') || (path[strlen(path)-1] == '\\')) + path[strlen(path)-1] = 0; + + compressor = (icomp) ? icomp : new SWCompress(); + + if (fileMode == -1) { // try read/write if possible + fileMode = FileMgr::RDWR; + } + + buf.setFormatted("%s/ot.%czs", path, uniqueIndexID[blockType]); + idxfp[0] = FileMgr::getSystemFileMgr()->open(buf, fileMode, true); + + buf.setFormatted("%s/nt.%czs", path, uniqueIndexID[blockType]); + idxfp[1] = FileMgr::getSystemFileMgr()->open(buf, fileMode, true); + + buf.setFormatted("%s/ot.%czz", path, uniqueIndexID[blockType]); + textfp[0] = FileMgr::getSystemFileMgr()->open(buf, fileMode, true); + + buf.setFormatted("%s/nt.%czz", path, uniqueIndexID[blockType]); + textfp[1] = FileMgr::getSystemFileMgr()->open(buf, fileMode, true); + + buf.setFormatted("%s/ot.%czv", path, uniqueIndexID[blockType]); + compfp[0] = FileMgr::getSystemFileMgr()->open(buf, fileMode, true); + + buf.setFormatted("%s/nt.%czv", path, uniqueIndexID[blockType]); + compfp[1] = FileMgr::getSystemFileMgr()->open(buf, fileMode, true); + + instance++; +} + + +/****************************************************************************** + * zVerse4 Destructor - Cleans up instance of zVerse4 + */ + +zVerse4::~zVerse4() +{ + int loop1; + + if (cacheBuf) { + flushCache(); + free(cacheBuf); + } + + if (path) + delete [] path; + + if (compressor) + delete compressor; + + --instance; + + for (loop1 = 0; loop1 < 2; loop1++) { + FileMgr::getSystemFileMgr()->close(idxfp[loop1]); + FileMgr::getSystemFileMgr()->close(textfp[loop1]); + FileMgr::getSystemFileMgr()->close(compfp[loop1]); + } +} + + +/****************************************************************************** + * zVerse4::findoffset - Finds the offset of the key verse from the indexes + * + * + * + * ENT: testmt - testament to find (0 - Bible/module introduction) + * book - book to find (0 - testament introduction) + * chapter - chapter to find (0 - book introduction) + * verse - verse to find (0 - chapter introduction) + * start - address to store the starting offset + * size - address to store the size of the entry + */ + +void zVerse4::findOffset(char testmt, long idxoff, long *start, unsigned long *size, unsigned long *buffnum) const +{ + __u32 ulBuffNum = 0; // buffer number + __u32 ulVerseStart = 0; // verse offset within buffer + __u32 usVerseSize = 0; // verse size + // set start to offset in + // set size to + // set + *start = *size = *buffnum = 0; + //fprintf(stderr, "Finding offset %ld\n", idxoff); + idxoff *= 12; // TODO: Is this the correct size? (throughout) + if (!testmt) { + testmt = ((idxfp[0]) ? 1:2); + } + + // assert we have and valid file descriptor + if (compfp[testmt-1]->getFd() < 1) + return; + + long newOffset = compfp[testmt-1]->seek(idxoff, SEEK_SET); + if (newOffset == idxoff) { + if (compfp[testmt-1]->read(&ulBuffNum, 4) != 4) { + fprintf(stderr, "Error reading ulBuffNum\n"); + return; + } + } + else return; + + if (compfp[testmt-1]->read(&ulVerseStart, 4) < 4) + { + fprintf(stderr, "Error reading ulVerseStart\n"); + return; + } + if (compfp[testmt-1]->read(&usVerseSize, 4) < 4) + { + fprintf(stderr, "Error reading usVerseSize\n"); + return; + } + + *buffnum = swordtoarch32(ulBuffNum); + *start = swordtoarch32(ulVerseStart); + *size = swordtoarch32(usVerseSize); + +} + + +/****************************************************************************** + * zVerse4::zreadtext - gets text at a given offset + * + * ENT: testmt - testament file to search in (0 - Old; 1 - New) + * start - starting offset where the text is located in the file + * size - size of text entry + 1 (null) + * buf - buffer to store text + * + */ + +void zVerse4::zReadText(char testmt, long start, unsigned long size, unsigned long ulBuffNum, SWBuf &inBuf) const { + __u32 ulCompOffset = 0; // compressed buffer start + __u32 ulCompSize = 0; // buffer size compressed + __u32 ulUnCompSize = 0; // buffer size uncompressed + + if (!testmt) { + testmt = ((idxfp[0]) ? 1:2); + } + + // assert we have and valid file descriptor + if (compfp[testmt-1]->getFd() < 1) + return; + + if (size && + !(((long) ulBuffNum == cacheBufIdx) && (testmt == cacheTestament) && (cacheBuf))) { + //fprintf(stderr, "Got buffer number{%ld} versestart{%ld} versesize{%d}\n", ulBuffNum, ulVerseStart, usVerseSize); + + if (idxfp[testmt-1]->seek(ulBuffNum*12, SEEK_SET)!=(long) ulBuffNum*12) + { + fprintf(stderr, "Error seeking compressed file index\n"); + return; + } + if (idxfp[testmt-1]->read(&ulCompOffset, 4)<4) + { + fprintf(stderr, "Error reading ulCompOffset\n"); + return; + } + if (idxfp[testmt-1]->read(&ulCompSize, 4)<4) + { + fprintf(stderr, "Error reading ulCompSize\n"); + return; + } + if (idxfp[testmt-1]->read(&ulUnCompSize, 4)<4) + { + fprintf(stderr, "Error reading ulUnCompSize\n"); + return; + } + + ulCompOffset = swordtoarch32(ulCompOffset); + ulCompSize = swordtoarch32(ulCompSize); + ulUnCompSize = swordtoarch32(ulUnCompSize); + + if (textfp[testmt-1]->seek(ulCompOffset, SEEK_SET)!=(long)ulCompOffset) + { + fprintf(stderr, "Error: could not seek to right place in compressed text\n"); + return; + } + SWBuf pcCompText; + pcCompText.setSize(ulCompSize+5); + + if (textfp[testmt-1]->read(pcCompText.getRawData(), ulCompSize)<(long)ulCompSize) { + fprintf(stderr, "Error reading compressed text\n"); + return; + } + pcCompText.setSize(ulCompSize); + rawZFilter(pcCompText, 0); // 0 = decipher + + unsigned long bufSize = ulCompSize; + compressor->zBuf(&bufSize, pcCompText.getRawData()); + + if (cacheBuf) { + flushCache(); + free(cacheBuf); + } + + unsigned long len = 0; + compressor->Buf(0, &len); + cacheBuf = (char *)calloc(len + 1, 1); + memcpy(cacheBuf, compressor->Buf(), len); + cacheBufSize = (int)strlen(cacheBuf); // TODO: can we just use len? + cacheTestament = testmt; + cacheBufIdx = ulBuffNum; + } + + inBuf = ""; + if ((size > 0) && cacheBuf && ((unsigned)start < cacheBufSize)) { + inBuf.setFillByte(0); + inBuf.setSize(size+1); + strncpy(inBuf.getRawData(), &(cacheBuf[start]), size); + inBuf.setSize(strlen(inBuf.c_str())); + } +} + + +/****************************************************************************** + * zVerse4::settext - Sets text for current offset + * + * ENT: testmt - testament to find (0 - Bible/module introduction) + * idxoff - offset into .vss + * buf - buffer to store + * len - length of buffer (0 - null terminated) + */ + +void zVerse4::doSetText(char testmt, long idxoff, const char *buf, long len) { + + len = (len < 0) ? strlen(buf) : len; + if (!testmt) + testmt = ((idxfp[0]) ? 1:2); + if ((!dirtyCache) || (cacheBufIdx < 0)) { + cacheBufIdx = idxfp[testmt-1]->seek(0, SEEK_END) / 12; + cacheTestament = testmt; + if (cacheBuf) + free(cacheBuf); + cacheBuf = (char *)calloc(len + 1, 1); + } + else cacheBuf = (char *)((cacheBuf)?realloc(cacheBuf, strlen(cacheBuf)+(len + 1)):calloc((len + 1), 1)); + + dirtyCache = true; + + __u32 start; + __u32 size; + __u32 outBufIdx = (__u32)cacheBufIdx; + + idxoff *= 12; + size = (__u32)len; + + start = (__u32)strlen(cacheBuf); + + if (!size) + start = outBufIdx = 0; + + outBufIdx = archtosword32(outBufIdx); + start = archtosword32(start); + size = archtosword32(size); + + compfp[testmt-1]->seek(idxoff, SEEK_SET); + compfp[testmt-1]->write(&outBufIdx, 4); + compfp[testmt-1]->write(&start, 4); + compfp[testmt-1]->write(&size, 4); + strcat(cacheBuf, buf); +} + + +void zVerse4::flushCache() const { + if (dirtyCache) { + __u32 idxoff; + __u32 start, outstart; + __u32 size, outsize; + __u32 zsize, outzsize; + + idxoff = (__u32)cacheBufIdx * 12; + if (cacheBuf) { + size = outsize = zsize = outzsize = (__u32)strlen(cacheBuf); + if (size) { + compressor->Buf(cacheBuf); + unsigned long tmpSize; + compressor->zBuf(&tmpSize); + outzsize = zsize = (__u32)tmpSize; + + SWBuf buf; + buf.setSize(zsize + 5); + memcpy(buf.getRawData(), compressor->zBuf(&tmpSize), tmpSize); + outzsize = zsize = (__u32)tmpSize; + buf.setSize(zsize); + rawZFilter(buf, 1); // 1 = encipher + + start = outstart = (__u32)textfp[cacheTestament-1]->seek(0, SEEK_END); + + outstart = archtosword32(start); + outsize = archtosword32(size); + outzsize = archtosword32(zsize); + + textfp[cacheTestament-1]->write(buf, zsize); + + idxfp[cacheTestament-1]->seek(idxoff, SEEK_SET); + idxfp[cacheTestament-1]->write(&outstart, 4); + idxfp[cacheTestament-1]->write(&outzsize, 4); + idxfp[cacheTestament-1]->write(&outsize, 4); + } + free(cacheBuf); + cacheBuf = 0; + } + dirtyCache = false; + } +} + +/****************************************************************************** + * RawVerse::linkentry - links one entry to another + * + * ENT: testmt - testament to find (0 - Bible/module introduction) + * destidxoff - dest offset into .vss + * srcidxoff - source offset into .vss + */ + +void zVerse4::doLinkEntry(char testmt, long destidxoff, long srcidxoff) { + __s32 bufidx; + __s32 start; + __u32 size; + + destidxoff *= 12; + srcidxoff *= 12; + + if (!testmt) + testmt = ((idxfp[1]) ? 1:2); + + // get source + compfp[testmt-1]->seek(srcidxoff, SEEK_SET); + compfp[testmt-1]->read(&bufidx, 4); + compfp[testmt-1]->read(&start, 4); + compfp[testmt-1]->read(&size, 4); + + // write dest + compfp[testmt-1]->seek(destidxoff, SEEK_SET); + compfp[testmt-1]->write(&bufidx, 4); + compfp[testmt-1]->write(&start, 4); + compfp[testmt-1]->write(&size, 4); +} + + +/****************************************************************************** + * RawVerse::CreateModule - Creates new module files + * + * ENT: path - directory to store module files + * RET: error status + */ + +char zVerse4::createModule(const char *ipath, int blockBound, const char *v11n) +{ + char *path = 0; + char *buf = new char [ strlen (ipath) + 20 ]; + char retVal = 0; + FileDesc *fd, *fd2; + __s32 offset = 0; + __s32 size = 0; + VerseKey vk; + + stdstr(&path, ipath); + + if ((path[strlen(path)-1] == '/') || (path[strlen(path)-1] == '\\')) + path[strlen(path)-1] = 0; + + sprintf(buf, "%s/ot.%czs", path, uniqueIndexID[blockBound]); + FileMgr::removeFile(buf); + fd = FileMgr::getSystemFileMgr()->open(buf, FileMgr::CREAT|FileMgr::WRONLY, FileMgr::IREAD|FileMgr::IWRITE); + if (fd->getFd() < 1) goto erroropen1; + FileMgr::getSystemFileMgr()->close(fd); + + sprintf(buf, "%s/nt.%czs", path, uniqueIndexID[blockBound]); + FileMgr::removeFile(buf); + fd = FileMgr::getSystemFileMgr()->open(buf, FileMgr::CREAT|FileMgr::WRONLY, FileMgr::IREAD|FileMgr::IWRITE); + if (fd->getFd() < 1) goto erroropen1; + FileMgr::getSystemFileMgr()->close(fd); + + sprintf(buf, "%s/ot.%czz", path, uniqueIndexID[blockBound]); + FileMgr::removeFile(buf); + fd = FileMgr::getSystemFileMgr()->open(buf, FileMgr::CREAT|FileMgr::WRONLY, FileMgr::IREAD|FileMgr::IWRITE); + if (fd->getFd() < 1) goto erroropen1; + FileMgr::getSystemFileMgr()->close(fd); + + sprintf(buf, "%s/nt.%czz", path, uniqueIndexID[blockBound]); + FileMgr::removeFile(buf); + fd = FileMgr::getSystemFileMgr()->open(buf, FileMgr::CREAT|FileMgr::WRONLY, FileMgr::IREAD|FileMgr::IWRITE); + if (fd->getFd() < 1) goto erroropen1; + FileMgr::getSystemFileMgr()->close(fd); + + sprintf(buf, "%s/ot.%czv", path, uniqueIndexID[blockBound]); + FileMgr::removeFile(buf); + fd = FileMgr::getSystemFileMgr()->open(buf, FileMgr::CREAT|FileMgr::WRONLY, FileMgr::IREAD|FileMgr::IWRITE); + if (fd->getFd() < 1) goto erroropen1; + + sprintf(buf, "%s/nt.%czv", path, uniqueIndexID[blockBound]); + FileMgr::removeFile(buf); + fd2 = FileMgr::getSystemFileMgr()->open(buf, FileMgr::CREAT|FileMgr::WRONLY, FileMgr::IREAD|FileMgr::IWRITE); + if (fd2->getFd() < 1) goto erroropen2; + + vk.setVersificationSystem(v11n); + vk.setIntros(true); + + offset = archtosword32(offset); + size = archtosword32(size); + + for (vk = TOP; !vk.popError(); vk++) { + if (vk.getTestament() < 2) { + if (fd->write(&offset, 4) != 4) goto writefailure; //compBufIdxOffset + if (fd->write(&offset, 4) != 4) goto writefailure; + if (fd->write(&size, 4) != 4) goto writefailure; + } + else { + if (fd2->write(&offset, 4) != 4) goto writefailure; //compBufIdxOffset + if (fd2->write(&offset, 4) != 4) goto writefailure; + if (fd2->write(&size, 4) != 4) goto writefailure; + } + } + fd2->write(&offset, 4); //compBufIdxOffset + fd2->write(&offset, 4); + fd2->write(&size, 4); + + goto cleanup; + +erroropen1: + retVal = -1; + goto cleanup1; + +erroropen2: + retVal = -1; + goto cleanup; + +writefailure: + retVal = -2; + +cleanup: + FileMgr::getSystemFileMgr()->close(fd2); +cleanup1: + FileMgr::getSystemFileMgr()->close(fd); + + delete [] path; + delete [] buf; + + return retVal; +} + + +SWORD_NAMESPACE_END -- cgit v1.2.3