diff options
Diffstat (limited to 'src/modules/texts/ztext/rawtxt2z.cpp')
| -rw-r--r-- | src/modules/texts/ztext/rawtxt2z.cpp | 457 |
1 files changed, 0 insertions, 457 deletions
diff --git a/src/modules/texts/ztext/rawtxt2z.cpp b/src/modules/texts/ztext/rawtxt2z.cpp deleted file mode 100644 index 7eafe2a..0000000 --- a/src/modules/texts/ztext/rawtxt2z.cpp +++ /dev/null @@ -1,457 +0,0 @@ -// Compression on variable granularity
-
-#include <fcntl.h>
-#include <iostream>
-#include <fstream>
-#include <string>
-
-#ifndef __GNUC__
-#include <io.h>
-#else
-#include <unistd.h>
-#endif
-
-#include <zlib.h>
-#include <versekey.h>
-
-int iBufSize, ulBuffNum;
-ofstream cfile;
-ofstream cfile2;
-
-int ofd[2], oxfd[2], ovxfd[2];
-int ifd[2], ixfd[2];
-int itestfd[2], itestxfd[2];
-unsigned long ulIOff=0, ulCOff=0, ulFOff=0, ulNone=0;
-string currbuff="";
-
-
-int openreadfile(char *buffer, char *path, const char *filename)
-{
- int filenum;
- sprintf(buffer, "%s/%s", path, filename);
- cfile << buffer << "\n";
- filenum = open(buffer, O_RDONLY|O_BINARY);
- if (filenum > 0)
- {
- return filenum;
- }
- else
- {
- cerr << "failed to open file to read\n";
- exit(-1);
- }
-}
-
-int openwritefile(char *buffer, char *path, const char *filename)
-{
- int filenum;
- sprintf(buffer, "%s/%s", path, filename);
- cfile << buffer << "\n";
- filenum = open(buffer, O_WRONLY|O_BINARY|O_CREAT|O_TRUNC);
- if (filenum > 0)
- {
- return filenum;
- }
- else
- {
- cerr << "failed to open file to read\n";
- exit(-1);
- }
-}
-
-int bytebound(unsigned long offset, VerseKey &thekey)
-{
- unsigned long bufferoff;
- cfile << "byteboundtest " << thekey << "\n";
- bufferoff = iBufSize * (ulBuffNum+1);
- if (offset > bufferoff)
- {
- return 1;
- }
- else
- {
- return 0;
- }
-}
-
-int versebound(unsigned long offset, VerseKey &thekey)
-{
- cfile << "verseboundtest " << thekey << "\n";
- return 1;
-}
-
-int chapterbound(unsigned long offset, VerseKey &thekey)
-{
- VerseKey testkey;
- testkey = thekey;
- testkey++;
- //cfile << "chapterboundtest " << testkey;
- if (testkey.Verse()==1 || (!thekey.compare("Revelation of John 22:21")))
- {
- //cfile << " 1\n";
- return 1;
- }
- else
- {
- //cfile << " 0\n";
- return 0;
- }
-}
-
-int bookbound(unsigned long offset, VerseKey &thekey)
-{
- VerseKey testkey;
- testkey = thekey;
- cfile << "bookboundtest " << testkey << "\n";
- testkey++;
- if (testkey.Chapter()==1 || (!thekey.compare("Revelation of John 22:21")))
- {
- return 1;
- }
- else
- {
- return 0;
- }
-}
-
-
-typedef int (*boundfunc)(unsigned long offset, VerseKey &thekey);
-
-int writeblock(int i)
-{
- char *destbuff=NULL;
- unsigned long compsize = 0, buffsize=0;
-
- cfile << "compressing block\n";
- // compress current buffer
- buffsize = currbuff.length();
- write(itestfd[i], currbuff.c_str(), buffsize);
- compsize = (unsigned long) (buffsize*1.01)+20; // at least 1% bigger than buffer + 12 bytes
- //cfile << "{" << compsize << "}";
- //destbuff = (char *) calloc(compsize + 1, 1);
- destbuff = new char[compsize];
- if (compress((Bytef*)destbuff, &compsize, (const Bytef*)currbuff.c_str(), buffsize)!=Z_OK)
- {
- cerr << "Could not compress buffer: exiting\n";
- delete[] destbuff;
- exit(-1);
- }
- //cout << "Compressed buffer{" << compsize << "}\n" << destbuff << "\n";
- //cout.flush();
- // write to compressed file index
- ulCOff = lseek(ofd[i], 0, SEEK_END);
- write(oxfd[i], &ulCOff, 4); // offset in compressed file
- write(oxfd[i], &compsize, 4); // compressed size
- write(oxfd[i], &buffsize, 4); // uncompressed size
- cfile << buffsize << " -> " << compsize << "\n";
- cfile2 << "Compressed{" << compsize << "}\n" << destbuff << "\n";
- cfile2.flush();
-
- //write compressed buffer to file
- write(ofd[i], destbuff, compsize);
-
- //free(destbuff);
- delete[] destbuff;
-
- currbuff = "";
- ulBuffNum++;
- ulIOff = 0;
- return 1;
-}
-
-
-
-int main(int argc, char **argv)
-{
- VerseKey key1, key2, key3;
- int i;
- char xbuff[64];
- unsigned long offset;
- unsigned short size=0;
- unsigned long ulsize=0;
- char *tmpbuf=NULL;
- int iType;
- boundfunc blockbound[4] = {bytebound, versebound, chapterbound, bookbound};
- bool newbook=true, newchapter=true, newtestament = true, newmodule = true, lasttodo=true;
-
- if ((argc < 2) || (argc > 4)) {
- cerr << "usage: " << argv[0] << " datapath [compression type [buffer size]]\n";
- exit(1);
- }
-
- if (argc>2)
- {
- iType = atoi(argv[2]);
- if (argc==4)
- {
- iBufSize = atoi(argv[3]);
- }
- else
- {
- iBufSize = 1;
- }
- }
- else
- {
- iType = 2;
- iBufSize = 1;
- }
-
- cfile.open("raw2z.log", ios::out);
- if (!cfile.is_open())
- {
- cerr << "Failed to open log file\n";
- exit(-1);
- }
- cfile2.open("raw2z.lg2", ios::out);
- if (!cfile2.is_open())
- {
- cerr << "Failed to open log file\n";
- exit(-1);
- }
- cfile << iType << " " << iBufSize << "\n";
-
- if ((iType<=0) || (iType > 4) || !iBufSize || !strcmp(argv[1], "-h") || !strcmp(argv[1], "--help") || !strcmp(argv[1], "/?") || !strcmp(argv[1], "-help"))
- {
- cfile << argv[0] << " - a tool to create compressed Sword modules\n";
- cfile << "version 0.1\n\n";
- cfile << "usage: "<< argv[0] << " datapath [compression type [buffer size]]\n\n";
- cfile << "datapath: the directory in which to find the raw module\n";
- cfile << "compression type: (default 2)\n" << " 1 - bytes\n" << " 2 - verses\n" << " 3 - chapters\n" << " 4 - books\n";
- cfile << "buffer size (default 1): the number of the compression type in each block\n";
- exit(1);
- }
-
- //zobj = new SWCompress();
- //rawdrv = new RawVerse(argv[1]);
-
-#ifndef O_BINARY
-#define O_BINARY 0
-#endif
- cfile << "opening files\n";
-
- tmpbuf = new char [ strlen(argv[1]) + 11 ];
-
- //original files
- ifd[0] = openreadfile(tmpbuf, argv[1], "ot");
- ixfd[0] = openreadfile(tmpbuf, argv[1], "ot.vss");
- ifd[1] = openreadfile(tmpbuf, argv[1], "nt");
- ixfd[1] = openreadfile(tmpbuf, argv[1], "nt.vss");
-
-switch ( iType) {
- case 1 :
- ofd[0] = openwritefile(tmpbuf, argv[1], "ot.rzz");
- oxfd[0] = openwritefile(tmpbuf, argv[1], "ot.rzs");
- ovxfd[0] = openwritefile(tmpbuf, argv[1], "ot.rzv");
- ofd[1] = openwritefile(tmpbuf, argv[1], "nt.rzz");
- oxfd[1] = openwritefile(tmpbuf, argv[1], "nt.rzs");
- ovxfd[1] = openwritefile(tmpbuf, argv[1], "nt.rzv");
- //boundfunc = bytebound;
- break;
- case 2 :
- ofd[0] = openwritefile(tmpbuf, argv[1], "ot.vzz");
- oxfd[0] = openwritefile(tmpbuf, argv[1], "ot.vzs");
- ovxfd[0] = openwritefile(tmpbuf, argv[1], "ot.vzv");
- ofd[1] = openwritefile(tmpbuf, argv[1], "nt.vzz");
- oxfd[1] = openwritefile(tmpbuf, argv[1], "nt.vzs");
- ovxfd[1] = openwritefile(tmpbuf, argv[1], "nt.vzv");
- break;
- case 3 :
- ofd[0] = openwritefile(tmpbuf, argv[1], "ot.czz");
- oxfd[0] = openwritefile(tmpbuf, argv[1], "ot.czs");
- ovxfd[0] = openwritefile(tmpbuf, argv[1], "ot.czv");
- ofd[1] = openwritefile(tmpbuf, argv[1], "nt.czz");
- oxfd[1] = openwritefile(tmpbuf, argv[1], "nt.czs");
- ovxfd[1] = openwritefile(tmpbuf, argv[1], "nt.czv");
- break;
- case 4 :
- ofd[0] = openwritefile(tmpbuf, argv[1], "ot.bzz");
- oxfd[0] = openwritefile(tmpbuf, argv[1], "ot.bzs");
- ovxfd[0] = openwritefile(tmpbuf, argv[1], "ot.bzv");
- ofd[1] = openwritefile(tmpbuf, argv[1], "nt.bzz");
- oxfd[1] = openwritefile(tmpbuf, argv[1], "nt.bzs");
- ovxfd[1] = openwritefile(tmpbuf, argv[1], "nt.bzv");
- break;
- default:
- cerr << "Unknown compression type\n";
- exit(-1);
-}
- itestfd[0] = openwritefile(tmpbuf, argv[1], "ot.tst");
- itestfd[1] = openwritefile(tmpbuf, argv[1], "nt.tst");
- itestxfd[0] = openwritefile(tmpbuf, argv[1], "ot.tdx");
- itestxfd[1] = openwritefile(tmpbuf, argv[1], "nt.tdx");
-
-
- delete [] tmpbuf;
-
- //cfile << "about to start\n";
-
-for ( i=0; i<2; i++)
-{
- ulIOff=0, ulBuffNum=0;
- currbuff = "";
- key1 = (i == 1) ? "Matthew 1:1" : "Genesis 1:1";
- key2 = key3 = key1;
- newtestament = true;
-
- cfile << "key: " << key1 << " Testament {" << key1.Testament()-1 << "}\n";
- //cfile << "Chapter {" << key.Chapter() << "}\n";
- //cfile << "Verse {" << key.Verse() << "}\n";
- //cfile << key.compare("Revelation of John 22:21") << "\n";
- //cfile << key.compare("Genesis 1:1") << "\n";
- do
- {
- //cfile << "ok";
- // read current verse offset
- if (read(ixfd[i], &offset, 4) != 4)
- {
- cfile << "Failed to read input verse offsets?\n";
- break;
- }
- if (read(ixfd[i], &size, 2) != 2)
- {
- cfile << "Failed to read input verse sizes?\n";
- break;
- }
- cfile << "key:" << key1 << " offset:" << offset << " size:" << size << "\n";
- sprintf(xbuff, "key{%s} offset{%ld} size{%d}\n", (const char *)key1, offset, size);
- write(itestxfd[i], &xbuff, strlen(xbuff));
- ulsize = size;
- if (!offset && !size)
- {
- //Check for module header
- if (read(ixfd[i], &ulIOff, 4) != 4)
- {
- cfile << "Failed to read input verse offsets?\n";
- break;
- }
- ulsize = ulIOff;
- ulIOff = 0;
- lseek(ixfd[i], 6, SEEK_SET);
- }
-
- if (ulsize)
- {
- // read current verse and add to current buffer
- tmpbuf = (char *) calloc(ulsize + 1, 1);
- lseek(ifd[i], offset, SEEK_SET);
- read(ifd[i], tmpbuf, ulsize);
- currbuff += tmpbuf;
- //cfile << currbuff << "\n";
-
- // write to verse index into compressed
- write(ovxfd[i], &ulBuffNum, 4); // current buffer number
- write(ovxfd[i], &ulIOff, 4); // offset within the buffer
- write(ovxfd[i], &size, 2); // verse size
-
- ulFOff = lseek(ofd[i], 0, SEEK_CUR) + size;
- if (key1.compare("Revelation of John 22:21")!=-1)
- {
- lasttodo = false;
- }
- if (blockbound[iType-1](ulFOff, key1)/*at block boudary*/)
- {
- writeblock(i);
- /*
- cfile << "compressing block\n";
- // compress current buffer
- buffsize = currbuff.length();
- write(itestfd[i], currbuff.c_str(), buffsize);
- compsize = (unsigned long) (buffsize*1.01)+20; // at least 1% bigger than buffer + 12 bytes
- //cfile << "{" << compsize << "}";
- //destbuff = (char *) calloc(compsize + 1, 1);
- destbuff = new char[compsize];
- if (compress((Bytef*)destbuff, &compsize, (const Bytef*)currbuff.c_str(), buffsize)!=Z_OK)
- {
- cerr << "Could not compress buffer: exiting\n";
- delete[] destbuff;
- exit(-1);
- }
- //cout << "Compressed buffer{" << compsize << "}\n" << destbuff << "\n";
- //cout.flush();
- // write to compressed file index
- ulCOff = lseek(ofd[i], 0, SEEK_END);
- write(oxfd[i], &ulCOff, 4); // offset in compressed file
- write(oxfd[i], &compsize, 4); // compressed size
- write(oxfd[i], &buffsize, 4); // uncompressed size
- cfile << buffsize << " -> " << compsize << "\n";
- cfile2 << "Compressed{" << compsize << "}\n" << destbuff << "\n";
- cfile2.flush();
-
- //write compressed buffer to file
- write(ofd[i], destbuff, compsize);
-
- //free(destbuff);
- delete[] destbuff;
-
- currbuff = "";
- ulBuffNum++;
- ulIOff = 0;
- */
- }
- else
- {
- ulIOff += ulsize;
- }
- free(tmpbuf);
-
- if (newmodule)
- {
- newmodule = false;
- cfile << "had a new module " << (const char *) key1 << "{" << offset << "}\n";
- writeblock(i);
- }
- else if (newtestament)
- {
- newtestament = false;
- cfile << "had a new testament " << (const char *) key1 << "{" << offset << "}\n";
- }
- else if (newbook)
- {
- newbook = false;
- cfile << "had a new book " << (const char *) key1 << "{" << offset << "}\n";
- }
- else if (newchapter)
- {
- newchapter = false;
- cfile << "had a new chapter " << (const char *) key1 << "{" << offset << "}\n";
- }
- else
- {
- key1++;
- }
-
- if (key1.Chapter()!=key2.Chapter() || (key1.Book()!=key2.Book()))
- {
- newchapter = true;
- cfile << "got a new chapter " << (const char *) key1 << "\n";
- }
- if (key1.Book()!=key2.Book())
- {
- newbook = true;
- cfile << "got a new book " << (const char *) key1 << "\n";
- }
- key2 = key1;
-
- }
- else
- {
- cfile << "empty offset\n";
- // write to verse index into compressed
- write(ovxfd[i], &ulNone, 4); // current buffer number
- write(ovxfd[i], &size, 2); // verse size
- write(ovxfd[i], &ulNone, 4); // offset within the buffer
- }
- }
- while ( (key1.Testament()==i+1) && ((key1.compare("Revelation of John 22:21")==-1) || (lasttodo)));
-
- close(ifd[i]);
- close(ofd[i]);
- close(ixfd[i]);
- close(oxfd[i]);
- close(ovxfd[i]);
- close(itestfd[i]);
- close(itestxfd[i]);
-}
- return 1;
-}
|