ViewVC Help
View File | Revision Log | Show Annotations | View Changeset | Root Listing
root/gclib/gclib/gcdb.h
Revision: 2
Committed: Mon Mar 22 22:03:27 2010 UTC (9 years, 7 months ago) by gpertea
File size: 12290 byte(s)
Log Message:
added my gclib source files

Line File contents
1 #ifndef __GCDB_H
2 #define __GCDB_H
3 #include <stdlib.h>
4 #include <stddef.h>
5 #include <fcntl.h>
6 #include <sys/stat.h>
7 #include "GBase.h"
8
9 #if defined(__WIN32__) || defined(WIN32)
10 #define PROT_READ 1
11 #define PROT_WRITE 2
12 #define PROT_READWRITE 3
13 #define MAP_SHARED 1
14 #define MAP_PRIVATE 2
15 #define F_OK 0
16 #define R_OK 4
17 #define W_OK 2
18 #define RW_OK 6
19
20 #if !defined(MAP_FAILED)
21 #define MAP_FAILED ((void *) -1)
22 #endif
23 void *mmap(char *,size_t,int,int,int,off_t);
24 int munmap(void *,size_t);
25 #else
26 #include <sys/mman.h>
27 #endif
28
29 #define MAX_UINT 0xFFFFFFFFUL
30
31
32 //=====================================================
33 //------------- buffer stuff -------------------
34 //=====================================================
35 #define GCDBUFFER_INSIZE 8192
36 #define GCDBUFFER_OUTSIZE 8192
37
38
39 typedef int (*opfunc)(int, char*, size_t);
40
41 //typedef unsigned long gcdb_seek_pos;
42 typedef off_t gcdb_seek_pos;
43 typedef unsigned int (*uint_conv)(void*); //uint conversion function pointer
44 typedef off_t (*offt_conv)(void*); //uint conversion function pointer
45
46
47 //conversion function --> to platform independent uint
48 extern uint_conv gcvt_uint;
49 extern offt_conv gcvt_offt;
50
51 int endian_test(void);
52 unsigned int uint32_sun(void* x86int);
53 unsigned int uint32_x86(void* x86int);
54 //for file offsets: off_t runtime conversions:
55 off_t offt_sun(void* offt);
56 off_t offt_x86(void* offt);
57
58
59 class GCDBuffer {
60 public:
61 char *x;
62 unsigned int p;
63 unsigned int n;
64 int fd;
65 opfunc op;
66 //methods:
67 GCDBuffer() {
68 x=NULL;
69 fd=0;
70 op=NULL;
71 n=0;
72 //check endianness
73 gcvt_uint=(endian_test())? &uint32_sun : &uint32_x86;
74 gcvt_offt=(endian_test())? &offt_sun : &offt_x86;
75 }
76 GCDBuffer(opfunc aop,int afd,char *buf,unsigned int len) {
77 //check endianness
78 gcvt_uint=(endian_test())? &uint32_sun : &uint32_x86;
79 gcvt_offt=(endian_test())? &offt_sun : &offt_x86;
80 init(aop, afd, buf, len);
81 }
82 void init(opfunc aop,int afd,char *buf,unsigned int len) {
83 x=buf;
84 fd=afd;
85 op=aop;
86 p=0;
87 n=len;
88 }
89 int flush();
90 int write_all(char* buf, unsigned int pt);
91 int put(char* buf,unsigned int len);
92 int putalign(char* buf,unsigned int len);
93 int putflush(char* buf,unsigned int len);
94 int puts(char *buf);
95 int putsalign(char *buf);
96 int putsflush(char *buf);
97 int oneRead(char* buf, unsigned int len);
98 int getthis(char* buf,unsigned int len);
99 int get(char* buf,unsigned int len);
100 int bget(char* buf,unsigned int len);
101 int feed();
102 char *peek();
103 void seek(unsigned int len);
104 int copy(GCDBuffer* bin);
105 };
106
107
108 //=====================================================
109 //------------- cdb utils -------------------
110 //=====================================================
111 #ifndef __WIN32__
112 extern int errno;
113 #endif
114 extern int error_intr;
115 extern int error_nomem;
116 extern int error_proto;
117
118 //additional data to be appended to the cdb file:
119 #define CDBMSK_OPT_MULTI 0x00000001
120 #define CDBMSK_OPT_C 0x00000002
121 #define CDBMSK_OPT_CADD 0x00000004
122 #define CDBMSK_OPT_COMPRESS 0x00000008
123 //creates a compressed version of the database
124 //uses plenty of unions for ensuring compatibility with
125 // the old 'CIDX' info structure
126
127 //damn, sun and 64bit machines
128 // align this to 64bit -- so sizeof() is misled!
129 #pragma pack(4)
130 // I wish, but stupid gcc 2.95.3 alpha-decosf version does not
131 // recognize this pragma directive !!?
132 //
133 struct cdbInfo {
134 uint32 num_keys;
135 union {
136 uint32 num_records;
137 char oldtag[4]; // 'CIDX' for old tag style
138 };
139 // data file size -- used to be uint32, now it could be 64bit
140 union {
141 off_t dbsize;
142 uint32 oldnum[2]; //num_keys, num_records
143 };
144 union {
145 uint32 idxflags;
146 uint32 old_dbsize;
147 };
148 union {
149 int dbnamelen;
150 int old_idxflags;
151 };
152 // -- the actual db name precedes this fixed-size record
153 union {
154 char tag[4]; //'CDBX' for new files with LFS
155 uint32 old_dbnamelen;
156 };
157 };
158 #pragma pack()
159
160 extern int cdbInfoSIZE;
161
162 void uint32_pack(char *,uint32);
163 void uint32_pack_big(char *,uint32);
164 void uint32_unpack(char *,uint32 *);
165 void uint32_unpack_big(char *,uint32 *);
166
167 //=====================================================
168 //------------- cdb index -------------------
169 //=====================================================
170
171 #define CDB_HPLIST 1000
172
173 struct cdb_hp { uint32 h; uint32 p; } ;
174
175 struct cdb_hplist {
176 struct cdb_hp hp[CDB_HPLIST];
177 struct cdb_hplist *next;
178 int num;
179 };
180
181 //the index file should always be smaller than 4GB !
182
183 class GCdbWrite {
184 GCDBuffer* cdbuf;
185 char bspace[8192];
186 char fname[1024];
187 char final[2048];
188 uint32 count[256];
189 uint32 start[256];
190 struct cdb_hplist *head;
191 struct cdb_hp *split; /* includes space for hash */
192 struct cdb_hp *hash;
193 uint32 numentries;
194 uint32 pos; //file position
195 int posplus(uint32 len);
196 int fd; //file descriptor
197 public:
198 //methods:
199 GCdbWrite(int afd); //was: init
200 GCdbWrite(char* fname);
201 ~GCdbWrite();
202 int addbegin(unsigned int keylen,unsigned int datalen);
203 int addend(unsigned int keylen,unsigned int datalen,uint32 h);
204 int addrec(const char *key,unsigned int keylen,char *data,unsigned int datalen);
205 int add(const char *key, char *data, unsigned int datalen);
206 int getNumEntries() { return numentries; }
207 int finish();
208 int close();
209 int getfd() { return fd; }
210 char* getfile() { return fname; }
211 };
212
213
214 //=====================================================
215 //------------- cdb -------------------
216 //=====================================================
217
218 #define CDB_HASHSTART 5381
219
220 uint32 cdb_hashadd(uint32,unsigned char);
221 uint32 cdb_hash(const char *,unsigned int);
222
223 class GCdbRead {
224 uint32 size; // initialized if map is nonzero
225 uint32 loop; // number of hash slots searched under this key
226 uint32 khash; // initialized if loop is nonzero
227 uint32 kpos; // initialized if loop is nonzero
228 uint32 hpos; // initialized if loop is nonzero
229 uint32 hslots; // initialized if loop is nonzero
230 uint32 dpos; // initialized if cdb_findnext() returns 1
231 uint32 dlen; // initialized if cdb_findnext() returns 1
232 char fname[1024];
233 char *map; // 0 if no map is available
234 int fd;
235 public:
236 //methods:
237 GCdbRead(int fd); //was cdb_init
238 GCdbRead(char* afname); //was cdb_init
239 ~GCdbRead(); //was cdb_free
240 int read(char *,unsigned int,uint32);
241 int match(const char *key, unsigned int len, uint32 pos);
242 void findstart() { loop =0; }
243 int findnext(const char *key,unsigned int len);
244 int find(const char *key);
245 int datapos() { return dpos; }
246 int datalen() { return dlen; }
247 int getfd() { return fd; }
248 char* getfile() { return fname; }
249 };
250
251 class GReadBuf {
252 protected:
253 FILE* f;
254 uchar* buf;
255 int buflen;
256 int bufused; //
257 int bufpos;
258 off_t fpos;
259 bool eof;
260 bool eob;
261
262 int refill(bool repos=false) {
263 //refill the buffer-----------
264 if (repos && bufpos==0) return 0; //no need to repos
265 if (eof) return 0;
266 int fr=0;
267 if (repos && bufpos<bufused) {
268 int kept=bufused-bufpos;
269 memmove((void*)buf, (void*)(buf+bufpos),kept);
270 fr=(int)fread((void *)(buf+kept), 1, buflen-kept, f);
271 if (fr<buflen-kept) eof=true;
272 buf[kept+fr]='\0';
273 bufused=kept+fr;
274 }
275 else {
276 fr=(int)fread((void *)buf, 1, buflen, f);
277 if (fr<buflen) eof=true;
278 buf[fr]='\0'; //only for text record parsers
279 bufused=fr;
280 }
281 if (feof(f)) eof=true;
282 if (ferror(f)) {
283 GMessage("GReadBuf::refill - error at fread!\n");
284 eof=true;
285 }
286 bufpos=0;
287 fpos+=fr; //bytes read from file so far
288 return fr;
289 }
290 public:
291 GReadBuf(FILE* fin, int bsize=4096) {
292 f=fin;
293 buflen=bsize;
294 GMALLOC(buf,buflen+1);
295 bufpos=0; //current pointer for get function
296 bufused=0;
297 fpos=0;
298 eof=false;
299 eob=false;
300 refill();
301 }
302 ~GReadBuf() { GFREE(buf); }
303
304 //reads len chars from stream into the outbuf
305 //updates bufpos
306 //->returns the number of bytes read
307 int get(uchar *outbuf, int len) {
308 if (eob) return 0;
309 int rd=0; //bytes read
310 while (!eob && rd<len) {
311 int to_read=GMIN((bufused-bufpos),(len-rd));
312 memcpy((void*)(outbuf+rd),(void*)(buf+bufpos), to_read);
313 bufpos+=to_read;
314 rd+=to_read;
315 if (bufpos>=bufused) {
316 if (eof) eob=true;
317 else refill();
318 }
319 }//while
320 return rd;
321 }
322
323 uchar* getStr(uchar *outbuf, int len) {
324 int rd=get(outbuf,len);
325 if (rd==0) return NULL;
326 else {
327 outbuf[rd]='\0';
328 return outbuf;
329 }
330 }
331
332 // getc equivalent
333 int getch() {
334 if (eob) return -1;
335 int ch=(int)(uchar)buf[bufpos];
336 bufpos++;
337 if (bufpos>=bufused) {
338 if (eof) eob=true;
339 else refill();
340 }
341 return ch;
342 }
343
344 //---
345 bool isEof() { return eob; }
346 bool ended() { return eob; }
347 off_t getPos() {
348 //returns the virtual file position
349 // = the actual file offset of the byte at bufpos
350 return fpos-(bufused-bufpos);
351 }
352 //skip into the stream the specified number of bytes
353 int skip(int skiplen) {
354 if (eob) return 0;
355 int r=0; //the actual number of bytes skipped
356 while (skiplen && !eob) {
357 int dif=GMIN(bufused-bufpos,skiplen);
358 skiplen-=dif;
359 bufpos+=dif;
360 r+=dif;
361 if (bufpos>=bufused) {
362 if (eof) { eob=true; return r; }
363 refill();
364 }
365 }
366 return r;
367 }
368 //look ahead without updating the read pointer (bufpos)
369 //Cannot peek more than buflen!
370 int peek(uchar* outbuf, int len) {
371 if (eob) return -1;
372 //if (eob || len>buflen) return -1;
373 if (len>bufused-bufpos) refill(true);
374 int mlen=GMIN((bufused-bufpos),len);
375 memcpy((void*)outbuf, (void*)(buf+bufpos), mlen);
376 return mlen;
377 }
378
379 uchar* peekStr(uchar* outbuf, int len) {
380 int rd=peek(outbuf,len);
381 if (rd>0) { outbuf[rd]='\0'; return outbuf; }
382 else return NULL;
383 }
384 //looks ahead to check if what follows matches
385 int peekCmp(char* cmpstr, int cmplen=0) {
386 if (eob) //GError("GReadBuf::peekcmp error: eob!\n");
387 return -2;
388 if (!cmplen) cmplen=strlen(cmpstr);
389 if (cmplen>bufused-bufpos) {
390 refill(true);
391 if (cmplen>bufused-bufpos) return -2;
392 }
393 //use memcmp
394 return memcmp((void*)(buf+bufpos), cmpstr, cmplen);
395 }
396
397 };
398
399 //circular line buffer, with read-ahead (peeking) capability
400 class GReadBufLine {
401 protected:
402 struct BufLine {
403 off_t fpos;
404 int len;
405 char* chars;
406 };
407 int bufcap; //total number of lines in the buf array
408 int bufidx; // the "current line" index in buf array
409 bool isEOF;
410 int lno;
411 FILE* file;
412 off_t filepos; //current file/stream offset for the first char of buf[bufidx]
413 BufLine* buf; //array of bufferred lines
414 char* readline(int idx);//read line from file into the buffer
415 int fillbuf();
416 bool isEOB;
417 public:
418 const char* line(); //gets current line and advances the "current line" pointer
419 //use putLine() to revert/undo this advancement
420 off_t fpos(); //gets current line's byte offset in the file
421 // does NOT advance the "current line" pointer
422 int len(); //gets current line's length
423 // does NOT advance the "current line" pointer
424 bool isEof() { return isEOB; }
425 bool eof() { return isEOB; }
426 off_t getfpos() { return fpos(); }
427 const char* getline() { return line(); }
428 const char* getLine() { return line(); }
429 int getLen() { return len(); }
430 int linenumber() { return lno; }
431 int lineno() { return lno; }
432 int getLineNo() { return lno; }
433 void putLine();
434 GReadBufLine(FILE* stream, int bcap=20) {
435 if (bcap<2) bcap=2; //at least 1 prev line is needed for putLine()
436 bufcap=bcap;
437 bufidx=-1;
438 isEOB=false;
439 isEOF=false;
440 lno=0;
441 GMALLOC(buf, bufcap * sizeof(BufLine));
442 for (int i=0;i<bufcap;i++) {
443 buf[i].chars=NULL;
444 buf[i].fpos=-1;
445 buf[i].len=0;
446 }
447 file=stream;
448 fillbuf();
449 }
450 ~GReadBufLine() {
451 for (int i=0;i<bufcap;i++) {
452 GFREE(buf[i].chars);
453 }
454 GFREE(buf);
455 }
456 };
457
458 #endif