ViewVC Help
View File | Revision Log | Show Annotations | View Changeset | Root Listing
root/gclib/gclib/gcdb.cpp
Revision: 310
Committed: Fri Mar 22 20:06:27 2013 UTC (6 years ago) by gpertea
File size: 24470 byte(s)
Log Message:
sync with igm repo

Line File contents
1 #include "gcdb.h"
2 #include <errno.h>
3
4 #ifdef __WIN32__
5 /* m m a p === from imagick sources
6 % Method mmap emulates the Unix method of the same name.
7 % The format of the mmap method is:
8 % void *mmap(char *address,size_t length,int protection,
9 % int access,int file,off_t offset)
10 */
11 void *mmap(char *address,size_t length,int protection,int access,
12 int file, off_t offset) {
13 void *map;
14 HANDLE handle;
15 map=(void *) NULL;
16 handle=INVALID_HANDLE_VALUE;
17 switch (protection)
18 {
19 case PROT_READ:
20 default:
21 {
22 handle=CreateFileMapping((HANDLE) _get_osfhandle(file),0,PAGE_READONLY,0,
23 length,0);
24 if (!handle)
25 break;
26 map=(void *) MapViewOfFile(handle,FILE_MAP_READ,0,0,length);
27 CloseHandle(handle);
28 break;
29 }
30 case PROT_WRITE:
31 {
32 handle=CreateFileMapping((HANDLE) _get_osfhandle(file),0,PAGE_READWRITE,0,
33 length,0);
34 if (!handle)
35 break;
36 map=(void *) MapViewOfFile(handle,FILE_MAP_WRITE,0,0,length);
37 CloseHandle(handle);
38 break;
39 }
40 case PROT_READWRITE:
41 {
42 handle=CreateFileMapping((HANDLE) _get_osfhandle(file),0,PAGE_READWRITE,0,
43 length,0);
44 if (!handle)
45 break;
46 map=(void *) MapViewOfFile(handle,FILE_MAP_ALL_ACCESS,0,0,length);
47 CloseHandle(handle);
48 break;
49 }
50 }
51 if (map == (void *) NULL)
52 return((void *) MAP_FAILED);
53 return((void *) ((char *) map+offset));
54 }
55
56 /* =========== m u n m a p ===========================
57 %
58 % Method munmap emulates the Unix method with the same name.
59 % The format of the munmap method is:
60 % int munmap(void *map,size_t length)
61 % A description of each parameter follows:
62 % > status: Method munmap returns 0 on success; otherwise, it
63 % returns -1 and sets errno to indicate the error.
64 % > map: The address of the binary large object.
65 % > length: The length of the binary large object.
66 %
67 */
68 int munmap(void *map,size_t length) {
69 if (!UnmapViewOfFile(map))
70 return(-1);
71 return(0);
72 }
73
74 #endif
75
76
77
78 int cdbInfoSIZE=offsetof(cdbInfo, tag)+4;
79 int IdxDataSIZE=offsetof(CIdxData, reclen)+sizeof(uint32);
80 int IdxDataSIZE32=offsetof(CIdxData32, reclen)+sizeof(uint32);
81 /*
82 int IdxSeqDataSIZE=offsetof(CIdxSeqData, elen)+sizeof(byte);
83 int IdxSeqDataSIZE32=offsetof(CIdxSeqData32, elen)+sizeof(byte);
84 */
85
86 //=====================================================
87 //------------- buffer stuff -------------------
88 //=====================================================
89
90 //-------------------------------------
91 //--------- misc utility functions -----
92
93 static int gcdb_seek_set(int fd,gcdb_seek_pos pos) {
94 if (lseek(fd, pos, 0) == -1)
95 return -1;
96 return 0;
97 }
98
99 #define gcdb_seek_begin(fd) (gcdb_seek_set((fd),(gcdb_seek_pos) 0))
100
101 static unsigned int gcdb_strlen(const char *s) {
102 register char *t;
103 t = (char*)s;
104 for (;;) {
105 if (!*t) return t - s;
106 ++t;
107 /*if (!*t) return t - s;
108 ++t;
109 if (!*t) return t - s;
110 ++t;
111 if (!*t) return t - s;
112 ++t;
113 */
114 }
115
116 }
117
118
119 static int byte_diff(char *s, unsigned int n,char *t) {
120 for (;;) {
121 if (!n) return 0; if (*s != *t) break; ++s; ++t; --n;
122 if (!n) return 0; if (*s != *t) break; ++s; ++t; --n;
123 if (!n) return 0; if (*s != *t) break; ++s; ++t; --n;
124 if (!n) return 0; if (*s != *t) break; ++s; ++t; --n;
125 }
126 return ((int)(unsigned int)(unsigned char) *s)
127 - ((int)(unsigned int)(unsigned char) *t);
128 }
129
130 static void gcdb_byte_copy(char *to, unsigned int n, char *from) {
131 for (;;) {
132 if (!n) return; *to++ = *from++; --n;
133 if (!n) return; *to++ = *from++; --n;
134 if (!n) return; *to++ = *from++; --n;
135 if (!n) return; *to++ = *from++; --n;
136 }
137 }
138
139 static void gcdb_byte_copyr(char *to, unsigned int n, char *from) {
140 to += n;
141 from += n;
142 for (;;) {
143 if (!n) return; *--to = *--from; --n;
144 if (!n) return; *--to = *--from; --n;
145 if (!n) return; *--to = *--from; --n;
146 if (!n) return; *--to = *--from; --n;
147 }
148 }
149
150 #define ALIGNMENT 16 /* XXX: assuming that this alignment is enough */
151 #define SPACE 4096 /* must be multiple of ALIGNMENT */
152
153 typedef union { char irrelevant[ALIGNMENT]; double d; } aligned;
154 static aligned realspace[SPACE / ALIGNMENT];
155 #define space ((char *) realspace)
156
157 static unsigned int avail = SPACE; /* multiple of ALIGNMENT; 0<=avail<=SPACE */
158
159 offt_conv_func gcvt_offt;
160 uint_conv_func gcvt_uint;
161 int16_conv_func gcvt_int16;
162
163 char *gcdb_alloc(unsigned int n) {
164 char *x;
165 n = ALIGNMENT + n - (n & (ALIGNMENT - 1)); /* XXX: could overflow */
166 if (n <= avail) { avail -= n; return space + avail; }
167 x = (char*) malloc(n);
168 if (!x) return NULL;
169 //if (!x) GError("Error: mgcdb_alloc(%d) failed !\n", n);
170 return x;
171 }
172
173
174 int GCDBuffer::write_all(char* buf, unsigned int len) {
175 int w;
176 while (len) {
177 w = op(fd,buf,len);
178 if (w == -1) {
179 if (errno == error_intr) continue;
180 return -1; /* note that some data may have been written */
181 }
182 /* if (w == 0) ; luser's fault */
183 buf += w;
184 len -= w;
185 }
186 return 0;
187 }
188
189 int GCDBuffer::flush() {
190 int pt=p;
191 if (!pt) return 0;
192 p = 0;
193 //return allwrite(op,fd,x,pt);
194 return write_all(x,pt);
195 }
196
197 int GCDBuffer::putalign(char *buf,unsigned int len) {
198 unsigned int bn;
199
200 while (len > (bn = n-p)) {
201 gcdb_byte_copy(x + p,bn,buf);
202 p += bn; buf += bn; len -= bn;
203 if (GCDBuffer::flush() == -1) return -1;
204 }
205
206 /* now len <= s->n - s->p */
207 gcdb_byte_copy(x + p,len,buf);
208 p += len;
209 return 0;
210 }
211
212 int GCDBuffer::put(char *buf,unsigned int len) {
213 unsigned int bn=n;
214 if (len > bn - p) {
215 if (GCDBuffer::flush() == -1) return -1;
216 /* now s->p == 0 */
217 if (bn < GCDBUFFER_OUTSIZE) bn = GCDBUFFER_OUTSIZE;
218 while (len > n) {
219 if (bn > len) bn = len;
220 if (write_all(buf, bn) == -1) return -1;
221 buf += bn;
222 len -= bn;
223 }
224 }
225 /* now len <= s->n - s->p */
226 gcdb_byte_copy(x + p,len,buf);
227 p += len;
228 return 0;
229 }
230
231 int GCDBuffer::putflush(char *buf,unsigned int len) {
232 if (flush() == -1) return -1;
233 return write_all(buf,len);
234 }
235
236 int GCDBuffer::putsalign(char *buf) {
237 return GCDBuffer::putalign(buf, gcdb_strlen(buf));
238 }
239
240 int GCDBuffer::puts(char *buf) {
241 return GCDBuffer::put(buf, gcdb_strlen(buf));
242 }
243
244 int GCDBuffer::putsflush(char *buf) {
245 return GCDBuffer::putflush(buf, gcdb_strlen(buf));
246 }
247
248 static int oneread(opfunc op,int fd, char *buf,unsigned int len) {
249 int r;
250 for (;;) {
251 r = op(fd,buf,len);
252 if (r == -1 && errno == error_intr) continue;
253 return r;
254 }
255 }
256
257 int GCDBuffer::oneRead(char* buf, unsigned int len) {
258 return op(fd,buf,len);
259 /*int r;
260 for (;;) {
261 r = op(fd,buf,len);
262 if (r == -1 && errno == error_intr) continue;
263 return r;
264 }*/
265 }
266
267 int GCDBuffer::getthis(char *buf,unsigned int len) {
268 if (len > p) len = p;
269 p -= len;
270 gcdb_byte_copy(buf, len,x + n);
271 n += len;
272 return len;
273 }
274
275 int GCDBuffer::feed() {
276 int r;
277 if (p) return p;
278 r = oneRead(x,n);
279 if (r <= 0)
280 return r;
281 p = r;
282 n -= r;
283 if (n > 0) gcdb_byte_copyr(x + n,r,x);
284 return r;
285 }
286
287 int GCDBuffer::bget(char *buf,unsigned int len) {
288 int r;
289 if (p > 0) return getthis(buf,len);
290 if (n <= len) return oneRead(buf,n);
291 r = GCDBuffer::feed(); if (r <= 0) return r;
292 return getthis(buf,len);
293 }
294
295 int GCDBuffer::get(char *buf,unsigned int len) {
296 int r;
297 if (p > 0) return getthis(buf,len);
298 if (n <= len) return oneread(op,fd,buf,len);
299 r = GCDBuffer::feed();
300 if (r <= 0)
301 return r;
302 return getthis(buf,len);
303 }
304
305 char* GCDBuffer::peek() {
306 return x + n;
307 }
308
309 void GCDBuffer::seek(unsigned int len) {
310 n += len;
311 p -= len;
312 }
313
314 int GCDBuffer::copy(GCDBuffer* bin) {
315 int n_in;
316 char *x_in;
317 for (;;) {
318 n_in = bin->feed();
319 if (n_in < 0) return -2;
320 if (!n_in) return 0;
321 x_in = bin->peek();
322 if (GCDBuffer::put(x_in,n_in) == -1) return -3;
323 bin->seek(n_in);
324 }
325 }
326
327 //=====================================================
328 //------------- cdb utils -------------------
329 //=====================================================
330
331 int error_intr =
332 #ifdef EINTR
333 EINTR;
334 #else
335 -1;
336 #endif
337
338 int error_nomem =
339 #ifdef ENOMEM
340 ENOMEM;
341 #else
342 -2;
343 #endif
344
345 int error_proto =
346 #ifdef EPROTO
347 EPROTO;
348 #else
349 -15;
350 #endif
351 //------------------------------------------------
352 //------------ allocation routines:
353
354 /* conversion of unsigned int offsets read from a file
355 can also be used to prepare unsigned integers to be written
356 into a file in an independent platform manner
357 */
358
359 union UInt32Bytes {
360 unsigned char b[4];
361 int32_t ui;
362 };
363
364 union UInt16Bytes {
365 unsigned char b[2];
366 int16_t ui;
367 };
368
369
370 unsigned int uint32_sun(void* x86int) {
371 UInt32Bytes ub;
372 ub.b[3]=((unsigned char*)x86int)[0];
373 ub.b[0]=((unsigned char*)x86int)[3];
374 ub.b[1]=((unsigned char*)x86int)[2];
375 ub.b[2]=((unsigned char*)x86int)[1];
376 return ub.ui;
377 }
378
379 int16_t int16_sun(void* x86int) {
380 UInt16Bytes ub;
381 ub.b[1]=((unsigned char*)x86int)[0];
382 ub.b[0]=((unsigned char*)x86int)[1];
383 return ub.ui;
384 }
385
386 /* unsigned int uint32_sun(void* x86int) {
387 unsigned char b[4];
388 b[3]=((unsigned char*)x86int)[0];
389 b[0]=((unsigned char*)x86int)[3];
390 b[1]=((unsigned char*)x86int)[2];
391 b[2]=((unsigned char*)x86int)[1];
392 return *((unsigned int*)b);
393 return *ub;
394 }*/
395
396 unsigned int uint32_x86(void* offt) {
397 return *((unsigned int*)offt);
398 }
399
400 int16_t int16_x86(void* v) {
401 return *((int16_t *)v);
402 }
403
404 //-------- 64bit types conversion :
405 union ULongBytes {
406 unsigned char b[8];
407 off_t ob;
408 };
409
410 off_t offt_sun(void* offt) {
411 //unsigned char b[8];
412 ULongBytes ub;
413 if (sizeof(off_t)==8) { //64 bit?
414 // upper words:
415 ub.b[3]=((unsigned char*)offt)[4];
416 ub.b[0]=((unsigned char*)offt)[7];
417 ub.b[1]=((unsigned char*)offt)[6];
418 ub.b[2]=((unsigned char*)offt)[5];
419 //--
420 ub.b[7]=((unsigned char*)offt)[0];
421 ub.b[4]=((unsigned char*)offt)[3];
422 ub.b[5]=((unsigned char*)offt)[2];
423 ub.b[6]=((unsigned char*)offt)[1];
424 }
425 else {
426 ub.b[3]=((unsigned char*)offt)[0];
427 ub.b[0]=((unsigned char*)offt)[3];
428 ub.b[1]=((unsigned char*)offt)[2];
429 ub.b[2]=((unsigned char*)offt)[1];
430 }
431 //return *((off_t*)b);
432 return ub.ob;
433 }
434
435 /*
436 off_t offt_sun(void* offt) {
437 unsigned char b[8];
438 if (sizeof(off_t)==8) { //64 bit?
439 // upper words:
440 b[3]=((unsigned char*)offt)[4];
441 b[0]=((unsigned char*)offt)[7];
442 b[1]=((unsigned char*)offt)[6];
443 b[2]=((unsigned char*)offt)[5];
444 //--
445 b[7]=((unsigned char*)offt)[0];
446 b[4]=((unsigned char*)offt)[3];
447 b[5]=((unsigned char*)offt)[2];
448 b[6]=((unsigned char*)offt)[1];
449 }
450 else {
451 b[3]=((unsigned char*)offt)[0];
452 b[0]=((unsigned char*)offt)[3];
453 b[1]=((unsigned char*)offt)[2];
454 b[2]=((unsigned char*)offt)[1];
455 }
456 return *((off_t*)b);
457 }
458 */
459
460
461 off_t offt_x86(void* offt) {
462 return *((off_t*)offt);
463 }
464
465
466
467 //------------------------ platform independent uint32 :
468
469 void uint32_pack(char s[4],uint32 u)
470 {
471 s[0] = u & 255;
472 u >>= 8;
473 s[1] = u & 255;
474 u >>= 8;
475 s[2] = u & 255;
476 s[3] = u >> 8;
477 }
478
479 void uint32_pack_big(char s[4],uint32 u)
480 {
481 s[3] = u & 255;
482 u >>= 8;
483 s[2] = u & 255;
484 u >>= 8;
485 s[1] = u & 255;
486 s[0] = u >> 8;
487 }
488
489 /* unpacking: */
490
491
492 void uint32_unpack(char s[4],uint32 *u)
493 {
494 uint32 result;
495
496 result = (unsigned char) s[3];
497 result <<= 8;
498 result += (unsigned char) s[2];
499 result <<= 8;
500 result += (unsigned char) s[1];
501 result <<= 8;
502 result += (unsigned char) s[0];
503
504 *u = result;
505 }
506
507 void uint32_unpack_big(char s[4],uint32 *u)
508 {
509 uint32 result;
510
511 result = (unsigned char) s[0];
512 result <<= 8;
513 result += (unsigned char) s[1];
514 result <<= 8;
515 result += (unsigned char) s[2];
516 result <<= 8;
517 result += (unsigned char) s[3];
518
519 *u = result;
520 }
521
522 /*
523 big/little endian check
524 */
525 int endian_test(void) {
526 unsigned short v=0x0001;
527 unsigned char* b = (unsigned char*)&v;
528 return b[1];
529 }
530
531 void gcvt_endian_setup() {
532 //check endianness
533 if (endian_test()) {
534 gcvt_uint = &uint32_sun;
535 gcvt_offt = &offt_sun;
536 gcvt_int16 = &int16_sun;
537 }
538 else {
539 gcvt_uint = &uint32_x86;
540 gcvt_offt = &offt_x86;
541 gcvt_int16 = &int16_x86;
542 }
543 }
544
545 //=====================================================
546 //------------- cdb index -------------------
547 //=====================================================
548
549 GCdbWrite::GCdbWrite(int afd) {
550 //check endianness :)
551 gcvt_endian_setup();
552 cdbuf=new GCDBuffer((opfunc)&write,(int) afd,(char*)bspace,sizeof bspace);
553 head = NULL;
554 split = 0;
555 hash = 0;
556 numentries = 0;
557 fd = afd;
558 pos = sizeof final;
559 gcdb_seek_set(fd, pos);
560
561 fname[0]='\0';
562 //should return and test the result of gcdb_seek_set!!!
563 }
564
565 GCdbWrite::GCdbWrite(char* afname) {
566 #ifdef __WIN32__
567 fd = open(afname,O_WRONLY | O_TRUNC | O_BINARY | O_CREAT, S_IREAD|S_IWRITE);
568 #else
569 fd = open(afname,O_WRONLY | O_NDELAY | O_TRUNC | O_CREAT, 0664);
570 #endif
571 if (fd == -1)
572 GError("GCdbWrite: Error creating file '%s'\n", fname);
573
574 gcvt_endian_setup();
575
576 cdbuf=new GCDBuffer((opfunc)&write,(int) fd,(char*)bspace,sizeof bspace);
577 head = NULL;
578 split = 0;
579 hash = 0;
580 numentries = 0;
581 pos = sizeof final;
582 gcdb_seek_set(fd, pos);
583 strcpy(fname, afname);
584
585 //should return and test the result of gcdb_seek_set!!!
586 }
587
588 GCdbWrite::~GCdbWrite() {
589 cdbuf->flush();
590 #ifndef __WIN32__
591 /* NFS silliness */
592 if (fsync(fd) == -1)
593 GError("GCdbWrite: Error at fsync() for file '%s'\n",
594 fname);
595 #endif
596 if (::close(fd) == -1)
597 GError("GCdbWrite: Error at closing file '%s'\n",
598 fname);
599 delete cdbuf;
600 if (head!=NULL) free(head);
601 }
602
603 int GCdbWrite::posplus(uint32 len) {
604 uint32 newpos = pos + len;
605 if (newpos < len) { //errno = error_nomem;
606 return -1; }
607 pos = newpos;
608 return 0;
609 }
610
611 int GCdbWrite::addend(unsigned int keylen,unsigned int datalen,uint32 h) {
612 struct cdb_hplist *chead = head;
613 if (!chead || (chead->num >= CDB_HPLIST)) {
614 chead = (struct cdb_hplist *) gcdb_alloc(sizeof(struct cdb_hplist));
615 if (!chead) return -1;
616 chead->num = 0;
617 chead->next = head;
618 head = chead;
619 }
620 chead->hp[head->num].h = h;
621 chead->hp[head->num].p = pos;
622 ++chead->num;
623 ++numentries;
624 if (posplus(8) == -1) return -1;
625 if (posplus(keylen) == -1) return -1;
626 if (posplus(datalen) == -1) return -1;
627 return 0;
628 }
629
630 int GCdbWrite::addbegin(unsigned int keylen,unsigned int datalen) {
631 char buf[8];
632 //if (keylen > MAX_UINT) { /* errno = error_nomem; */return -1; }
633 // if (datalen > MAX_UINT) { /*errno = error_nomem;*/ return -1; }
634 uint32_pack(buf,keylen);
635 uint32_pack(buf + 4,datalen);
636 if (cdbuf->putalign(buf,8) == -1) return -1;
637 return 0;
638 }
639
640 #define cdbuffer_PUTC(s,c) \
641 ( ((s).n != (s).p) \
642 ? ( (s).x[(s).p++] = (c), 0 ) \
643 : (s).put(&(c),1) \
644 )
645
646 int GCdbWrite::add(const char* key, char* recdata, unsigned int datalen) {
647 unsigned int i;
648 unsigned int klen=strlen(key);
649 if (klen<1) {
650 GMessage("Warning: zero length key found\n");
651 return 0;
652 }
653 //------------ adding record -----------------
654 if (addbegin(klen,datalen)==-1)
655 GError("GCdbWrite: Error at addbegin(%d, %d)\n",klen, datalen);
656 uint32 h=CDB_HASHSTART;
657 for (i = 0;i < klen; ++i) {
658 //if (cdbuffer_PUTC(c.cdbuf,key[i]) == -1)
659 if ( ((cdbuf->n!=cdbuf->p) ? (cdbuf->x[cdbuf->p++]=(key[i]),0 )
660 : cdbuf->put((char*)&(key[i]),1) )==-1)
661 GError("GCdbWrite: Error at cdbbuf.put, key '%s'\n", key);
662 h = cdb_hashadd(h,key[i]);
663 }
664 if (cdbuf->put(recdata,datalen) == -1)
665 GError("GCdbWrite: Error at final cdbuf.put() at key='%s', datalen=%d\n",
666 key, datalen);
667 if (addend(klen,datalen,h) == -1)
668 GError("GCdbWrite: Error at addend(%d, %d, h)\n", klen, datalen);
669 return 1;
670 }
671
672 int GCdbWrite::addrec(const char *key,unsigned int keylen,char *data,unsigned int datalen) {
673 if (GCdbWrite::addbegin(keylen,datalen) == -1) return -1;
674 if (cdbuf->putalign((char*)key,keylen) == -1) return -1;
675 if (cdbuf->putalign(data,datalen) == -1) return -1;
676 return GCdbWrite::addend(keylen,datalen,cdb_hash(key,keylen));
677 }
678
679
680 int GCdbWrite::finish() {
681 char buf[8];
682 int i;
683 uint32 len;
684 uint32 u;
685 uint32 memsize;
686 uint32 icount;
687 uint32 where;
688 struct cdb_hplist *x;
689 struct cdb_hp *hp;
690
691 for (i = 0;i < 256;++i)
692 count[i] = 0;
693
694 for (x = head;x;x = x->next) {
695 i = x->num;
696 while (i--)
697 ++count[255 & x->hp[i].h];
698 }
699
700 memsize = 1;
701 for (i = 0;i < 256;++i) {
702 u = count[i] * 2;
703 if (u > memsize)
704 memsize = u;
705 }
706
707 memsize += numentries; /* no overflow possible up to now */
708 u = (uint32) 0 - (uint32) 1;
709 u /= sizeof(struct cdb_hp);
710 if (memsize > u) { /* errno = error_nomem;*/ return -1; }
711
712 split = (struct cdb_hp *) gcdb_alloc(memsize * sizeof(struct cdb_hp));
713 if (!split) return -1;
714
715 hash = split + numentries;
716
717 u = 0;
718 for (i = 0;i < 256;++i) {
719 u += count[i]; /* bounded by numentries, so no overflow */
720 start[i] = u;
721 }
722
723 for (x = head;x;x = x->next) {
724 i = x->num;
725 while (i--)
726 split[--start[255 & x->hp[i].h]] = x->hp[i];
727 }
728
729 for (i = 0;i < 256;++i) {
730 icount = count[i];
731
732 len = icount + icount; /* no overflow possible */
733 uint32_pack(final + 8 * i,pos);
734 uint32_pack(final + 8 * i + 4,len);
735
736 for (u = 0;u < len;++u)
737 hash[u].h = hash[u].p = 0;
738
739 hp = split + start[i];
740 for (u = 0;u < icount;++u) {
741 where = (hp->h >> 8) % len;
742 while (hash[where].p)
743 if (++where == len)
744 where = 0;
745 hash[where] = *hp++;
746 }
747
748 for (u = 0;u < len;++u) {
749 uint32_pack(buf,hash[u].h);
750 uint32_pack(buf + 4,hash[u].p);
751 if (cdbuf->putalign(buf,8) == -1) return -1;
752 if (posplus(8) == -1) return -1;
753 }
754 }
755
756 if (cdbuf->flush() == -1) return -1;
757 if (gcdb_seek_begin(fd) == -1) return -1;
758 return cdbuf->putflush(final,sizeof(final));
759 }
760
761 //=====================================================
762 //------------- cdb -------------------
763 //=====================================================
764 uint32 cdb_hashadd(uint32 h,unsigned char c) {
765 h += (h << 5);
766 return h ^ c;
767 }
768
769 uint32 cdb_hash(const char *buf,unsigned int len) {
770 uint32 h;
771 h = CDB_HASHSTART;
772 while (len) {
773 h = cdb_hashadd(h,*buf++);
774 --len;
775 }
776 return h;
777 }
778
779 //---------------------------------------------------------------
780 //-------------------------- cdb methods ------------------------
781
782 GCdbRead::GCdbRead(int afd) {
783 struct stat st;
784 char *x;
785 map=NULL;
786 gcvt_endian_setup();
787
788 findstart();
789 fd = afd;
790 if (fstat(fd,&st) == 0) {
791 if (st.st_size <= MAX_UINT) {
792 #ifndef NO_MMAP
793 x = (char *) mmap(0,st.st_size,PROT_READ,MAP_SHARED,fd,0);
794 if (x + 1) {
795 size = st.st_size;
796 map = x;
797 }
798 else {
799 GError("Error mapping the file (size=%ld)!\n",st.st_size);
800 }
801 #endif
802 }
803 else {
804 GError("Error mapping the file (size %ld > MAX_UINT)\n",
805 st.st_size);
806 }
807 }
808 }
809
810 GCdbRead::GCdbRead(char* afname) {
811 struct stat st;
812 char *x;
813 map=NULL;
814 gcvt_endian_setup();
815
816 findstart();
817 #ifdef __WIN32__
818 fd = open(afname, O_RDONLY|O_BINARY);
819 #else
820 fd = open(afname, O_RDONLY);
821 #endif
822 if (fd == -1)
823 GError("Error: cannot open file %s\n", afname);
824 strcpy(fname, afname);
825 if (fstat(fd,&st) == 0) {
826 if (st.st_size <= MAX_UINT) {
827 #ifndef NO_MMAP
828 x = (char *) mmap(0,st.st_size,PROT_READ,MAP_SHARED,fd,0);
829 if (x + 1) {
830 size = st.st_size;
831 map = x;
832 }
833 else {
834 GError("GCdbRead: Error mapping the file (size=%ld)!\n",st.st_size);
835 }
836 #endif
837 }
838 else {
839 GError("GCdbRead: Error mapping the file (size %ld > MAX_UINT)\n",
840 st.st_size);
841 }
842 }
843 }
844
845
846 GCdbRead::~GCdbRead() {
847 if (map!=NULL) {
848 munmap(map,size);
849 map = 0;
850 }
851 }
852
853 int GCdbRead::read(char *buf,unsigned int len, uint32 pos) {
854 #ifndef NO_MMAP
855 if (map) {
856 if ((pos > size) || (size - pos < len)) {
857 /* errno = error_proto; */
858 return -1;
859 }
860 gcdb_byte_copy(buf, len, map + pos);
861 }
862 else
863 #endif
864 {
865 if (gcdb_seek_set(fd,pos) == -1) return -1;
866 while (len > 0) {
867 int r;
868 do {
869 r = ::read(fd,buf,len);
870 } while ((r == -1) && (errno == error_intr));
871 if (r == -1) return -1;
872 if (r == 0) {
873 //errno = error_proto;
874 return -1;
875 }
876 buf += r;
877 len -= r;
878 }
879 }
880 return 0;
881 }
882
883 int GCdbRead::match(const char *key, unsigned int len, uint32 pos) {
884 char buf[32];
885 unsigned int n;
886 while (len > 0) {
887 n = sizeof buf;
888 if (n > len) n = len;
889 if (GCdbRead::read(buf,n,pos) == -1) return -1;
890 if (byte_diff(buf,n,(char*)key)) return 0;
891 pos += n;
892 key += n;
893 len -= n;
894 }
895 return 1;
896 }
897
898 int GCdbRead::findnext(const char *key,unsigned int len) {
899 char buf[8];
900 uint32 pos;
901 uint32 u;
902 if (!loop) {
903 u = cdb_hash(key,len);
904 if (GCdbRead::read(buf,8,(u << 3) & 2047) == -1) return -1;
905 uint32_unpack(buf + 4,&hslots);
906 if (!hslots) return 0;
907 uint32_unpack(buf,&hpos);
908 khash = u;
909 u >>= 8;
910 u %= hslots;
911 u <<= 3;
912 kpos = hpos + u;
913 }
914 while (loop < hslots) {
915 if (GCdbRead::read(buf,8,kpos) == -1) return - 1;
916 uint32_unpack(buf + 4, &pos);
917 if (!pos) return 0;
918 loop += 1;
919 kpos += 8;
920 if (kpos == hpos + (hslots << 3)) kpos = hpos;
921 uint32_unpack(buf,&u);
922 if (u == khash) {
923 if (GCdbRead::read(buf,8,pos) == -1) return -1;
924 uint32_unpack(buf,&u);
925 if (u == len)
926 switch(GCdbRead::match(key,len,pos + 8)) {
927 case -1:
928 return -1;
929 case 1:
930 uint32_unpack(buf + 4,&dlen);
931 dpos = pos + 8 + len;
932 return 1;
933 }
934 }
935 }
936 return 0;
937 }
938
939 int GCdbRead::find(const char *key) {
940 GCdbRead::findstart();
941 return GCdbRead::findnext(key,gcdb_strlen(key));
942 }
943
944 //----- GReadBuf and GReadBufLine
945
946 char* GReadBufLine::readline(int idx) {
947 //reads a char at a time until \n and/or \r are encountered
948 GFREE(buf[idx].chars);
949 buf[idx].len=0;
950 if (isEOF) return NULL;
951 int len=0;
952 buf[idx].fpos=filepos;
953 int c=0;
954 int allocated=256;
955 GMALLOC(buf[idx].chars, allocated);
956 while ((c=getc(file))!=EOF) {
957 if (len>=allocated-1) {
958 allocated+=256;
959 GREALLOC(buf[idx].chars, allocated);
960 }
961 if (c=='\n' || c=='\r') {
962 buf[idx].chars[len]='\0';
963 if (c=='\r') { //DOS file -- special case
964 if ((c=getc(file))!='\n') ungetc(c,file);
965 else filepos++;
966 }
967 filepos++;
968 buf[idx].len=len;
969 return buf[idx].chars;
970 }
971 filepos++;
972 buf[idx].chars[len]=(char)c;
973 len++;
974 } //while i<buf_cap-1
975 if (c==EOF) { //end of file reached while reading chars
976 isEOF=true;
977 }
978 buf[idx].len=len;
979 if (len==0 && isEOF) {
980 GFREE(buf[idx].chars);
981 }
982 else {
983 buf[idx].chars[len]='\0';
984 }
985 return buf[idx].chars;
986 }
987
988
989
990 int GReadBufLine::fillbuf() {
991 if (isEOF) return -1;
992 if (bufidx==0 || bufidx==1) return 0; //buffer was just filled!
993 int bufstart=0;
994 GASSERT( (bufidx<=bufcap) );
995 if (bufidx>0) { //preserve the lines already in buffer
996 int bidx=bufidx-1;//always leave room for PREVIOUS line, for putLine()
997 for (int i=0;i<bufcap-bidx;i++) {
998 GFREE(buf[i].chars);
999 buf[i]=buf[bidx+i];
1000 buf[bidx+i].chars=NULL;
1001 }
1002 //memmove((void*)&buf[0], (void*)&buf[bidx], (bufcap-bidx)*sizeof(BufLine));
1003 bufstart=bufcap-bidx;
1004 bufidx=1;
1005 }
1006 else {
1007 bufidx=0; //only the first time, before the first getLine()
1008 }
1009 int rlines=0;
1010 for (int i=bufstart;i<bufcap;i++) {
1011 if (readline(i)!=NULL) rlines++;
1012 }
1013 return rlines;
1014 }
1015
1016 //get a line from the buffer, update "current line" pointer
1017 const char* GReadBufLine::line() {
1018 if (isEOB) return NULL;
1019 GASSERT( (bufidx>=0 && bufidx<bufcap) );
1020 char* r=buf[bufidx].chars;
1021 lno++;
1022 if (r==NULL) {
1023 isEOB=true;
1024 return NULL;
1025 }
1026 bufidx++;
1027 if (bufidx==bufcap) {
1028 if (isEOF) isEOB=true;
1029 else fillbuf();
1030 }
1031 return r;
1032 }
1033
1034 off_t GReadBufLine::fpos() {
1035 if (isEOB || bufidx==0) return -1;
1036 GASSERT( (bufidx>0 && bufidx<bufcap) );
1037 return buf[bufidx-1].fpos;
1038 }
1039
1040 int GReadBufLine::len() {
1041 if (isEOB || bufidx==0) return -1;
1042 GASSERT( (bufidx>0 && bufidx<bufcap) );
1043 return buf[bufidx-1].len;
1044 }
1045
1046 void GReadBufLine::putLine() {
1047 if (bufidx==0) GError("Error: calling putLine() before getLine()!\n");
1048 bufidx--;
1049 isEOB=false;
1050 lno--;
1051 }