ViewVC Help
View File | Revision Log | Show Annotations | View Changeset | Root Listing
root/gclib/gclib/gcdb.cpp
Revision: 264
Committed: Mon Aug 27 18:21:11 2012 UTC (7 years, 1 month ago) by gpertea
File size: 24469 byte(s)
Log Message:
minor refactoring etc

Line User Rev File contents
1 gpertea 2 #include "gcdb.h"
2     #include <errno.h>
3 gpertea 16
4     #ifdef __WIN32__
5     /* m m a p === from imagick sources
6 gpertea 2 % Method mmap emulates the Unix method of the same name.
7     % The format of the mmap method is:
8     % void *mmap(char *address,size_t length,int protection,
9     % int access,int file,off_t offset)
10     */
11     void *mmap(char *address,size_t length,int protection,int access,
12     int file, off_t offset) {
13     void *map;
14     HANDLE handle;
15     map=(void *) NULL;
16     handle=INVALID_HANDLE_VALUE;
17     switch (protection)
18     {
19     case PROT_READ:
20     default:
21     {
22     handle=CreateFileMapping((HANDLE) _get_osfhandle(file),0,PAGE_READONLY,0,
23     length,0);
24     if (!handle)
25     break;
26     map=(void *) MapViewOfFile(handle,FILE_MAP_READ,0,0,length);
27     CloseHandle(handle);
28     break;
29     }
30     case PROT_WRITE:
31     {
32     handle=CreateFileMapping((HANDLE) _get_osfhandle(file),0,PAGE_READWRITE,0,
33     length,0);
34     if (!handle)
35     break;
36     map=(void *) MapViewOfFile(handle,FILE_MAP_WRITE,0,0,length);
37     CloseHandle(handle);
38     break;
39     }
40     case PROT_READWRITE:
41     {
42     handle=CreateFileMapping((HANDLE) _get_osfhandle(file),0,PAGE_READWRITE,0,
43     length,0);
44     if (!handle)
45     break;
46     map=(void *) MapViewOfFile(handle,FILE_MAP_ALL_ACCESS,0,0,length);
47     CloseHandle(handle);
48     break;
49     }
50     }
51     if (map == (void *) NULL)
52     return((void *) MAP_FAILED);
53     return((void *) ((char *) map+offset));
54     }
55    
56     /* =========== m u n m a p ===========================
57     %
58     % Method munmap emulates the Unix method with the same name.
59     % The format of the munmap method is:
60     % int munmap(void *map,size_t length)
61     % A description of each parameter follows:
62     % > status: Method munmap returns 0 on success; otherwise, it
63     % returns -1 and sets errno to indicate the error.
64     % > map: The address of the binary large object.
65     % > length: The length of the binary large object.
66     %
67     */
68     int munmap(void *map,size_t length) {
69     if (!UnmapViewOfFile(map))
70     return(-1);
71     return(0);
72     }
73    
74     #endif
75    
76    
77    
78     int cdbInfoSIZE=offsetof(cdbInfo, tag)+4;
79 gpertea 16 int IdxDataSIZE=offsetof(CIdxData, reclen)+sizeof(uint32);
80     int IdxDataSIZE32=offsetof(CIdxData32, reclen)+sizeof(uint32);
81     /*
82     int IdxSeqDataSIZE=offsetof(CIdxSeqData, elen)+sizeof(byte);
83     int IdxSeqDataSIZE32=offsetof(CIdxSeqData32, elen)+sizeof(byte);
84     */
85 gpertea 2
86     //=====================================================
87     //------------- buffer stuff -------------------
88     //=====================================================
89    
90     //-------------------------------------
91     //--------- misc utility functions -----
92    
93     static int gcdb_seek_set(int fd,gcdb_seek_pos pos) {
94     if (lseek(fd, pos, 0) == -1)
95     return -1;
96     return 0;
97     }
98    
99     #define gcdb_seek_begin(fd) (gcdb_seek_set((fd),(gcdb_seek_pos) 0))
100    
101     static unsigned int gcdb_strlen(const char *s) {
102     register char *t;
103     t = (char*)s;
104     for (;;) {
105 gpertea 264 if (!*t) return t - s;
106     ++t;
107     /*if (!*t) return t - s;
108     ++t;
109     if (!*t) return t - s;
110     ++t;
111     if (!*t) return t - s;
112     ++t;
113     */
114 gpertea 2 }
115 gpertea 264
116 gpertea 2 }
117    
118    
119     static int byte_diff(char *s, unsigned int n,char *t) {
120     for (;;) {
121     if (!n) return 0; if (*s != *t) break; ++s; ++t; --n;
122     if (!n) return 0; if (*s != *t) break; ++s; ++t; --n;
123     if (!n) return 0; if (*s != *t) break; ++s; ++t; --n;
124     if (!n) return 0; if (*s != *t) break; ++s; ++t; --n;
125     }
126     return ((int)(unsigned int)(unsigned char) *s)
127     - ((int)(unsigned int)(unsigned char) *t);
128     }
129    
130     static void gcdb_byte_copy(char *to, unsigned int n, char *from) {
131     for (;;) {
132     if (!n) return; *to++ = *from++; --n;
133     if (!n) return; *to++ = *from++; --n;
134     if (!n) return; *to++ = *from++; --n;
135     if (!n) return; *to++ = *from++; --n;
136     }
137     }
138    
139     static void gcdb_byte_copyr(char *to, unsigned int n, char *from) {
140     to += n;
141     from += n;
142     for (;;) {
143     if (!n) return; *--to = *--from; --n;
144     if (!n) return; *--to = *--from; --n;
145     if (!n) return; *--to = *--from; --n;
146     if (!n) return; *--to = *--from; --n;
147     }
148     }
149    
150     #define ALIGNMENT 16 /* XXX: assuming that this alignment is enough */
151     #define SPACE 4096 /* must be multiple of ALIGNMENT */
152    
153     typedef union { char irrelevant[ALIGNMENT]; double d; } aligned;
154     static aligned realspace[SPACE / ALIGNMENT];
155     #define space ((char *) realspace)
156    
157     static unsigned int avail = SPACE; /* multiple of ALIGNMENT; 0<=avail<=SPACE */
158    
159 gpertea 16 offt_conv_func gcvt_offt;
160     uint_conv_func gcvt_uint;
161     int16_conv_func gcvt_int16;
162 gpertea 2
163     char *gcdb_alloc(unsigned int n) {
164     char *x;
165     n = ALIGNMENT + n - (n & (ALIGNMENT - 1)); /* XXX: could overflow */
166     if (n <= avail) { avail -= n; return space + avail; }
167     x = (char*) malloc(n);
168     if (!x) return NULL;
169     //if (!x) GError("Error: mgcdb_alloc(%d) failed !\n", n);
170     return x;
171     }
172    
173    
174     int GCDBuffer::write_all(char* buf, unsigned int len) {
175     int w;
176     while (len) {
177     w = op(fd,buf,len);
178     if (w == -1) {
179     if (errno == error_intr) continue;
180     return -1; /* note that some data may have been written */
181     }
182     /* if (w == 0) ; luser's fault */
183     buf += w;
184     len -= w;
185     }
186     return 0;
187     }
188    
189     int GCDBuffer::flush() {
190     int pt=p;
191     if (!pt) return 0;
192     p = 0;
193     //return allwrite(op,fd,x,pt);
194     return write_all(x,pt);
195     }
196    
197     int GCDBuffer::putalign(char *buf,unsigned int len) {
198     unsigned int bn;
199    
200     while (len > (bn = n-p)) {
201     gcdb_byte_copy(x + p,bn,buf);
202     p += bn; buf += bn; len -= bn;
203     if (GCDBuffer::flush() == -1) return -1;
204     }
205    
206     /* now len <= s->n - s->p */
207     gcdb_byte_copy(x + p,len,buf);
208     p += len;
209     return 0;
210     }
211    
212     int GCDBuffer::put(char *buf,unsigned int len) {
213     unsigned int bn=n;
214     if (len > bn - p) {
215     if (GCDBuffer::flush() == -1) return -1;
216     /* now s->p == 0 */
217     if (bn < GCDBUFFER_OUTSIZE) bn = GCDBUFFER_OUTSIZE;
218     while (len > n) {
219     if (bn > len) bn = len;
220     if (write_all(buf, bn) == -1) return -1;
221     buf += bn;
222     len -= bn;
223     }
224     }
225     /* now len <= s->n - s->p */
226     gcdb_byte_copy(x + p,len,buf);
227     p += len;
228     return 0;
229     }
230    
231     int GCDBuffer::putflush(char *buf,unsigned int len) {
232     if (flush() == -1) return -1;
233     return write_all(buf,len);
234     }
235    
236     int GCDBuffer::putsalign(char *buf) {
237     return GCDBuffer::putalign(buf, gcdb_strlen(buf));
238     }
239    
240     int GCDBuffer::puts(char *buf) {
241     return GCDBuffer::put(buf, gcdb_strlen(buf));
242     }
243    
244     int GCDBuffer::putsflush(char *buf) {
245     return GCDBuffer::putflush(buf, gcdb_strlen(buf));
246     }
247    
248     static int oneread(opfunc op,int fd, char *buf,unsigned int len) {
249     int r;
250     for (;;) {
251     r = op(fd,buf,len);
252     if (r == -1 && errno == error_intr) continue;
253     return r;
254     }
255     }
256    
257     int GCDBuffer::oneRead(char* buf, unsigned int len) {
258     return op(fd,buf,len);
259     /*int r;
260     for (;;) {
261     r = op(fd,buf,len);
262     if (r == -1 && errno == error_intr) continue;
263     return r;
264     }*/
265     }
266    
267     int GCDBuffer::getthis(char *buf,unsigned int len) {
268     if (len > p) len = p;
269     p -= len;
270     gcdb_byte_copy(buf, len,x + n);
271     n += len;
272     return len;
273     }
274    
275     int GCDBuffer::feed() {
276     int r;
277     if (p) return p;
278     r = oneRead(x,n);
279     if (r <= 0)
280     return r;
281     p = r;
282     n -= r;
283     if (n > 0) gcdb_byte_copyr(x + n,r,x);
284     return r;
285     }
286    
287     int GCDBuffer::bget(char *buf,unsigned int len) {
288     int r;
289     if (p > 0) return getthis(buf,len);
290     if (n <= len) return oneRead(buf,n);
291     r = GCDBuffer::feed(); if (r <= 0) return r;
292     return getthis(buf,len);
293     }
294    
295     int GCDBuffer::get(char *buf,unsigned int len) {
296     int r;
297     if (p > 0) return getthis(buf,len);
298     if (n <= len) return oneread(op,fd,buf,len);
299     r = GCDBuffer::feed();
300     if (r <= 0)
301     return r;
302     return getthis(buf,len);
303     }
304    
305     char* GCDBuffer::peek() {
306     return x + n;
307     }
308    
309     void GCDBuffer::seek(unsigned int len) {
310     n += len;
311     p -= len;
312     }
313    
314     int GCDBuffer::copy(GCDBuffer* bin) {
315     int n_in;
316     char *x_in;
317     for (;;) {
318     n_in = bin->feed();
319     if (n_in < 0) return -2;
320     if (!n_in) return 0;
321     x_in = bin->peek();
322     if (GCDBuffer::put(x_in,n_in) == -1) return -3;
323     bin->seek(n_in);
324     }
325     }
326    
327     //=====================================================
328     //------------- cdb utils -------------------
329     //=====================================================
330    
331     int error_intr =
332     #ifdef EINTR
333     EINTR;
334     #else
335     -1;
336     #endif
337    
338     int error_nomem =
339     #ifdef ENOMEM
340     ENOMEM;
341     #else
342     -2;
343     #endif
344    
345     int error_proto =
346     #ifdef EPROTO
347     EPROTO;
348     #else
349     -15;
350     #endif
351     //------------------------------------------------
352     //------------ allocation routines:
353    
354     /* conversion of unsigned int offsets read from a file
355     can also be used to prepare unsigned integers to be written
356     into a file in an independent platform manner
357     */
358    
359 gpertea 16 union UInt32Bytes {
360     unsigned char b[4];
361     int32_t ui;
362     };
363    
364     union UInt16Bytes {
365     unsigned char b[2];
366     int16_t ui;
367     };
368    
369    
370 gpertea 2 unsigned int uint32_sun(void* x86int) {
371 gpertea 16 UInt32Bytes ub;
372     ub.b[3]=((unsigned char*)x86int)[0];
373     ub.b[0]=((unsigned char*)x86int)[3];
374     ub.b[1]=((unsigned char*)x86int)[2];
375     ub.b[2]=((unsigned char*)x86int)[1];
376     return ub.ui;
377     }
378    
379     int16_t int16_sun(void* x86int) {
380     UInt16Bytes ub;
381     ub.b[1]=((unsigned char*)x86int)[0];
382     ub.b[0]=((unsigned char*)x86int)[1];
383     return ub.ui;
384     }
385    
386     /* unsigned int uint32_sun(void* x86int) {
387 gpertea 2 unsigned char b[4];
388     b[3]=((unsigned char*)x86int)[0];
389     b[0]=((unsigned char*)x86int)[3];
390     b[1]=((unsigned char*)x86int)[2];
391     b[2]=((unsigned char*)x86int)[1];
392     return *((unsigned int*)b);
393 gpertea 16 return *ub;
394     }*/
395 gpertea 2
396     unsigned int uint32_x86(void* offt) {
397     return *((unsigned int*)offt);
398     }
399    
400 gpertea 16 int16_t int16_x86(void* v) {
401     return *((int16_t *)v);
402     }
403 gpertea 2
404 gpertea 16 //-------- 64bit types conversion :
405     union ULongBytes {
406     unsigned char b[8];
407     off_t ob;
408     };
409    
410 gpertea 2 off_t offt_sun(void* offt) {
411 gpertea 16 //unsigned char b[8];
412     ULongBytes ub;
413     if (sizeof(off_t)==8) { //64 bit?
414     // upper words:
415     ub.b[3]=((unsigned char*)offt)[4];
416     ub.b[0]=((unsigned char*)offt)[7];
417     ub.b[1]=((unsigned char*)offt)[6];
418     ub.b[2]=((unsigned char*)offt)[5];
419     //--
420     ub.b[7]=((unsigned char*)offt)[0];
421     ub.b[4]=((unsigned char*)offt)[3];
422     ub.b[5]=((unsigned char*)offt)[2];
423     ub.b[6]=((unsigned char*)offt)[1];
424     }
425     else {
426     ub.b[3]=((unsigned char*)offt)[0];
427     ub.b[0]=((unsigned char*)offt)[3];
428     ub.b[1]=((unsigned char*)offt)[2];
429     ub.b[2]=((unsigned char*)offt)[1];
430     }
431     //return *((off_t*)b);
432     return ub.ob;
433     }
434    
435     /*
436     off_t offt_sun(void* offt) {
437 gpertea 2 unsigned char b[8];
438     if (sizeof(off_t)==8) { //64 bit?
439     // upper words:
440     b[3]=((unsigned char*)offt)[4];
441     b[0]=((unsigned char*)offt)[7];
442     b[1]=((unsigned char*)offt)[6];
443     b[2]=((unsigned char*)offt)[5];
444     //--
445     b[7]=((unsigned char*)offt)[0];
446     b[4]=((unsigned char*)offt)[3];
447     b[5]=((unsigned char*)offt)[2];
448     b[6]=((unsigned char*)offt)[1];
449     }
450     else {
451     b[3]=((unsigned char*)offt)[0];
452     b[0]=((unsigned char*)offt)[3];
453     b[1]=((unsigned char*)offt)[2];
454     b[2]=((unsigned char*)offt)[1];
455     }
456     return *((off_t*)b);
457     }
458 gpertea 16 */
459 gpertea 2
460    
461     off_t offt_x86(void* offt) {
462     return *((off_t*)offt);
463     }
464    
465    
466    
467     //------------------------ platform independent uint32 :
468    
469     void uint32_pack(char s[4],uint32 u)
470     {
471     s[0] = u & 255;
472     u >>= 8;
473     s[1] = u & 255;
474     u >>= 8;
475     s[2] = u & 255;
476     s[3] = u >> 8;
477     }
478    
479     void uint32_pack_big(char s[4],uint32 u)
480     {
481     s[3] = u & 255;
482     u >>= 8;
483     s[2] = u & 255;
484     u >>= 8;
485     s[1] = u & 255;
486     s[0] = u >> 8;
487     }
488    
489     /* unpacking: */
490    
491    
492     void uint32_unpack(char s[4],uint32 *u)
493     {
494     uint32 result;
495    
496     result = (unsigned char) s[3];
497     result <<= 8;
498     result += (unsigned char) s[2];
499     result <<= 8;
500     result += (unsigned char) s[1];
501     result <<= 8;
502     result += (unsigned char) s[0];
503    
504     *u = result;
505     }
506    
507     void uint32_unpack_big(char s[4],uint32 *u)
508     {
509     uint32 result;
510    
511     result = (unsigned char) s[0];
512     result <<= 8;
513     result += (unsigned char) s[1];
514     result <<= 8;
515     result += (unsigned char) s[2];
516     result <<= 8;
517     result += (unsigned char) s[3];
518    
519     *u = result;
520     }
521    
522 gpertea 16 /*
523     big/little endian check
524     */
525     int endian_test(void) {
526     unsigned short v=0x0001;
527     unsigned char* b = (unsigned char*)&v;
528     return b[1];
529     }
530    
531     void gcvt_endian_setup() {
532     //check endianness
533     if (endian_test()) {
534     gcvt_uint = &uint32_sun;
535     gcvt_offt = &offt_sun;
536     gcvt_int16 = &int16_sun;
537     }
538     else {
539     gcvt_uint = &uint32_x86;
540     gcvt_offt = &offt_x86;
541     gcvt_int16 = &int16_x86;
542     }
543     }
544    
545 gpertea 2 //=====================================================
546     //------------- cdb index -------------------
547     //=====================================================
548    
549     GCdbWrite::GCdbWrite(int afd) {
550     //check endianness :)
551 gpertea 16 gcvt_endian_setup();
552 gpertea 2 cdbuf=new GCDBuffer((opfunc)&write,(int) afd,(char*)bspace,sizeof bspace);
553     head = NULL;
554     split = 0;
555     hash = 0;
556     numentries = 0;
557     fd = afd;
558     pos = sizeof final;
559     gcdb_seek_set(fd, pos);
560    
561     fname[0]='\0';
562     //should return and test the result of gcdb_seek_set!!!
563     }
564    
565     GCdbWrite::GCdbWrite(char* afname) {
566 gpertea 16 #ifdef __WIN32__
567 gpertea 2 fd = open(afname,O_WRONLY | O_TRUNC | O_BINARY | O_CREAT, S_IREAD|S_IWRITE);
568     #else
569     fd = open(afname,O_WRONLY | O_NDELAY | O_TRUNC | O_CREAT, 0664);
570     #endif
571     if (fd == -1)
572     GError("GCdbWrite: Error creating file '%s'\n", fname);
573    
574 gpertea 16 gcvt_endian_setup();
575 gpertea 2
576     cdbuf=new GCDBuffer((opfunc)&write,(int) fd,(char*)bspace,sizeof bspace);
577     head = NULL;
578     split = 0;
579     hash = 0;
580     numentries = 0;
581     pos = sizeof final;
582     gcdb_seek_set(fd, pos);
583     strcpy(fname, afname);
584    
585     //should return and test the result of gcdb_seek_set!!!
586     }
587    
588     GCdbWrite::~GCdbWrite() {
589     cdbuf->flush();
590 gpertea 16 #ifndef __WIN32__
591 gpertea 2 /* NFS silliness */
592     if (fsync(fd) == -1)
593     GError("GCdbWrite: Error at fsync() for file '%s'\n",
594     fname);
595     #endif
596     if (::close(fd) == -1)
597     GError("GCdbWrite: Error at closing file '%s'\n",
598     fname);
599     delete cdbuf;
600     if (head!=NULL) free(head);
601     }
602    
603     int GCdbWrite::posplus(uint32 len) {
604     uint32 newpos = pos + len;
605     if (newpos < len) { //errno = error_nomem;
606     return -1; }
607     pos = newpos;
608     return 0;
609     }
610    
611     int GCdbWrite::addend(unsigned int keylen,unsigned int datalen,uint32 h) {
612     struct cdb_hplist *chead = head;
613     if (!chead || (chead->num >= CDB_HPLIST)) {
614     chead = (struct cdb_hplist *) gcdb_alloc(sizeof(struct cdb_hplist));
615     if (!chead) return -1;
616     chead->num = 0;
617     chead->next = head;
618     head = chead;
619     }
620     chead->hp[head->num].h = h;
621     chead->hp[head->num].p = pos;
622     ++chead->num;
623     ++numentries;
624     if (posplus(8) == -1) return -1;
625     if (posplus(keylen) == -1) return -1;
626     if (posplus(datalen) == -1) return -1;
627     return 0;
628     }
629    
630     int GCdbWrite::addbegin(unsigned int keylen,unsigned int datalen) {
631     char buf[8];
632     //if (keylen > MAX_UINT) { /* errno = error_nomem; */return -1; }
633     // if (datalen > MAX_UINT) { /*errno = error_nomem;*/ return -1; }
634     uint32_pack(buf,keylen);
635     uint32_pack(buf + 4,datalen);
636     if (cdbuf->putalign(buf,8) == -1) return -1;
637     return 0;
638     }
639    
640     #define cdbuffer_PUTC(s,c) \
641     ( ((s).n != (s).p) \
642     ? ( (s).x[(s).p++] = (c), 0 ) \
643     : (s).put(&(c),1) \
644     )
645    
646     int GCdbWrite::add(const char* key, char* recdata, unsigned int datalen) {
647     unsigned int i;
648     unsigned int klen=strlen(key);
649     if (klen<1) {
650     GMessage("Warning: zero length key found\n");
651     return 0;
652     }
653     //------------ adding record -----------------
654     if (addbegin(klen,datalen)==-1)
655     GError("GCdbWrite: Error at addbegin(%d, %d)\n",klen, datalen);
656     uint32 h=CDB_HASHSTART;
657     for (i = 0;i < klen; ++i) {
658     //if (cdbuffer_PUTC(c.cdbuf,key[i]) == -1)
659     if ( ((cdbuf->n!=cdbuf->p) ? (cdbuf->x[cdbuf->p++]=(key[i]),0 )
660     : cdbuf->put((char*)&(key[i]),1) )==-1)
661     GError("GCdbWrite: Error at cdbbuf.put, key '%s'\n", key);
662     h = cdb_hashadd(h,key[i]);
663     }
664     if (cdbuf->put(recdata,datalen) == -1)
665     GError("GCdbWrite: Error at final cdbuf.put() at key='%s', datalen=%d\n",
666     key, datalen);
667     if (addend(klen,datalen,h) == -1)
668     GError("GCdbWrite: Error at addend(%d, %d, h)\n", klen, datalen);
669     return 1;
670     }
671    
672     int GCdbWrite::addrec(const char *key,unsigned int keylen,char *data,unsigned int datalen) {
673     if (GCdbWrite::addbegin(keylen,datalen) == -1) return -1;
674     if (cdbuf->putalign((char*)key,keylen) == -1) return -1;
675     if (cdbuf->putalign(data,datalen) == -1) return -1;
676     return GCdbWrite::addend(keylen,datalen,cdb_hash(key,keylen));
677     }
678    
679    
680     int GCdbWrite::finish() {
681     char buf[8];
682     int i;
683     uint32 len;
684     uint32 u;
685     uint32 memsize;
686     uint32 icount;
687     uint32 where;
688     struct cdb_hplist *x;
689     struct cdb_hp *hp;
690    
691     for (i = 0;i < 256;++i)
692     count[i] = 0;
693    
694     for (x = head;x;x = x->next) {
695     i = x->num;
696     while (i--)
697     ++count[255 & x->hp[i].h];
698     }
699    
700     memsize = 1;
701     for (i = 0;i < 256;++i) {
702     u = count[i] * 2;
703     if (u > memsize)
704     memsize = u;
705     }
706    
707     memsize += numentries; /* no overflow possible up to now */
708     u = (uint32) 0 - (uint32) 1;
709     u /= sizeof(struct cdb_hp);
710     if (memsize > u) { /* errno = error_nomem;*/ return -1; }
711    
712     split = (struct cdb_hp *) gcdb_alloc(memsize * sizeof(struct cdb_hp));
713     if (!split) return -1;
714    
715     hash = split + numentries;
716    
717     u = 0;
718     for (i = 0;i < 256;++i) {
719     u += count[i]; /* bounded by numentries, so no overflow */
720     start[i] = u;
721     }
722    
723     for (x = head;x;x = x->next) {
724     i = x->num;
725     while (i--)
726     split[--start[255 & x->hp[i].h]] = x->hp[i];
727     }
728    
729     for (i = 0;i < 256;++i) {
730     icount = count[i];
731    
732     len = icount + icount; /* no overflow possible */
733     uint32_pack(final + 8 * i,pos);
734     uint32_pack(final + 8 * i + 4,len);
735    
736     for (u = 0;u < len;++u)
737     hash[u].h = hash[u].p = 0;
738    
739     hp = split + start[i];
740     for (u = 0;u < icount;++u) {
741     where = (hp->h >> 8) % len;
742     while (hash[where].p)
743     if (++where == len)
744     where = 0;
745     hash[where] = *hp++;
746     }
747    
748     for (u = 0;u < len;++u) {
749     uint32_pack(buf,hash[u].h);
750     uint32_pack(buf + 4,hash[u].p);
751     if (cdbuf->putalign(buf,8) == -1) return -1;
752     if (posplus(8) == -1) return -1;
753     }
754     }
755    
756     if (cdbuf->flush() == -1) return -1;
757     if (gcdb_seek_begin(fd) == -1) return -1;
758     return cdbuf->putflush(final,sizeof final);
759     }
760    
761     //=====================================================
762     //------------- cdb -------------------
763     //=====================================================
764     uint32 cdb_hashadd(uint32 h,unsigned char c) {
765     h += (h << 5);
766     return h ^ c;
767     }
768    
769     uint32 cdb_hash(const char *buf,unsigned int len) {
770     uint32 h;
771     h = CDB_HASHSTART;
772     while (len) {
773     h = cdb_hashadd(h,*buf++);
774     --len;
775     }
776     return h;
777     }
778    
779     //---------------------------------------------------------------
780     //-------------------------- cdb methods ------------------------
781    
782     GCdbRead::GCdbRead(int afd) {
783     struct stat st;
784     char *x;
785     map=NULL;
786 gpertea 16 gcvt_endian_setup();
787 gpertea 2
788     findstart();
789     fd = afd;
790 gpertea 16 if (fstat(fd,&st) == 0) {
791 gpertea 2 if (st.st_size <= MAX_UINT) {
792     #ifndef NO_MMAP
793     x = (char *) mmap(0,st.st_size,PROT_READ,MAP_SHARED,fd,0);
794     if (x + 1) {
795     size = st.st_size;
796     map = x;
797     }
798     else {
799     GError("Error mapping the file (size=%ld)!\n",st.st_size);
800     }
801     #endif
802     }
803     else {
804     GError("Error mapping the file (size %ld > MAX_UINT)\n",
805     st.st_size);
806     }
807 gpertea 16 }
808 gpertea 2 }
809    
810     GCdbRead::GCdbRead(char* afname) {
811     struct stat st;
812     char *x;
813     map=NULL;
814 gpertea 16 gcvt_endian_setup();
815 gpertea 2
816     findstart();
817     #ifdef __WIN32__
818     fd = open(afname, O_RDONLY|O_BINARY);
819     #else
820     fd = open(afname, O_RDONLY);
821     #endif
822     if (fd == -1)
823     GError("Error: cannot open file %s\n", afname);
824     strcpy(fname, afname);
825 gpertea 16 if (fstat(fd,&st) == 0) {
826 gpertea 2 if (st.st_size <= MAX_UINT) {
827     #ifndef NO_MMAP
828     x = (char *) mmap(0,st.st_size,PROT_READ,MAP_SHARED,fd,0);
829     if (x + 1) {
830     size = st.st_size;
831     map = x;
832     }
833     else {
834     GError("GCdbRead: Error mapping the file (size=%ld)!\n",st.st_size);
835     }
836     #endif
837     }
838     else {
839     GError("GCdbRead: Error mapping the file (size %ld > MAX_UINT)\n",
840     st.st_size);
841     }
842 gpertea 16 }
843 gpertea 2 }
844    
845    
846     GCdbRead::~GCdbRead() {
847     if (map!=NULL) {
848     munmap(map,size);
849     map = 0;
850     }
851     }
852    
853     int GCdbRead::read(char *buf,unsigned int len, uint32 pos) {
854     #ifndef NO_MMAP
855     if (map) {
856     if ((pos > size) || (size - pos < len)) {
857     /* errno = error_proto; */
858     return -1;
859     }
860     gcdb_byte_copy(buf, len, map + pos);
861     }
862     else
863     #endif
864     {
865     if (gcdb_seek_set(fd,pos) == -1) return -1;
866     while (len > 0) {
867     int r;
868     do {
869     r = ::read(fd,buf,len);
870     } while ((r == -1) && (errno == error_intr));
871     if (r == -1) return -1;
872     if (r == 0) {
873     //errno = error_proto;
874     return -1;
875     }
876     buf += r;
877     len -= r;
878     }
879     }
880     return 0;
881     }
882    
883     int GCdbRead::match(const char *key, unsigned int len, uint32 pos) {
884     char buf[32];
885     unsigned int n;
886     while (len > 0) {
887     n = sizeof buf;
888     if (n > len) n = len;
889     if (GCdbRead::read(buf,n,pos) == -1) return -1;
890     if (byte_diff(buf,n,(char*)key)) return 0;
891     pos += n;
892     key += n;
893     len -= n;
894     }
895     return 1;
896     }
897    
898     int GCdbRead::findnext(const char *key,unsigned int len) {
899     char buf[8];
900     uint32 pos;
901     uint32 u;
902     if (!loop) {
903     u = cdb_hash(key,len);
904     if (GCdbRead::read(buf,8,(u << 3) & 2047) == -1) return -1;
905     uint32_unpack(buf + 4,&hslots);
906     if (!hslots) return 0;
907     uint32_unpack(buf,&hpos);
908     khash = u;
909     u >>= 8;
910     u %= hslots;
911     u <<= 3;
912     kpos = hpos + u;
913     }
914     while (loop < hslots) {
915     if (GCdbRead::read(buf,8,kpos) == -1) return - 1;
916     uint32_unpack(buf + 4, &pos);
917     if (!pos) return 0;
918     loop += 1;
919     kpos += 8;
920     if (kpos == hpos + (hslots << 3)) kpos = hpos;
921     uint32_unpack(buf,&u);
922     if (u == khash) {
923     if (GCdbRead::read(buf,8,pos) == -1) return -1;
924     uint32_unpack(buf,&u);
925     if (u == len)
926     switch(GCdbRead::match(key,len,pos + 8)) {
927     case -1:
928     return -1;
929     case 1:
930     uint32_unpack(buf + 4,&dlen);
931     dpos = pos + 8 + len;
932     return 1;
933     }
934     }
935     }
936     return 0;
937     }
938    
939     int GCdbRead::find(const char *key) {
940     GCdbRead::findstart();
941     return GCdbRead::findnext(key,gcdb_strlen(key));
942     }
943    
944     //----- GReadBuf and GReadBufLine
945    
946     char* GReadBufLine::readline(int idx) {
947     //reads a char at a time until \n and/or \r are encountered
948     GFREE(buf[idx].chars);
949     buf[idx].len=0;
950     if (isEOF) return NULL;
951     int len=0;
952     buf[idx].fpos=filepos;
953     int c=0;
954     int allocated=256;
955     GMALLOC(buf[idx].chars, allocated);
956     while ((c=getc(file))!=EOF) {
957     if (len>=allocated-1) {
958     allocated+=256;
959     GREALLOC(buf[idx].chars, allocated);
960     }
961     if (c=='\n' || c=='\r') {
962     buf[idx].chars[len]='\0';
963     if (c=='\r') { //DOS file -- special case
964     if ((c=getc(file))!='\n') ungetc(c,file);
965     else filepos++;
966     }
967     filepos++;
968     buf[idx].len=len;
969     return buf[idx].chars;
970     }
971     filepos++;
972     buf[idx].chars[len]=(char)c;
973     len++;
974     } //while i<buf_cap-1
975     if (c==EOF) { //end of file reached while reading chars
976     isEOF=true;
977     }
978     buf[idx].len=len;
979     if (len==0 && isEOF) {
980     GFREE(buf[idx].chars);
981     }
982     else {
983     buf[idx].chars[len]='\0';
984     }
985     return buf[idx].chars;
986     }
987    
988    
989    
990     int GReadBufLine::fillbuf() {
991     if (isEOF) return -1;
992     if (bufidx==0 || bufidx==1) return 0; //buffer was just filled!
993     int bufstart=0;
994     GASSERT( (bufidx<=bufcap) );
995     if (bufidx>0) { //preserve the lines already in buffer
996     int bidx=bufidx-1;//always leave room for PREVIOUS line, for putLine()
997     for (int i=0;i<bufcap-bidx;i++) {
998     GFREE(buf[i].chars);
999     buf[i]=buf[bidx+i];
1000     buf[bidx+i].chars=NULL;
1001     }
1002     //memmove((void*)&buf[0], (void*)&buf[bidx], (bufcap-bidx)*sizeof(BufLine));
1003     bufstart=bufcap-bidx;
1004     bufidx=1;
1005     }
1006     else {
1007     bufidx=0; //only the first time, before the first getLine()
1008     }
1009     int rlines=0;
1010     for (int i=bufstart;i<bufcap;i++) {
1011     if (readline(i)!=NULL) rlines++;
1012     }
1013     return rlines;
1014     }
1015    
1016     //get a line from the buffer, update "current line" pointer
1017     const char* GReadBufLine::line() {
1018     if (isEOB) return NULL;
1019     GASSERT( (bufidx>=0 && bufidx<bufcap) );
1020     char* r=buf[bufidx].chars;
1021     lno++;
1022     if (r==NULL) {
1023     isEOB=true;
1024     return NULL;
1025     }
1026     bufidx++;
1027     if (bufidx==bufcap) {
1028     if (isEOF) isEOB=true;
1029     else fillbuf();
1030     }
1031     return r;
1032     }
1033    
1034     off_t GReadBufLine::fpos() {
1035     if (isEOB || bufidx==0) return -1;
1036     GASSERT( (bufidx>0 && bufidx<bufcap) );
1037     return buf[bufidx-1].fpos;
1038     }
1039    
1040     int GReadBufLine::len() {
1041     if (isEOB || bufidx==0) return -1;
1042     GASSERT( (bufidx>0 && bufidx<bufcap) );
1043     return buf[bufidx-1].len;
1044     }
1045    
1046     void GReadBufLine::putLine() {
1047     if (bufidx==0) GError("Error: calling putLine() before getLine()!\n");
1048     bufidx--;
1049     isEOB=false;
1050     lno--;
1051     }