ViewVC Help
View File | Revision Log | Show Annotations | View Changeset | Root Listing
root/gclib/fqtrim/fqtrim.cpp
(Generate patch)
# Line 3 | Line 3
3   #include "GHash.hh"
4   #include "GList.hh"
5   #include <ctype.h>
6 + #include "GAlnExtend.h"
7  
8   #define USAGE "Usage:\n\
9 < fqtrim [{-5 <5adapter> -3 <3adapter>|-f <adapters_file>}] [-a <min_matchlen>]\\\n\
10 <   [-q <minq> [-t <trim_max_len>]] [-p {64|33}] [-o <outsuffix>]\\\n\
9 > fqtrim [{-5 <5adaptor> -3 <3adaptor>|-f <adaptors_file>}] [-a <min_matchlen>]\\\n\
10 >   [-R] [-q <minq> [-t <trim_max_len>]] [-p {64|33}] [-o <outsuffix>]\\\n\
11     [-l <minlen>] [-C] [-D] [-Q] [-n <rename_prefix>] [-r <discarded.lst>]\\\n\
12      <input.fq>[,<input_mates.fq>\n\
13   \n\
14 < Trim low quality bases at the 3' end and can trim adapter sequence(s), filter\n\
14 > Trim low quality bases at the 3' end and can trim adaptor sequence(s), filter\n\
15   for low complexity and collapse duplicate reads.\n\
16   If read pairs should be trimmed and kept together (i.e. without discarding\n\
17   one read in a pair), the two file names should be given delimited by a comma\n\
# Line 24 | Line 25
25      file(s) named <input>.<outsuffix> which will be created in the \n\
26      current (working) directory; (writes to stdout if -o- is given);\n\
27      a suffix ending with .gz, .gzip or .bz2 will enforce compression\n\
28 < -f  file with adapter sequences to trim, each line having this format:\n\
29 <    <5'-adapter-sequence> <3'-adapter-sequence>\n\
30 < -5  trim the given adapter or primer sequence at the 5' end of each read\n\
28 > -f  file with adaptor sequences to trim, each line having this format:\n\
29 >    <5'-adaptor-sequence> <3'-adaptor-sequence>\n\
30 > -5  trim the given adaptor or primer sequence at the 5' end of each read\n\
31      (e.g. -5 CGACAGGTTCAGAGTTCTACAGTCCGACGATC)\n\
32 < -3  trim the given adapter sequence at the 3' end of each read\n\
32 > -3  trim the given adaptor sequence at the 3' end of each read\n\
33      (e.g. -3 TCGTATGCCGTCTTCTGCTTG)\n\
34 < -a  minimum length of exact match to adaptor sequence at the proper end (6)\n\
34 > -A  disable polyA/T trimming (enabled by default)\n\
35 > -y  minimum length of exact match to adaptor sequence at the proper end (6)\n\
36   -q  trim bases with quality value lower than <minq> (starting at the 3' end)\n\
37   -t  for -q option, maximum trimming at the 3' end is limited to <trim_max_len>\n\
38   -m  maximum percentage of Ns allowed in a read after trimming (default 7)\n\
# Line 46 | Line 48
48   -Q  convert quality values to the other Phred qv type\n\
49   -V  verbose processing\n\
50   "
51 <
51 >
52   //-z  for -o option, the output stream(s) will be first piped into the given\n
53   //   <zcmd> command, which must output to stdout (e.g. -z 'bzip2 -9 -c')\n
54  
55 <
56 < // example 3' adapter for miRNAs: TCGTATGCCGTCTTCTGCTTG
55 >
56 > // example 3' adaptor for miRNAs: TCGTATGCCGTCTTCTGCTTG
57  
58   //For paired reads sequencing:
59   //3' : ACACTCTTTCCCTACACGACGCTCTTCCGATCT
# Line 60 | Line 62
62   //FILE* f_out2=NULL; //for paired reads
63   //FILE* f_in=NULL; //input fastq (stdin if not provided)
64   //FILE* f_in2=NULL; //for paired reads
65 +
66   FILE* freport=NULL;
67  
68   bool debug=false;
69   bool verbose=false;
70   bool doCollapse=false;
71   bool doDust=false;
72 + bool doPolyTrim=true;
73   bool fastaOutput=false;
74   bool trashReport=false;
75 < //bool rawFormat=false;
75 > bool revCompl=false; //also reverse complement adaptor sequences
76   int min_read_len=16;
77   double max_perc_N=7.0;
78   int dust_cutoff=16;
79   bool isfasta=false;
80   bool convert_phred=false;
81 < GStr outsuffix; // -o
78 < //GStr adapter3;
79 < //GStr adapter5;
81 > GStr outsuffix; // -o
82   GStr prefix;
83   GStr zcmd;
84   int num_trimmed5=0;
# Line 91 | Line 93
93   int qv_phredtype=0; // could be 64 or 33 (0 means undetermined yet)
94   int qv_cvtadd=0; //could be -31 or +31
95  
96 < int a3len=0;
97 < int a5len=0;
98 < // adaptor matching metrics -- for extendMatch() function
99 < const int a_m_score=2; //match score
100 < const int a_mis_score=-3; //mismatch
101 < const int a_dropoff_score=7;
102 < int a_min_score=12; //an exact match of 6 bases at the proper ends WILL be trimmed
103 < const int a_min_chain_score=15; //for gapped alignments
104 <
105 < class CSegChain;
106 <
107 < class CSegPair {
108 <  public:
109 <   GSeg a;
110 <   GSeg b; //the adapter segment
111 <   int score;
112 <   int flags;
113 <   CSegChain* chain;
114 <   CSegPair(int astart=0, int aend=0, int bstart=0, int bend=0, int mscore=0):a(astart,aend),b(bstart, bend) {
115 <      score=mscore;
116 <      if (score==0) score=a.len()*a_m_score;
117 <      flags=0;
118 <      chain=NULL;
119 <      }
120 <   int len() { return  a.len(); }
121 <   bool operator==(CSegPair& d){
122 <      //return (a.start==d.a.start && a.end==d.a.end && b.start==d.b.start && b.end==d.b.end);
123 <      //make equal even segments that are included into one another:
124 <      return (d.a.start>=a.start && d.a.end<=a.end && d.b.start>=b.start && d.b.end<=b.end);
125 <      }
126 <   bool operator>(CSegPair& d){ //ordering based on b (adaptor) start coord and score
125 <     if (b.start==d.b.start) {
126 <        if (score==d.score) {
127 <           //just try to be consistent:
128 <           if (b.end==d.b.end) {
129 <             return (a.start==d.a.start)?(a.end<d.a.end):(a.start<d.a.start);
130 <             }
131 <           return (b.end>d.b.end);
132 <           }
133 <         else return (score<d.score);
96 > // adaptor matching metrics -- for X-drop ungapped extension
97 > //const int match_reward=2;
98 > //const int mismatch_penalty=3;
99 > const int match_reward=2;
100 > const int mismatch_penalty=4;
101 > const int Xdrop=10;
102 >
103 > const int poly_m_score=2; //match score
104 > const int poly_mis_score=-3; //mismatch
105 > const int poly_dropoff_score=7;
106 > int poly_minScore=12; //i.e. an exact match of 6 bases at the proper ends WILL be trimmed
107 >
108 > const char *polyA_seed="AAAA";
109 > const char *polyT_seed="TTTT";
110 >
111 > struct CASeqData {
112 >   //positional data for every possible hexamer in an adaptor
113 >   GVec<uint16>* pz[4096]; //0-based coordinates of all possible hexamers in the adaptor sequence
114 >   GVec<uint16>* pzr[4096]; //0-based coordinates of all possible hexamers for the reverse complement of the adaptor sequence
115 >   GStr seq; //actual adaptor sequence data
116 >   GStr seqr; //reverse complement sequence
117 >   int amlen; //fraction of adaptor length matching that's
118 >              //enough to consider the alignment
119 >   GAlnTrimType trim_type;
120 >   bool use_reverse;
121 >   CASeqData(bool rev=false):seq(),seqr(),
122 >             amlen(0), use_reverse(rev) {
123 >     trim_type=galn_None; //should be updated later!
124 >     for (int i=0;i<4096;i++) {
125 >        pz[i]=NULL;
126 >        pzr[i]=NULL;
127          }
135     return (b.start>d.b.start);
128       }
137   bool operator<(CSegPair& d){ //ordering based on b (adaptor) coord
138     /*if (b.start==d.b.start && b.end==d.b.end) {
139          return (a.start==d.a.start)?(a.end<d.a.end):(a.start<d.a.start);
140          }
141     return (b.start==d.b.start)?(b.end<d.b.end):(b.start<d.b.start);*/
142     if (b.start==d.b.start) {
143        if (score==d.score) {
144           //just try to be consistent:
145           if (b.end==d.b.end) {
146             return (a.start==d.a.start)?(a.end>d.a.end):(a.start>d.a.start);
147             }
148           return (b.end<d.b.end);
149           }
150         else return (score>d.score);
151        }
152     return (b.start<d.b.start);
153     }
154 };
129  
130 < int cmpSegEnds(pointer sa, pointer sb) { //sort by adaptor seg ends AND score
131 < CSegPair& x = *(CSegPair *)sa;
132 < CSegPair& y = *(CSegPair *)sb;
133 < /*
134 < if (x.b.end==y.b.end) {
135 <     if (x.b.start==y.b.start) {
136 <         if (x.a.end==y.a.end) {
137 <            if (x.a.start==y.a.start) return 0;
138 <            return ((x.a.start>y.a.start) ? -1 : 1);
139 <            }
140 <          else {
141 <            return ((x.a.end>y.a.end) ? -1 : 1);
142 <            }
143 <          }
144 <      else {
145 <       return ((x.b.start>y.b.start) ? -1 : 1);
146 <       }
147 <     }
148 <    else {
175 <     return ((x.b.end>y.b.end) ? -1 : 1);
176 <     }
177 < */
178 <  if (x.b.end==y.b.end) {
179 <     if (x.score==y.score) {
180 <     if (x.b.start==y.b.start) {
181 <         if (x.a.end==y.a.end) {
182 <            if (x.a.start==y.a.start) return 0;
183 <            return ((x.a.start<y.a.start) ? -1 : 1);
184 <            }
185 <          else {
186 <            return ((x.a.end<y.a.end) ? -1 : 1);
187 <            }
188 <          }
189 <      else {
190 <       return ((x.b.start<y.b.start) ? -1 : 1);
191 <       }
192 <      } else return ((x.score>y.score) ? -1 : 1);
193 <     }
194 <    else {
195 <     return ((x.b.end>y.b.end) ? -1 : 1);
130 >   void update(const char* s) {
131 >         seq=s;
132 >         table6mers(seq.chars(), seq.length(), pz);
133 >         amlen=iround(double(seq.length())*0.8);
134 >         if (amlen<12)
135 >                amlen=12;
136 >         if (!use_reverse) return;
137 >         //reverse complement
138 >         seqr=s;
139 >         int slen=seq.length();
140 >         for (int i=0;i<slen;i++)
141 >           seqr[i]=ntComplement(seq[slen-i-1]);
142 >         table6mers(seqr.chars(), seqr.length(), pzr);
143 >   }
144 >
145 >   ~CASeqData() {
146 >     for (int i=0;i<4096;i++) {
147 >       delete pz[i];
148 >       delete pzr[i];
149       }
150 +   }
151 + };
152  
153 < }
153 > GVec<CASeqData> adaptors5;
154 > GVec<CASeqData> adaptors3;
155  
156 < class CSegChain:public GList<CSegPair> {
157 < public:
202 <   uint astart;
203 <   uint aend;
204 <   uint bstart;
205 <   uint bend;
206 <   int score;
207 <   bool endSort;
208 <  CSegChain(bool aln5=false):GList<CSegPair>(true,true,true) {//sorted, free elements, unique
209 <   //as SegPairs are inserted, they will be sorted by a.start coordinate
210 <   score=0;
211 <   astart=MAX_UINT;
212 <   aend=0;
213 <   bstart=MAX_UINT;
214 <   bend=0;
215 <   endSort=aln5;
216 <   if (aln5) { setSorted(cmpSegEnds); }
217 <   }
218 < bool operator==(CSegChain& d) {
219 <   //return (score==d.score);
220 <    return (astart==d.astart && aend==d.aend && bstart==d.bstart && bend==d.bend);
221 <   }
222 < bool operator>(CSegChain& d) { // order based on b (adaptor) coordinate
223 <   //return (score<d.score);
224 <   if (bstart==d.bstart && bend==d.bend) {
225 <          return (astart==d.astart)?(aend>d.aend):(astart>d.astart);
226 <          }
227 <     return (bstart==d.bstart)?(bend>d.bend):(bstart>d.bstart);
228 <   }
229 < bool operator<(CSegChain& d) {
230 <   //return (score>d.score);
231 <   if (bstart==d.bstart && bend==d.bend) {
232 <          return (astart==d.astart)?(aend<d.aend):(astart<d.astart);
233 <          }
234 <     return (bstart==d.bstart)?(bend<d.bend):(bstart<d.bstart);
235 <   }
236 < void addSegPair(CSegPair* segp) {
237 <   if (AddIfNew(segp)!=segp) return;
238 <   score+=segp->score;
239 <   if (astart>segp->a.start) astart=segp->a.start;
240 <   if (aend<segp->a.end) aend=segp->a.end;
241 <   if (bstart>segp->b.start) bstart=segp->b.start;
242 <   if (bend<segp->b.end) bend=segp->b.end;
243 <   }
244 < //for building actual chains:
245 < bool extendChain(CSegPair* segp) { //segp expected to be "Greater Than" current chain
246 <   int bgap=0;
247 <   int agap=0;
248 <   //if (endSort) {
249 <   if (bstart>segp->b.start) {
250 <      bgap = (int)(bstart-segp->b.end);
251 <      if (abs(bgap)>2) return false;
252 <      agap = (int)(astart-segp->a.end);
253 <      if (abs(agap)>2) return false;
254 <      }
255 <     else {
256 <      bgap = (int) (segp->b.start-bend);
257 <      if (abs(bgap)>2) return false;
258 <      agap = (int)(segp->a.start-aend);
259 <      if (abs(agap)>2) return false;
260 <      }
261 <   if (agap*bgap<0) return false;
262 <   addSegPair(segp);
263 <   score-=abs(agap)+abs(bgap);
264 <   return true;
265 <   }
266 < };
156 > CGreedyAlignData* gxmem_l=NULL;
157 > CGreedyAlignData* gxmem_r=NULL;
158  
159   // element in dhash:
160   class FqDupRec {
# Line 313 | Line 204
204  
205   GHash<FqDupRec> dhash; //hash to keep track of duplicates
206  
207 < void setupFiles(FILE*& f_in, FILE*& f_in2, FILE*& f_out, FILE*& f_out2,
208 <                       GStr& s, GStr& infname, GStr& infname2);
207 > void addAdaptor(GVec<CASeqData>& adaptors, GStr& seq, GAlnTrimType trim_type);
208 > int loadAdaptors(const char* fname);
209 >
210 > void setupFiles(FILE*& f_in, FILE*& f_in2, FILE*& f_out, FILE*& f_out2,
211 >                       GStr& s, GStr& infname, GStr& infname2);
212   // uses outsuffix to generate output file names and open file handles as needed
213 <
213 >
214   void writeRead(FILE* f_out, GStr& rname, GStr& rinfo, GStr& rseq, GStr& rqv, int& outcounter);
215   void trash_report(char trashcode, GStr& rname, FILE* freport);
216  
217 < bool getFastxRec(GLineReader& fq, GStr& rseq, GStr& rqv,
217 > bool getFastxRec(GLineReader& fq, GStr& rseq, GStr& rqv,
218            GStr& rname, GStr& rinfo, GStr& infname);
219  
220 < char process_read(GStr& rname, GStr& rseq, GStr& rqv, int &l5, int &l3);
220 > char process_read(GStr& rname, GStr& rseq, GStr& rqv, int &l5, int &l3);
221   //returns 0 if the read was untouched, 1 if it was trimmed and a trash code if it was trashed
222  
223   bool ntrim(GStr& rseq, int &l5, int &l3); //returns true if any trimming occured
224   bool qtrim(GStr& qvs, int &l5, int &l3); //return true if any trimming occured
225   int dust(GStr& seq);
226 < bool trim_adapter3(GStr& seq, int &l5, int &l3); //returns true if any trimming occured
227 < bool trim_adapter5(GStr& seq, int &l5, int &l3); //returns true if any trimming occured
226 > bool trim_poly5(GStr &seq, int &l5, int &l3, const char* poly_seed); //returns true if any trimming occured
227 > bool trim_poly3(GStr &seq, int &l5, int &l3, const char* poly_seed);
228 > bool trim_adaptor5(GStr& seq, int &l5, int &l3); //returns true if any trimming occured
229 > bool trim_adaptor3(GStr& seq, int &l5, int &l3);
230  
231   void convertPhred(char* q, int len);
232   void convertPhred(GStr& q);
233  
234   int main(int argc, char * const argv[]) {
235 <  GArgs args(argc, argv, "YQDCVl:d:3:5:m:n:r:p:q:f:t:o:z:a:");
235 >  GArgs args(argc, argv, "YQDCRVAl:d:3:5:m:n:r:p:q:f:t:o:z:a:");
236    int e;
237    if ((e=args.isError())>0) {
238        GMessage("%s\nInvalid argument: %s\n", USAGE, argv[e]);
# Line 347 | Line 243
243    convert_phred=(args.getOpt('Q')!=NULL);
244    doCollapse=(args.getOpt('C')!=NULL);
245    doDust=(args.getOpt('D')!=NULL);
246 +  revCompl=(args.getOpt('R')!=NULL);
247 +  if (args.getOpt('A')) doPolyTrim=false;
248    /*
249    rawFormat=(args.getOpt('R')!=NULL);
250    if (rawFormat) {
# Line 355 | Line 253
253    */
254    prefix=args.getOpt('n');
255    GStr s=args.getOpt('l');
256 <  if (!s.is_empty())
256 >  if (!s.is_empty())
257       min_read_len=s.asInt();
258    s=args.getOpt('m');
259 <  if (!s.is_empty())
259 >  if (!s.is_empty())
260       max_perc_N=s.asDouble();
261    s=args.getOpt('d');
262    if (!s.is_empty()) {
# Line 384 | Line 282
282          qv_phredtype=64;
283          qv_cvtadd=-31;
284          }
285 <       else
285 >       else
286           GMessage("%s\nInvalid value for -p option (can only be 64 or 33)!\n",USAGE);
287       }
288 <  if (args.getOpt('3')!=NULL) {
289 <    adapter3=args.getOpt('3');
290 <    adapter3.upper();
291 <    a3len=adapter3.length();
292 <    }
293 <  if (args.getOpt('5')!=NULL) {
294 <    adapter5=args.getOpt('5');
295 <    adapter5.upper();
296 <    a5len=adapter5.length();
288 >  s=args.getOpt('f');
289 >  if (!s.is_empty()) {
290 >   loadAdaptors(s.chars());
291 >   }
292 >  bool fileAdaptors=adaptors5.Count()+adaptors3.Count();
293 >  s=args.getOpt('5');
294 >  if (!s.is_empty()) {
295 >    if (fileAdaptors)
296 >      GError("Error: options -5 and -f cannot be used together!\n");
297 >    s.upper();
298 >    addAdaptor(adaptors5, s, galn_TrimLeft);
299 >    }
300 >  s=args.getOpt('3');
301 >  if (!s.is_empty()) {
302 >    if (fileAdaptors)
303 >      GError("Error: options -3 and -f cannot be used together!\n");
304 >      s.upper();
305 >      addAdaptor(adaptors3, s, galn_TrimRight);
306      }
307 <  s=args.getOpt('a');
307 >  s=args.getOpt('y');
308    if (!s.is_empty()) {
309 <     int a_minmatch=s.asInt();
310 <     a_min_score=a_minmatch<<1;
309 >     int minmatch=s.asInt();
310 >     poly_minScore=minmatch*poly_m_score;
311       }
312 <  
312 >
313    if (args.getOpt('o')!=NULL) outsuffix=args.getOpt('o');
314                           else outsuffix="-";
315    trashReport=  (args.getOpt('r')!=NULL);
# Line 419 | Line 326
326    if (trashReport)
327      openfw(freport, args, 'r');
328    char* infile=NULL;
329 +
330 +  if (adaptors5.Count()>0)
331 +    //gxmem_l=new CGreedyAlignData(match_reward, mismatch_penalty, Xdrop-2);
332 +        gxmem_l=new CGreedyAlignData(match_reward, mismatch_penalty, Xdrop);
333 +  if (adaptors3.Count()>0)
334 +    gxmem_r=new CGreedyAlignData(match_reward, mismatch_penalty, Xdrop);
335 +
336    while ((infile=args.nextNonOpt())!=NULL) {
337 +    //for each input file
338      int incounter=0; //counter for input reads
339      int outcounter=0; //counter for output reads
340      int trash_s=0; //too short from the get go
341      int trash_Q=0;
342      int trash_N=0;
343      int trash_D=0;
344 +    int trash_poly=0;
345      int trash_A3=0;
346      int trash_A5=0;
347      s=infile;
# Line 450 | Line 366
366         int a5=0, a3=0, b5=0, b3=0;
367         char tcode=0, tcode2=0;
368         tcode=process_read(seqid, rseq, rqv, a5, a3);
369 <       //if (!doCollapse) {
454 <         if (fq2!=NULL) {
369 >       if (fq2!=NULL) {
370              getFastxRec(*fq2, rseq2, rqv2, seqid2, seqinfo2, infname2);
371              if (seqid.substr(0,seqid.length()-1)!=seqid2.substr(0,seqid2.length()-1)) {
372                 GError("Error: no paired match for read %s vs %s (%s,%s)\n",
# Line 483 | Line 398
398                 int nocounter=0;
399                 writeRead(f_out2, seqid2, seqinfo2, rseq2, rqv2, nocounter);
400                 }
401 <            } //paired read
487 <       // }
401 >            } //pair read
402         if (tcode>1) { //trashed
403 +         #ifdef GDEBUG
404 +         GMessage(" !!!!TRASH code = %c\n",tcode);
405 +         #endif
406            if (tcode=='s') trash_s++;
407 +          else if (tcode=='A' || tcode=='T') trash_poly++;
408              else if (tcode=='Q') trash_Q++;
409                else if (tcode=='N') trash_N++;
410                 else if (tcode=='D') trash_D++;
# Line 499 | Line 417
417              rseq=rseq.substr(a5,a3-a5+1);
418              if (!rqv.is_empty()) rqv=rqv.substr(a5,a3-a5+1);
419              }
420 +         #ifdef GDEBUG
421 +            GMessage("  After trimming:\n");
422 +            GMessage("%s\n",rseq.chars());
423 +         #endif
424            writeRead(f_out, seqid, seqinfo, rseq, rqv, outcounter);
425            }
426         } //for each fastq record
# Line 526 | Line 448
448                 }
449              }
450           outcounter++;
451 <         if (qd->count>maxdup_count) {
451 >         if (qd->count>maxdup_count) {
452              maxdup_count=qd->count;
453              maxdup_seq=seq;
454              }
455           if (isfasta) {
456             if (prefix.is_empty()) {
457 <             fprintf(f_out, ">%s_x%d\n%s\n", qd->firstname, qd->count,
457 >             fprintf(f_out, ">%s_x%d\n%s\n", qd->firstname, qd->count,
458                             rseq.chars());
459               }
460             else { //use custom read name
# Line 543 | Line 465
465           else { //fastq format
466            if (convert_phred) convertPhred(qd->qv, qd->len);
467            if (prefix.is_empty()) {
468 <            fprintf(f_out, "@%s_x%d\n%s\n+\n%s\n", qd->firstname, qd->count,
468 >            fprintf(f_out, "@%s_x%d\n%s\n+\n%s\n", qd->firstname, qd->count,
469                             rseq.chars(), qd->qv);
470              }
471            else { //use custom read name
# Line 579 | Line 501
501           GMessage("         Trashed by N%%:%9d\n", trash_N);
502         if (trash_Q>0)
503           GMessage("Trashed by low quality:%9d\n", trash_Q);
504 +       if (trash_poly>0)
505 +         GMessage("   Trashed by poly-A/T:%9d\n", trash_poly);
506         if (trash_A5>0)
507 <         GMessage(" Trashed by 5' adapter:%9d\n", trash_A5);
507 >         GMessage(" Trashed by 5' adaptor:%9d\n", trash_A5);
508         if (trash_A3>0)
509 <         GMessage(" Trashed by 3' adapter:%9d\n", trash_A3);
509 >         GMessage(" Trashed by 3' adaptor:%9d\n", trash_A3);
510         }
511      if (trashReport) {
512            FWCLOSE(freport);
# Line 590 | Line 514
514      FWCLOSE(f_out);
515      FWCLOSE(f_out2);
516     } //while each input file
517 <
517 > delete gxmem_l;
518 > delete gxmem_r;
519   //getc(stdin);
520   }
521  
# Line 605 | Line 530
530     const char* seq;
531     bool valid;
532     NData() {
533 +    seqlen=0;
534      NCount=0;
535      end5=0;
536      end3=0;
# Line 635 | Line 561
561       perc_N=(n*100.0)/(end5-end3+1);
562       }
563   };
564 <
564 >
565   static NData feat;
566   int perc_lenN=12; // incremental distance from ends, in percentage of
567            // sequence length, where N-trimming is done (default:12 %) (autolimited to 20)
568 <          
568 >
569   void N_analyze(int l5, int l3, int p5, int p3) {
570   /* assumes feat was filled properly */
571   int old_dif, t5,t3,v;
572   if (l3<l5+2 || p5>p3 ) {
573     feat.end5=l5+1;
574     feat.end3=l3+1;
575 <   return;
575 >   return;
576     }
577  
578   t5=feat.NPos[p5]-l5;
579   t3=l3-feat.NPos[p3];
580   old_dif=p3-p5;
581   v=(int)((((double)(l3-l5))*perc_lenN)/100);
582 < if (v>20) v=20; /* enforce N-search limit for very long reads */
582 > if (v>20) v=20; /* enforce N-search limit for very long reads */
583   if (t5 < v ) {
584     l5=feat.NPos[p5]+1;
585     p5++;
# Line 670 | Line 596
596             feat.end3=l3+1;
597             return;
598             }
599 <    else
599 >    else
600        N_analyze(l5,l3, p5,p3);
601   }
602  
# Line 711 | Line 637
637   feat.init(rseq);
638   l5=feat.end5-1;
639   l3=feat.end3-1;
640 < N_analyze(feat.end5-1, feat.end3-1, 0, feat.NCount-1);
640 > N_analyze(feat.end5-1, feat.end3-1, 0, feat.NCount-1);
641   if (l5==feat.end5-1 && l3==feat.end3-1) {
642      if (feat.perc_N>max_perc_N) {
643             feat.valid=false;
# Line 729 | Line 655
655     return true;
656     }
657   feat.N_calc();
658 <
658 >
659   if (feat.perc_N>max_perc_N) {
660        feat.valid=false;
661        l3=l5+1;
# Line 741 | Line 667
667   //--------------- dust functions ----------------
668   class DNADuster {
669   public:
670 <  int dustword;
671 <  int dustwindow;
672 <  int dustwindow2;
670 >  int dustword;
671 >  int dustwindow;
672 >  int dustwindow2;
673    int dustcutoff;
674    int mv, iv, jv;
675    int counts[32*32*32];
# Line 838 | Line 764
764                      }
765             }
766           }
767 < //return first;
767 > //return first;
768   }
769   };
770  
# Line 856 | Line 782
782   return ncount;
783   }
784  
785 + struct SLocScore {
786 +  int pos;
787 +  int score;
788 +  SLocScore(int p=0,int s=0) {
789 +    pos=p;
790 +    score=s;
791 +    }
792 +  void set(int p, int s) {
793 +    pos=p;
794 +    score=s;
795 +    }
796 +  void add(int p, int add) {
797 +    pos=p;
798 +    score+=add;
799 +    }
800 + };
801  
802 < // ------------------ adapter matching - simple k-mer seed & extend, no indels for now
803 < //when a k-mer match is found, simply try to extend the alignment using a drop-off scheme
804 < //check minimum score and
805 < //for 3' adapter trimming:
806 < //     require that the right end of the alignment for either the adaptor OR the read must be
807 < //     < 3 distance from its right end
808 < // for 5' adapter trimming:
809 < //     require that the left end of the alignment for either the adaptor OR the read must
810 < //     be at coordinate < 3 from start
811 <
812 < bool extendMatch(const char* a, int alen, int ai,
813 <                 const char* b, int blen, int bi, int mlen, int& l5, int& l3, CSegChain& segs, bool end5=false) {
814 < //so the alignment starts at ai in a, bi in b, with a perfect match of length mlen
815 < #ifdef DEBUG
816 < GStr dbg(b);
875 < #endif
876 < //if (debug) {
877 < //  GMessage(">> in %s\n\textending hit: %s at position %d\n", a, (dbg.substr(bi, mlen)).chars(), ai);
878 < //  }
879 < int a_l=ai; //alignment coordinates on a
880 < int a_r=ai+mlen-1;
881 < int b_l=bi; //alignment coordinates on b
882 < int b_r=bi+mlen-1;
883 < int ai_maxscore=ai;
884 < int bi_maxscore=bi;
885 < int score=mlen*a_m_score;
886 < int maxscore=score;
887 < int mism5score=a_mis_score;
888 < if (end5 && ai<(alen>>1)) mism5score-=2; // increase penalty for mismatches at 5' end
889 < //try to extend to the left first, if possible
890 < while (ai>0 && bi>0) {
891 <   ai--;
892 <   bi--;
893 <   score+= (a[ai]==b[bi])? a_m_score : mism5score;
894 <   if (score>maxscore) {
895 <       ai_maxscore=ai;
896 <       bi_maxscore=bi;
897 <       maxscore=score;
898 <       }
899 <     else if (maxscore-score>a_dropoff_score) break;
900 <   }
901 < a_l=ai_maxscore;
902 < b_l=bi_maxscore;
903 < //if (debug) GMessage("  after l-extend: %*s%s\t\t(score=%d)\n",a_l," ",dbg.substr(b_l,b_r-b_l+1).chars(),maxscore);
904 < //now extend to the right
905 < ai_maxscore=a_r;
906 < bi_maxscore=b_r;
907 < ai=a_r;
908 < bi=b_r;
909 < score=maxscore;
910 < //sometimes there are extra AAAAs at the end of the read, ignore those
911 < if (strcmp(&a[alen-4],"AAAA")==0) {
912 <    alen-=3;
913 <    while (a[alen-1]=='A' && alen>ai) alen--;
914 <    }
915 < while (ai<alen-1 && bi<blen-1) {
916 <   ai++;
917 <   bi++;
918 <   //score+= (a[ai]==b[bi])? a_m_score : a_mis_score;
919 <   if (a[ai]==b[bi]) { //match
920 <      score+=a_m_score;
921 <      if (ai>=alen-2) {
922 <           score+=a_m_score-(alen-ai-1);
923 <           }
802 > bool trim_poly3(GStr &seq, int &l5, int &l3, const char* poly_seed) {
803 > if (!doPolyTrim) return false;
804 > int rlen=seq.length();
805 > l5=0;
806 > l3=rlen-1;
807 > int32 seedVal=*(int32*)poly_seed;
808 > char polyChar=poly_seed[0];
809 > //assumes N trimming was already done
810 > //so a poly match should be very close to the end of the read
811 > // -- find the initial match (seed)
812 > int lmin=GMAX((rlen-16), 0);
813 > int li;
814 > for (li=rlen-4;li>lmin;li--) {
815 >   if (seedVal==*(int*)&(seq[li])) {
816 >      break;
817        }
925    else { //mismatch
926      score+=a_mis_score;
927      }  
928   if (score>maxscore) {
929       ai_maxscore=ai;
930       bi_maxscore=bi;
931       maxscore=score;
932       }
933     else if (maxscore-score>a_dropoff_score) break;
818     }
819 <  a_r=ai_maxscore;
820 <  b_r=bi_maxscore;
821 <  int a_ovh3=alen-a_r-1;
822 <  int b_ovh3=blen-b_r-1;
823 <  int mmovh3=(a_ovh3<b_ovh3)? a_ovh3 : b_ovh3;
824 <  int mmovh5=(a_l<b_l)? a_l : b_l;
825 <  //if (debug) GMessage("  after r-extend: %*s%s\t\t(score=%d)\n",a_l," ",dbg.substr(b_l,b_r-b_l+1).chars(),maxscore);
826 < #ifdef DEBUG
827 <  if (debug) GMessage("     extended to: %*s\n",a_r+1,dbg.substr(b_l,b_r-b_l+1).chars());
828 < #endif
829 <  if (maxscore>=a_min_score && mmovh3<2 && mmovh5<2) {
830 <     if (a_l<a_ovh3) {
831 <        //adapter closer to the left end (typical for 5' adapter)
832 <        l5=a_r+1;
833 <        l3=alen-1;
834 <        }
835 <      else {
836 <        //adapter matching at the right end (typical for 3' adapter)
953 <        l5=0;
954 <        l3=a_l-1;
819 > if (li<=lmin) return false;
820 > //seed found, try to extend it both ways
821 > //extend right
822 > int ri=li+3;
823 > SLocScore loc(ri, poly_m_score<<2);
824 > SLocScore maxloc(loc);
825 > //extend right
826 > while (ri<rlen-1) {
827 >   ri++;
828 >   if (seq[ri]==polyChar) {
829 >                loc.add(ri,poly_m_score);
830 >                }
831 >   else if (seq[ri]=='N') {
832 >                loc.add(ri,0);
833 >                }
834 >   else { //mismatch
835 >        loc.add(ri,poly_mis_score);
836 >        if (maxloc.score-loc.score>poly_dropoff_score) break;
837          }
838 <     return true;
839 <     }
840 < else { //keep this segment pair for later (gapped alignment)
959 <   segs.addSegPair(new CSegPair(a_l+1, a_r+1, b_l+1, b_r+1, maxscore));
960 <   //this will also update min & max coordinates in segs (segs.astart, .aend, .bstart, .bend)
961 <   }
962 <  //do not trim:
963 <  l5=0;
964 <  l3=alen-1;
965 <  return false;
966 < }
967 <
968 < /*
969 < int getWordValue(const char* s, int wlen) {
970 < int r=0;
971 < while (wlen--) { r+=(((int)s[wlen])<<wlen) }
972 < return r;
973 < }
974 < */
975 < int get3mer_value(const char* s) {
976 < return (s[0]<<16)+(s[1]<<8)+s[2];
977 < }
978 <
979 < int w3_match(int qv, const char* str, int slen, int start_index=0) {
980 < if (start_index>=slen || start_index<0) return -1;
981 < for (int i=start_index;i<slen-3;i++) {
982 <   int rv=get3mer_value(str+i);
983 <   if (rv==qv) return i;
984 <   }
985 < return -1;
986 < }
987 <
988 < int w3_rmatch(int qv, const char* str, int slen, int end_index=-1) {
989 < if (end_index>=slen) return -1;
990 < if (end_index<0) end_index=slen-1;
991 < for (int i=end_index-2;i>=0;i--) {
992 <   int rv=get3mer_value(str+i);
993 <   if (rv==qv) return i;
994 <   }
995 < return -1;
996 < }
997 <
998 < int fast4match(int32 qv, const char* str, int slen, int start_index=0) {
999 < if (start_index>=slen || start_index<0) return -1;
1000 < for (int i=start_index;i<slen-4;i++) {
1001 <   int32* rv=(int32*)(str+i);
1002 <   if (*rv==qv) return i;
1003 <   }
1004 < return -1;
1005 < }
1006 <
1007 < int fast4rmatch(int32 qv, const char* str, int slen, int end_index=-1) {
1008 < if (end_index>=slen) return -1;
1009 < if (end_index<0) end_index=slen-1;
1010 < for (int i=end_index-3;i>=0;i--) {
1011 <   int32* rv=(int32*)(str+i);
1012 <   if (*rv==qv) return i;
1013 <   }
1014 < return -1;
1015 < }
1016 <
1017 < #ifdef DEBUG
1018 < void dbgPrintChain(CSegChain& chain, const char* aseq) {
1019 <  GStr s(aseq);
1020 <  for (int i=0;i<chain.Count();i++) {
1021 <   CSegPair& seg=*chain[i];
1022 <   GMessage("  dbg chain seg%d: %*s [%d-%d:%d-%d]\n",i,seg.a.start-1+seg.len(),
1023 <            s.substr(seg.b.start-1, seg.len()).chars(), seg.b.start,seg.b.end,seg.a.start,seg.a.end);
838 >   if (maxloc.score<=loc.score) {
839 >      maxloc=loc;
840 >      }
841     }
842 + ri=maxloc.pos;
843 + if (ri<rlen-6) return false; //no trimming wanted, too far from 3' end
844 + //ri = right boundary for the poly match
845 + //extend left
846 + loc.set(li, maxloc.score);
847 + maxloc.pos=li;
848 + while (li>0) {
849 +    li--;
850 +    if (seq[li]==polyChar) {
851 +                 loc.add(li,poly_m_score);
852 +                 }
853 +    else if (seq[li]=='N') {
854 +                 loc.add(li,0);
855 +                 }
856 +    else { //mismatch
857 +         loc.add(li,poly_mis_score);
858 +         if (maxloc.score-loc.score>poly_dropoff_score) break;
859 +         }
860 +    if (maxloc.score<=loc.score) {
861 +       maxloc=loc;
862 +       }
863 +    }
864 + li=maxloc.pos;
865 + if ((maxloc.score==poly_minScore && ri==rlen-1) ||
866 +    (maxloc.score>poly_minScore && ri>=rlen-3) ||
867 +    (maxloc.score>(poly_minScore*3) && ri>=rlen-8)) {
868 +  //trimming this li-ri match at 3' end
869 +    l3=li-1;
870 +    if (l3<0) l3=0;
871 +    return true;
872 +    }
873 + return false;
874   }
1026 #endif
875  
876 < bool trim_adapter3(GStr& seq, int&l5, int &l3) {
876 > bool trim_poly5(GStr &seq, int &l5, int &l3, const char* poly_seed) {
877 > if (!doPolyTrim) return false;
878   int rlen=seq.length();
879   l5=0;
880   l3=rlen-1;
881 < //first try a full match, we might get lucky
882 < int fi=-1;
883 < if ((fi=seq.index(adapter3))>=0) {
884 <   if (fi<rlen-fi-a3len) {//match is closer to the right end
885 <      l5=fi+a3len;
886 <      l3=rlen-1;
887 <      }
888 <    else {
889 <      l5=0;
890 <      l3=fi-1;
881 > int32 seedVal=*(int32*)poly_seed;
882 > char polyChar=poly_seed[0];
883 > //assumes N trimming was already done
884 > //so a poly match should be very close to the end of the read
885 > // -- find the initial match (seed)
886 > int lmax=GMIN(12, rlen-4);//how far from 5' end to look for 4-mer seeds
887 > int li;
888 > for (li=0;li<=lmax;li++) {
889 >   if (seedVal==*(int*)&(seq[li])) {
890 >      break;
891        }
1043   return true;
892     }
893 < #ifdef DEBUG
894 < if (debug) GMessage(">TRIM3 >>   Read: %s\n",seq.chars());
895 < #endif
896 <
897 < //also, for fast detection of other adapter-only reads that start past
898 < // the beginning of the adapter sequence, try to see if the first a3len-4
899 < // bases of the read are a substring of the adapter
900 < if (rlen>a3len-3) {
901 <   GStr rstart=seq.substr(1,a3len-4);
902 <   if ((fi=adapter3.index(rstart))>=0) {
903 <     l3=rlen-1;
904 <     l5=a3len-4;
905 <     while (fi+l5<a3len && l5<l3 && adapter3[fi+l5]==seq[l5]) l5++;
906 <     return true;
907 <     }
908 <  }
909 < CSegChain a3segs; //no chains here, just an ordered collection of segment pairs
910 <  //check the easy cases - 11 bases exact match at the end
911 < int fdlen=11;
912 <  if (a3len<16) {
1065 <   fdlen=a3len>>1;
1066 <   }
1067 < if (fdlen>4) {
1068 <     //check if we're lucky enough to have the last 11 bases of the read a part of the adapter
1069 <     GStr rstart=seq.substr(-fdlen-3,fdlen);
1070 <     if ((fi=adapter3.index(rstart))>=0) {
1071 < #ifdef DEBUG
1072 <       if (debug) GMessage("  W11match found: %*s\n", rlen-3, (adapter3.substr(fi,fdlen)).chars());
1073 < #endif
1074 <       if (extendMatch(seq.chars(), rlen, rlen-fdlen-3,
1075 <                     adapter3.chars(), a3len, fi,  fdlen, l5,l3, a3segs))
1076 <            return true;
1077 <       }
1078 <     //another easy case: first 11 characters of the adaptor found as a substring of the read
1079 <     GStr bstr=adapter3.substr(0, fdlen);
1080 <     if ((fi=seq.rindex(bstr))>=0) {
1081 < #ifdef DEBUG
1082 <       if (debug) GMessage("  A11match found: %*s\n", fi+fdlen, bstr.chars());
1083 < #endif
1084 <       if (extendMatch(seq.chars(), rlen, fi,
1085 <                     adapter3.chars(), a3len, 0,  fdlen, l5,l3, a3segs))
1086 <            return true;
893 > if (li>lmax) return false;
894 > //seed found, try to extend it both ways
895 > //extend left
896 > int ri=li+3; //save rightmost base of the seed
897 > SLocScore loc(li, poly_m_score<<2);
898 > SLocScore maxloc(loc);
899 > while (li>0) {
900 >    li--;
901 >    if (seq[li]==polyChar) {
902 >                 loc.add(li,poly_m_score);
903 >                 }
904 >    else if (seq[li]=='N') {
905 >                 loc.add(li,0);
906 >                 }
907 >    else { //mismatch
908 >         loc.add(li,poly_mis_score);
909 >         if (maxloc.score-loc.score>poly_dropoff_score) break;
910 >         }
911 >    if (maxloc.score<=loc.score) {
912 >       maxloc=loc;
913         }
914 <     } //tried to match 11 bases first
915 <    
916 < //no easy cases, so let's do the wmer hashing for the first 12 bases of the adaptor
917 < //-- only extend if the match is in the 3' (ending) region of the read
918 < int wordSize=3;
919 < int hlen=12;
920 < if (hlen>a3len-wordSize) hlen=a3len-wordSize;
921 < int imin=rlen>>1; //last half of the read, left boundary for the wmer match
922 < if (imin<a3len) { imin=GMIN(a3len, rlen-wordSize); }
923 < imin=rlen-imin;
924 < for (int iw=0;iw<hlen;iw++) {
925 <   //int32* qv=(int32*)(adapter3.chars()+iw);
926 <   int qv=get3mer_value(adapter3.chars()+iw);
927 <   fi=-1;
928 <   //while ((fi=fast4rmatch(*qv, seq.chars(), rlen, fi))>=0 && fi>=imin) {
929 <   while ((fi=w3_rmatch(qv, seq.chars(), rlen, fi))>=0 && fi>=imin) {
930 <     //GMessage(" ... fi=%d after w3_rmatch() (imin=%d)\n", fi, imin);
914 >    }
915 > li=maxloc.pos;
916 > if (li>5) return false; //no trimming wanted, too far from 5' end
917 > //li = right boundary for the poly match
918 >
919 > //extend right
920 > loc.set(ri, maxloc.score);
921 > maxloc.pos=ri;
922 > while (ri<rlen-1) {
923 >   ri++;
924 >   if (seq[ri]==polyChar) {
925 >                loc.add(ri,poly_m_score);
926 >                }
927 >   else if (seq[ri]=='N') {
928 >                loc.add(ri,0);
929 >                }
930 >   else { //mismatch
931 >        loc.add(ri,poly_mis_score);
932 >        if (maxloc.score-loc.score>poly_dropoff_score) break;
933 >        }
934 >   if (maxloc.score<=loc.score) {
935 >      maxloc=loc;
936 >      }
937 >   }
938 > ri=maxloc.pos;
939 > if ((maxloc.score==poly_minScore && li==0) ||
940 >     (maxloc.score>poly_minScore && li<2)
941 >     || (maxloc.score>(poly_minScore*3) && li<8)) {
942 >    //adjust l5 to reflect this trimming of 5' end
943 >    l5=ri+1;
944 >    if (l5>rlen-1) l5=rlen-1;
945 >    return true;
946 >    }
947 > return false;
948 > }
949  
950 < #ifdef DEBUG
951 <     if (debug) GMessage("    Wmatch found: %*s\n", fi+wordSize, (adapter3.substr(iw,wordSize)).chars());
952 < #endif
953 <     if (extendMatch(seq.chars(), rlen, fi, adapter3.chars(),
954 <                   a3len, iw, wordSize, l5,l3, a3segs)) return true;
955 <     fi--;
956 <     if (fi<imin) break;
957 <     }
958 <   } //for each wmer in the first hlen bases of the adaptor
959 < /*
960 < //couldn't find a good trimming extension, hash 12 more bases of the adapter to collect more segment pairs there
961 < //but only do this if we already have segment pairs collected in the last 12 bases of the adapter
962 < if (a3segs.bstart>3 || a3segs.bend<(uint)(hlen-wordSize)) return false;
963 < int hlen2=a3len-wordSize;
964 < //if (hlen2>a3len-4) hlen2=a3len-4;
965 < if (hlen2>hlen) {
1122 < #ifdef DEBUG
1123 <     if (debug && a3segs.Count()>0) {
1124 <        GMessage("  >>>>>2nd. hash: %s\n",seq.chars());
950 > bool trim_adaptor3(GStr& seq, int&l5, int &l3) {
951 > if (adaptors3.Count()==0) return false;
952 > int rlen=seq.length();
953 > l5=0;
954 > l3=rlen-1;
955 > bool trimmed=false;
956 > GStr wseq(seq);
957 > int wlen=rlen;
958 > GXSeqData seqdata;
959 > int numruns=revCompl ? 2 : 1;
960 > GList<GXAlnInfo> bestalns(true, true, false);
961 > for (int ai=0;ai<adaptors3.Count();ai++) {
962 >   for (int r=0;r<numruns;r++) {
963 >     if (r) {
964 >          seqdata.update(adaptors3[ai].seqr.chars(), adaptors3[ai].seqr.length(),
965 >                 adaptors3[ai].pzr, wseq.chars(), wlen, adaptors3[ai].amlen);
966          }
967 < #endif
968 <     for (int iw=hlen;iw<hlen2;iw++) {
969 <         //int* qv=(int32 *)(adapter3.chars()+iw);
970 <         int qv=get3mer_value(adapter3.chars()+iw);
971 <         fi=-1;
972 <         //while ((fi=fast4rmatch(*qv, seq.chars(), rlen, fi))>=0 && fi>=imin) {
973 <         while ((fi=w3_rmatch(qv, seq.chars(), rlen, fi))>=0 && fi>=imin) {
974 <           extendMatch(seq.chars(), rlen, fi, adapter3.chars(),
975 <                         a3len, iw, wordSize, l5,l3, a3segs);
976 <           fi--;
977 <           if (fi<imin) break;
978 <           }
979 <         } //for each wmer between hlen2 and hlen bases of the adaptor
967 >     else {
968 >            seqdata.update(adaptors3[ai].seq.chars(), adaptors3[ai].seq.length(),
969 >                 adaptors3[ai].pz, wseq.chars(), wlen, adaptors3[ai].amlen);
970 >        }
971 >     GXAlnInfo* aln=match_adaptor(seqdata, adaptors3[ai].trim_type, gxmem_r, 86);
972 >         if (aln) {
973 >           if (aln->strong) {
974 >                   trimmed=true;
975 >                   bestalns.Add(aln);
976 >                   break; //will check the rest next time
977 >                   }
978 >            else bestalns.Add(aln);
979 >           }
980 >   }//forward and reverse adaptors
981 >   if (trimmed) break; //will check the rest in the next cycle
982 >  }//for each 3' adaptor
983 > if (bestalns.Count()>0) {
984 >           GXAlnInfo* aln=bestalns[0];
985 >           if (aln->sl-1 > wlen-aln->sr) {
986 >                   //keep left side
987 >                   l3-=(wlen-aln->sl+1);
988 >                   if (l3<0) l3=0;
989 >                   }
990 >           else { //keep right side
991 >                   l5+=aln->sr;
992 >                   if (l5>=rlen) l5=rlen-1;
993 >                   }
994 >           //delete aln;
995 >           //if (l3-l5+1<min_read_len) return true;
996 >           wseq=seq.substr(l5,l3-l5+1);
997 >           wlen=wseq.length();
998 >           return true; //break the loops here to report a good find
999       }
1000 < //lastly, analyze collected a3segs for a possible gapped alignment:
1141 < GList<CSegChain> segchains(false,true,false);
1142 < #ifdef DEBUG
1143 < if (debug && a3segs.Count()>0) {
1144 <   GMessage(">>>>>>>>>   Read: %s\n",seq.chars());
1145 <   }
1146 < #endif
1147 < for (int i=0;i<a3segs.Count();i++) {
1148 <   if (a3segs[i]->chain==NULL) {
1149 <       if (a3segs[i]->b.start>3) continue; //don't start a hopeless chain
1150 <       CSegChain* newchain=new CSegChain();
1151 <       newchain->setFreeItem(false);
1152 <       newchain->addSegPair(a3segs[i]);
1153 <       a3segs[i]->chain=newchain;
1154 <       segchains.Add(newchain); //just to free them when done
1155 <       }
1156 <   for (int j=i+1;j<a3segs.Count();j++) {
1157 <      CSegChain* chain=a3segs[i]->chain;
1158 <      if (chain->extendChain(a3segs[j])) {
1159 <          a3segs[j]->chain=chain;
1160 < #ifdef DEBUG
1161 <          if (debug) dbgPrintChain(*chain, adapter3.chars());
1162 < #endif      
1163 <          //save time by checking here if the extended chain is already acceptable for trimming
1164 <          if (chain->aend>(uint)(rlen-4) && chain->bstart<4 && chain->score>a_min_chain_score) {
1165 <            l5=0;
1166 <            l3=chain->astart-2;
1167 < #ifdef DEBUG
1168 <          if (debug && a3segs.Count()>0) {
1169 <            GMessage(">>> >> trimmed-3: %*s\n",l3-l5+1,seq.substr(l5,l3-l5+1).chars());
1170 <            }
1171 < #endif
1172 <            return true;
1173 <            }
1174 <          } //chain can be extended
1175 <      }
1176 <   } //collect segment alignments into chains
1177 < */  
1178 < return false; //no adapter parts found
1000 >  return false;
1001   }
1002  
1003 < bool trim_adapter5(GStr& seq, int&l5, int &l3) {
1004 < //if (debug) GMessage("trim_adapter5 on: %s\n", seq.chars());
1003 > bool trim_adaptor5(GStr& seq, int&l5, int &l3) {
1004 > if (adaptors5.Count()==0) return false;
1005   int rlen=seq.length();
1006   l5=0;
1007   l3=rlen-1;
1008 < //try to see if adapter is fully included in the read
1009 < int fi=-1;
1010 < if ((fi=seq.index(adapter5))>=0) {
1011 <   if (fi<rlen-fi-a5len) {//match is closer to the right end
1012 <      l5=fi+a5len;
1013 <      l3=rlen-1;
1014 <      }
1015 <    else {
1016 <      l5=0;
1017 <      l3=fi-1;
1018 <      }
1197 <   return true;
1198 <   }
1199 < #ifdef DEBUG
1200 < if (debug) GMessage(">TRIM5 >>   Read: %s\n",seq.chars());
1201 < #endif
1202 <
1203 < CSegChain a5segs(true); //list of segment pairs to analyze later if no extendMatch succeeded
1204 <
1205 < //try the easy way out first - look for an exact match of 11 bases
1206 < int fdlen=11;
1207 <  if (a5len<16) {
1208 <   fdlen=a5len>>1;
1209 <   }
1210 < if (fdlen>4) {
1211 <     GStr rstart=seq.substr(1,fdlen); //skip the first base as it's sometimes bogus
1212 <     if ((fi=adapter5.index(rstart))>=0) {
1213 < #ifdef DEBUG
1214 <       if (debug) GMessage("  W11match found: %*s\n", 1+fdlen, (adapter3.substr(fi,fdlen)).chars());
1215 < #endif
1216 <       if (extendMatch(seq.chars(), rlen, 1,
1217 <                     adapter5.chars(), a5len, fi,  fdlen, l5,l3, a5segs, true))
1218 <           return true;
1219 <       }
1220 <     //another easy case: last 11 characters of the adaptor found as a substring of the read
1221 <     GStr bstr=adapter5.substr(-fdlen);
1222 <     if ((fi=seq.index(bstr))>=0) {
1223 < #ifdef DEBUG
1224 <       if (debug) GMessage("  A11match found: %*s\n", fi+fdlen, bstr.chars());
1225 < #endif
1226 <       if (extendMatch(seq.chars(), rlen, fi,
1227 <                     adapter5.chars(), a5len, a5len-fdlen,  fdlen, l5,l3,a5segs,true))
1228 <          return true;
1229 <       }
1230 <     } //tried to matching at most 11 bases first
1231 <    
1232 < //-- no easy cases, do the wmer hashing for the last 12 bases of the adaptor
1233 < //-- only extend a wmer if it matches in the 5' (beginning) region of the read
1234 < int wordSize=3;
1235 < int hlen=12;
1236 < if (hlen>a5len-wordSize) hlen=a5len-wordSize;
1237 < int imax=rlen>>1; //first half of the read, right boundary for the wmer match
1238 < if (imax<a5len) { imax=GMIN(a5len, rlen-wordSize); }
1239 < for (int iw=0;iw<=hlen;iw++) {
1240 <   int apstart=a5len-iw-wordSize;
1241 <   fi=0;
1242 <   //int* qv=(int32 *)(adapter5.chars()+apstart);
1243 <   int qv=get3mer_value(adapter5.chars()+apstart);
1244 <   //while ((fi=fast4match(*qv, seq.chars(), rlen, fi))>=0 && fi<=imax) {
1245 <   while ((fi=w3_match(qv, seq.chars(), rlen, fi))>=0 && fi<=imax) {
1246 < #ifdef DEBUG
1247 <     if (debug) GMessage("    Wmatch found: %*s\n", fi+wordSize, (adapter5.substr(apstart,wordSize)).chars());
1248 < #endif
1249 <     if (extendMatch(seq.chars(), rlen, fi, adapter5.chars(),
1250 <                a5len, apstart, wordSize, l5,l3, a5segs, true)) return true;
1251 <     fi++;
1252 <     if (fi>imax) break;
1253 <     }
1254 <   } //for each wmer in the last hlen bases of the adaptor
1255 < /*
1256 <
1257 < //couldn't find a good trimming extension, hash 12 more bases of the adapter to collect more segment pairs there
1258 < //but only do this if we already have segment pairs collected in the last 12 bases of the adapter
1259 < if (a5segs.bend<(uint)(a5len-3) || a5segs.bstart>(uint)(a5len-hlen+4)) return false;
1260 < int hlen2=a5len-wordSize;
1261 < //if (hlen2>a5len-wordSize) hlen2=a5len-wordSize;
1262 < #ifdef DEBUG
1263 <      if (debug && a5segs.Count()>0) {
1264 <        GMessage("  >>>>>2nd. hash: %s\n",seq.chars());
1008 > bool trimmed=false;
1009 > GStr wseq(seq);
1010 > int wlen=rlen;
1011 > GXSeqData seqdata;
1012 > int numruns=revCompl ? 2 : 1;
1013 > GList<GXAlnInfo> bestalns(true, true, false);
1014 > for (int ai=0;ai<adaptors5.Count();ai++) {
1015 >   for (int r=0;r<numruns;r++) {
1016 >     if (r) {
1017 >          seqdata.update(adaptors5[ai].seqr.chars(), adaptors5[ai].seqr.length(),
1018 >                 adaptors5[ai].pzr, wseq.chars(), wlen, adaptors5[ai].amlen);
1019          }
1020 < #endif
1021 < if (hlen2>hlen) {
1022 <     for (int iw=hlen+1;iw<=hlen2;iw++) {
1023 <         int apstart=a5len-iw-wordSize;
1024 <         fi=0;
1025 <         //int* qv=(int32 *)(adapter5.chars()+apstart);
1026 <         int qv=get3mer_value(adapter5.chars()+apstart);
1027 <         //while ((fi=fast4match(*qv, seq.chars(), rlen, fi))>=0 && fi<=imax) {
1028 <         while ((fi=w3_match(qv, seq.chars(), rlen, fi))>=0 && fi<=imax) {
1029 <           extendMatch(seq.chars(), rlen, fi, adapter5.chars(),
1030 <                      a5len, apstart, wordSize, l5,l3, a5segs, true);
1031 <           fi++;
1032 <           if (fi>imax) break;
1033 <           }
1034 <         } //for each wmer between hlen2 and hlen bases of the adaptor
1020 >     else {
1021 >            seqdata.update(adaptors5[ai].seq.chars(), adaptors5[ai].seq.length(),
1022 >                 adaptors5[ai].pz, wseq.chars(), wlen, adaptors5[ai].amlen);
1023 >        }
1024 >         GXAlnInfo* aln=match_adaptor(seqdata, adaptors5[ai].trim_type, gxmem_l, 90);
1025 >         if (aln) {
1026 >           if (aln->strong) {
1027 >                   trimmed=true;
1028 >                   bestalns.Add(aln);
1029 >                   break; //will check the rest next time
1030 >                   }
1031 >            else bestalns.Add(aln);
1032 >           }
1033 >         } //forward and reverse?
1034 >   if (trimmed) break; //will check the rest in the next cycle
1035 >  }//for each 5' adaptor
1036 >  if (bestalns.Count()>0) {
1037 >           GXAlnInfo* aln=bestalns[0];
1038 >           if (aln->sl-1 > wlen-aln->sr) {
1039 >                   //keep left side
1040 >                   l3-=(wlen-aln->sl+1);
1041 >                   if (l3<0) l3=0;
1042 >                   }
1043 >           else { //keep right side
1044 >                   l5+=aln->sr;
1045 >                   if (l5>=rlen) l5=rlen-1;
1046 >                   }
1047 >           //delete aln;
1048 >           //if (l3-l5+1<min_read_len) return true;
1049 >           wseq=seq.substr(l5,l3-l5+1);
1050 >           wlen=wseq.length();
1051 >           return true; //break the loops here to report a good find
1052       }
1053 < if (a5segs.bend<(uint)(a5len-3) || a5segs.bstart>(uint)(a5len-hlen+4)) return false;
1054 < // lastly, analyze collected a5segs for a possible gapped alignment:
1284 < GList<CSegChain> segchains(false,true,false);
1285 < #ifdef DEBUG
1286 < if (debug && a5segs.Count()>0) {
1287 <   GMessage(">>>>>>>>>   Read: %s\n",seq.chars());
1288 <   }
1289 < #endif
1290 < for (int i=0;i<a5segs.Count();i++) {
1291 <   if (a5segs[i]->chain==NULL) {
1292 <       if (a5segs[i]->b.end<(int)(a5len-4)) continue; //don't start a hopeless chain
1293 <       CSegChain* newchain=new CSegChain(true);
1294 <       newchain->setFreeItem(false);
1295 <       newchain->addSegPair(a5segs[i]);
1296 <       a5segs[i]->chain=newchain;
1297 <       segchains.Add(newchain); //just to free them when done
1298 <       }
1299 <   for (int j=i+1;j<a5segs.Count();j++) {
1300 <      CSegChain* chain=a5segs[i]->chain;
1301 <      if (chain->extendChain(a5segs[j])) {
1302 <         a5segs[j]->chain=chain;
1303 < #ifdef DEBUG
1304 <         if (debug) dbgPrintChain(*chain, adapter5.chars());
1305 < #endif      
1306 <      //save time by checking here if the extended chain is already acceptable for trimming
1307 <         if (chain->bend>(uint)(a5len-3) && chain->astart<4 && chain->score>a_min_chain_score) {
1308 <            l5=chain->aend;
1309 <            l3=rlen-1;
1310 <            return true;
1311 <            }
1312 <         } //chain can be extended
1313 <      }
1314 <   } //collect segment alignments into chains
1315 < */
1316 < return false; //no adapter parts found
1317 < }
1053 >  return false;
1054 > }
1055  
1056 < //convert qvs to/from phred64 from/to phread33
1056 > //convert qvs to/from phred64 from/to phread33
1057   void convertPhred(GStr& q) {
1058   for (int i=0;i<q.length();i++) q[i]+=qv_cvtadd;
1059   }
# Line 1325 | Line 1062
1062   for (int i=0;i<len;i++) q[i]+=qv_cvtadd;
1063   }
1064  
1065 < bool getFastxRec(GLineReader& fq, GStr& rseq, GStr& rqv,
1065 > bool getFastxRec(GLineReader& fq, GStr& rseq, GStr& rqv,
1066            GStr& rname, GStr& rinfo, GStr& infname) {
1067   rseq="";
1068   rqv="";
# Line 1341 | Line 1078
1078        } //raw qseq format
1079   else { // FASTQ or FASTA */
1080   isfasta=(l[0]=='>');
1081 < if (!isfasta && l[0]!='@') GError("Error: fasta/fastq record marker not found(%s)\n%s\n",
1081 > if (!isfasta && l[0]!='@') GError("Error: fasta/fastq record marker not found(%s)\n%s\n",
1082        infname.chars(), l);
1083   GStr s(l);
1084   rname=&(l[1]);
1085   for (int i=0;i<rname.length();i++)
1086 <    if (rname[i]<=' ') {
1087 <       if (i<rname.length()-2) rinfo=rname.substr(i+1);
1088 <       rname.cut(i);
1089 <       break;
1086 >    if (rname[i]<=' ') {
1087 >       if (i<rname.length()-2) rinfo=rname.substr(i+1);
1088 >       rname.cut(i);
1089 >       break;
1090         }
1091    //now get the sequence
1092 < if ((l=fq.getLine())==NULL)
1092 > if ((l=fq.getLine())==NULL)
1093        GError("Error: unexpected EOF after header for read %s (%s)\n",
1094                     rname.chars(), infname.chars());
1095   rseq=l; //this must be the DNA line
1096   while ((l=fq.getLine())!=NULL) {
1097        //seq can span multiple lines
1098        if (l[0]=='>' || l[0]=='+') {
1099 <           fq.pushBack();
1099 >           fq.pushBack();
1100             break; //
1101             }
1102        rseq+=l;
1103 <      } //check for multi-line seq
1103 >      } //check for multi-line seq
1104   if (!isfasta) { //reading fastq quality values, which can also be multi-line
1105      if ((l=fq.getLine())==NULL)
1106          GError("Error: unexpected EOF after sequence for %s\n", rname.chars());
1107      if (l[0]!='+') GError("Error: fastq qv header marker not detected!\n");
1108 <    if ((l=fq.getLine())==NULL)
1108 >    if ((l=fq.getLine())==NULL)
1109          GError("Error: unexpected EOF after qv header for %s\n", rname.chars());
1110      rqv=l;
1111 <    //if (rqv.length()!=rseq.length())
1111 >    //if (rqv.length()!=rseq.length())
1112      //  GError("Error: qv len != seq len for %s\n", rname.chars());
1113      while (rqv.length()<rseq.length() && ((l=fq.getLine())!=NULL)) {
1114        rqv+=l; //append to qv string
1115        }
1116      }// fastq
1117   // } //<-- FASTA or FASTQ
1118 < rseq.upper(); //TODO: what if we care about masking?
1118 > rseq.upper();
1119   return true;
1120   }
1121  
1122 + #ifdef GDEBUG
1123 + void showTrim(GStr& s, int l5, int l3) {
1124 +  if (l5>0 || l3==0) {
1125 +    color_bg(c_red);
1126 +    }
1127 +  for (int i=0;i<s.length()-1;i++) {
1128 +    if (i && i==l5) color_resetbg();
1129 +    fprintf(stderr, "%c", s[i]);
1130 +    if (i && i==l3) color_bg(c_red);
1131 +   }
1132 +  fprintf(stderr, "%c", s[s.length()-1]);
1133 +  color_reset();
1134 +  fprintf(stderr, "\n");
1135 + }
1136 + #endif
1137 +
1138   char process_read(GStr& rname, GStr& rseq, GStr& rqv, int &l5, int &l3) {
1139 < //returns 0 if the read was untouched, 1 if it was just trimmed
1139 > //returns 0 if the read was untouched, 1 if it was just trimmed
1140   // and a trash code if it was trashed
1141   l5=0;
1142   l3=rseq.length()-1;
1143 + #ifdef GDEBUG
1144 +   //rseq.reverse();
1145 +   GMessage(">%s\n", rname.chars());
1146 +   GMessage("%s\n",rseq.chars());
1147 + #endif
1148   if (l3-l5+1<min_read_len) {
1149     return 's';
1150     }
# Line 1422 | Line 1180
1180     w5=0;
1181     w3=wseq.length()-1;
1182     }
1183 < if (a3len>0) {
1184 <  if (trim_adapter3(wseq, w5, w3)) {
1183 > char trim_code;
1184 > //clean the more dirty end first - 3'
1185 > int prev_w5=0;
1186 > int prev_w3=0;
1187 > bool w3upd=true;
1188 > bool w5upd=true;
1189 > do {
1190 >  trim_code=0;
1191 >  if (w3upd && trim_poly3(wseq, w5, w3, polyA_seed)) {
1192 >      trim_code='A';
1193 >      }
1194 >  else if (w3upd && trim_poly3(wseq, w5, w3, polyT_seed)) {
1195 >      trim_code='T';
1196 >      }
1197 >  else if (w5upd && trim_poly5(wseq, w5, w3, polyA_seed)) {
1198 >      trim_code='A';
1199 >      }
1200 >  else if (w5upd && trim_poly5(wseq, w5, w3, polyT_seed)) {
1201 >      trim_code='T';
1202 >      }
1203 >  else if (trim_adaptor5(wseq, w5, w3)) {
1204 >      trim_code='5';
1205 >      }
1206 >  else if (trim_adaptor3(wseq, w5, w3)) {
1207 >      trim_code='3';
1208 >      }
1209 >  if (trim_code) {
1210 >     w3upd=(w3!=prev_w3);
1211 >         w5upd=(w5!=prev_w5);
1212 >         if (w3upd) prev_w3=w3;
1213 >         if (w5upd) prev_w5=w5;
1214 >   #ifdef GDEBUG
1215 >     GMessage("#### TRIM by '%c' code ( w5-w3 = %d-%d ):\n",trim_code, w5,w3);
1216 >     showTrim(wseq, w5, w3);
1217 >   #endif
1218       int trimlen=wseq.length()-(w3-w5+1);
1219       num_trimmed3++;
1220 <     if (trimlen<min_trimmed3)
1220 >     if (trimlen<min_trimmed3)
1221           min_trimmed3=trimlen;
1222       l5+=w5;
1223       l3-=(wseq.length()-1-w3);
1224       if (w3-w5+1<min_read_len) {
1225 <         return '3';
1435 <         }
1436 <      //-- keep only the w5..w3 range
1437 <      wseq=wseq.substr(w5, w3-w5+1);
1438 <      if (!wqv.is_empty())
1439 <         wqv=wqv.substr(w5, w3-w5+1);
1440 <      }//some adapter was trimmed
1441 <   } //adapter trimming
1442 < if (a5len>0) {
1443 <  if (trim_adapter5(wseq, w5, w3)) {
1444 <     int trimlen=wseq.length()-(w3-w5+1);
1445 <     num_trimmed5++;
1446 <     if (trimlen<min_trimmed5)
1447 <         min_trimmed5=trimlen;
1448 <     l5+=w5;
1449 <     l3-=(wseq.length()-1-w3);
1450 <     if (w3-w5+1<min_read_len) {
1451 <         return '5';
1225 >         return trim_code;
1226           }
1227        //-- keep only the w5..w3 range
1228        wseq=wseq.substr(w5, w3-w5+1);
1229        if (!wqv.is_empty())
1230           wqv=wqv.substr(w5, w3-w5+1);
1231 <      }//some adapter was trimmed
1232 <   } //adapter trimming
1231 >      }//trimming at 3' end
1232 > } while (trim_code);
1233 >
1234   if (doCollapse) {
1235     //keep read for later
1236     FqDupRec* dr=dhash.Find(wseq.chars());
1237     if (dr==NULL) { //new entry
1238 <          //if (prefix.is_empty())
1239 <             dhash.Add(wseq.chars(),
1238 >          //if (prefix.is_empty())
1239 >             dhash.Add(wseq.chars(),
1240                    new FqDupRec(&wqv, rname.chars()));
1241            //else dhash.Add(wseq.chars(), new FqDupRec(wqv.chars(),wqv.length()));
1242           }
# Line 1495 | Line 1270
1270         fprintf(f_out, "%s\n", rseq.chars()); //plain one-line fasta for now
1271         }
1272        else {
1273 <       fprintf(f_out, ">%s%08d\n%s\n", prefix.chars(), outcounter,
1273 >       fprintf(f_out, ">%s%08d\n%s\n", prefix.chars(), outcounter,
1274                            rseq.chars());
1275         }
1276       }
# Line 1506 | Line 1281
1281         fprintf(f_out, "%s\n+\n%s\n", rseq.chars(), rqv.chars());
1282         }
1283        else
1284 <       fprintf(f_out, "@%s_%08d\n%s\n+\n%s\n", prefix.chars(), outcounter,
1284 >       fprintf(f_out, "@%s_%08d\n%s\n+\n%s\n", prefix.chars(), outcounter,
1285                            rseq.chars(),rqv.chars() );
1286       }
1287   }
# Line 1514 | Line 1289
1289   void trash_report(char trashcode, GStr& rname, FILE* freport) {
1290   if (freport==NULL || trashcode<=' ') return;
1291   if (trashcode=='3' || trashcode=='5') {
1292 <   fprintf(freport, "%s\tA%c\n",rname.chars(),trashcode);
1292 >   fprintf(freport, "%s\ta%c\n",rname.chars(),trashcode);
1293     }
1294   else {
1295     fprintf(freport, "%s\t%c\n",rname.chars(),trashcode);
# Line 1606 | Line 1381
1381      }
1382   }
1383  
1384 < void setupFiles(FILE*& f_in, FILE*& f_in2, FILE*& f_out, FILE*& f_out2,
1385 <                       GStr& s, GStr& infname, GStr& infname2) {
1384 > void addAdaptor(GVec<CASeqData>& adaptors, GStr& seq, GAlnTrimType trim_type) {
1385 > //TODO: prepare CASeqData here, and collect hexamers as well
1386 >  if (seq.is_empty() || seq=="-" ||
1387 >          seq=="N/A" || seq==".") return;
1388 >
1389 > CASeqData adata(revCompl);
1390 > int idx=adaptors.Add(adata);
1391 > if (idx<0) GError("Error: failed to add adaptor!\n");
1392 > adaptors[idx].trim_type=trim_type;
1393 > adaptors[idx].update(seq.chars());
1394 > }
1395 >
1396 >
1397 > int loadAdaptors(const char* fname) {
1398 >  GLineReader lr(fname);
1399 >  char* l;
1400 >  while ((l=lr.nextLine())!=NULL) {
1401 >   if (lr.length()<=3 || l[0]=='#') continue;
1402 >   if ( l[0]==' ' || l[0]=='\t' || l[0]==',' ||
1403 >        l[0]==';'|| l[0]==':' ) {
1404 >      int i=1;
1405 >      while (l[i]!=0 && isspace(l[i])) {
1406 >        i++;
1407 >        }
1408 >      if (l[i]!=0) {
1409 >        GStr s(&(l[i]));
1410 >      #ifdef GDEBUG
1411 >          //s.reverse();
1412 >      #endif
1413 >        addAdaptor(adaptors3, s, galn_TrimRight);
1414 >        continue;
1415 >        }
1416 >      }
1417 >    else {
1418 >      GStr s(l);
1419 >      s.startTokenize("\t ;,:");
1420 >      GStr a5,a3;
1421 >      if (s.nextToken(a5))
1422 >            s.nextToken(a3);
1423 >        else continue; //no tokens on this line
1424 >      GAlnTrimType ttype5=galn_TrimLeft;
1425 >      a5.upper();
1426 >      a3.upper();
1427 >      if (a3.is_empty() || a3==a5 || a3=="=") {
1428 >         a3.clear();
1429 >         ttype5=galn_TrimEither;
1430 >         }
1431 >     #ifdef GDEBUG
1432 >     //   a5.reverse();
1433 >     //   a3.reverse();
1434 >     #endif
1435 >      addAdaptor(adaptors5, a5, ttype5);
1436 >      addAdaptor(adaptors3, a3, galn_TrimRight);
1437 >      }
1438 >   }
1439 >   return adaptors5.Count()+adaptors3.Count();
1440 > }
1441 >
1442 > void setupFiles(FILE*& f_in, FILE*& f_in2, FILE*& f_out, FILE*& f_out2,
1443 >                       GStr& s, GStr& infname, GStr& infname2) {
1444   // uses outsuffix to generate output file names and open file handles as needed
1445   infname="";
1446   infname2="";
# Line 1635 | Line 1468
1468   s.startTokenize(",:");
1469   s.nextToken(infname);
1470   bool paired=s.nextToken(infname2);
1471 < if (fileExists(infname.chars())==0)
1471 > if (fileExists(infname.chars())==0)
1472      GError("Error: cannot find file %s!\n",infname.chars());
1473   GStr fname(getFileName(infname.chars()));
1474   GStr picmd;
# Line 1657 | Line 1490
1490   if (!paired) return;
1491   if (doCollapse) GError("Error: sorry, -C option cannot be used with paired reads!\n");
1492   // ---- paired reads:-------------
1493 < if (fileExists(infname2.chars())==0)
1493 > if (fileExists(infname2.chars())==0)
1494       GError("Error: cannot find file %s!\n",infname2.chars());
1495   picmd="";
1496   GStr fname2(getFileName(infname2.chars()));

Diff Legend

Removed lines
+ Added lines
< Changed lines
> Changed lines