Implement the "rightchar" feature of match data.

doc/pcre2test.1

@@ -1,4 +1,4 @@
-.TH PCRE2TEST 1 "12 August 2014" "PCRE 10.00"
+.TH PCRE2TEST 1 "19 August 2014" "PCRE 10.00"
 .SH NAME
 pcre2test - a program for testing Perl-compatible regular expressions.
 .SH SYNOPSIS
@@ -633,6 +633,7 @@ not affect the compilation process.
       aftertext                 show text after match
       allaftertext              show text after captures
       allcaptures               show all captures
+      allusedtext               show all consulted text 
   /g  global                    global matching
       jitverify                 verify JIT usage
       mark                      show mark values
@@ -691,6 +692,7 @@ pattern.
       aftertext                 show text after match
       allaftertext              show text after captures
       allcaptures               show all captures
+      allusedtext               show all consulted text 
       altglobal                 alternative global matching
       bsr=[anycrlf|unicode]     specify \eR handling
       callout_capture           show captures at callout time
@@ -735,6 +737,21 @@ contains multiple copies of the same substring. The \fBallaftertext\fP modifier
 requests the same action for captured substrings as well as the main matched
 substring. In each case the remainder is output on the following line with a
 plus character following the capture number.
+.P
+The \fBallusedtext\fP modifier requests that all the text that was consulted 
+during a successful pattern match be shown. This affects the output if there 
+is a lookbehind at the start of a match, or a lookahead at the end, or if \eK 
+is used in the pattern. Characters that precede or follow the start and end of 
+the actual match are indicated in the output by '<' or '>' characters 
+underneath them. Here is an example:
+.sp
+  /(?<=pqr)abc(?=xyz)/
+      123pqrabcxyz456\=allusedtext
+   0: pqrabcxyz
+      <<<   >>>
+.sp
+This shows that the matched string is "abc", with the preceding and following
+strings "pqr" and "xyz" also consulted during the match.
 .
 .
 .SS "Showing the value of all capture groups"
@@ -1142,6 +1159,6 @@ Cambridge CB2 3QH, England.
 .rs
 .sp
 .nf
-Last updated: 12 August 2014
+Last updated: 19 August 2014
 Copyright (c) 1997-2014 University of Cambridge.
 .fi

src/pcre2_dfa_match.c

@@ -544,6 +544,8 @@ for (;;)
   BOOL could_continue = reset_could_continue;
   reset_could_continue = FALSE;
   
+  if (ptr > mb->last_used_ptr) mb->last_used_ptr = ptr; 
+
   /* Make the new state list into the active state list and empty the
   new state list. */
 
@@ -967,6 +969,14 @@ for (;;)
 
         if (clen > 0)
           {
+          if (ptr >= mb->last_used_ptr)
+            {
+            PCRE2_SPTR temp = ptr + 1;
+#if defined SUPPORT_UTF && PCRE2_CODE_UNIT_WIDTH != 32
+            if (utf) { FORWARDCHAR(temp); }
+#endif
+            mb->last_used_ptr = temp;
+            } 
 #ifdef SUPPORT_UTF
           if ((mb->poptions & PCRE2_UCP) != 0)
             {
@@ -3447,6 +3457,7 @@ for (;;)
   /* OK, now we can do the business */
 
   mb->start_used_ptr = start_match;
+  mb->last_used_ptr = start_match;
   mb->recursive = NULL;
 
   rc = internal_dfa_match(
@@ -3471,7 +3482,7 @@ for (;;)
       match_data->ovector[1] = (PCRE2_SIZE)(end_subject - subject);
       }
     match_data->leftchar = (PCRE2_SIZE)(mb->start_used_ptr - subject);
-    match_data->rightchar = 0; /* FIXME */
+    match_data->rightchar = mb->last_used_ptr - subject;
     match_data->startchar = (PCRE2_SIZE)(start_match - subject);
     match_data->rc = rc; 
     return rc;

src/pcre2_intmodedep.h

@@ -752,6 +752,7 @@ typedef struct match_block {
   PCRE2_SPTR start_match_ptr;     /* Start of matched string */
   PCRE2_SPTR end_match_ptr;       /* Subject position at end match */
   PCRE2_SPTR start_used_ptr;      /* Earliest consulted character */
+  PCRE2_SPTR last_used_ptr;       /* Latest consulted character */ 
   PCRE2_SPTR mark;                /* Mark pointer to pass back on success */
   PCRE2_SPTR nomatch_mark;        /* Mark pointer to pass back on failure */
   PCRE2_SPTR once_target;         /* Where to back up to for atomic groups */
@@ -783,6 +784,7 @@ typedef struct dfa_match_block {
   PCRE2_SPTR start_subject ;      /* Start of the subject string */
   PCRE2_SPTR end_subject;         /* End of subject string */
   PCRE2_SPTR start_used_ptr;      /* Earliest consulted character */
+  PCRE2_SPTR last_used_ptr;       /* Latest consulted character */ 
   const uint8_t *tables;          /* Character tables */
   PCRE2_SIZE start_offset;        /* The start offset value */
   uint32_t moptions;              /* Match options */

src/pcre2_match.c

@@ -1450,11 +1450,14 @@ for (;;)
     break;
 
 
-    /* End of the pattern, either real or forced. */
+    /* End of the pattern, either real or forced. In an assertion ACCEPT,
+    update the last used pointer. */
 
-    case OP_END:
-    case OP_ACCEPT:
     case OP_ASSERT_ACCEPT:
+    if (eptr > mb->last_used_ptr) mb->last_used_ptr = eptr; 
+
+    case OP_ACCEPT:
+    case OP_END:
 
     /* If we have matched an empty string, fail if not in an assertion and not
     in a recursion if either PCRE2_NOTEMPTY is set, or if PCRE2_NOTEMPTY_ATSTART
@@ -1918,6 +1921,7 @@ for (;;)
       mb->end_match_ptr = eptr;      /* For ONCE_NC */
       mb->end_offset_top = offset_top;
       mb->start_match_ptr = mstart;
+      if (eptr > mb->last_used_ptr) mb->last_used_ptr = eptr; 
       RRETURN(MATCH_MATCH);         /* Sets mb->mark */
       }
 
@@ -1941,6 +1945,7 @@ for (;;)
         {
         mb->end_match_ptr = eptr;
         mb->start_match_ptr = mstart;
+        if (eptr > mb->last_used_ptr) mb->last_used_ptr = eptr; 
         RRETURN(MATCH_MATCH);
         }
 
@@ -1984,6 +1989,7 @@ for (;;)
       mb->start_match_ptr = mstart;    /* In case \K reset it */
       mb->end_match_ptr = eptr;
       mb->end_offset_top = offset_top;
+      if (eptr > mb->last_used_ptr) mb->last_used_ptr = eptr; 
       RRETURN(MATCH_KETRPOS);
       }
 
@@ -2202,6 +2208,9 @@ for (;;)
           }
         else
           {
+          PCRE2_SPTR nextptr = eptr + 1;
+          FORWARDCHAR(nextptr);  
+          if (nextptr > mb->last_used_ptr) mb->last_used_ptr = nextptr; 
           GETCHAR(c, eptr);
           if ((mb->poptions & PCRE2_UCP) != 0)
             {
@@ -2251,20 +2260,23 @@ for (;;)
           cur_is_word = FALSE;
           }
         else
-#ifdef SUPPORT_UTF
-        if ((mb->poptions & PCRE2_UCP) != 0)
           {
-          c = *eptr;
+          if (eptr >= mb->last_used_ptr) mb->last_used_ptr = eptr + 1; 
-          if (c == '_') cur_is_word = TRUE; else
+#ifdef SUPPORT_UTF
+          if ((mb->poptions & PCRE2_UCP) != 0)
             {
-            int cat = UCD_CATEGORY(c);
+            c = *eptr;
-            cur_is_word = (cat == ucp_L || cat == ucp_N);
+            if (c == '_') cur_is_word = TRUE; else
+              {
+              int cat = UCD_CATEGORY(c);
+              cur_is_word = (cat == ucp_L || cat == ucp_N);
+              }
             }
-          }
+          else
-        else
 #endif
-        cur_is_word = MAX_255(*eptr)
+          cur_is_word = MAX_255(*eptr)
-          && ((mb->ctypes[*eptr] & ctype_word) != 0);
+            && ((mb->ctypes[*eptr] & ctype_word) != 0);
+          }   
         }
 
       /* Now see if the situation is what we want */
@@ -6780,6 +6792,7 @@ for(;;)
 
   mb->start_match_ptr = start_match;
   mb->start_used_ptr = start_match;
+  mb->last_used_ptr = start_match; 
   mb->match_call_count = 0;
   mb->match_function_type = 0;
   mb->end_offset_top = 0;
@@ -6985,9 +6998,10 @@ if (rc == MATCH_MATCH || rc == MATCH_ACCEPT)
     
   /* Set the remaining returned values */
   
-  match_data->leftchar = mb->start_used_ptr - subject;
-  match_data->rightchar = 0;  /* FIXME */
   match_data->startchar = start_match - subject;
+  match_data->leftchar = mb->start_used_ptr - subject;
+  match_data->rightchar = ((mb->last_used_ptr > mb->end_match_ptr)?
+    mb->last_used_ptr : mb->end_match_ptr) - subject;
   return match_data->rc;
   }
 
@@ -7011,9 +7025,9 @@ else if (match_partial != NULL)
     match_data->ovector[0] = match_partial - subject;
     match_data->ovector[1] = end_subject - subject;
     }
-  match_data->leftchar = start_partial - subject;
-  match_data->rightchar = 0;  /* FIXME */
   match_data->startchar = match_partial - subject;
+  match_data->leftchar = start_partial - subject;
+  match_data->rightchar = end_subject - subject;
   match_data->rc = PCRE2_ERROR_PARTIAL;
   }
 

src/pcre2test.c

@@ -321,25 +321,26 @@ either on a pattern or a data line, so they must all be distinct. */
 #define CTL_AFTERTEXT        0x00000001u
 #define CTL_ALLAFTERTEXT     0x00000002u
 #define CTL_ALLCAPTURES      0x00000004u
-#define CTL_ALTGLOBAL        0x00000008u
+#define CTL_ALLUSEDTEXT      0x00000008u
-#define CTL_BINCODE          0x00000010u
+#define CTL_ALTGLOBAL        0x00000010u
-#define CTL_CALLOUT_CAPTURE  0x00000020u
+#define CTL_BINCODE          0x00000020u
-#define CTL_CALLOUT_NONE     0x00000040u
+#define CTL_CALLOUT_CAPTURE  0x00000040u
-#define CTL_DFA              0x00000080u
+#define CTL_CALLOUT_NONE     0x00000080u
-#define CTL_FINDLIMITS       0x00000100u
+#define CTL_DFA              0x00000100u
-#define CTL_FULLBINCODE      0x00000200u
+#define CTL_FINDLIMITS       0x00000200u
-#define CTL_GETALL           0x00000400u
+#define CTL_FULLBINCODE      0x00000400u
-#define CTL_GLOBAL           0x00000800u
+#define CTL_GETALL           0x00000800u
-#define CTL_HEXPAT           0x00001000u
+#define CTL_GLOBAL           0x00001000u
-#define CTL_INFO             0x00002000u
+#define CTL_HEXPAT           0x00002000u
-#define CTL_JITVERIFY        0x00004000u
+#define CTL_INFO             0x00004000u
-#define CTL_MARK             0x00008000u
+#define CTL_JITVERIFY        0x00008000u
-#define CTL_MEMORY           0x00010000u
+#define CTL_MARK             0x00010000u
-#define CTL_PATLEN           0x00020000u
+#define CTL_MEMORY           0x00020000u
-#define CTL_POSIX            0x00040000u
+#define CTL_PATLEN           0x00040000u
+#define CTL_POSIX            0x00080000u
 
-#define CTL_BSR_SET          0x00080000u  /* This is informational */
+#define CTL_BSR_SET          0x00100000u  /* This is informational */
-#define CTL_NL_SET           0x00100000u  /* This is informational */
+#define CTL_NL_SET           0x00200000u  /* This is informational */
 
 #define CTL_DEBUG            (CTL_FULLBINCODE|CTL_INFO)  /* For setting */
 #define CTL_ANYINFO          (CTL_DEBUG|CTL_BINCODE)     /* For testing */
@@ -348,9 +349,15 @@ either on a pattern or a data line, so they must all be distinct. */
 /* These are all the controls that may be set either on a pattern or on a
 data line. */
 
-#define CTL_ALLPD            (CTL_AFTERTEXT|CTL_ALLAFTERTEXT|CTL_ALLCAPTURES|\
+#define CTL_ALLPD  (CTL_AFTERTEXT|\
-                              CTL_ALTGLOBAL|CTL_GLOBAL|CTL_JITVERIFY|CTL_MARK|\
+                    CTL_ALLAFTERTEXT|\
-                              CTL_MEMORY)
+                    CTL_ALLCAPTURES|\
+                    CTL_ALLUSEDTEXT|\
+                    CTL_ALTGLOBAL|\
+                    CTL_GLOBAL|\
+                    CTL_JITVERIFY|\
+                    CTL_MARK|\
+                    CTL_MEMORY)
 
 typedef struct patctl {    /* Structure for pattern modifiers. */
   uint32_t  options;       /* Must be in same position as datctl */
@@ -409,6 +416,7 @@ static modstruct modlist[] = {
   { "allaftertext",        MOD_PNDP, MOD_CTL, CTL_ALLAFTERTEXT,          PO(control) },
   { "allcaptures",         MOD_PND,  MOD_CTL, CTL_ALLCAPTURES,           PO(control) },
   { "allow_empty_class",   MOD_PAT,  MOD_OPT, PCRE2_ALLOW_EMPTY_CLASS,   PO(options) },
+  { "allusedtext",         MOD_PNDP, MOD_CTL, CTL_ALLUSEDTEXT,           PO(control) },
   { "alt_bsux",            MOD_PAT,  MOD_OPT, PCRE2_ALT_BSUX,            PO(options) },
   { "altglobal",           MOD_PND,  MOD_CTL, CTL_ALTGLOBAL,             PO(control) },
   { "anchored",            MOD_PD,   MOD_OPT, PCRE2_ANCHORED,            PD(options) },
@@ -4417,7 +4425,10 @@ for (gmatched = 0;; gmatched++)
     {
     int i;
     uint8_t *nptr;
+    BOOL showallused; 
     PCRE2_SIZE *ovector;
+    PCRE2_SIZE leftchar = FLD(match_data, leftchar);
+    PCRE2_SIZE rightchar = FLD(match_data, rightchar);
 
     /* This is a check against a lunatic return value. */
 
@@ -4453,6 +4464,7 @@ for (gmatched = 0;; gmatched++)
     ovector = FLD(match_data, ovector);
     for (i = 0; i < 2*capcount; i += 2)
       {
+      PCRE2_SIZE lleft, lmiddle, lright; 
       PCRE2_SIZE start = ovector[i];
       PCRE2_SIZE end = ovector[i+1];
 
@@ -4470,12 +4482,40 @@ for (gmatched = 0;; gmatched++)
         fprintf(outfile, "<unset>\n");
         continue;
         }
-      PCHARSV(pp, start, end - start, utf, outfile);
+         
+      /* For the whole matched string, if ALLUSEDTEXT is set, and if the
+      leftmost consulted character is before the start of the match or the
+      rightmost consulted character is past the end of the match, we want to
+      show all consulted characters, and indicate which were lookarounds. */
+         
+      showallused = i == 0 && (dat_datctl.control & CTL_ALLUSEDTEXT) != 0 &&
+        (leftchar < start || rightchar > end); 
+      if (showallused)
+        { 
+        PCHARS(lleft, pp, leftchar, start - leftchar, utf, outfile);
+        PCHARS(lmiddle, pp, start, end - start, utf, outfile);
+        PCHARS(lright, pp, end, rightchar - end, utf, outfile);
+        }
+      else
+        {     
+        PCHARSV(pp, start, end - start, utf, outfile);
+        } 
+       
 #ifdef FIXME
       if (verify_jit && jit_was_used) fprintf(outfile, " (JIT)");
 #endif
       fprintf(outfile, "\n");
        
+      if (showallused)
+        {
+        PCRE2_SIZE j; 
+        fprintf(outfile, "    ");
+        for (j = 0; j < lleft; j++) fprintf(outfile, "<");
+        for (j = 0; j < lmiddle; j++) fprintf(outfile, " ");
+        for (j = 0; j < lright; j++) fprintf(outfile, ">");
+        fprintf(outfile, "\n");    
+        }  
+ 
       /* Note: don't use the start/end variables here because we want to
       show the text from what is reported as the end. */
 

testdata/testinput2

@@ -4049,4 +4049,25 @@ a random value. /Ix
     aaaabcde
     aaaabcde\=ovector=100
 
+/abc(?=xyz)/allusedtext
+    abcxyzpqr
+    abcxyzpqr\=aftertext
+    
+/(?<=pqr)abc(?=xyz)/allusedtext
+    xyzpqrabcxyzpqr
+    xyzpqrabcxyzpqr\=aftertext
+    
+/a\b/
+    a.\=allusedtext
+    a\=allusedtext  
+
+/abc\Kxyz/
+    abcxyz\=allusedtext
+
+/abc(?=xyz(*ACCEPT))/
+    abcxyz\=allusedtext
+
+/abc(?=abcde)(?=ab)/allusedtext
+    abcabcdefg
+
 # End of testinput2 

testdata/testinput5

@@ -1630,4 +1630,7 @@
 /\X?abc/utf,no_start_optimize
 \xff\x7f\x00\x00\x03\x00\x41\xcc\x80\x41\x{300}\x61\x62\x63\x00\=no_utf_check,offset=06
 
+/(?<=\x{100})\x{200}(?=\x{300})/utf,allusedtext
+    \x{100}\x{200}\x{300}
+
 # End of testinput5 

testdata/testinput6

@@ -4783,4 +4783,19 @@
 '\A(?:[^\"]++|\"(?:[^\"]++|\"\")*+\")++'
     NON QUOTED \"QUOT\"\"ED\" AFTER \"NOT MATCHED
 
-# End of testinput8 
+/abc(?=xyz)/allusedtext
+    abcxyzpqr
+    abcxyzpqr\=aftertext
+    
+/(?<=pqr)abc(?=xyz)/allusedtext
+    xyzpqrabcxyzpqr
+    xyzpqrabcxyzpqr\=aftertext
+    
+/a\b/
+    a.\=allusedtext
+    a\=allusedtext  
+
+/abc(?=abcde)(?=ab)/allusedtext
+    abcabcdefg
+
+# End of testinput6

testdata/testinput7

@@ -2123,4 +2123,7 @@
     A\x{2005}Z
     A\x{85}\x{180e}\x{2005}Z
 
+/(?<=\x{100})\x{200}(?=\x{300})/utf,allusedtext
+    \x{100}\x{200}\x{300}
+
 # End of testinput7

testdata/testoutput2

@@ -13680,4 +13680,44 @@ No match
     aaaabcde\=ovector=100
  0: aaaab
 
+/abc(?=xyz)/allusedtext
+    abcxyzpqr
+ 0: abcxyz
+       >>>
+    abcxyzpqr\=aftertext
+ 0: abcxyz
+       >>>
+ 0+ xyzpqr
+    
+/(?<=pqr)abc(?=xyz)/allusedtext
+    xyzpqrabcxyzpqr
+ 0: pqrabcxyz
+    <<<   >>>
+    xyzpqrabcxyzpqr\=aftertext
+ 0: pqrabcxyz
+    <<<   >>>
+ 0+ xyzpqr
+    
+/a\b/
+    a.\=allusedtext
+ 0: a.
+     >
+    a\=allusedtext  
+ 0: a
+
+/abc\Kxyz/
+    abcxyz\=allusedtext
+ 0: abcxyz
+    <<<   
+
+/abc(?=xyz(*ACCEPT))/
+    abcxyz\=allusedtext
+ 0: abcxyz
+       >>>
+
+/abc(?=abcde)(?=ab)/allusedtext
+    abcabcdefg
+ 0: abcabcde
+       >>>>>
+
 # End of testinput2 

testdata/testoutput5

@@ -4001,4 +4001,9 @@ Subject length lower bound = 1
 \xff\x7f\x00\x00\x03\x00\x41\xcc\x80\x41\x{300}\x61\x62\x63\x00\=no_utf_check,offset=06
  0: A\x{300}abc
 
+/(?<=\x{100})\x{200}(?=\x{300})/utf,allusedtext
+    \x{100}\x{200}\x{300}
+ 0: \x{100}\x{200}\x{300}
+    <<<<<<<       >>>>>>>
+
 # End of testinput5 

testdata/testoutput6

@@ -7659,4 +7659,34 @@ Matched, but offsets vector is too small to show all matches
     NON QUOTED \"QUOT\"\"ED\" AFTER \"NOT MATCHED
  0: NON QUOTED "QUOT""ED" AFTER 
 
-# End of testinput8 
+/abc(?=xyz)/allusedtext
+    abcxyzpqr
+ 0: abcxyz
+       >>>
+    abcxyzpqr\=aftertext
+ 0: abcxyz
+       >>>
+ 0+ xyzpqr
+    
+/(?<=pqr)abc(?=xyz)/allusedtext
+    xyzpqrabcxyzpqr
+ 0: pqrabcxyz
+    <<<   >>>
+    xyzpqrabcxyzpqr\=aftertext
+ 0: pqrabcxyz
+    <<<   >>>
+ 0+ xyzpqr
+    
+/a\b/
+    a.\=allusedtext
+ 0: a.
+     >
+    a\=allusedtext  
+ 0: a
+
+/abc(?=abcde)(?=ab)/allusedtext
+    abcabcdefg
+ 0: abcabcde
+       >>>>>
+
+# End of testinput6

testdata/testoutput7

@@ -3771,4 +3771,9 @@ No match
     A\x{85}\x{180e}\x{2005}Z
  0: A\x{85}\x{180e}\x{2005}Z
 
+/(?<=\x{100})\x{200}(?=\x{300})/utf,allusedtext
+    \x{100}\x{200}\x{300}
+ 0: \x{100}\x{200}\x{300}
+    <<<<<<<       >>>>>>>
+
 # End of testinput7