Fix stack overflow bug, copying fix from PCRE1.

src/pcre2_compile.c

@@ -1195,6 +1195,7 @@ for (code = first_significant_code(code + PRIV(OP_lengths)[*code], TRUE);
   if (c == OP_RECURSE)
     {
     PCRE2_SPTR scode = cb->start_code + GET(code, 1);
+    PCRE2_SPTR endgroup = scode;
     BOOL empty_branch;
 
     /* Test for forward reference or uncompleted reference. This is disabled
@@ -1209,20 +1210,16 @@ for (code = first_significant_code(code + PRIV(OP_lengths)[*code], TRUE);
       if (GET(scode, 1) == 0) return TRUE;    /* Unclosed */
       }
 
-    /* If we are scanning a completed pattern, there are no forward references
-    and all groups are complete. We need to detect whether this is a recursive
-    call, as otherwise there will be an infinite loop. If it is a recursion,
-    just skip over it. Simple recursions are easily detected. For mutual
-    recursions we keep a chain on the stack. */
-
-    else
-      {
-      recurse_check *r = recurses;
-      PCRE2_SPTR endgroup = scode;
+    /* If the reference is to a completed group, we need to detect whether this
+    is a recursive call, as otherwise there will be an infinite loop. If it is
+    a recursion, just skip over it. Simple recursions are easily detected. For
+    mutual recursions we keep a chain on the stack. */
 
     do endgroup += GET(endgroup, 1); while (*endgroup == OP_ALT);
     if (code >= scode && code <= endgroup) continue;  /* Simple recursion */
-
+    else
+      {
+      recurse_check *r = recurses;
       for (r = recurses; r != NULL; r = r->prev)
         if (r->group == scode) break;
       if (r != NULL) continue;   /* Mutual recursion */

testdata/testinput1

@@ -4913,6 +4913,12 @@
 /((?(R1)a+|(?1)b))/
     aaaabcde
     
+/((?(R)a|(?1)))*/
+    aaa
+
+/((?(R)a|(?1)))+/
+    aaa 
+
 /a(*:any 
 name)/mark
     abc

testdata/testoutput1

@@ -8200,6 +8200,16 @@ MK: M
  0: aaaab
  1: aaaab
     
+/((?(R)a|(?1)))*/
+    aaa
+ 0: aaa
+ 1: a
+
+/((?(R)a|(?1)))+/
+    aaa 
+ 0: aaa
+ 1: a
+
 /a(*:any 
 name)/mark
     abc