walkero
/
pcre2
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Rework early fail optimization in JIT.

This commit is contained in:
Zoltán Herczeg 2020-03-05 07:58:49 +00:00
parent 106d9d3a25
commit 21c40e638b
1 changed files with 303 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

512
src/pcre2_jit_compile.c
View File

@ -223,6 +223,12 @@ enum control_types {
type_then_trap = 1 type_then_trap = 1
}; };
enum early_fail_types {
type_skip = 0,
type_fail = 1,
type_fail_range = 2
};
typedef int (SLJIT_FUNC *jit_function)(jit_arguments *args); typedef int (SLJIT_FUNC *jit_function)(jit_arguments *args);
/* The following structure is the key data type for the recursive /* The following structure is the key data type for the recursive
@ -405,8 +411,8 @@ typedef struct compiler_common {
/* Fast forward skipping byte code pointer. */ /* Fast forward skipping byte code pointer. */
PCRE2_SPTR fast_forward_bc_ptr; PCRE2_SPTR fast_forward_bc_ptr;
/* Locals used by fast fail optimization. */ /* Locals used by fast fail optimization. */
sljit_s32 fast_fail_start_ptr; sljit_s32 early_fail_start_ptr;
sljit_s32 fast_fail_end_ptr; sljit_s32 early_fail_end_ptr;
/* Flipped and lower case tables. */ /* Flipped and lower case tables. */
const sljit_u8 *fcc; const sljit_u8 *fcc;
@ -1220,141 +1226,39 @@ while (cc < ccend)
return TRUE; return TRUE;
} }
static BOOL is_accelerated_repeat(PCRE2_SPTR cc) #define EARLY_FAIL_ENHANCE_MAX (1 + 3)
{
switch(*cc)
{
case OP_TYPESTAR:
case OP_TYPEMINSTAR:
case OP_TYPEPLUS:
case OP_TYPEMINPLUS:
case OP_TYPEPOSSTAR:
case OP_TYPEPOSPLUS:
return (cc[1] != OP_ANYNL && cc[1] != OP_EXTUNI);
case OP_STAR: /*
case OP_MINSTAR: start:
case OP_PLUS: 0 - skip / early fail allowed
case OP_MINPLUS: 1 - only early fail with range allowed
case OP_POSSTAR: >1 - (start - 1) early fail is processed
case OP_POSPLUS:
case OP_STARI: return: current number of iterators enhanced with fast fail
case OP_MINSTARI: */
case OP_PLUSI: static int detect_early_fail(compiler_common *common, PCRE2_SPTR cc, int *private_data_start, sljit_s32 depth, int start)
case OP_MINPLUSI:
case OP_POSSTARI:
case OP_POSPLUSI:
case OP_NOTSTAR:
case OP_NOTMINSTAR:
case OP_NOTPLUS:
case OP_NOTMINPLUS:
case OP_NOTPOSSTAR:
case OP_NOTPOSPLUS:
case OP_NOTSTARI:
case OP_NOTMINSTARI:
case OP_NOTPLUSI:
case OP_NOTMINPLUSI:
case OP_NOTPOSSTARI:
case OP_NOTPOSPLUSI:
return TRUE;
case OP_CLASS:
case OP_NCLASS:
#if defined SUPPORT_UNICODE || PCRE2_CODE_UNIT_WIDTH != 8
case OP_XCLASS:
cc += (*cc == OP_XCLASS) ? GET(cc, 1) : (int)(1 + (32 / sizeof(PCRE2_UCHAR)));
#else
cc += (1 + (32 / sizeof(PCRE2_UCHAR)));
#endif
/* Only these types are supported. */
switch(*cc)
{
case OP_CRSTAR:
case OP_CRMINSTAR:
case OP_CRPLUS:
case OP_CRMINPLUS:
case OP_CRPOSSTAR:
case OP_CRPOSPLUS:
return TRUE;
}
break;
}
return FALSE;
}
static SLJIT_INLINE BOOL detect_fast_forward_skip(compiler_common *common, int *private_data_start)
{
PCRE2_SPTR cc = common->start;
PCRE2_SPTR end;
/* Skip not repeated brackets. */
while (TRUE)
{
switch(*cc)
{
case OP_SOD:
case OP_SOM:
case OP_SET_SOM:
case OP_NOT_WORD_BOUNDARY:
case OP_WORD_BOUNDARY:
case OP_EODN:
case OP_EOD:
case OP_CIRC:
case OP_CIRCM:
case OP_DOLL:
case OP_DOLLM:
/* Zero width assertions. */
cc++;
continue;
}
if (*cc != OP_BRA && *cc != OP_CBRA)
break;
end = cc + GET(cc, 1);
/* Iterated brackets are skipped. */
if (*end != OP_KET || PRIVATE_DATA(end) != 0)
return FALSE;
if (*cc == OP_CBRA)
{
if (common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0)
return FALSE;
cc += IMM2_SIZE;
}
cc += 1 + LINK_SIZE;
}
if (is_accelerated_repeat(cc))
{
common->fast_forward_bc_ptr = cc;
common->private_data_ptrs[(cc + 1) - common->start] = *private_data_start;
*private_data_start += sizeof(sljit_sw);
return TRUE;
}
return FALSE;
}
static void detect_fast_fail(compiler_common *common, PCRE2_SPTR cc, int *private_data_start, sljit_s32 depth)
{ {
PCRE2_SPTR next_alt; PCRE2_SPTR next_alt;
PCRE2_SPTR end; PCRE2_SPTR end;
PCRE2_SPTR accelerated_start;
int result = 0;
int count;
BOOL fast_forward_allowed = TRUE;
SLJIT_ASSERT(*cc == OP_BRA || *cc == OP_CBRA); SLJIT_ASSERT(*cc == OP_ONCE || *cc == OP_BRA || *cc == OP_CBRA);
SLJIT_ASSERT(*cc != OP_CBRA || common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] != 0); SLJIT_ASSERT(*cc != OP_CBRA || common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] != 0);
SLJIT_ASSERT(start < EARLY_FAIL_ENHANCE_MAX);
do do
{ {
count = start;
next_alt = cc + GET(cc, 1); next_alt = cc + GET(cc, 1);
cc += 1 + LINK_SIZE + ((*cc == OP_CBRA) ? IMM2_SIZE : 0); cc += 1 + LINK_SIZE + ((*cc == OP_CBRA) ? IMM2_SIZE : 0);
while (TRUE) while (TRUE)
{ {
accelerated_start = NULL;
switch(*cc) switch(*cc)
{ {
case OP_SOD: case OP_SOD:
@ -1362,6 +1266,16 @@ do
case OP_SET_SOM: case OP_SET_SOM:
case OP_NOT_WORD_BOUNDARY: case OP_NOT_WORD_BOUNDARY:
case OP_WORD_BOUNDARY: case OP_WORD_BOUNDARY:
case OP_EODN:
case OP_EOD:
case OP_CIRC:
case OP_CIRCM:
case OP_DOLL:
case OP_DOLLM:
/* Zero width assertions. */
cc++;
continue;
case OP_NOT_DIGIT: case OP_NOT_DIGIT:
case OP_DIGIT: case OP_DIGIT:
case OP_NOT_WHITESPACE: case OP_NOT_WHITESPACE:
@ -1376,17 +1290,14 @@ do
case OP_HSPACE: case OP_HSPACE:
case OP_NOT_VSPACE: case OP_NOT_VSPACE:
case OP_VSPACE: case OP_VSPACE:
case OP_EODN: case OP_EXTUNI:
case OP_EOD: fast_forward_allowed = FALSE;
case OP_CIRC:
case OP_CIRCM:
case OP_DOLL:
case OP_DOLLM:
cc++; cc++;
continue; continue;
case OP_NOTPROP: case OP_NOTPROP:
case OP_PROP: case OP_PROP:
fast_forward_allowed = FALSE;
cc += 1 + 2; cc += 1 + 2;
continue; continue;
@ -1394,6 +1305,116 @@ do
case OP_CHARI: case OP_CHARI:
case OP_NOT: case OP_NOT:
case OP_NOTI: case OP_NOTI:
fast_forward_allowed = FALSE;
cc += 2;
#ifdef SUPPORT_UNICODE
if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
#endif
continue;
case OP_TYPESTAR:
case OP_TYPEMINSTAR:
case OP_TYPEPLUS:
case OP_TYPEMINPLUS:
case OP_TYPEPOSSTAR:
case OP_TYPEPOSPLUS:
/* The type or prop opcode is skipped in the next iteration. */
cc += 1;
if (cc[0] != OP_ANYNL && cc[0] != OP_EXTUNI)
{
accelerated_start = cc - 1;
break;
}
if (count == 0)
count = 1;
fast_forward_allowed = FALSE;
continue;
case OP_TYPEUPTO:
case OP_TYPEMINUPTO:
case OP_TYPEEXACT:
case OP_TYPEPOSUPTO:
cc += IMM2_SIZE;
case OP_TYPEQUERY:
case OP_TYPEMINQUERY:
case OP_TYPEPOSQUERY:
/* The type or prop opcode is skipped in the next iteration. */
fast_forward_allowed = FALSE;
if (count == 0)
count = 1;
cc += 1;
continue;
case OP_STAR:
case OP_MINSTAR:
case OP_PLUS:
case OP_MINPLUS:
case OP_POSSTAR:
case OP_POSPLUS:
case OP_STARI:
case OP_MINSTARI:
case OP_PLUSI:
case OP_MINPLUSI:
case OP_POSSTARI:
case OP_POSPLUSI:
case OP_NOTSTAR:
case OP_NOTMINSTAR:
case OP_NOTPLUS:
case OP_NOTMINPLUS:
case OP_NOTPOSSTAR:
case OP_NOTPOSPLUS:
case OP_NOTSTARI:
case OP_NOTMINSTARI:
case OP_NOTPLUSI:
case OP_NOTMINPLUSI:
case OP_NOTPOSSTARI:
case OP_NOTPOSPLUSI:
accelerated_start = cc;
cc += 2;
#ifdef SUPPORT_UNICODE
if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
#endif
break;
case OP_UPTO:
case OP_MINUPTO:
case OP_EXACT:
case OP_POSUPTO:
case OP_UPTOI:
case OP_MINUPTOI:
case OP_EXACTI:
case OP_POSUPTOI:
case OP_NOTUPTO:
case OP_NOTMINUPTO:
case OP_NOTEXACT:
case OP_NOTPOSUPTO:
case OP_NOTUPTOI:
case OP_NOTMINUPTOI:
case OP_NOTEXACTI:
case OP_NOTPOSUPTOI:
cc += IMM2_SIZE;
case OP_QUERY:
case OP_MINQUERY:
case OP_POSQUERY:
case OP_QUERYI:
case OP_MINQUERYI:
case OP_POSQUERYI:
case OP_NOTQUERY:
case OP_NOTMINQUERY:
case OP_NOTPOSQUERY:
case OP_NOTQUERYI:
case OP_NOTMINQUERYI:
case OP_NOTPOSQUERYI:
fast_forward_allowed = FALSE;
if (count == 0)
count = 1;
cc += 2; cc += 2;
#ifdef SUPPORT_UNICODE #ifdef SUPPORT_UNICODE
if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]); if (common->utf && HAS_EXTRALEN(cc[-1])) cc += GET_EXTRALEN(cc[-1]);
@ -1409,12 +1430,25 @@ do
end = cc + (1 + (32 / sizeof(PCRE2_UCHAR))); end = cc + (1 + (32 / sizeof(PCRE2_UCHAR)));
#endif #endif
if (*end >= OP_CRSTAR && *end <= OP_CRPOSRANGE) if ((*end >= OP_CRSTAR && *end <= OP_CRMINPLUS) || (*end >= OP_CRPOSSTAR && *end <= OP_CRPOSPLUS))
{
accelerated_start = cc;
cc = end + 1;
break; break;
}
fast_forward_allowed = FALSE;
if (count == 0)
count = 1;
cc = end; cc = end;
if (*end == OP_CRQUERY || *end == OP_CRMINQUERY || *end == OP_CRPOSQUERY)
cc++;
else if (*end == OP_CRRANGE || *end == OP_CRMINRANGE || *end == OP_CRPOSRANGE)
cc += 1 + 2 * IMM2_SIZE;
continue; continue;
case OP_ONCE:
case OP_BRA: case OP_BRA:
case OP_CBRA: case OP_CBRA:
end = cc + GET(cc, 1); end = cc + GET(cc, 1);
@ -1432,7 +1466,8 @@ do
continue; continue;
} }
if (depth == 0) fast_forward_allowed = FALSE;
if (depth >= 4)
break; break;
end = bracketend(cc) - (1 + LINK_SIZE); end = bracketend(cc) - (1 + LINK_SIZE);
@ -1442,7 +1477,12 @@ do
if (*cc == OP_CBRA && common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0) if (*cc == OP_CBRA && common->optimized_cbracket[GET2(cc, 1 + LINK_SIZE)] == 0)
break; break;
detect_fast_fail(common, cc, private_data_start, depth - 1); count = detect_early_fail(common, cc, private_data_start, depth + 1, count);
if (count < EARLY_FAIL_ENHANCE_MAX)
{
cc = end + (1 + LINK_SIZE);
continue;
}
break; break;
case OP_KET: case OP_KET:
@ -1452,25 +1492,67 @@ do
cc += 1 + LINK_SIZE; cc += 1 + LINK_SIZE;
continue; continue;
} }
if (accelerated_start != NULL)
{
if (count == 0)
{
count++;
if (fast_forward_allowed && *next_alt == OP_KET)
{
common->fast_forward_bc_ptr = accelerated_start;
common->private_data_ptrs[(accelerated_start + 1) - common->start] = ((*private_data_start) << 3) | type_skip;
*private_data_start += sizeof(sljit_sw);
}
else
{
common->private_data_ptrs[(accelerated_start + 1) - common->start] = ((*private_data_start) << 3) | type_fail;
if (common->early_fail_start_ptr == 0)
common->early_fail_start_ptr = *private_data_start;
*private_data_start += sizeof(sljit_sw);
common->early_fail_end_ptr = *private_data_start;
if (*private_data_start > SLJIT_MAX_LOCAL_SIZE)
return EARLY_FAIL_ENHANCE_MAX;
}
}
else
{
common->private_data_ptrs[(accelerated_start + 1) - common->start] = ((*private_data_start) << 3) | type_fail_range;
if (common->early_fail_start_ptr == 0)
common->early_fail_start_ptr = *private_data_start;
*private_data_start += 2 * sizeof(sljit_sw);
common->early_fail_end_ptr = *private_data_start;
if (*private_data_start > SLJIT_MAX_LOCAL_SIZE)
return EARLY_FAIL_ENHANCE_MAX;
}
count++;
if (count < EARLY_FAIL_ENHANCE_MAX)
continue;
}
break; break;
} }
if (is_accelerated_repeat(cc)) if (*cc != OP_ALT && *cc != OP_KET)
{ result = EARLY_FAIL_ENHANCE_MAX;
common->private_data_ptrs[(cc + 1) - common->start] = *private_data_start; else if (result < count)
result = count;
if (common->fast_fail_start_ptr == 0) fast_forward_allowed = FALSE;
common->fast_fail_start_ptr = *private_data_start;
*private_data_start += sizeof(sljit_sw);
common->fast_fail_end_ptr = *private_data_start;
if (*private_data_start > SLJIT_MAX_LOCAL_SIZE)
return;
}
cc = next_alt; cc = next_alt;
} }
while (*cc == OP_ALT); while (*cc == OP_ALT);
return result;
} }
static int get_class_iterator_size(PCRE2_SPTR cc) static int get_class_iterator_size(PCRE2_SPTR cc)
@ -3045,19 +3127,20 @@ else
} }
} }
static SLJIT_INLINE void reset_fast_fail(compiler_common *common) static SLJIT_INLINE void reset_early_fail(compiler_common *common)
{ {
DEFINE_COMPILER; DEFINE_COMPILER;
sljit_s32 size = common->fast_fail_end_ptr - common->fast_fail_start_ptr; sljit_u32 size = (sljit_u32)(common->early_fail_end_ptr - common->early_fail_start_ptr);
sljit_u32 uncleared_size;
sljit_s32 src = SLJIT_IMM; sljit_s32 src = SLJIT_IMM;
sljit_s32 i; sljit_s32 i;
struct sljit_label *loop; struct sljit_label *loop;
SLJIT_ASSERT(common->fast_fail_start_ptr < common->fast_fail_end_ptr); SLJIT_ASSERT(common->early_fail_start_ptr < common->early_fail_end_ptr);
if (size == sizeof(sljit_sw)) if (size == sizeof(sljit_sw))
{ {
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->fast_fail_start_ptr, SLJIT_IMM, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->early_fail_start_ptr, SLJIT_IMM, 0);
return; return;
} }
@ -3069,16 +3152,16 @@ if (sljit_get_register_index(TMP3) >= 0 && !sljit_has_cpu_feature(SLJIT_HAS_ZERO
if (size <= 6 * sizeof(sljit_sw)) if (size <= 6 * sizeof(sljit_sw))
{ {
for (i = common->fast_fail_start_ptr; i < common->fast_fail_end_ptr; i += sizeof(sljit_sw)) for (i = common->early_fail_start_ptr; i < common->early_fail_end_ptr; i += sizeof(sljit_sw))
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), i, src, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), i, src, 0);
return; return;
} }
GET_LOCAL_BASE(TMP1, 0, common->fast_fail_start_ptr); GET_LOCAL_BASE(TMP1, 0, common->early_fail_start_ptr);
i = ((size / (sljit_s32)sizeof(sljit_sw)) % 3) * sizeof(sljit_sw); uncleared_size = ((size / sizeof(sljit_sw)) % 3) * sizeof(sljit_sw);
OP2(SLJIT_ADD, TMP2, 0, TMP1, 0, SLJIT_IMM, size - i); OP2(SLJIT_ADD, TMP2, 0, TMP1, 0, SLJIT_IMM, size - uncleared_size);
loop = LABEL(); loop = LABEL();
OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), 0, src, 0); OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), 0, src, 0);
@ -3087,10 +3170,10 @@ OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), -2 * (sljit_sw)sizeof(sljit_sw), src, 0);
OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), -1 * (sljit_sw)sizeof(sljit_sw), src, 0); OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), -1 * (sljit_sw)sizeof(sljit_sw), src, 0);
CMPTO(SLJIT_LESS, TMP1, 0, TMP2, 0, loop); CMPTO(SLJIT_LESS, TMP1, 0, TMP2, 0, loop);
if (i >= (sljit_sw)sizeof(sljit_sw)) if (uncleared_size >= sizeof(sljit_sw))
OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), 0, src, 0); OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), 0, src, 0);
if (i >= 2 * (sljit_sw)sizeof(sljit_sw)) if (uncleared_size >= 2 * sizeof(sljit_sw))
OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), sizeof(sljit_sw), src, 0); OP1(SLJIT_MOV, SLJIT_MEM1(TMP1), sizeof(sljit_sw), src, 0);
} }
@ -11002,8 +11085,8 @@ backtrack_common *backtrack;
PCRE2_UCHAR opcode; PCRE2_UCHAR opcode;
PCRE2_UCHAR type; PCRE2_UCHAR type;
sljit_u32 max = 0, exact; sljit_u32 max = 0, exact;
BOOL fast_fail; sljit_s32 early_fail_ptr = PRIVATE_DATA(cc + 1);
sljit_s32 fast_str_ptr; sljit_s32 early_fail_type;
BOOL charpos_enabled; BOOL charpos_enabled;
PCRE2_UCHAR charpos_char; PCRE2_UCHAR charpos_char;
unsigned int charpos_othercasebit; unsigned int charpos_othercasebit;
@ -11020,18 +11103,21 @@ int tmp_base, tmp_offset;
PUSH_BACKTRACK(sizeof(char_iterator_backtrack), cc, NULL); PUSH_BACKTRACK(sizeof(char_iterator_backtrack), cc, NULL);
fast_str_ptr = PRIVATE_DATA(cc + 1); early_fail_type = (early_fail_ptr & 0x7);
fast_fail = TRUE; early_fail_ptr >>= 3;
SLJIT_ASSERT(common->fast_forward_bc_ptr == NULL || fast_str_ptr == 0 || cc == common->fast_forward_bc_ptr); /* During recursion, these optimizations are disabled. */
if (common->early_fail_start_ptr == 0)
{
early_fail_ptr = 0;
early_fail_type = type_skip;
}
if (cc == common->fast_forward_bc_ptr) SLJIT_ASSERT(common->fast_forward_bc_ptr != NULL || early_fail_ptr == 0
fast_fail = FALSE; || (early_fail_ptr >= common->early_fail_start_ptr && early_fail_ptr <= common->early_fail_end_ptr));
else if (common->fast_fail_start_ptr == 0)
fast_str_ptr = 0;
SLJIT_ASSERT(common->fast_forward_bc_ptr != NULL || fast_str_ptr == 0 if (early_fail_type == type_fail)
|| (fast_str_ptr >= common->fast_fail_start_ptr && fast_str_ptr <= common->fast_fail_end_ptr)); add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), early_fail_ptr));
cc = get_iterator_parameters(common, cc, &opcode, &type, &max, &exact, &end); cc = get_iterator_parameters(common, cc, &opcode, &type, &max, &exact, &end);
@ -11046,13 +11132,11 @@ else
tmp_offset = POSSESSIVE0; tmp_offset = POSSESSIVE0;
} }
if (fast_fail && fast_str_ptr != 0)
add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_LESS_EQUAL, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), fast_str_ptr));
/* Handle fixed part first. */ /* Handle fixed part first. */
if (exact > 1) if (exact > 1)
{ {
SLJIT_ASSERT(fast_str_ptr == 0); SLJIT_ASSERT(early_fail_ptr == 0);
if (common->mode == PCRE2_JIT_COMPLETE if (common->mode == PCRE2_JIT_COMPLETE
#ifdef SUPPORT_UNICODE #ifdef SUPPORT_UNICODE
&& !common->utf && !common->utf
@ -11077,18 +11161,31 @@ if (exact > 1)
} }
} }
else if (exact == 1) else if (exact == 1)
{
compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, TRUE); compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, TRUE);
if (early_fail_type == type_fail_range)
{
OP1(SLJIT_MOV, TMP1, 0, SLJIT_MEM1(SLJIT_SP), early_fail_ptr);
OP1(SLJIT_MOV, TMP2, 0, SLJIT_MEM1(SLJIT_SP), early_fail_ptr + (int)sizeof(sljit_sw));
OP2(SLJIT_SUB, TMP1, 0, TMP1, 0, TMP2, 0);
OP2(SLJIT_SUB, TMP2, 0, STR_PTR, 0, TMP2, 0);
add_jump(compiler, &backtrack->topbacktracks, CMP(SLJIT_LESS_EQUAL, TMP2, 0, TMP1, 0));
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr + (int)sizeof(sljit_sw), STR_PTR, 0);
}
}
switch(opcode) switch(opcode)
{ {
case OP_STAR: case OP_STAR:
case OP_UPTO: case OP_UPTO:
SLJIT_ASSERT(fast_str_ptr == 0 || opcode == OP_STAR); SLJIT_ASSERT(early_fail_ptr == 0 || opcode == OP_STAR);
if (type == OP_ANYNL || type == OP_EXTUNI) if (type == OP_ANYNL || type == OP_EXTUNI)
{ {
SLJIT_ASSERT(private_data_ptr == 0); SLJIT_ASSERT(private_data_ptr == 0);
SLJIT_ASSERT(fast_str_ptr == 0); SLJIT_ASSERT(early_fail_ptr == 0);
allocate_stack(common, 2); allocate_stack(common, 2);
OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0); OP1(SLJIT_MOV, SLJIT_MEM1(STACK_TOP), STACK(0), STR_PTR, 0);
@ -11133,8 +11230,8 @@ switch(opcode)
OP1(SLJIT_MOV, STR_PTR, 0, STR_END, 0); OP1(SLJIT_MOV, STR_PTR, 0, STR_END, 0);
process_partial_match(common); process_partial_match(common);
if (fast_str_ptr != 0) if (early_fail_ptr != 0)
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_END, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_END, 0);
BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL();
break; break;
} }
@ -11164,8 +11261,8 @@ switch(opcode)
OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
if (fast_str_ptr != 0) if (early_fail_ptr != 0)
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_PTR, 0);
BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL();
break; break;
} }
@ -11224,8 +11321,8 @@ switch(opcode)
add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_ZERO)); add_jump(compiler, &backtrack->topbacktracks, JUMP(SLJIT_ZERO));
} }
compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, FALSE); compile_char1_matchingpath(common, type, cc, &backtrack->topbacktracks, FALSE);
if (fast_str_ptr != 0) if (early_fail_ptr != 0)
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_PTR, 0);
JUMPHERE(jump); JUMPHERE(jump);
detect_partial_match(common, &backtrack->topbacktracks); detect_partial_match(common, &backtrack->topbacktracks);
@ -11247,8 +11344,8 @@ switch(opcode)
/* Search the last instance of charpos_char. */ /* Search the last instance of charpos_char. */
label = LABEL(); label = LABEL();
compile_char1_matchingpath(common, type, cc, &no_match, FALSE); compile_char1_matchingpath(common, type, cc, &no_match, FALSE);
if (fast_str_ptr != 0) if (early_fail_ptr != 0)
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_PTR, 0);
detect_partial_match(common, &no_match); detect_partial_match(common, &no_match);
OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0)); OP1(MOV_UCHAR, TMP1, 0, SLJIT_MEM1(STR_PTR), IN_UCHARS(0));
if (charpos_othercasebit != 0) if (charpos_othercasebit != 0)
@ -11306,8 +11403,8 @@ switch(opcode)
set_jumps(no_match, LABEL()); set_jumps(no_match, LABEL());
OP1(SLJIT_MOV, STR_PTR, 0, base, offset0); OP1(SLJIT_MOV, STR_PTR, 0, base, offset0);
if (fast_str_ptr != 0) if (early_fail_ptr != 0)
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_PTR, 0);
} }
#endif #endif
else else
@ -11335,8 +11432,8 @@ switch(opcode)
OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
set_jumps(no_match, LABEL()); set_jumps(no_match, LABEL());
OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
if (fast_str_ptr != 0) if (early_fail_ptr != 0)
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_PTR, 0);
} }
BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL();
@ -11347,12 +11444,12 @@ switch(opcode)
allocate_stack(common, 1); allocate_stack(common, 1);
OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL(); BACKTRACK_AS(char_iterator_backtrack)->matchingpath = LABEL();
if (fast_str_ptr != 0) if (early_fail_ptr != 0)
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_PTR, 0);
break; break;
case OP_MINUPTO: case OP_MINUPTO:
SLJIT_ASSERT(fast_str_ptr == 0); SLJIT_ASSERT(early_fail_ptr == 0);
if (private_data_ptr == 0) if (private_data_ptr == 0)
allocate_stack(common, 2); allocate_stack(common, 2);
OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
@ -11362,7 +11459,7 @@ switch(opcode)
case OP_QUERY: case OP_QUERY:
case OP_MINQUERY: case OP_MINQUERY:
SLJIT_ASSERT(fast_str_ptr == 0); SLJIT_ASSERT(early_fail_ptr == 0);
if (private_data_ptr == 0) if (private_data_ptr == 0)
allocate_stack(common, 1); allocate_stack(common, 1);
OP1(SLJIT_MOV, base, offset0, STR_PTR, 0); OP1(SLJIT_MOV, base, offset0, STR_PTR, 0);
@ -11383,8 +11480,8 @@ switch(opcode)
{ {
OP1(SLJIT_MOV, STR_PTR, 0, STR_END, 0); OP1(SLJIT_MOV, STR_PTR, 0, STR_END, 0);
process_partial_match(common); process_partial_match(common);
if (fast_str_ptr != 0) if (early_fail_ptr != 0)
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_END, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_END, 0);
break; break;
} }
@ -11400,12 +11497,12 @@ switch(opcode)
set_jumps(no_match, LABEL()); set_jumps(no_match, LABEL());
OP1(SLJIT_MOV, STR_PTR, 0, tmp_base, tmp_offset); OP1(SLJIT_MOV, STR_PTR, 0, tmp_base, tmp_offset);
if (fast_str_ptr != 0) if (early_fail_ptr != 0)
{ {
if (tmp_base == TMP3) if (tmp_base == TMP3)
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, TMP3, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, TMP3, 0);
else else
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_PTR, 0);
} }
break; break;
} }
@ -11419,12 +11516,12 @@ switch(opcode)
set_jumps(no_char1_match, LABEL()); set_jumps(no_char1_match, LABEL());
OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1)); OP2(SLJIT_SUB, STR_PTR, 0, STR_PTR, 0, SLJIT_IMM, IN_UCHARS(1));
set_jumps(no_match, LABEL()); set_jumps(no_match, LABEL());
if (fast_str_ptr != 0) if (early_fail_ptr != 0)
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_PTR, 0);
break; break;
case OP_POSUPTO: case OP_POSUPTO:
SLJIT_ASSERT(fast_str_ptr == 0); SLJIT_ASSERT(early_fail_ptr == 0);
#if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32 #if defined SUPPORT_UNICODE && PCRE2_CODE_UNIT_WIDTH != 32
if (common->utf) if (common->utf)
{ {
@ -11461,8 +11558,8 @@ switch(opcode)
JUMPHERE(jump); JUMPHERE(jump);
} }
if (fast_str_ptr != 0) if (early_fail_ptr != 0)
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), fast_str_ptr, STR_PTR, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), early_fail_ptr, STR_PTR, 0);
break; break;
} }
@ -11482,7 +11579,7 @@ switch(opcode)
break; break;
case OP_POSQUERY: case OP_POSQUERY:
SLJIT_ASSERT(fast_str_ptr == 0); SLJIT_ASSERT(early_fail_ptr == 0);
OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0); OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0);
compile_char1_matchingpath(common, type, cc, &no_match, TRUE); compile_char1_matchingpath(common, type, cc, &no_match, TRUE);
OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0); OP1(SLJIT_MOV, tmp_base, tmp_offset, STR_PTR, 0);
@ -13449,13 +13546,10 @@ memset(common->private_data_ptrs, 0, total_length * sizeof(sljit_s32));
private_data_size = common->cbra_ptr + (re->top_bracket + 1) * sizeof(sljit_sw); private_data_size = common->cbra_ptr + (re->top_bracket + 1) * sizeof(sljit_sw);
set_private_data_ptrs(common, &private_data_size, ccend); set_private_data_ptrs(common, &private_data_size, ccend);
if ((re->overall_options & PCRE2_ANCHORED) == 0 && (re->overall_options & PCRE2_NO_START_OPTIMIZE) == 0) if ((re->overall_options & PCRE2_ANCHORED) == 0 && (re->overall_options & PCRE2_NO_START_OPTIMIZE) == 0 && !common->has_skip_in_assert_back)
{ detect_early_fail(common, common->start, &private_data_size, 0, 0);
if (!detect_fast_forward_skip(common, &private_data_size) && !common->has_skip_in_assert_back)
detect_fast_fail(common, common->start, &private_data_size, 4);
}
SLJIT_ASSERT(common->fast_fail_start_ptr <= common->fast_fail_end_ptr); SLJIT_ASSERT(common->early_fail_start_ptr <= common->early_fail_end_ptr);
if (private_data_size > SLJIT_MAX_LOCAL_SIZE) if (private_data_size > SLJIT_MAX_LOCAL_SIZE)
{ {
@ -13499,8 +13593,8 @@ OP1(SLJIT_MOV, STACK_LIMIT, 0, SLJIT_MEM1(TMP2), SLJIT_OFFSETOF(struct sljit_sta
OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1); OP2(SLJIT_ADD, TMP1, 0, TMP1, 0, SLJIT_IMM, 1);
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LIMIT_MATCH, TMP1, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), LIMIT_MATCH, TMP1, 0);
if (common->fast_fail_start_ptr < common->fast_fail_end_ptr) if (common->early_fail_start_ptr < common->early_fail_end_ptr)
reset_fast_fail(common); reset_early_fail(common);
if (mode == PCRE2_JIT_PARTIAL_SOFT) if (mode == PCRE2_JIT_PARTIAL_SOFT)
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, -1); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->hit_start, SLJIT_IMM, -1);
@ -13546,7 +13640,7 @@ OP1(SLJIT_MOV, COUNT_MATCH, 0, SLJIT_MEM1(SLJIT_SP), LIMIT_MATCH);
if (common->capture_last_ptr != 0) if (common->capture_last_ptr != 0)
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, SLJIT_IMM, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->capture_last_ptr, SLJIT_IMM, 0);
if (common->fast_forward_bc_ptr != NULL) if (common->fast_forward_bc_ptr != NULL)
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), PRIVATE_DATA(common->fast_forward_bc_ptr + 1), STR_PTR, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), PRIVATE_DATA(common->fast_forward_bc_ptr + 1) >> 3, STR_PTR, 0);
if (common->start_ptr != OVECTOR(0)) if (common->start_ptr != OVECTOR(0))
OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_ptr, STR_PTR, 0); OP1(SLJIT_MOV, SLJIT_MEM1(SLJIT_SP), common->start_ptr, STR_PTR, 0);
@ -13647,7 +13741,7 @@ if ((re->overall_options & PCRE2_ANCHORED) == 0 && common->match_end_ptr != 0)
} }
OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP), OP1(SLJIT_MOV, STR_PTR, 0, SLJIT_MEM1(SLJIT_SP),
(common->fast_forward_bc_ptr != NULL) ? (PRIVATE_DATA(common->fast_forward_bc_ptr + 1)) : common->start_ptr); (common->fast_forward_bc_ptr != NULL) ? (PRIVATE_DATA(common->fast_forward_bc_ptr + 1) >> 3) : common->start_ptr);
if ((re->overall_options & PCRE2_ANCHORED) == 0) if ((re->overall_options & PCRE2_ANCHORED) == 0)
{ {
@ -13698,8 +13792,8 @@ if (common->might_be_empty)
} }
common->fast_forward_bc_ptr = NULL; common->fast_forward_bc_ptr = NULL;
common->fast_fail_start_ptr = 0; common->early_fail_start_ptr = 0;
common->fast_fail_end_ptr = 0; common->early_fail_end_ptr = 0;
common->currententry = common->entries; common->currententry = common->entries;
common->local_quit_available = TRUE; common->local_quit_available = TRUE;
quit_label = common->quit_label; quit_label = common->quit_label;