walkero
/
harfbuzz
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[Indic-like] Disable automatic joiner handling for basic shaping features 

Not for Arabic, but for Indic-like scripts.  ZWJ/ZWNJ have special
meanings in those scripts, so let font lookups take full control.

This undoes the regression caused by automatic-joiners handling
introduced two commits ago.

We only disable automatic joiner handling for the "basic shaping
features" of Indic, Myanmar, and SEAsian shapers.  The "presentation
forms" and other features are still applied with automatic-joiner
handling.

This change also changes the test suite failure statistics, such that
a few scripts show more "failures".  The most affected is Kannada.
However, upon inspection, we believe that in most, if not all, of the
new failures, we are producing results superior to Uniscribe.  Hard to
count those!

Here's an example of what is fixed by the recent joiner-handling
changes:

  https://bugs.freedesktop.org/show_bug.cgi?id=58714

New numbers, for future reference:

BENGALI: 353892 out of 354188 tests passed. 296 failed (0.0835714%)
DEVANAGARI: 707336 out of 707394 tests passed. 58 failed (0.00819911%)
GUJARATI: 366262 out of 366457 tests passed. 195 failed (0.0532122%)
GURMUKHI: 60706 out of 60747 tests passed. 41 failed (0.067493%)
KANNADA: 950680 out of 951913 tests passed. 1233 failed (0.129529%)
KHMER: 299074 out of 299124 tests passed. 50 failed (0.0167155%)
LAO: 53611 out of 53644 tests passed. 33 failed (0.0615167%)
MALAYALAM: 1047983 out of 1048334 tests passed. 351 failed (0.0334817%)
ORIYA: 42320 out of 42329 tests passed. 9 failed (0.021262%)
SINHALA: 271539 out of 271847 tests passed. 308 failed (0.113299%)
TAMIL: 1091753 out of 1091754 tests passed. 1 failed (9.15957e-05%)
TELUGU: 970555 out of 970573 tests passed. 18 failed (0.00185457%)
TIBETAN: 208469 out of 208469 tests passed. 0 failed (0%)
This commit is contained in:
Behdad Esfahbod 2013-02-14 10:40:12 -05:00
parent 0b45479198
commit cfc507c543
9 changed files with 85 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
src/hb-ot-layout-gsubgpos-private.hh
View File

@ -261,6 +261,7 @@ struct hb_apply_context_t
hb_buffer_t *buffer; hb_buffer_t *buffer;
hb_direction_t direction; hb_direction_t direction;
hb_mask_t lookup_mask; hb_mask_t lookup_mask;
bool auto_joiners;
recurse_func_t recurse_func; recurse_func_t recurse_func;
unsigned int nesting_level_left; unsigned int nesting_level_left;
unsigned int lookup_props; unsigned int lookup_props;
@ -272,11 +273,13 @@ struct hb_apply_context_t
hb_apply_context_t (unsigned int table_index_, hb_apply_context_t (unsigned int table_index_,
hb_font_t *font_, hb_font_t *font_,
hb_buffer_t *buffer_, hb_buffer_t *buffer_,
hb_mask_t lookup_mask_) : hb_mask_t lookup_mask_,
bool auto_joiners_) :
table_index (table_index_), table_index (table_index_),
font (font_), face (font->face), buffer (buffer_), font (font_), face (font->face), buffer (buffer_),
direction (buffer_->props.direction), direction (buffer_->props.direction),
lookup_mask (lookup_mask_), lookup_mask (lookup_mask_),
auto_joiners (auto_joiners_),
recurse_func (NULL), recurse_func (NULL),
nesting_level_left (MAX_NESTING_LEVEL), nesting_level_left (MAX_NESTING_LEVEL),
lookup_props (0), lookup_props (0),
@ -292,8 +295,8 @@ struct hb_apply_context_t
{ {
inline matcher_t (void) : inline matcher_t (void) :
lookup_props (0), lookup_props (0),
ignore_zwnj (true), ignore_zwnj (false),
ignore_zwj (true), ignore_zwj (false),
mask (-1), mask (-1),
#define arg1(arg) (arg) /* Remove the macro to see why it's needed! */ #define arg1(arg) (arg) /* Remove the macro to see why it's needed! */
syllable arg1(0), syllable arg1(0),
@ -369,9 +372,12 @@ struct hb_apply_context_t
end (c->buffer->len) end (c->buffer->len)
{ {
matcher.set_lookup_props (c->lookup_props); matcher.set_lookup_props (c->lookup_props);
if (c->auto_joiners)
{
/* Ignore ZWNJ if we are matching GSUB context, or matching GPOS. */ /* Ignore ZWNJ if we are matching GSUB context, or matching GPOS. */
matcher.set_ignore_zwnj (context_match || c->table_index == 1); matcher.set_ignore_zwnj (context_match || c->table_index == 1);
matcher.set_ignore_zwj (true); matcher.set_ignore_zwj (true);
}
if (!context_match) if (!context_match)
{ {
matcher.set_mask (c->lookup_mask); matcher.set_mask (c->lookup_mask);
@ -439,9 +445,12 @@ struct hb_apply_context_t
num_items (num_items_) num_items (num_items_)
{ {
matcher.set_lookup_props (c->lookup_props); matcher.set_lookup_props (c->lookup_props);
if (c->auto_joiners)
{
/* Ignore ZWNJ if we are matching GSUB context, or matching GPOS. */ /* Ignore ZWNJ if we are matching GSUB context, or matching GPOS. */
matcher.set_ignore_zwnj (context_match || c->table_index == 1); matcher.set_ignore_zwnj (context_match || c->table_index == 1);
matcher.set_ignore_zwj (true); matcher.set_ignore_zwj (true);
}
if (!context_match) if (!context_match)
{ {
matcher.set_mask (c->lookup_mask); matcher.set_mask (c->lookup_mask);

6
src/hb-ot-layout-private.hh
View File

@ -206,7 +206,8 @@ HB_INTERNAL hb_bool_t
hb_ot_layout_substitute_lookup (hb_font_t *font, hb_ot_layout_substitute_lookup (hb_font_t *font,
hb_buffer_t *buffer, hb_buffer_t *buffer,
unsigned int lookup_index, unsigned int lookup_index,
hb_mask_t mask); hb_mask_t mask,
hb_bool_t auto_joiners);
/* Should be called after all the substitute_lookup's are done */ /* Should be called after all the substitute_lookup's are done */
HB_INTERNAL void HB_INTERNAL void
@ -223,7 +224,8 @@ HB_INTERNAL hb_bool_t
hb_ot_layout_position_lookup (hb_font_t *font, hb_ot_layout_position_lookup (hb_font_t *font,
hb_buffer_t *buffer, hb_buffer_t *buffer,
unsigned int lookup_index, unsigned int lookup_index,
hb_mask_t mask); hb_mask_t mask,
hb_bool_t auto_joiners);
/* Should be called after all the position_lookup's are done */ /* Should be called after all the position_lookup's are done */
HB_INTERNAL void HB_INTERNAL void

10
src/hb-ot-layout.cc
View File

@ -666,11 +666,12 @@ hb_bool_t
hb_ot_layout_substitute_lookup (hb_font_t *font, hb_ot_layout_substitute_lookup (hb_font_t *font,
hb_buffer_t *buffer, hb_buffer_t *buffer,
unsigned int lookup_index, unsigned int lookup_index,
hb_mask_t mask) hb_mask_t mask,
hb_bool_t auto_joiners)
{ {
if (unlikely (lookup_index >= hb_ot_layout_from_face (font->face)->gsub_lookup_count)) return false; if (unlikely (lookup_index >= hb_ot_layout_from_face (font->face)->gsub_lookup_count)) return false;
OT::hb_apply_context_t c (0, font, buffer, mask); OT::hb_apply_context_t c (0, font, buffer, mask, auto_joiners);
const OT::SubstLookup& l = hb_ot_layout_from_face (font->face)->gsub->get_lookup (lookup_index); const OT::SubstLookup& l = hb_ot_layout_from_face (font->face)->gsub->get_lookup (lookup_index);
@ -715,11 +716,12 @@ hb_bool_t
hb_ot_layout_position_lookup (hb_font_t *font, hb_ot_layout_position_lookup (hb_font_t *font,
hb_buffer_t *buffer, hb_buffer_t *buffer,
unsigned int lookup_index, unsigned int lookup_index,
hb_mask_t mask) hb_mask_t mask,
hb_bool_t auto_joiners)
{ {
if (unlikely (lookup_index >= hb_ot_layout_from_face (font->face)->gpos_lookup_count)) return false; if (unlikely (lookup_index >= hb_ot_layout_from_face (font->face)->gpos_lookup_count)) return false;
OT::hb_apply_context_t c (1, font, buffer, mask); OT::hb_apply_context_t c (1, font, buffer, mask, auto_joiners);
const OT::PosLookup& l = hb_ot_layout_from_face (font->face)->gpos->get_lookup (lookup_index); const OT::PosLookup& l = hb_ot_layout_from_face (font->face)->gpos->get_lookup (lookup_index);

12
src/hb-ot-map-private.hh
View File

@ -49,14 +49,16 @@ struct hb_ot_map_t
unsigned int shift; unsigned int shift;
hb_mask_t mask; hb_mask_t mask;
hb_mask_t _1_mask; /* mask for value=1, for quick access */ hb_mask_t _1_mask; /* mask for value=1, for quick access */
hb_bool_t needs_fallback; unsigned int needs_fallback : 1;
unsigned int auto_joiners : 1;
static int cmp (const feature_map_t *a, const feature_map_t *b) static int cmp (const feature_map_t *a, const feature_map_t *b)
{ return a->tag < b->tag ? -1 : a->tag > b->tag ? 1 : 0; } { return a->tag < b->tag ? -1 : a->tag > b->tag ? 1 : 0; }
}; };
struct lookup_map_t { struct lookup_map_t {
unsigned int index; unsigned short index;
unsigned short auto_joiners : 1;
hb_mask_t mask; hb_mask_t mask;
static int cmp (const lookup_map_t *a, const lookup_map_t *b) static int cmp (const lookup_map_t *a, const lookup_map_t *b)
@ -136,7 +138,8 @@ struct hb_ot_map_t
HB_INTERNAL void add_lookups (hb_face_t *face, HB_INTERNAL void add_lookups (hb_face_t *face,
unsigned int table_index, unsigned int table_index,
unsigned int feature_index, unsigned int feature_index,
hb_mask_t mask); hb_mask_t mask,
bool auto_joiners);
hb_mask_t global_mask; hb_mask_t global_mask;
@ -148,7 +151,8 @@ struct hb_ot_map_t
enum hb_ot_map_feature_flags_t { enum hb_ot_map_feature_flags_t {
F_NONE = 0x0000, F_NONE = 0x0000,
F_GLOBAL = 0x0001, F_GLOBAL = 0x0001,
F_HAS_FALLBACK = 0x0002 F_HAS_FALLBACK = 0x0002,
F_MANUAL_JOINERS = 0x0004
}; };
inline hb_ot_map_feature_flags_t inline hb_ot_map_feature_flags_t
operator | (hb_ot_map_feature_flags_t l, hb_ot_map_feature_flags_t r) operator | (hb_ot_map_feature_flags_t l, hb_ot_map_feature_flags_t r)

32
src/hb-ot-map.cc
View File

@ -33,7 +33,8 @@ void
hb_ot_map_t::add_lookups (hb_face_t *face, hb_ot_map_t::add_lookups (hb_face_t *face,
unsigned int table_index, unsigned int table_index,
unsigned int feature_index, unsigned int feature_index,
hb_mask_t mask) hb_mask_t mask,
bool auto_joiners)
{ {
unsigned int lookup_indices[32]; unsigned int lookup_indices[32];
unsigned int offset, len; unsigned int offset, len;
@ -53,6 +54,7 @@ hb_ot_map_t::add_lookups (hb_face_t *face,
return; return;
lookup->mask = mask; lookup->mask = mask;
lookup->index = lookup_indices[i]; lookup->index = lookup_indices[i];
lookup->auto_joiners = auto_joiners;
} }
offset += len; offset += len;
@ -108,7 +110,10 @@ void hb_ot_map_t::substitute (const hb_ot_shape_plan_t *plan, hb_font_t *font, h
for (unsigned int pause_index = 0; pause_index < pauses[table_index].len; pause_index++) { for (unsigned int pause_index = 0; pause_index < pauses[table_index].len; pause_index++) {
const pause_map_t *pause = &pauses[table_index][pause_index]; const pause_map_t *pause = &pauses[table_index][pause_index];
for (; i < pause->num_lookups; i++) for (; i < pause->num_lookups; i++)
hb_ot_layout_substitute_lookup (font, buffer, lookups[table_index][i].index, lookups[table_index][i].mask); hb_ot_layout_substitute_lookup (font, buffer,
lookups[table_index][i].index,
lookups[table_index][i].mask,
lookups[table_index][i].auto_joiners);
buffer->clear_output (); buffer->clear_output ();
@ -117,7 +122,9 @@ void hb_ot_map_t::substitute (const hb_ot_shape_plan_t *plan, hb_font_t *font, h
} }
for (; i < lookups[table_index].len; i++) for (; i < lookups[table_index].len; i++)
hb_ot_layout_substitute_lookup (font, buffer, lookups[table_index][i].index, lookups[table_index][i].mask); hb_ot_layout_substitute_lookup (font, buffer, lookups[table_index][i].index,
lookups[table_index][i].mask,
lookups[table_index][i].auto_joiners);
} }
void hb_ot_map_t::position (const hb_ot_shape_plan_t *plan, hb_font_t *font, hb_buffer_t *buffer) const void hb_ot_map_t::position (const hb_ot_shape_plan_t *plan, hb_font_t *font, hb_buffer_t *buffer) const
@ -128,14 +135,18 @@ void hb_ot_map_t::position (const hb_ot_shape_plan_t *plan, hb_font_t *font, hb_
for (unsigned int pause_index = 0; pause_index < pauses[table_index].len; pause_index++) { for (unsigned int pause_index = 0; pause_index < pauses[table_index].len; pause_index++) {
const pause_map_t *pause = &pauses[table_index][pause_index]; const pause_map_t *pause = &pauses[table_index][pause_index];
for (; i < pause->num_lookups; i++) for (; i < pause->num_lookups; i++)
hb_ot_layout_position_lookup (font, buffer, lookups[table_index][i].index, lookups[table_index][i].mask); hb_ot_layout_position_lookup (font, buffer, lookups[table_index][i].index,
lookups[table_index][i].mask,
lookups[table_index][i].auto_joiners);
if (pause->callback) if (pause->callback)
pause->callback (plan, font, buffer); pause->callback (plan, font, buffer);
} }
for (; i < lookups[table_index].len; i++) for (; i < lookups[table_index].len; i++)
hb_ot_layout_position_lookup (font, buffer, lookups[table_index][i].index, lookups[table_index][i].mask); hb_ot_layout_position_lookup (font, buffer, lookups[table_index][i].index,
lookups[table_index][i].mask,
lookups[table_index][i].auto_joiners);
} }
void hb_ot_map_t::collect_lookups (unsigned int table_index, hb_set_t *lookups_out) const void hb_ot_map_t::collect_lookups (unsigned int table_index, hb_set_t *lookups_out) const
@ -232,6 +243,7 @@ hb_ot_map_builder_t::compile (hb_ot_map_t &m)
map->index[1] = feature_index[1]; map->index[1] = feature_index[1];
map->stage[0] = info->stage[0]; map->stage[0] = info->stage[0];
map->stage[1] = info->stage[1]; map->stage[1] = info->stage[1];
map->auto_joiners = !(info->flags & F_MANUAL_JOINERS);
if ((info->flags & F_GLOBAL) && info->max_value == 1) { if ((info->flags & F_GLOBAL) && info->max_value == 1) {
/* Uses the global bit */ /* Uses the global bit */
map->shift = 0; map->shift = 0;
@ -264,7 +276,7 @@ hb_ot_map_builder_t::compile (hb_ot_map_t &m)
script_index[table_index], script_index[table_index],
language_index[table_index], language_index[table_index],
&required_feature_index)) &required_feature_index))
m.add_lookups (face, table_index, required_feature_index, 1); m.add_lookups (face, table_index, required_feature_index, 1, true);
unsigned int pause_index = 0; unsigned int pause_index = 0;
unsigned int last_num_lookups = 0; unsigned int last_num_lookups = 0;
@ -272,7 +284,10 @@ hb_ot_map_builder_t::compile (hb_ot_map_t &m)
{ {
for (unsigned i = 0; i < m.features.len; i++) for (unsigned i = 0; i < m.features.len; i++)
if (m.features[i].stage[table_index] == stage) if (m.features[i].stage[table_index] == stage)
m.add_lookups (face, table_index, m.features[i].index[table_index], m.features[i].mask); m.add_lookups (face, table_index,
m.features[i].index[table_index],
m.features[i].mask,
m.features[i].auto_joiners);
/* Sort lookups and merge duplicates */ /* Sort lookups and merge duplicates */
if (last_num_lookups < m.lookups[table_index].len) if (last_num_lookups < m.lookups[table_index].len)
@ -284,7 +299,10 @@ hb_ot_map_builder_t::compile (hb_ot_map_t &m)
if (m.lookups[table_index][i].index != m.lookups[table_index][j].index) if (m.lookups[table_index][i].index != m.lookups[table_index][j].index)
m.lookups[table_index][++j] = m.lookups[table_index][i]; m.lookups[table_index][++j] = m.lookups[table_index][i];
else else
{
m.lookups[table_index][j].mask |= m.lookups[table_index][i].mask; m.lookups[table_index][j].mask |= m.lookups[table_index][i].mask;
m.lookups[table_index][j].auto_joiners &= m.lookups[table_index][i].auto_joiners;
}
m.lookups[table_index].shrink (j + 1); m.lookups[table_index].shrink (j + 1);
} }

2
src/hb-ot-shape-complex-arabic-fallback.hh
View File

@ -244,7 +244,7 @@ arabic_fallback_plan_shape (arabic_fallback_plan_t *fallback_plan,
{ {
for (unsigned int i = 0; i < ARABIC_NUM_FALLBACK_FEATURES; i++) for (unsigned int i = 0; i < ARABIC_NUM_FALLBACK_FEATURES; i++)
if (fallback_plan->lookup_array[i]) { if (fallback_plan->lookup_array[i]) {
OT::hb_apply_context_t c (0, font, buffer, fallback_plan->mask_array[i]); OT::hb_apply_context_t c (0, font, buffer, fallback_plan->mask_array[i], true/*auto_joiners*/);
fallback_plan->lookup_array[i]->apply_string (&c, &fallback_plan->digest_array[i]); fallback_plan->lookup_array[i]->apply_string (&c, &fallback_plan->digest_array[i]);
} }
} }

29
src/hb-ot-shape-complex-indic.cc
View File

@ -339,18 +339,18 @@ indic_features[] =
* Basic features. * Basic features.
* These features are applied in order, one at a time, after initial_reordering. * These features are applied in order, one at a time, after initial_reordering.
*/ */
{HB_TAG('n','u','k','t'), F_GLOBAL}, {HB_TAG('n','u','k','t'), F_MANUAL_JOINERS | F_GLOBAL},
{HB_TAG('a','k','h','n'), F_GLOBAL}, {HB_TAG('a','k','h','n'), F_MANUAL_JOINERS | F_GLOBAL},
{HB_TAG('r','p','h','f'), F_NONE}, {HB_TAG('r','p','h','f'), F_MANUAL_JOINERS},
{HB_TAG('r','k','r','f'), F_GLOBAL}, {HB_TAG('r','k','r','f'), F_MANUAL_JOINERS | F_GLOBAL},
{HB_TAG('p','r','e','f'), F_NONE}, {HB_TAG('p','r','e','f'), F_MANUAL_JOINERS},
{HB_TAG('b','l','w','f'), F_NONE}, {HB_TAG('b','l','w','f'), F_MANUAL_JOINERS},
{HB_TAG('h','a','l','f'), F_NONE}, {HB_TAG('h','a','l','f'), F_MANUAL_JOINERS},
{HB_TAG('a','b','v','f'), F_NONE}, {HB_TAG('a','b','v','f'), F_MANUAL_JOINERS},
{HB_TAG('p','s','t','f'), F_NONE}, {HB_TAG('p','s','t','f'), F_MANUAL_JOINERS},
{HB_TAG('c','f','a','r'), F_NONE}, {HB_TAG('c','f','a','r'), F_MANUAL_JOINERS},
{HB_TAG('v','a','t','u'), F_GLOBAL}, {HB_TAG('v','a','t','u'), F_MANUAL_JOINERS | F_GLOBAL},
{HB_TAG('c','j','c','t'), F_GLOBAL}, {HB_TAG('c','j','c','t'), F_MANUAL_JOINERS | F_GLOBAL},
/* /*
* Other features. * Other features.
* These features are applied all at once, after final_reordering. * These features are applied all at once, after final_reordering.
@ -1003,8 +1003,9 @@ initial_reordering_consonant_syllable (const hb_ot_shape_plan_t *plan,
do { do {
j--; j--;
/* A ZWJ disables CJCT, however, it's mere presence is enough /* ZWJ/ZWNJ should disable CJCT. They do that by simply
* to disable ligation. No explicit action needed. */ * being there, since we don't skip them for the CJCT
* feature (ie. F_MANUAL_JOINERS) */
/* A ZWNJ disables HALF. */ /* A ZWNJ disables HALF. */
if (non_joiner) if (non_joiner)

4
src/hb-ot-shape-complex-myanmar.cc
View File

@ -107,12 +107,12 @@ collect_features_myanmar (hb_ot_shape_planner_t *plan)
unsigned int i = 0; unsigned int i = 0;
map->add_gsub_pause (initial_reordering); map->add_gsub_pause (initial_reordering);
for (; i < MYANMAR_BASIC_FEATURES; i++) { for (; i < MYANMAR_BASIC_FEATURES; i++) {
map->add_global_bool_feature (myanmar_features[i]); map->add_feature (myanmar_features[i], 1, F_GLOBAL | F_MANUAL_JOINERS);
map->add_gsub_pause (NULL); map->add_gsub_pause (NULL);
} }
map->add_gsub_pause (final_reordering); map->add_gsub_pause (final_reordering);
for (; i < MYANMAR_NUM_FEATURES; i++) { for (; i < MYANMAR_NUM_FEATURES; i++) {
map->add_global_bool_feature (myanmar_features[i]); map->add_feature (myanmar_features[i], 1, F_GLOBAL);
} }
} }

4
src/hb-ot-shape-complex-sea.cc
View File

@ -109,12 +109,12 @@ collect_features_sea (hb_ot_shape_planner_t *plan)
unsigned int i = 0; unsigned int i = 0;
map->add_gsub_pause (initial_reordering); map->add_gsub_pause (initial_reordering);
for (; i < SEA_BASIC_FEATURES; i++) { for (; i < SEA_BASIC_FEATURES; i++) {
map->add_global_bool_feature (sea_features[i]); map->add_feature (sea_features[i], 1, F_GLOBAL | F_MANUAL_JOINERS);
map->add_gsub_pause (NULL); map->add_gsub_pause (NULL);
} }
map->add_gsub_pause (final_reordering); map->add_gsub_pause (final_reordering);
for (; i < SEA_NUM_FEATURES; i++) { for (; i < SEA_NUM_FEATURES; i++) {
map->add_global_bool_feature (sea_features[i]); map->add_feature (sea_features[i], 1, F_GLOBAL | F_MANUAL_JOINERS);
} }
} }