walkero
/
harfbuzz
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

[Indic] Futher adjust ZWJ handling in Indic-like shapers 

After the Ngapi hackfest work, we were assuming that fonts
won't use presentation features to choose specific forms
(eg. conjuncts).  As such, we were using auto-joiner behavior
for such features.  It proved to be troublesome as many fonts
used presentation forms ('pres') for example to form conjuncts,
which need to be disabled when a ZWJ is inserted.

Two examples:

	U+0D2F,U+200D,U+0D4D,U+0D2F with kartika.ttf
	U+0995,U+09CD,U+200D,U+09B7 with vrinda.ttf

What we do now is to never do magic to ZWJ during GSUB's main input
match for Indic-style shapers.  Note that backtrack/lookahead are still
matched liberally, as is GPOS.  This seems to be an acceptable
compromise.

As to the bug that initially started this work, that one needs to
be fixed differently:

  Bug 58714 - Kannada u+0cb0 u+200d u+0ccd u+0c95 u+0cbe does not
  provide same results as Windows8
  https://bugs.freedesktop.org/show_bug.cgi?id=58714

New numbers:

BENGALI: 353689 out of 354188 tests passed. 499 failed (0.140886%)
DEVANAGARI: 707305 out of 707394 tests passed. 89 failed (0.0125814%)
GUJARATI: 366349 out of 366457 tests passed. 108 failed (0.0294714%)
GURMUKHI: 60706 out of 60747 tests passed. 41 failed (0.067493%)
KANNADA: 951030 out of 951913 tests passed. 883 failed (0.0927606%)
KHMER: 299070 out of 299124 tests passed. 54 failed (0.0180527%)
LAO: 53611 out of 53644 tests passed. 33 failed (0.0615167%)
MALAYALAM: 1048102 out of 1048334 tests passed. 232 failed (0.0221304%)
ORIYA: 42320 out of 42329 tests passed. 9 failed (0.021262%)
SINHALA: 271666 out of 271847 tests passed. 181 failed (0.0665816%)
TAMIL: 1091753 out of 1091754 tests passed. 1 failed (9.15957e-05%)
TELUGU: 970555 out of 970573 tests passed. 18 failed (0.00185457%)
TIBETAN: 208469 out of 208469 tests passed. 0 failed (0%)
This commit is contained in:
Behdad Esfahbod 2013-03-19 05:53:26 -04:00
parent 8226fb56f1
commit a8cf7b43fa
12 changed files with 53 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
src/hb-ot-layout-gsubgpos-private.hh
View File

@ -261,7 +261,7 @@ struct hb_apply_context_t
hb_buffer_t *buffer;
hb_direction_t direction;
hb_mask_t lookup_mask;
bool auto_joiners;
bool auto_zwj;
recurse_func_t recurse_func;
unsigned int nesting_level_left;
unsigned int lookup_props;
@ -274,12 +274,12 @@ struct hb_apply_context_t
hb_font_t *font_,
hb_buffer_t *buffer_,
hb_mask_t lookup_mask_,
bool auto_joiners_) :
bool auto_zwj_) :
table_index (table_index_),
font (font_), face (font->face), buffer (buffer_),
direction (buffer_->props.direction),
lookup_mask (lookup_mask_),
auto_joiners (auto_joiners_),
auto_zwj (auto_zwj_),
recurse_func (NULL),
nesting_level_left (MAX_NESTING_LEVEL),
lookup_props (0),
@ -383,16 +383,12 @@ struct hb_apply_context_t
end (c->buffer->len)
{
matcher.set_lookup_props (c->lookup_props);
if (c->auto_joiners)
{
/* Ignore ZWNJ if we are matching GSUB context, or matching GPOS. */
matcher.set_ignore_zwnj (context_match || c->table_index == 1);
matcher.set_ignore_zwj (true);
}
/* Ignore ZWNJ if we are matching GSUB context, or matching GPOS. */
matcher.set_ignore_zwnj (context_match || c->table_index == 1);
/* Ignore ZWJ if we are matching GSUB context, or matching GPOS, or if asked to. */
matcher.set_ignore_zwj (context_match || c->table_index == 1 || c->auto_zwj);
if (!context_match)
{
matcher.set_mask (c->lookup_mask);
}
matcher.set_syllable (start_index_ == c->buffer->idx ? c->buffer->cur().syllable () : 0);
}
inline void set_lookup_props (unsigned int lookup_props) { matcher.set_lookup_props (lookup_props); }
@ -457,16 +453,12 @@ struct hb_apply_context_t
num_items (num_items_)
{
matcher.set_lookup_props (c->lookup_props);
if (c->auto_joiners)
{
/* Ignore ZWNJ if we are matching GSUB context, or matching GPOS. */
matcher.set_ignore_zwnj (context_match || c->table_index == 1);
matcher.set_ignore_zwj (true);
}
/* Ignore ZWNJ if we are matching GSUB context, or matching GPOS. */
matcher.set_ignore_zwnj (context_match || c->table_index == 1);
/* Ignore ZWJ if we are matching GSUB context, or matching GPOS, or if asked to. */
matcher.set_ignore_zwj (context_match || c->table_index == 1 || c->auto_zwj);
if (!context_match)
{
matcher.set_mask (c->lookup_mask);
}
matcher.set_syllable (start_index_ == c->buffer->idx ? c->buffer->cur().syllable () : 0);
}
inline void set_lookup_props (unsigned int lookup_props) { matcher.set_lookup_props (lookup_props); }

4
src/hb-ot-layout-private.hh
View File

@ -207,7 +207,7 @@ hb_ot_layout_substitute_lookup (hb_font_t *font,
hb_buffer_t *buffer,
unsigned int lookup_index,
hb_mask_t mask,
hb_bool_t auto_joiners);
hb_bool_t auto_zwj);
/* Should be called after all the substitute_lookup's are done */
HB_INTERNAL void
@ -225,7 +225,7 @@ hb_ot_layout_position_lookup (hb_font_t *font,
hb_buffer_t *buffer,
unsigned int lookup_index,
hb_mask_t mask,
hb_bool_t auto_joiners);
hb_bool_t auto_zwj);
/* Should be called after all the position_lookup's are done */
HB_INTERNAL void

8
src/hb-ot-layout.cc
View File

@ -667,11 +667,11 @@ hb_ot_layout_substitute_lookup (hb_font_t *font,
hb_buffer_t *buffer,
unsigned int lookup_index,
hb_mask_t mask,
hb_bool_t auto_joiners)
hb_bool_t auto_zwj)
{
if (unlikely (lookup_index >= hb_ot_layout_from_face (font->face)->gsub_lookup_count)) return false;
OT::hb_apply_context_t c (0, font, buffer, mask, auto_joiners);
OT::hb_apply_context_t c (0, font, buffer, mask, auto_zwj);
const OT::SubstLookup& l = hb_ot_layout_from_face (font->face)->gsub->get_lookup (lookup_index);
@ -717,11 +717,11 @@ hb_ot_layout_position_lookup (hb_font_t *font,
hb_buffer_t *buffer,
unsigned int lookup_index,
hb_mask_t mask,
hb_bool_t auto_joiners)
hb_bool_t auto_zwj)
{
if (unlikely (lookup_index >= hb_ot_layout_from_face (font->face)->gpos_lookup_count)) return false;
OT::hb_apply_context_t c (1, font, buffer, mask, auto_joiners);
OT::hb_apply_context_t c (1, font, buffer, mask, auto_zwj);
const OT::PosLookup& l = hb_ot_layout_from_face (font->face)->gpos->get_lookup (lookup_index);

8
src/hb-ot-map-private.hh
View File

@ -50,7 +50,7 @@ struct hb_ot_map_t
hb_mask_t mask;
hb_mask_t _1_mask; /* mask for value=1, for quick access */
unsigned int needs_fallback : 1;
unsigned int auto_joiners : 1;
unsigned int auto_zwj : 1;
static int cmp (const feature_map_t *a, const feature_map_t *b)
{ return a->tag < b->tag ? -1 : a->tag > b->tag ? 1 : 0; }
@ -58,7 +58,7 @@ struct hb_ot_map_t
struct lookup_map_t {
unsigned short index;
unsigned short auto_joiners : 1;
unsigned short auto_zwj : 1;
hb_mask_t mask;
static int cmp (const lookup_map_t *a, const lookup_map_t *b)
@ -139,7 +139,7 @@ struct hb_ot_map_t
unsigned int table_index,
unsigned int feature_index,
hb_mask_t mask,
bool auto_joiners);
bool auto_zwj);
hb_mask_t global_mask;
@ -152,7 +152,7 @@ enum hb_ot_map_feature_flags_t {
F_NONE = 0x0000,
F_GLOBAL = 0x0001,
F_HAS_FALLBACK = 0x0002,
F_MANUAL_JOINERS = 0x0004
F_MANUAL_ZWJ = 0x0004
};
/* Macro version for where const is desired. */
#define F_COMBINE(l,r) (hb_ot_map_feature_flags_t ((unsigned int) (l) | (unsigned int) (r)))

18
src/hb-ot-map.cc
View File

@ -34,7 +34,7 @@ hb_ot_map_t::add_lookups (hb_face_t *face,
unsigned int table_index,
unsigned int feature_index,
hb_mask_t mask,
bool auto_joiners)
bool auto_zwj)
{
unsigned int lookup_indices[32];
unsigned int offset, len;
@ -54,7 +54,7 @@ hb_ot_map_t::add_lookups (hb_face_t *face,
return;
lookup->mask = mask;
lookup->index = lookup_indices[i];
lookup->auto_joiners = auto_joiners;
lookup->auto_zwj = auto_zwj;
}
offset += len;
@ -113,7 +113,7 @@ void hb_ot_map_t::substitute (const hb_ot_shape_plan_t *plan, hb_font_t *font, h
hb_ot_layout_substitute_lookup (font, buffer,
lookups[table_index][i].index,
lookups[table_index][i].mask,
lookups[table_index][i].auto_joiners);
lookups[table_index][i].auto_zwj);
buffer->clear_output ();
@ -124,7 +124,7 @@ void hb_ot_map_t::substitute (const hb_ot_shape_plan_t *plan, hb_font_t *font, h
for (; i < lookups[table_index].len; i++)
hb_ot_layout_substitute_lookup (font, buffer, lookups[table_index][i].index,
lookups[table_index][i].mask,
lookups[table_index][i].auto_joiners);
lookups[table_index][i].auto_zwj);
}
void hb_ot_map_t::position (const hb_ot_shape_plan_t *plan, hb_font_t *font, hb_buffer_t *buffer) const
@ -137,7 +137,7 @@ void hb_ot_map_t::position (const hb_ot_shape_plan_t *plan, hb_font_t *font, hb_
for (; i < pause->num_lookups; i++)
hb_ot_layout_position_lookup (font, buffer, lookups[table_index][i].index,
lookups[table_index][i].mask,
lookups[table_index][i].auto_joiners);
lookups[table_index][i].auto_zwj);
if (pause->callback)
pause->callback (plan, font, buffer);
@ -146,7 +146,7 @@ void hb_ot_map_t::position (const hb_ot_shape_plan_t *plan, hb_font_t *font, hb_
for (; i < lookups[table_index].len; i++)
hb_ot_layout_position_lookup (font, buffer, lookups[table_index][i].index,
lookups[table_index][i].mask,
lookups[table_index][i].auto_joiners);
lookups[table_index][i].auto_zwj);
}
void hb_ot_map_t::collect_lookups (unsigned int table_index, hb_set_t *lookups_out) const
@ -243,7 +243,7 @@ hb_ot_map_builder_t::compile (hb_ot_map_t &m)
map->index[1] = feature_index[1];
map->stage[0] = info->stage[0];
map->stage[1] = info->stage[1];
map->auto_joiners = !(info->flags & F_MANUAL_JOINERS);
map->auto_zwj = !(info->flags & F_MANUAL_ZWJ);
if ((info->flags & F_GLOBAL) && info->max_value == 1) {
/* Uses the global bit */
map->shift = 0;
@ -286,7 +286,7 @@ hb_ot_map_builder_t::compile (hb_ot_map_t &m)
m.add_lookups (face, table_index,
m.features[i].index[table_index],
m.features[i].mask,
m.features[i].auto_joiners);
m.features[i].auto_zwj);
/* Sort lookups and merge duplicates */
if (last_num_lookups < m.lookups[table_index].len)
@ -300,7 +300,7 @@ hb_ot_map_builder_t::compile (hb_ot_map_t &m)
else
{
m.lookups[table_index][j].mask |= m.lookups[table_index][i].mask;
m.lookups[table_index][j].auto_joiners &= m.lookups[table_index][i].auto_joiners;
m.lookups[table_index][j].auto_zwj &= m.lookups[table_index][i].auto_zwj;
}
m.lookups[table_index].shrink (j + 1);
}

2
src/hb-ot-shape-complex-arabic-fallback.hh
View File

@ -244,7 +244,7 @@ arabic_fallback_plan_shape (arabic_fallback_plan_t *fallback_plan,
{
for (unsigned int i = 0; i < ARABIC_NUM_FALLBACK_FEATURES; i++)
if (fallback_plan->lookup_array[i]) {
OT::hb_apply_context_t c (0, font, buffer, fallback_plan->mask_array[i], true/*auto_joiners*/);
OT::hb_apply_context_t c (0, font, buffer, fallback_plan->mask_array[i], true/*auto_zwj*/);
fallback_plan->lookup_array[i]->apply_string (&c, &fallback_plan->digest_array[i]);
}
}

30
src/hb-ot-shape-complex-indic.cc
View File

@ -339,18 +339,18 @@ indic_features[] =
* Basic features.
* These features are applied in order, one at a time, after initial_reordering.
*/
{HB_TAG('n','u','k','t'), F_COMBINE (F_MANUAL_JOINERS, F_GLOBAL)},
{HB_TAG('a','k','h','n'), F_COMBINE (F_MANUAL_JOINERS, F_GLOBAL)},
{HB_TAG('r','p','h','f'), F_COMBINE (F_MANUAL_JOINERS, F_NONE )},
{HB_TAG('r','k','r','f'), F_COMBINE (F_MANUAL_JOINERS, F_GLOBAL)},
{HB_TAG('p','r','e','f'), F_COMBINE (F_MANUAL_JOINERS, F_NONE )},
{HB_TAG('b','l','w','f'), F_COMBINE (F_MANUAL_JOINERS, F_NONE )},
{HB_TAG('h','a','l','f'), F_COMBINE (F_MANUAL_JOINERS, F_NONE )},
{HB_TAG('a','b','v','f'), F_COMBINE (F_MANUAL_JOINERS, F_NONE )},
{HB_TAG('p','s','t','f'), F_COMBINE (F_MANUAL_JOINERS, F_NONE )},
{HB_TAG('c','f','a','r'), F_COMBINE (F_MANUAL_JOINERS, F_NONE )},
{HB_TAG('v','a','t','u'), F_COMBINE (F_MANUAL_JOINERS, F_GLOBAL)},
{HB_TAG('c','j','c','t'), F_COMBINE (F_MANUAL_JOINERS, F_GLOBAL)},
{HB_TAG('n','u','k','t'), F_GLOBAL},
{HB_TAG('a','k','h','n'), F_GLOBAL},
{HB_TAG('r','p','h','f'), F_NONE},
{HB_TAG('r','k','r','f'), F_GLOBAL},
{HB_TAG('p','r','e','f'), F_NONE},
{HB_TAG('b','l','w','f'), F_NONE},
{HB_TAG('h','a','l','f'), F_NONE},
{HB_TAG('a','b','v','f'), F_NONE},
{HB_TAG('p','s','t','f'), F_NONE},
{HB_TAG('c','f','a','r'), F_NONE},
{HB_TAG('v','a','t','u'), F_GLOBAL},
{HB_TAG('c','j','c','t'), F_GLOBAL},
/*
* Other features.
* These features are applied all at once, after final_reordering.
@ -428,12 +428,12 @@ collect_features_indic (hb_ot_shape_planner_t *plan)
unsigned int i = 0;
map->add_gsub_pause (initial_reordering);
for (; i < INDIC_BASIC_FEATURES; i++) {
map->add_feature (indic_features[i].tag, 1, indic_features[i].flags);
map->add_feature (indic_features[i].tag, 1, indic_features[i].flags | F_MANUAL_ZWJ);
map->add_gsub_pause (NULL);
}
map->add_gsub_pause (final_reordering);
for (; i < INDIC_NUM_FEATURES; i++) {
map->add_feature (indic_features[i].tag, 1, indic_features[i].flags);
map->add_feature (indic_features[i].tag, 1, indic_features[i].flags | F_MANUAL_ZWJ);
}
}
@ -1021,7 +1021,7 @@ initial_reordering_consonant_syllable (const hb_ot_shape_plan_t *plan,
/* ZWJ/ZWNJ should disable CJCT. They do that by simply
* being there, since we don't skip them for the CJCT
* feature (ie. F_MANUAL_JOINERS) */
* feature (ie. F_MANUAL_ZWJ) */
/* A ZWNJ disables HALF. */
if (non_joiner)

4
src/hb-ot-shape-complex-myanmar.cc
View File

@ -92,12 +92,12 @@ collect_features_myanmar (hb_ot_shape_planner_t *plan)
map->add_gsub_pause (initial_reordering);
for (unsigned int i = 0; i < ARRAY_LENGTH (basic_features); i++)
{
map->add_feature (basic_features[i], 1, F_GLOBAL | F_MANUAL_JOINERS);
map->add_feature (basic_features[i], 1, F_GLOBAL | F_MANUAL_ZWJ);
map->add_gsub_pause (NULL);
}
map->add_gsub_pause (final_reordering);
for (unsigned int i = 0; i < ARRAY_LENGTH (other_features); i++)
map->add_feature (other_features[i], 1, F_GLOBAL);
map->add_feature (other_features[i], 1, F_GLOBAL | F_MANUAL_ZWJ);
}
static void

4
src/hb-ot-shape-complex-sea.cc
View File

@ -93,12 +93,12 @@ collect_features_sea (hb_ot_shape_planner_t *plan)
map->add_gsub_pause (initial_reordering);
for (unsigned int i = 0; i < ARRAY_LENGTH (basic_features); i++)
{
map->add_feature (basic_features[i], 1, F_GLOBAL | F_MANUAL_JOINERS);
map->add_feature (basic_features[i], 1, F_GLOBAL | F_MANUAL_ZWJ);
map->add_gsub_pause (NULL);
}
map->add_gsub_pause (final_reordering);
for (unsigned int i = 0; i < ARRAY_LENGTH (other_features); i++)
map->add_feature (other_features[i], 1, F_GLOBAL);
map->add_feature (other_features[i], 1, F_GLOBAL | F_MANUAL_ZWJ);
}
static void

2
src/hb-ot-shape-fallback.cc
View File

@ -420,7 +420,7 @@ _hb_ot_shape_fallback_kern (const hb_ot_shape_plan_t *plan,
hb_mask_t kern_mask = plan->map.get_1_mask (HB_DIRECTION_IS_HORIZONTAL (buffer->props.direction) ?
HB_TAG ('k','e','r','n') : HB_TAG ('v','k','r','n'));
OT::hb_apply_context_t c (1, font, buffer, kern_mask, true/*auto_joiners*/);
OT::hb_apply_context_t c (1, font, buffer, kern_mask, true/*auto_zwj*/);
c.set_lookup_props (OT::LookupFlag::IgnoreMarks);
for (buffer->idx = 0; buffer->idx < count;)

1
test/shaping/texts/in-tree/shaper-indic/indic/script-bengali/misc/misc.txt
View File

@ -50,3 +50,4 @@
ন্ত্র
ত্যু
চ্য্র
ক্‍ষ

1
test/shaping/texts/in-tree/shaper-indic/indic/script-malayalam/misc/misc.txt
View File

@ -61,3 +61,4 @@
ല്‍പ്പേ
ശിം‌
കോം‌
യ‍്യ