/* * Copyright © 2020 Google, Inc. * * This is part of HarfBuzz, a text shaping library. * * Permission is hereby granted, without written agreement and without * license or royalty fees, to use, copy, modify, and distribute this * software and its documentation for any purpose, provided that the * above copyright notice and the following two paragraphs appear in * all copies of this software. * * IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR * DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES * ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN * IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH * DAMAGE. * * THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, * BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND * FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS * ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO * PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. * * Google Author(s): Garret Rieger */ #ifndef HB_REPACKER_HH #define HB_REPACKER_HH #include "hb-open-type.hh" #include "hb-map.hh" #include "hb-priority-queue.hh" #include "hb-serialize.hh" #include "hb-vector.hh" struct graph_t { struct vertex_t { vertex_t () : distance (0), space (0), incoming_edges (0), start (0), end (0), priority(0) {} void fini () { obj.fini (); } hb_serialize_context_t::object_t obj; int64_t distance; int64_t space; unsigned incoming_edges; unsigned start; unsigned end; unsigned priority; bool is_shared () const { return incoming_edges > 1; } bool is_leaf () const { return !obj.links.length; } void raise_priority () { priority++; } int64_t modified_distance (unsigned order) const { // TODO(garretrieger): once priority is high enough, should try // setting distance = 0 which will force to sort immediately after // it's parent where possible. int64_t modified_distance = hb_min (hb_max(distance + distance_modifier (), 0), 0x7FFFFFFFFF); return (modified_distance << 24) | (0x00FFFFFF & order); } int64_t distance_modifier () const { if (!priority) return 0; int64_t table_size = obj.tail - obj.head; return -(table_size - table_size / (1 << hb_min(priority, 16u))); } }; struct overflow_record_t { unsigned parent; unsigned child; }; struct clone_buffer_t { clone_buffer_t () : head (nullptr), tail (nullptr) {} bool copy (const hb_serialize_context_t::object_t& object) { fini (); unsigned size = object.tail - object.head; head = (char*) hb_malloc (size); if (!head) return false; memcpy (head, object.head, size); tail = head + size; return true; } char* head; char* tail; void fini () { if (!head) return; hb_free (head); head = nullptr; } }; /* * A topological sorting of an object graph. Ordered * in reverse serialization order (first object in the * serialization is at the end of the list). This matches * the 'packed' object stack used internally in the * serializer */ graph_t (const hb_vector_t& objects) : edge_count_invalid (true), distance_invalid (true), positions_invalid (true), successful (true) { bool removed_nil = false; for (unsigned i = 0; i < objects.length; i++) { // TODO(grieger): check all links point to valid objects. // If this graph came from a serialization buffer object 0 is the // nil object. We don't need it for our purposes here so drop it. if (i == 0 && !objects[i]) { removed_nil = true; continue; } vertex_t* v = vertices_.push (); if (check_success (!vertices_.in_error ())) v->obj = *objects[i]; if (!removed_nil) continue; for (unsigned i = 0; i < v->obj.links.length; i++) // Fix indices to account for removed nil object. v->obj.links[i].objidx--; } } ~graph_t () { vertices_.fini_deep (); clone_buffers_.fini_deep (); } bool in_error () const { return !successful || vertices_.in_error () || clone_buffers_.in_error (); } const vertex_t& root () const { return vertices_[root_idx ()]; } unsigned root_idx () const { // Object graphs are in reverse order, the first object is at the end // of the vector. Since the graph is topologically sorted it's safe to // assume the first object has no incoming edges. return vertices_.length - 1; } const hb_serialize_context_t::object_t& object(unsigned i) const { return vertices_[i].obj; } /* * serialize graph into the provided serialization buffer. */ void serialize (hb_serialize_context_t* c) const { c->start_serialize (); for (unsigned i = 0; i < vertices_.length; i++) { c->push (); size_t size = vertices_[i].obj.tail - vertices_[i].obj.head; char* start = c->allocate_size (size); if (!start) return; memcpy (start, vertices_[i].obj.head, size); for (const auto& link : vertices_[i].obj.links) serialize_link (link, start, c); // All duplications are already encoded in the graph, so don't // enable sharing during packing. c->pop_pack (false); } c->end_serialize (); } /* * Generates a new topological sorting of graph using Kahn's * algorithm: https://en.wikipedia.org/wiki/Topological_sorting#Algorithms */ void sort_kahn () { positions_invalid = true; if (vertices_.length <= 1) { // Graph of 1 or less doesn't need sorting. return; } hb_vector_t queue; hb_vector_t sorted_graph; hb_vector_t id_map; if (unlikely (!check_success (id_map.resize (vertices_.length)))) return; hb_vector_t removed_edges; if (unlikely (!check_success (removed_edges.resize (vertices_.length)))) return; update_incoming_edge_count (); queue.push (root_idx ()); int new_id = vertices_.length - 1; while (!queue.in_error () && queue.length) { unsigned next_id = queue[0]; queue.remove (0); vertex_t& next = vertices_[next_id]; sorted_graph.push (next); id_map[next_id] = new_id--; for (const auto& link : next.obj.links) { removed_edges[link.objidx]++; if (!(vertices_[link.objidx].incoming_edges - removed_edges[link.objidx])) queue.push (link.objidx); } } check_success (!queue.in_error ()); check_success (!sorted_graph.in_error ()); if (!check_success (new_id == -1)) DEBUG_MSG (SUBSET_REPACK, nullptr, "Graph is not fully connected."); remap_obj_indices (id_map, &sorted_graph); sorted_graph.as_array ().reverse (); vertices_.fini_deep (); vertices_ = sorted_graph; sorted_graph.fini_deep (); } /* * Generates a new topological sorting of graph ordered by the shortest * distance to each node. */ void sort_shortest_distance () { positions_invalid = true; if (vertices_.length <= 1) { // Graph of 1 or less doesn't need sorting. return; } update_distances (); hb_priority_queue_t queue; hb_vector_t sorted_graph; hb_vector_t id_map; if (unlikely (!check_success (id_map.resize (vertices_.length)))) return; hb_vector_t removed_edges; if (unlikely (!check_success (removed_edges.resize (vertices_.length)))) return; update_incoming_edge_count (); queue.insert (root ().modified_distance (0), root_idx ()); int new_id = root_idx (); unsigned order = 1; while (!queue.in_error () && !queue.is_empty ()) { unsigned next_id = queue.pop_minimum().second; vertex_t& next = vertices_[next_id]; sorted_graph.push (next); id_map[next_id] = new_id--; for (const auto& link : next.obj.links) { removed_edges[link.objidx]++; if (!(vertices_[link.objidx].incoming_edges - removed_edges[link.objidx])) // Add the order that the links were encountered to the priority. // This ensures that ties between priorities objects are broken in a consistent // way. More specifically this is set up so that if a set of objects have the same // distance they'll be added to the topological order in the order that they are // referenced from the parent object. queue.insert (vertices_[link.objidx].modified_distance (order++), link.objidx); } } check_success (!queue.in_error ()); check_success (!sorted_graph.in_error ()); if (!check_success (new_id == -1)) DEBUG_MSG (SUBSET_REPACK, nullptr, "Graph is not fully connected."); remap_obj_indices (id_map, &sorted_graph); sorted_graph.as_array ().reverse (); vertices_.fini_deep (); vertices_ = sorted_graph; sorted_graph.fini_deep (); } /* * Finds any links using 32 bits and isolates the subgraphs they point too. */ bool isolate_32bit_links () { bool made_changes = false; hb_set_t target_links; unsigned root_index = root_idx (); int64_t next_space = 0; for (unsigned i = 0; i <= root_index; i++) { if (i == root_index && root_idx () > i) // root index may have moved due to graph modifications. i = root_idx (); for (auto& l : vertices_[i].obj.links) { if (l.width == 4 && !l.is_signed) { isolate_subgraph (l.objidx); vertices_[l.objidx].space = next_space++; distance_invalid = true; made_changes = true; } } } return made_changes; } /* * Isolates the subgraph of nodes reachable from root. Any links to nodes in the subgraph * that originate from outside of the subgraph will be removed by duplicating the linked to * object. */ bool isolate_subgraph (unsigned root_idx) { hb_hashmap_t subgraph; hb_hashmap_t index_map; find_subgraph(root_idx, subgraph); bool made_changes = false; for (auto entry : subgraph.iter ()) { const auto& node = vertices_[entry.first]; unsigned subgraph_incoming_edges = entry.second; if (subgraph_incoming_edges < node.incoming_edges) { // Only de-dup objects with incoming links from outside the subgraph. made_changes = true; duplicate_subgraph (entry.first, index_map); } } if (!made_changes) return false; remap_obj_indices (index_map, subgraph.keys ()); return true; } void find_subgraph (unsigned node_idx, hb_hashmap_t& subgraph) { if (subgraph.has (node_idx)) { subgraph.set (node_idx, subgraph[node_idx] + 1); return; } subgraph.set (node_idx, 1); for (const auto& link : vertices_[node_idx].obj.links) find_subgraph (link.objidx, subgraph); } /* * duplicates all nodes in the subgraph reachable from node_idx. Does not re-assign * links. index_map is updated with mappings from old id to new id. If a duplication has already * been performed for a given index, then it will be skipped. */ void duplicate_subgraph (unsigned node_idx, hb_hashmap_t& index_map) { if (index_map.has (node_idx)) return; index_map.set (node_idx, duplicate (node_idx)); for (const auto& l : object (node_idx).links) { duplicate_subgraph (l.objidx, index_map); } } /* * Creates a copy of node_idx and returns it's new index. */ unsigned duplicate (unsigned node_idx) { positions_invalid = true; distance_invalid = true; // TODO(grieger): compute new distance instead of invalidation. auto* clone = vertices_.push (); auto& child = vertices_[node_idx]; clone_buffer_t* buffer = clone_buffers_.push (); if (vertices_.in_error () || clone_buffers_.in_error () || !check_success (buffer->copy (child.obj))) { return -1; } clone->obj.head = buffer->head; clone->obj.tail = buffer->tail; clone->distance = child.distance; clone->space = child.space; for (const auto& l : child.obj.links) clone->obj.links.push (l); check_success (!clone->obj.links.in_error ()); // The last object is the root of the graph, so swap back the root to the end. // The root's obj idx does change, however since it's root nothing else refers to it. // all other obj idx's will be unaffected. vertex_t root = vertices_[vertices_.length - 2]; vertices_[vertices_.length - 2] = *clone; vertices_[vertices_.length - 1] = root; return vertices_.length - 2; } /* * Creates a copy of child and re-assigns the link from * parent to the clone. The copy is a shallow copy, objects * linked from child are not duplicated. */ void duplicate (unsigned parent_idx, unsigned child_idx) { DEBUG_MSG (SUBSET_REPACK, nullptr, " Duplicating %d => %d", parent_idx, child_idx); unsigned clone_idx = duplicate (child_idx); if (clone_idx == (unsigned) -1) return; // duplicate shifts the root node idx, so if parent_idx was root update it. if (parent_idx == clone_idx) parent_idx++; auto& clone = vertices_[clone_idx]; auto& child = vertices_[child_idx]; auto& parent = vertices_[parent_idx]; for (unsigned i = 0; i < parent.obj.links.length; i++) { auto& l = parent.obj.links[i]; if (l.objidx == child_idx) { l.objidx = clone_idx; clone.incoming_edges++; child.incoming_edges--; } } } /* * Raises the sorting priority of all children. */ void raise_childrens_priority (unsigned parent_idx) { DEBUG_MSG (SUBSET_REPACK, nullptr, " Raising priority of all children of %d", parent_idx); // This operation doesn't change ordering until a sort is run, so no need // to invalidate positions. It does not change graph structure so no need // to update distances or edge counts. auto& parent = vertices_[parent_idx].obj; for (unsigned i = 0; i < parent.links.length; i++) vertices_[parent.links[i].objidx].raise_priority (); } /* * Will any offsets overflow on graph when it's serialized? */ bool will_overflow (hb_vector_t* overflows = nullptr) { if (overflows) overflows->resize (0); update_positions (); for (int parent_idx = vertices_.length - 1; parent_idx >= 0; parent_idx--) { for (const auto& link : vertices_[parent_idx].obj.links) { int64_t offset = compute_offset (parent_idx, link); if (is_valid_offset (offset, link)) continue; if (!overflows) return true; overflow_record_t r; r.parent = parent_idx; r.child = link.objidx; overflows->push (r); } } if (!overflows) return false; return overflows->length; } void print_overflows (const hb_vector_t& overflows) { if (!DEBUG_ENABLED(SUBSET_REPACK)) return; update_incoming_edge_count (); for (const auto& o : overflows) { const auto& parent = vertices_[o.parent]; const auto& child = vertices_[o.child]; DEBUG_MSG (SUBSET_REPACK, nullptr, " overflow from %d (%d in, %d out) => %d (%d in, %d out)", o.parent, parent.incoming_edges, parent.obj.links.length, o.child, child.incoming_edges, child.obj.links.length); } } void err_other_error () { this->successful = false; } private: bool check_success (bool success) { return this->successful && (success || (err_other_error (), false)); } /* * Creates a map from objid to # of incoming edges. */ void update_incoming_edge_count () { if (!edge_count_invalid) return; for (unsigned i = 0; i < vertices_.length; i++) vertices_[i].incoming_edges = 0; for (const vertex_t& v : vertices_) { for (auto& l : v.obj.links) { vertices_[l.objidx].incoming_edges++; } } edge_count_invalid = false; } /* * compute the serialized start and end positions for each vertex. */ void update_positions () { if (!positions_invalid) return; unsigned current_pos = 0; for (int i = root_idx (); i >= 0; i--) { auto& v = vertices_[i]; v.start = current_pos; current_pos += v.obj.tail - v.obj.head; v.end = current_pos; } positions_invalid = false; } /* * Finds the distance to each object in the graph * from the initial node. */ void update_distances () { if (!distance_invalid) return; // Uses Dijkstra's algorithm to find all of the shortest distances. // https://en.wikipedia.org/wiki/Dijkstra%27s_algorithm // // Implementation Note: // Since our priority queue doesn't support fast priority decreases // we instead just add new entries into the queue when a priority changes. // Redundant ones are filtered out later on by the visited set. // According to https://www3.cs.stonybrook.edu/~rezaul/papers/TR-07-54.pdf // for practical performance this is faster then using a more advanced queue // (such as a fibonaacci queue) with a fast decrease priority. for (unsigned i = 0; i < vertices_.length; i++) { if (i == vertices_.length - 1) vertices_[i].distance = 0; else vertices_[i].distance = hb_int_max (int64_t); } hb_priority_queue_t queue; queue.insert (0, vertices_.length - 1); hb_set_t visited; while (!queue.in_error () && !queue.is_empty ()) { unsigned next_idx = queue.pop_minimum ().second; if (visited.has (next_idx)) continue; const auto& next = vertices_[next_idx]; int64_t next_distance = vertices_[next_idx].distance; visited.add (next_idx); for (const auto& link : next.obj.links) { if (visited.has (link.objidx)) continue; const auto& child = vertices_[link.objidx].obj; int64_t child_weight = (child.tail - child.head) + ((int64_t) 1 << (link.width * 8)) * (vertices_[link.objidx].space + 1); int64_t child_distance = next_distance + child_weight; if (child_distance < vertices_[link.objidx].distance) { vertices_[link.objidx].distance = child_distance; queue.insert (child_distance, link.objidx); } } } check_success (!queue.in_error ()); if (!check_success (queue.is_empty ())) { DEBUG_MSG (SUBSET_REPACK, nullptr, "Graph is not fully connected."); return; } distance_invalid = false; } int64_t compute_offset ( unsigned parent_idx, const hb_serialize_context_t::object_t::link_t& link) const { const auto& parent = vertices_[parent_idx]; const auto& child = vertices_[link.objidx]; int64_t offset = 0; switch ((hb_serialize_context_t::whence_t) link.whence) { case hb_serialize_context_t::whence_t::Head: offset = child.start - parent.start; break; case hb_serialize_context_t::whence_t::Tail: offset = child.start - parent.end; break; case hb_serialize_context_t::whence_t::Absolute: offset = child.start; break; } assert (offset >= link.bias); offset -= link.bias; return offset; } bool is_valid_offset (int64_t offset, const hb_serialize_context_t::object_t::link_t& link) const { if (link.is_signed) { if (link.width == 4) return offset >= -((int64_t) 1 << 31) && offset < ((int64_t) 1 << 31); else return offset >= -(1 << 15) && offset < (1 << 15); } else { if (link.width == 4) return offset >= 0 && offset < ((int64_t) 1 << 32); else if (link.width == 3) return offset >= 0 && offset < ((int32_t) 1 << 24); else return offset >= 0 && offset < (1 << 16); } } /* * Updates all objidx's in all links using the provided mapping. Corrects incoming edge counts. */ template void remap_obj_indices (const hb_hashmap_t& id_map, Iterator subgraph) { if (!id_map) return; for (unsigned i : subgraph) { for (unsigned j = 0; j < vertices_[i].obj.links.length; j++) { auto& link = vertices_[i].obj.links[j]; if (!id_map.has (link.objidx)) continue; vertices_[link.objidx].incoming_edges--; link.objidx = id_map[link.objidx]; vertices_[link.objidx].incoming_edges++; } } } /* * Updates all objidx's in all links using the provided mapping. */ void remap_obj_indices (const hb_vector_t& id_map, hb_vector_t* sorted_graph) const { for (unsigned i = 0; i < sorted_graph->length; i++) { for (unsigned j = 0; j < (*sorted_graph)[i].obj.links.length; j++) { auto& link = (*sorted_graph)[i].obj.links[j]; link.objidx = id_map[link.objidx]; } } } template void serialize_link_of_type (const hb_serialize_context_t::object_t::link_t& link, char* head, hb_serialize_context_t* c) const { OT::Offset* offset = reinterpret_cast*> (head + link.position); *offset = 0; c->add_link (*offset, // serializer has an extra nil object at the start of the // object array. So all id's are +1 of what our id's are. link.objidx + 1, (hb_serialize_context_t::whence_t) link.whence, link.bias); } void serialize_link (const hb_serialize_context_t::object_t::link_t& link, char* head, hb_serialize_context_t* c) const { switch (link.width) { case 4: if (link.is_signed) { serialize_link_of_type (link, head, c); } else { serialize_link_of_type (link, head, c); } return; case 2: if (link.is_signed) { serialize_link_of_type (link, head, c); } else { serialize_link_of_type (link, head, c); } return; case 3: serialize_link_of_type (link, head, c); return; default: // Unexpected link width. assert (0); } } public: // TODO(garretrieger): make private, will need to move most of offset overflow code into graph. hb_vector_t vertices_; private: hb_vector_t clone_buffers_; bool edge_count_invalid; bool distance_invalid; bool positions_invalid; bool successful; }; static bool _process_overflows (const hb_vector_t& overflows, hb_set_t& priority_bumped_parents, graph_t& sorted_graph) { bool resolution_attempted = false; // Try resolving the furthest overflows first. for (int i = overflows.length - 1; i >= 0; i--) { const graph_t::overflow_record_t& r = overflows[i]; const auto& child = sorted_graph.vertices_[r.child]; if (child.is_shared ()) { // The child object is shared, we may be able to eliminate the overflow // by duplicating it. sorted_graph.duplicate (r.parent, r.child); return true; } if (child.is_leaf () && !priority_bumped_parents.has (r.parent)) { // This object is too far from it's parent, attempt to move it closer. // // TODO(garretrieger): initially limiting this to leaf's since they can be // moved closer with fewer consequences. However, this can // likely can be used for non-leafs as well. // TODO(garretrieger): add a maximum priority, don't try to raise past this. // TODO(garretrieger): also try lowering priority of the parent. Make it // get placed further up in the ordering, closer to it's children. // this is probably preferable if the total size of the parent object // is < then the total size of the children (and the parent can be moved). // Since in that case moving the parent will cause a smaller increase in // the length of other offsets. sorted_graph.raise_childrens_priority (r.parent); priority_bumped_parents.add (r.parent); resolution_attempted = true; continue; } // TODO(garretrieger): add additional offset resolution strategies // - Promotion to extension lookups. // - Isolate the sub graphs of extension sub tables. // - Table splitting. } return resolution_attempted; } /* * Attempts to modify the topological sorting of the provided object graph to * eliminate offset overflows in the links between objects of the graph. If a * non-overflowing ordering is found the updated graph is serialized it into the * provided serialization context. * * If necessary the structure of the graph may be modified in ways that do not * affect the functionality of the graph. For example shared objects may be * duplicated. */ inline void hb_resolve_overflows (const hb_vector_t& packed, hb_tag_t table_tag, hb_serialize_context_t* c) { // Kahn sort is ~twice as fast as shortest distance sort and works for many fonts // so try it first to save time. graph_t sorted_graph (packed); sorted_graph.sort_kahn (); if (!sorted_graph.will_overflow ()) { sorted_graph.serialize (c); return; } sorted_graph.sort_shortest_distance (); if ((table_tag == HB_OT_TAG_GPOS || table_tag == HB_OT_TAG_GSUB) && sorted_graph.will_overflow ()) { if (sorted_graph.isolate_32bit_links ()) { DEBUG_MSG (SUBSET_REPACK, nullptr, "Isolated extension sub tables."); sorted_graph.sort_shortest_distance (); } } unsigned round = 0; hb_vector_t overflows; // TODO(garretrieger): select a good limit for max rounds. while (!sorted_graph.in_error () && sorted_graph.will_overflow (&overflows) && round++ < 10) { DEBUG_MSG (SUBSET_REPACK, nullptr, "=== Overflow resolution round %d ===", round); sorted_graph.print_overflows (overflows); hb_set_t priority_bumped_parents; if (!_process_overflows (overflows, priority_bumped_parents, sorted_graph)) { DEBUG_MSG (SUBSET_REPACK, nullptr, "No resolution available :("); break; } sorted_graph.sort_shortest_distance (); } if (sorted_graph.in_error ()) { c->err (HB_SERIALIZE_ERROR_OTHER); return; } if (sorted_graph.will_overflow ()) { c->err (HB_SERIALIZE_ERROR_OFFSET_OVERFLOW); DEBUG_MSG (SUBSET_REPACK, nullptr, "Offset overflow resolution failed."); return; } sorted_graph.serialize (c); } #endif /* HB_REPACKER_HH */