diff --git a/src/hb-face.cc b/src/hb-face.cc
index e7deb31dd..89901d202 100644
--- a/src/hb-face.cc
+++ b/src/hb-face.cc
@@ -784,10 +784,10 @@ hb_face_builder_create ()
 hb_bool_t
 hb_face_builder_add_table (hb_face_t *face, hb_tag_t tag, hb_blob_t *blob)
 {
-  if (tag == HB_MAP_VALUE_INVALID)
+  if (unlikely (face->destroy != (hb_destroy_func_t) _hb_face_builder_data_destroy))
     return false;
 
-  if (unlikely (face->destroy != (hb_destroy_func_t) _hb_face_builder_data_destroy))
+  if (tag == HB_MAP_VALUE_INVALID)
     return false;
 
   hb_face_builder_data_t *data = (hb_face_builder_data_t *) face->user_data;
@@ -819,6 +819,9 @@ void
 hb_face_builder_sort_tables (hb_face_t *face,
                              const hb_tag_t  *tags)
 {
+  if (unlikely (face->destroy != (hb_destroy_func_t) _hb_face_builder_data_destroy))
+    return;
+
   hb_face_builder_data_t *data = (hb_face_builder_data_t *) face->user_data;
 
   // Sort all unspecified tables after any specified tables.