diff --git a/util/helper-cairo-ft.hh b/util/helper-cairo-ft.hh
index 5152c7cac..8ec372812 100644
--- a/util/helper-cairo-ft.hh
+++ b/util/helper-cairo-ft.hh
@@ -43,6 +43,13 @@ void free_ft_library ()
 }
 #endif
 
+static void
+_release_blob (void *arg)
+{
+  FT_Face ft_face = (FT_Face) arg;
+  hb_blob_destroy ((hb_blob_t *) ft_face->generic.data);
+}
+
 static inline cairo_font_face_t *
 helper_cairo_create_ft_font_face (const font_options_t *font_opts)
 {
@@ -80,6 +87,9 @@ helper_cairo_create_ft_font_face (const font_options_t *font_opts)
   }
   else
   {
+    ft_face->generic.data = hb_blob_reference (font_opts->blob);
+    ft_face->generic.finalizer = _release_blob;
+
 #if !defined(HB_NO_VAR) && defined(HAVE_FT_SET_VAR_BLEND_COORDINATES)
     unsigned int num_coords;
     const float *coords = hb_font_get_var_coords_design (font_opts->font, &num_coords);
diff --git a/util/helper-cairo.hh b/util/helper-cairo.hh
index 556856e9b..3505794eb 100644
--- a/util/helper-cairo.hh
+++ b/util/helper-cairo.hh
@@ -134,13 +134,6 @@ helper_cairo_create_scaled_font (const font_options_t *font_opts)
   cairo_font_options_destroy (font_options);
   cairo_font_face_destroy (cairo_face);
 
-  static cairo_user_data_key_t key;
-  if (cairo_scaled_font_set_user_data (scaled_font,
-				       &key,
-				       (void *) hb_font_reference (font),
-				       (cairo_destroy_func_t) hb_font_destroy))
-    hb_font_destroy (font);
-
   return scaled_font;
 }