diff --git a/util/helper-cairo.hh b/util/helper-cairo.hh
index 19bdb4242..4d9ffbdaa 100644
--- a/util/helper-cairo.hh
+++ b/util/helper-cairo.hh
@@ -113,7 +113,8 @@ helper_cairo_create_scaled_font (const font_options_t *font_opts)
   cairo_matrix_init_scale (&font_matrix,
 			   font_opts->font_size_x,
 			   font_opts->font_size_y);
-  font_matrix.xy = -font_opts->slant * font_opts->font_size_x;
+  if (!use_hb_draw)
+    font_matrix.xy = -font_opts->slant * font_opts->font_size_x;
 
   font_options = cairo_font_options_create ();
   cairo_font_options_set_hint_style (font_options, CAIRO_HINT_STYLE_NONE);