diff --git a/configure.ac b/configure.ac
index 03155372b..989de12b4 100644
--- a/configure.ac
+++ b/configure.ac
@@ -102,9 +102,6 @@ if test "x$GCC" = "xyes"; then
 	# by overriding CXXFLAGS.
 	CXXFLAGS="-fno-rtti $CXXFLAGS -fno-exceptions -fno-threadsafe-statics"
 
-	# Assorted warnings
-	CXXFLAGS="$CXXFLAGS -Wcast-align"
-
 	case "$host" in
 		*-*-mingw*)
 		;;
diff --git a/src/hb.hh b/src/hb.hh
index 1aec868fc..7d0fd4d85 100644
--- a/src/hb.hh
+++ b/src/hb.hh
@@ -29,6 +29,10 @@
 #ifndef HB_HH
 #define HB_HH
 
+#ifndef HB_NO_GCC_PRAGMAS
+#pragma GCC diagnostic warning "-Wcast-align"
+#endif
+
 #ifdef HAVE_CONFIG_H
 #include "config.h"
 #endif