diff --git a/.ci/build-freetype.sh b/.ci/build-freetype.sh
new file mode 100644
index 000000000..0f09f92e9
--- /dev/null
+++ b/.ci/build-freetype.sh
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -x
+set -o errexit -o nounset
+
+# 22.0.16 is the libtool version of 2.9.0
+if pkg-config --atleast-version 22.0.16 freetype2; then exit; fi
+
+pushd $HOME
+wget http://download.savannah.gnu.org/releases/freetype/freetype-2.9.tar.bz2
+tar xf freetype-2.9.tar.bz2
+pushd freetype-2.9
+./autogen.sh
+./configure --prefix=$HOME/.local
+make -j4 install
+popd
+popd
diff --git a/.travis.yml b/.travis.yml
index 88967ddd4..681471dc0 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -16,9 +16,10 @@ matrix:
     - os: linux
       compiler: gcc
       script:
-        # Remove these two lines when Travis updated its distro
-        - wget http://download.savannah.gnu.org/releases/freetype/freetype-2.9.tar.bz2 && tar xf freetype-2.9.tar.bz2 && cd freetype-2.9 && ./autogen.sh && ./configure && make -j4 && cd ..
-        - export LD_LIBRARY_PATH="$PWD/freetype-2.9/objs/.libs"
+        # Remove the following three lines when Travis updates its distro
+        - export PKG_CONFIG_PATH="$HOME/.local/lib/pkgconfig"
+        - export LD_LIBRARY_PATH="$HOME/.local/lib"
+        - bash .ci/build-freetype.sh
 
         - ./autogen.sh
         - ./configure $CONFIGURE_OPTS --enable-gtk-doc --enable-code-coverage
@@ -34,9 +35,10 @@ matrix:
     - os: linux
       compiler: clang
       script:
-        # Remove these two lines when Travis updated its distro
-        - wget http://download.savannah.gnu.org/releases/freetype/freetype-2.9.tar.bz2 && tar xf freetype-2.9.tar.bz2 && cd freetype-2.9 && ./autogen.sh && ./configure && make -j4 && cd ..
-        - export LD_LIBRARY_PATH="$PWD/freetype-2.9/objs/.libs"
+        # Remove the following three lines when Travis updates its distro
+        - export PKG_CONFIG_PATH="$HOME/.local/lib/pkgconfig"
+        - export LD_LIBRARY_PATH="$HOME/.local/lib"
+        - bash .ci/build-freetype.sh
 
         - ./autogen.sh
         - ./configure $CONFIGURE_OPTS
@@ -57,6 +59,10 @@ notifications:
   irc: "irc.freenode.org#harfbuzz"
   email: harfbuzz-bots-chatter@googlegroups.com
 
+cache:
+  directories:
+    - /home/travis/.local
+
 addons:
   apt:
     packages: