diff --git a/src/fccache.c b/src/fccache.c
index 69a29f2..77a303d 100644
--- a/src/fccache.c
+++ b/src/fccache.c
@@ -476,6 +476,15 @@ FcDirCacheMapFd (int fd, struct stat *fd_stat)
     return cache;
 }
 
+void
+FcDirCacheReference (FcCache *cache, int nref)
+{
+    FcCacheSkip *skip = FcCacheFindByAddr (cache);
+
+    if (skip)
+	skip->ref += nref;
+}
+
 void
 FcDirCacheUnload (FcCache *cache)
 {
diff --git a/src/fccfg.c b/src/fccfg.c
index 1139744..d9502f0 100644
--- a/src/fccfg.c
+++ b/src/fccfg.c
@@ -239,6 +239,8 @@ FcConfigAddCache (FcConfig *config, FcCache *cache)
     fs = FcCacheSet (cache);
     if (fs)
     {
+	int	nref = 0;
+	
 	for (i = 0; i < fs->nfont; i++)
 	{
 	    FcPattern	*font = FcFontSetFont (fs, i);
@@ -260,9 +262,10 @@ FcConfigAddCache (FcConfig *config, FcCache *cache)
 	    if (!FcConfigAcceptFont (config, font))
 		continue;
 		
-	    FcPatternReference (font);
+	    nref++;
 	    FcFontSetAdd (config->fonts[FcSetSystem], font);
 	}
+	FcDirCacheReference (cache, nref);
     }
 
     /*
diff --git a/src/fcint.h b/src/fcint.h
index a054219..7dab169 100644
--- a/src/fcint.h
+++ b/src/fcint.h
@@ -513,6 +513,9 @@ FcCacheObjectDereference (void *object);
 FcPrivate void
 FcCacheFini (void);
     
+void
+FcDirCacheReference (FcCache *cache, int nref);
+
 /* fccfg.c */
 
 FcPrivate FcBool