diff --git a/src/hb-object-private.hh b/src/hb-object-private.hh
index 77e008910..ffe87b44d 100644
--- a/src/hb-object-private.hh
+++ b/src/hb-object-private.hh
@@ -106,37 +106,41 @@ typedef struct {
 
 /* XXX make this thread-safe, somehow! */
 
-typedef struct {
+struct hb_user_data_t {
+  hb_user_data_key_t *key;
   void *data;
   hb_destroy_func_t destroy;
 
+  inline bool operator == (hb_user_data_key_t *other_key) const { return key == other_key; }
+  inline bool operator == (hb_user_data_t &other) const { return key == other.key; }
+
   void finish (void) { if (destroy) destroy (data); }
-} hb_user_data_t;
+};
 
 struct hb_user_data_array_t {
 
-  hb_map_t<hb_user_data_key_t *, hb_user_data_t> map;
+  hb_set_t<hb_user_data_t> items;
 
   inline bool set (hb_user_data_key_t *key,
 		   void *              data,
 		   hb_destroy_func_t   destroy)
   {
-    if (!data && !destroy) {
-      map.unset (key);
-      return true;
-    }
     if (!key)
       return false;
-    hb_user_data_t user_data = {data, destroy};
-    return map.set (key, user_data);
+    if (!data && !destroy) {
+      items.remove (key);
+      return true;
+    }
+    hb_user_data_t user_data = {key, data, destroy};
+    return items.insert (user_data);
   }
 
   inline void *get (hb_user_data_key_t *key) {
-    hb_user_data_t *user_data = map.get (key);
+    hb_user_data_t *user_data = items.get (key);
     return user_data ? user_data->data : NULL;
   }
 
-  void finish (void) { map.finish (); }
+  void finish (void) { items.finish (); }
 };
 
 
diff --git a/src/hb-private.hh b/src/hb-private.hh
index 9a23cd3ef..28c33b79b 100644
--- a/src/hb-private.hh
+++ b/src/hb-private.hh
@@ -235,8 +235,6 @@ struct hb_static_array_t {
   Type *array;
   Type static_array[StaticSize];
 
-  void finish (void) { for (unsigned i = 0; i < len; i++) array[i].finish (); }
-
   inline Type& operator [] (unsigned int i)
   {
     return array[i];
@@ -285,25 +283,17 @@ template <typename Type>
 struct hb_array_t : hb_static_array_t<Type, 2> {};
 
 
-template <typename Key, typename Value>
-struct hb_map_t
+template <typename item_t>
+struct hb_set_t
 {
-  struct item_t {
-    Key key;
-    /* unsigned int hash; */
-    Value value;
-
-    void finish (void) { value.finish (); }
-  };
-
   hb_array_t <item_t> items;
 
   private:
 
   template <typename T>
-  inline item_t *find (T key) {
+  inline item_t *find (T v) {
     for (unsigned int i = 0; i < items.len; i++)
-      if (items[i].key == key)
+      if (items[i] == v)
 	return &items[i];
     return NULL;
   }
@@ -311,25 +301,22 @@ struct hb_map_t
   public:
 
   template <typename T>
-  inline bool set (T     key,
-		   Value &value)
+  inline bool insert (T v)
   {
-    item_t *item;
-    item = find (key);
+    item_t *item = find (v);
     if (item)
       item->finish ();
     else
       item = items.push ();
     if (unlikely (!item)) return false;
-    item->key = key;
-    item->value = value;
+    *item = v;
     return true;
   }
 
-  inline void unset (Key &key)
+  template <typename T>
+  inline void remove (T v)
   {
-    item_t *item;
-    item = find (key);
+    item_t *item = find (v);
     if (!item) return;
 
     item->finish ();
@@ -338,13 +325,16 @@ struct hb_map_t
   }
 
   template <typename T>
-  inline Value *get (T key)
+  inline item_t *get (T v)
   {
-    item_t *item = find (key);
-    return item ? &item->value : NULL;
+    return find (v);
+  }
+
+  void finish (void) {
+    for (unsigned i = 0; i < items.len; i++)
+      items[i].finish ();
   }
 
-  void finish (void) { items.finish (); }
 };
 
 
diff --git a/test/test-object.c b/test/test-object.c
index 297aa29f4..3662d3000 100644
--- a/test/test-object.c
+++ b/test/test-object.c
@@ -269,6 +269,10 @@ test_object (void)
 	g_assert (o->get_user_data (obj, &key[i]) == &data[i]);
       for (i = 100; i < 1000; i++)
 	g_assert (o->set_user_data (obj, &key[i], NULL, NULL));
+      for (i = 2; i < 100; i++)
+	g_assert (o->get_user_data (obj, &key[i]) == &data[i]);
+      for (i = 100; i < 1000; i++)
+	g_assert (!o->get_user_data (obj, &key[i]));
       g_assert_cmpuint (global_data, ==, 900);