diff --git a/src/hb-bit-set-invertible.hh b/src/hb-bit-set-invertible.hh
index 978d589ab..0832b0fc2 100644
--- a/src/hb-bit-set-invertible.hh
+++ b/src/hb-bit-set-invertible.hh
@@ -44,6 +44,8 @@ struct hb_bit_set_invertible_t
   hb_bit_set_invertible_t& operator= (hb_bit_set_invertible_t&& o) = default;
   friend void swap (hb_bit_set_invertible_t &a, hb_bit_set_invertible_t &b)
   {
+    if (likely (!a.s.successful || !b.s.successful))
+      return;
     hb_swap (a.inverted, b.inverted);
     hb_swap (a.s, b.s);
   }
diff --git a/src/hb-bit-set.hh b/src/hb-bit-set.hh
index 9f76e81ce..fd793696b 100644
--- a/src/hb-bit-set.hh
+++ b/src/hb-bit-set.hh
@@ -44,6 +44,8 @@ struct hb_bit_set_t
   hb_bit_set_t& operator= (hb_bit_set_t&& other) { hb_swap (*this, other); return *this; }
   friend void swap (hb_bit_set_t &a, hb_bit_set_t &b)
   {
+    if (likely (!a.successful || !b.successful))
+      return;
     hb_swap (a.successful, b.successful);
     hb_swap (a.population, b.population);
     hb_swap (a.last_page_lookup, b.last_page_lookup);