diff --git a/src/layout.rs b/src/layout.rs
index 6d85ef6a..75a5c942 100644
--- a/src/layout.rs
+++ b/src/layout.rs
@@ -1186,7 +1186,7 @@ mod seat {
                         key_id,
                         modifier, time,
                     ),
-                    false => submission.handle_drop_modifier(key_id, time),
+                    false => submission.handle_drop_modifier(key_id, modifier, time),
                 }
             }
             // only show when UI is present
diff --git a/src/submission.rs b/src/submission.rs
index 673f757b..a75aae5b 100644
--- a/src/submission.rs
+++ b/src/submission.rs
@@ -229,10 +229,11 @@ impl Submission {
 
     pub fn handle_drop_modifier(
         &mut self,
-        key_id: KeyStateId,
+        _key_id: KeyStateId,
+        modifier: Modifier,
         _time: Timestamp,
     ) {
-        vec_remove(&mut self.modifiers_active, |(id, _)| *id == key_id);
+        vec_remove(&mut self.modifiers_active, |(_, m)| *m == modifier);
         self.update_modifiers();
     }