diff --git a/eek/eek-key.c b/eek/eek-key.c
index 58a38030..b3e6361b 100644
--- a/eek/eek-key.c
+++ b/eek/eek-key.c
@@ -469,6 +469,23 @@ get_keyboard (EekKey *key)
  */
 EekSymbol *
 eek_key_get_symbol (EekKey *key)
+{
+    return eek_key_get_symbol_with_fallback (key, -1, -1);
+}
+
+/**
+ * eek_key_get_symbol_with_fallback:
+ * @key: an #EekKey
+ * @fallback_group: fallback group index
+ * @fallback_level: fallback level index
+ *
+ * Get the current symbol of @key.
+ * Returns: an #EekSymbol or %NULL on failure
+ */
+EekSymbol *
+eek_key_get_symbol_with_fallback (EekKey *key,
+                                  gint    fallback_group,
+                                  gint    fallback_level)
 {
     gint group, level;
     EekKeyboard *keyboard;
@@ -479,8 +496,11 @@ eek_key_get_symbol (EekKey *key)
     g_return_val_if_fail (keyboard, NULL);
 
     eek_keyboard_get_symbol_index (keyboard, &group, &level);
-
-    return eek_key_get_symbol_at_index (key, group, level);
+    return eek_key_get_symbol_at_index (key,
+                                        group,
+                                        level,
+                                        fallback_group,
+                                        fallback_level);
 }
 
 /**
@@ -488,6 +508,8 @@ eek_key_get_symbol (EekKey *key)
  * @key: an #EekKey
  * @group: group index of the symbol matrix
  * @level: level index of the symbol matrix
+ * @fallback_group: fallback group index
+ * @fallback_level: fallback level index
  *
  * Get the symbol at (@group, @level) in the symbol matrix of @key.
  * Returns: an #EekSymbol or %NULL on failure
@@ -495,11 +517,16 @@ eek_key_get_symbol (EekKey *key)
 EekSymbol *
 eek_key_get_symbol_at_index (EekKey *key,
                              gint    group,
-                             gint    level)
+                             gint    level,
+                             gint    fallback_group,
+                             gint    fallback_level)
 {
     EekKeyPrivate *priv = EEK_KEY_GET_PRIVATE(key);
     gint num_symbols;
 
+    g_return_val_if_fail (group >= 0, NULL);
+    g_return_val_if_fail (level >= 0, NULL);
+
     if (!priv->symbol_matrix)
         return NULL;
 
@@ -508,13 +535,20 @@ eek_key_get_symbol_at_index (EekKey *key,
     if (num_symbols == 0)
         return NULL;
 
-    if (group >= priv->symbol_matrix->num_groups)
-        return NULL;
-    if (level >= priv->symbol_matrix->num_levels)
-        return NULL;
+    if (group >= priv->symbol_matrix->num_groups) {
+        if (fallback_group < 0)
+            return NULL;
+        group = fallback_group;
+    }
+
+    if (level >= priv->symbol_matrix->num_levels) {
+        if (fallback_level < 0)
+            return NULL;
+        level = fallback_level;
+    }
 
     return priv->symbol_matrix->data[group * priv->symbol_matrix->num_levels +
-                                    level];
+                                     level];
 }
 
 /**
diff --git a/eek/eek-key.h b/eek/eek-key.h
index c46faf21..38cb6b55 100644
--- a/eek/eek-key.h
+++ b/eek/eek-key.h
@@ -104,9 +104,15 @@ void             eek_key_set_symbol_matrix   (EekKey          *key,
                                               EekSymbolMatrix *matrix);
 EekSymbolMatrix *eek_key_get_symbol_matrix   (EekKey          *key);
 EekSymbol       *eek_key_get_symbol          (EekKey          *key);
+EekSymbol *      eek_key_get_symbol_with_fallback
+                                             (EekKey          *key,
+                                              gint             fallback_group,
+                                              gint             fallback_level);
 EekSymbol       *eek_key_get_symbol_at_index (EekKey          *key,
                                               gint             group,
-                                              gint             level);
+                                              gint             level,
+                                              gint             fallback_group,
+                                              gint             fallback_level);
 
 void             eek_key_set_index           (EekKey          *key,
                                               gint             column,
diff --git a/eek/eek-keyboard.c b/eek/eek-keyboard.c
index 459457f3..9874b9f8 100644
--- a/eek/eek-keyboard.c
+++ b/eek/eek-keyboard.c
@@ -258,20 +258,25 @@ eek_keyboard_real_key_pressed (EekKeyboard *self,
     EekSymbol *symbol;
     EekModifierType modifier;
 
-    if (priv->modifier_behavior == EEK_MODIFIER_BEHAVIOR_LATCH)
-        priv->modifiers = 0;
-
-    symbol = eek_key_get_symbol_at_index (key, priv->group, priv->level);
+    symbol = eek_key_get_symbol_at_index (key,
+                                          priv->group,
+                                          priv->level,
+                                          -1,
+                                          0);
     if (!symbol)
         return;
 
     modifier = eek_symbol_get_modifier_mask (symbol);
-    if (modifier != 0) {
-        if (priv->modifier_behavior == EEK_MODIFIER_BEHAVIOR_NONE ||
-            priv->modifier_behavior == EEK_MODIFIER_BEHAVIOR_LATCH)
-            priv->modifiers |= modifier;
-        else if (priv->modifier_behavior == EEK_MODIFIER_BEHAVIOR_LOCK)
-            priv->modifiers ^= modifier;
+    switch (priv->modifier_behavior) {
+    case EEK_MODIFIER_BEHAVIOR_NONE:
+        priv->modifiers |= modifier;
+        break;
+    case EEK_MODIFIER_BEHAVIOR_LOCK:
+        priv->modifiers ^= modifier;
+        break;
+    case EEK_MODIFIER_BEHAVIOR_LATCH:
+        priv->modifiers = (priv->modifiers ^ modifier) & modifier;
+        break;
     }
     set_level_from_modifiers (self);
 }
@@ -284,7 +289,11 @@ eek_keyboard_real_key_released (EekKeyboard *self,
     EekSymbol *symbol;
     EekModifierType modifier;
 
-    symbol = eek_key_get_symbol_at_index (key, priv->group, priv->level);
+    symbol = eek_key_get_symbol_at_index (key,
+                                          priv->group,
+                                          priv->level,
+                                          -1,
+                                          0);
     if (!symbol)
         return;
 
diff --git a/eek/eek-renderer.c b/eek/eek-renderer.c
index 7e170fb8..6f9a42a9 100644
--- a/eek/eek-renderer.c
+++ b/eek/eek-renderer.c
@@ -442,7 +442,7 @@ eek_renderer_real_render_key_label (EekRenderer *self,
     gdouble size, scale;
     gint i;
 
-    symbol = eek_key_get_symbol (key);
+    symbol = eek_key_get_symbol_with_fallback (key, -1, 0);
     if (!symbol)
         return;
 
diff --git a/src/eekboard.c b/src/eekboard.c
index 2781de1d..7987e569 100644
--- a/src/eekboard.c
+++ b/src/eekboard.c
@@ -399,7 +399,10 @@ a11y_keystroke_listener (const AccessibleKeystroke *stroke,
 
     /* XXX: Ignore modifier keys since there is no way to receive
        SPI_KEY_RELEASED event for them. */
-    symbol = eek_key_get_symbol (key);
+    symbol = eek_key_get_symbol_with_fallback (key, -1, 0);
+    if (!symbol)
+        return FALSE;
+
     for (i = 0; i < G_N_ELEMENTS(ignored_keysyms); i++)
         if (EEK_IS_KEYSYM(symbol) &&
             eek_keysym_get_xkeysym (EEK_KEYSYM(symbol)) == ignored_keysyms[i])
@@ -434,7 +437,10 @@ on_key_pressed (EekKeyboard *keyboard,
     Eekboard *eekboard = user_data;
     EekSymbol *symbol;
 
-    symbol = eek_key_get_symbol (key);
+    symbol = eek_key_get_symbol_with_fallback (key, -1, 0);
+    if (!symbol)
+        return;
+
     EEKBOARD_NOTE("%s %X",
                   eek_symbol_get_name (symbol),
                   eek_keyboard_get_modifiers (keyboard));