diff --git a/data/keyboards/geometry/compact.xml b/data/keyboards/geometry/compact.xml
index 6a766c6c..5781a444 100644
--- a/data/keyboards/geometry/compact.xml
+++ b/data/keyboards/geometry/compact.xml
@@ -103,7 +103,7 @@
   <section angle="0">
     <bounds x="0" y="0" width="608.7804" height="201.3658"/>
     <row orientation="1">
-      <key keycode="66" name="ALTGR" oref="altline">
+      <key keycode="0" name="ABC123" oref="altline">
 	<bounds x="62.43902" y="162.3414" width="48.39024" height="37.46341"/>
       </key>
       <key keycode="150" name="I150" oref="altline">
diff --git a/data/keyboards/symbols/us.xml b/data/keyboards/symbols/us.xml
index 4c1ed959..1a584c13 100644
--- a/data/keyboards/symbols/us.xml
+++ b/data/keyboards/symbols/us.xml
@@ -165,11 +165,11 @@
     <keysym keyval="92">backslash</keysym>
     <keysym keyval="124">bar</keysym>
   </key>
-  <key keycode="66" name="ALTGR">
-    <keysym keyval="65027">ISO_Level3_Shift</keysym>
-    <keysym keyval="65027">ISO_Level3_Shift</keysym>
-    <keysym keyval="65027" label="ABC">ISO_Level3_Shift</keysym>
-    <keysym keyval="65027" label="ABC">ISO_Level3_Shift</keysym>
+  <key keycode="0" name="ABC123">
+    <symbol label="123">show-numbers</symbol>
+    <symbol label="123">show-numbers</symbol>
+    <symbol label="ABC">show-letters</symbol>
+    <symbol label="ABC">show-letters</symbol>
   </key>
   <key keycode="38" name="AC01">
     <keysym keyval="97">a</keysym>
diff --git a/eek/eek-keyboard.c b/eek/eek-keyboard.c
index 6f4ae0ec..fd890167 100644
--- a/eek/eek-keyboard.c
+++ b/eek/eek-keyboard.c
@@ -211,13 +211,23 @@ eek_keyboard_get_property (GObject    *object,
 }
 
 static void
-set_level_from_modifiers (EekKeyboard *self)
+set_level_from_modifiers (EekKeyboard *self, EekKey *key)
 {
     EekKeyboardPrivate *priv = EEK_KEYBOARD_GET_PRIVATE(self);
 
     /* The levels are: 0 Letters, 1 Upper case letters, 2 Numbers, 3 Symbols */
-    gint level = ((priv->modifiers & priv->alt_gr_mask) ? 2 : 0) |
-                 ((priv->modifiers & EEK_SHIFT_MASK) ? 1 : 0);
+
+    /* Use the numbers/letters bit from the old level */
+    gint level = priv->old_level & 2;
+
+    /* Handle non-emitting keys */
+    if (key && (eek_key_get_keycode(key) == 0)) {
+        const gchar *name = eek_element_get_name(EEK_ELEMENT(key));
+        if (g_strcmp0(name, "ABC123") == 0)
+            level ^= 2;
+    }
+
+    level |= ((priv->modifiers & EEK_SHIFT_MASK) ? 1 : 0);
 
     switch (priv->old_level) {
     case VIEW_LETTERS_UPPER:
@@ -249,6 +259,8 @@ set_level_from_modifiers (EekKeyboard *self)
 
     if (level == VIEW_NUMBERS || level == VIEW_SYMBOLS)
         priv->modifier_behavior = EEK_MODIFIER_BEHAVIOR_LOCK;
+    else if (level == VIEW_LETTERS_UPPER)
+        priv->modifier_behavior = EEK_MODIFIER_BEHAVIOR_LATCH;
 
     priv->old_level = level;
     eek_element_set_level (EEK_ELEMENT(self), level);
@@ -306,7 +318,7 @@ void eek_keyboard_press_key(EekKeyboard *keyboard, EekKey *key, guint32 timestam
     EekModifierType modifier = eek_symbol_get_modifier_mask (symbol);
     if (priv->modifier_behavior == EEK_MODIFIER_BEHAVIOR_NONE) {
         set_modifiers_with_key (keyboard, key, priv->modifiers | modifier);
-        set_level_from_modifiers (keyboard);
+        set_level_from_modifiers (keyboard, key);
     }
 
     // "Borrowed" from eek-context-service; doesn't influence the state but forwards the event
@@ -355,7 +367,7 @@ void eek_keyboard_release_key( EekKeyboard *keyboard,
                                     (priv->modifiers ^ modifier) & modifier);
         break;
     }
-    set_level_from_modifiers (keyboard);
+    set_level_from_modifiers (keyboard, key);
 
     // "Borrowed" from eek-context-service; doesn't influence the state but forwards the event
 
@@ -621,7 +633,7 @@ eek_keyboard_set_modifiers (EekKeyboard    *keyboard,
 {
     g_return_if_fail (EEK_IS_KEYBOARD(keyboard));
     keyboard->priv->modifiers = modifiers;
-    set_level_from_modifiers (keyboard);
+    set_level_from_modifiers (keyboard, NULL);
 }
 
 /**