diff --git a/eek/eek-renderer.c b/eek/eek-renderer.c
index 3e341318..a8a46dca 100644
--- a/eek/eek-renderer.c
+++ b/eek/eek-renderer.c
@@ -978,20 +978,10 @@ eek_renderer_find_button_by_position (EekRenderer *renderer,
     g_return_val_if_fail (EEK_IS_RENDERER(renderer), NULL);
 
     EekRendererPrivate *priv = eek_renderer_get_instance_private (renderer);
-    EekBounds bounds = squeek_view_get_bounds (view);
 
-    /* Transform from widget coordinates to keyboard coordinates */
-    x = (x - priv->origin_x)/priv->scale - bounds.x;
-    y = (y - priv->origin_y)/priv->scale - bounds.y;
-
-    if (x < 0 ||
-        y < 0 ||
-        x > bounds.width ||
-        y > bounds.height)
-        return NULL;
     EekPoint point = {
-        .x = x,
-        .y = y,
+        .x = (x - priv->origin_x)/priv->scale,
+        .y = (y - priv->origin_y)/priv->scale,
     };
     return squeek_view_find_button_by_position(view, point);
 }
diff --git a/src/layout.rs b/src/layout.rs
index bd70272c..b21d5d23 100644
--- a/src/layout.rs
+++ b/src/layout.rs
@@ -568,7 +568,7 @@ pub struct Button {
     pub state: Rc<RefCell<KeyState>>,
 }
 
-
+// FIXME: derive from the style/margin/padding
 const BUTTON_SPACING: f64 = 4.0;
 const ROW_SPACING: f64 = 7.0;
 
@@ -710,10 +710,17 @@ impl View {
     }
 
     /// Finds the first button that covers the specified point
-    /// relative to view's origin
+    /// relative to view's position's origin
     fn find_button_by_position(&mut self, point: c::Point)
         -> Option<&mut Box<Button>>
     {
+        // make point relative to the inside of the view,
+        // which is the origin of all rows
+        let point = c::Point {
+            x: point.x - self.bounds.x,
+            y: point.y - self.bounds.y,
+        };
+
         self.rows.iter_mut().find_map(
             |row| row.find_button_by_position(point.clone())
         )