diff --git a/Sortable.js b/Sortable.js
index 5c4672c..6f1d4f0 100644
--- a/Sortable.js
+++ b/Sortable.js
@@ -275,12 +275,12 @@
 				return; // only left button or enabled
 			}
 
-			if (options.handle) {
-				target = _closest(target, options.handle, el);
-			}
-
 			target = _closest(target, options.draggable, el);
 
+			if (!target) {
+				return;
+			}
+
 			// get the index of the dragged element within its parent
 			oldIndex = _index(target);
 
@@ -308,6 +308,12 @@
 				}
 			}
 
+
+			if (options.handle && !_closest(originalTarget, options.handle, el)) {
+				return;
+			}
+
+
 			// Prepare `dragstart`
 			if (target && !dragEl && (target.parentNode === el)) {
 				tapEvt = evt;