diff --git a/src/Panzoom/Panzoom.js b/src/Panzoom/Panzoom.js
index 1be96a6..8a7dfb0 100644
--- a/src/Panzoom/Panzoom.js
+++ b/src/Panzoom/Panzoom.js
@@ -365,7 +365,7 @@ export class Panzoom extends Base {
           }
 
           // Allow text selection
-          if (this.option("textSelection") && getTextNodeFromPoint(target, event.clientX, event.clientY)) {
+          if (this.option("textSelection") && getTextNodeFromPoint(target, pointer.clientX, pointer.clientY)) {
             return false;
           }
         }
diff --git a/src/shared/utils/PointerTracker.js b/src/shared/utils/PointerTracker.js
index b2dc8a1..67cd8f7 100644
--- a/src/shared/utils/PointerTracker.js
+++ b/src/shared/utils/PointerTracker.js
@@ -39,7 +39,7 @@ class PointerTracker {
     this.currentPointers = [];
 
     this._pointerStart = (event) => {
-      if (!(event.buttons & 1)) {
+      if (event.buttons > 0 && event.button !== 0) {
         return;
       }
 
@@ -53,8 +53,8 @@ class PointerTracker {
         return;
       }
 
-      window.addEventListener("mousemove", this._move, { passive: false });
-      window.addEventListener("mouseup", this._pointerEnd, { passive: false });
+      window.addEventListener("mousemove", this._move);
+      window.addEventListener("mouseup", this._pointerEnd);
     };
 
     this._touchStart = (event) => {
@@ -87,10 +87,6 @@ class PointerTracker {
     };
 
     this._triggerPointerEnd = (pointer, event) => {
-      if (!isTouchEvent(event) && event.buttons & 1) {
-        return false;
-      }
-
       const index = this.currentPointers.findIndex((p) => p.id === pointer.id);
 
       if (index < 0) {
@@ -106,12 +102,16 @@ class PointerTracker {
     };
 
     this._pointerEnd = (event) => {
+      if (event.buttons > 0 && event.button !== 0) {
+        return;
+      }
+
       if (!this._triggerPointerEnd(new Pointer(event), event)) {
         return;
       }
 
-      window.removeEventListener("mousemove", this._move);
-      window.removeEventListener("mouseup", this._pointerEnd);
+      window.removeEventListener("mousemove", this._move, { passive: false });
+      window.removeEventListener("mouseup", this._pointerEnd, { passive: false });
     };
 
     this._touchEnd = (event) => {
@@ -138,8 +138,8 @@ class PointerTracker {
     this._element.removeEventListener("touchend", this._touchEnd);
     this._element.removeEventListener("touchcancel", this._touchEnd);
 
-    window.removeEventListener("mousemove", this._move, { passive: false });
-    window.removeEventListener("mouseup", this._pointerEnd, { passive: false });
+    window.removeEventListener("mousemove", this._move);
+    window.removeEventListener("mouseup", this._pointerEnd);
   }
 
   _triggerPointerStart(pointer, event) {
diff --git a/tests/2_panzoom_test.js b/tests/2_panzoom_test.js
index f76fb80..4bc2c0f 100644
--- a/tests/2_panzoom_test.js
+++ b/tests/2_panzoom_test.js
@@ -237,6 +237,7 @@ describe("Panzoom", function () {
     const y = instance.$content.getClientRects()[0].top;
 
     triggerEvent(instance.$container, "mousedown", {
+      button: 0,
       buttons: 1,
       clientX: x + 60,
       clientY: y + 60,
@@ -315,6 +316,7 @@ describe("Panzoom", function () {
     const y = instance.$content.getClientRects()[0].top;
 
     triggerEvent(instance.$container, "mousedown", {
+      button: 0,
       buttons: 1,
       clientX: x,
       clientY: y + 50,
@@ -361,6 +363,7 @@ describe("Panzoom", function () {
     const y = instance.$content.getClientRects()[0].top;
 
     triggerEvent(instance.$container, "mousedown", {
+      button: 0,
       buttons: 1,
       clientX: x + 50,
       clientY: y + 50,
@@ -391,6 +394,7 @@ describe("Panzoom", function () {
     const y = instance.$content.getClientRects()[0].top;
 
     triggerEvent(instance.$container, "mousedown", {
+      button: 0,
       buttons: 1,
       clientX: x + 50,
       clientY: y + 50,
@@ -531,6 +535,7 @@ describe("Panzoom", function () {
     const y = instance.$content.getClientRects()[0].top;
 
     triggerEvent(instance.$content, "mousedown", {
+      button: 0,
       buttons: 1,
       clientX: x + 40,
       clientY: y + 40,
diff --git a/tests/3_carousel_test.js b/tests/3_carousel_test.js
index a40ff40..34ddcbe 100644
--- a/tests/3_carousel_test.js
+++ b/tests/3_carousel_test.js
@@ -581,6 +581,7 @@ describe("Carousel", function () {
     const y = instance.Panzoom.$content.getClientRects()[0].top;
 
     triggerEvent(instance.Panzoom.$container, "mousedown", {
+      button: 0,
       buttons: 1,
       clientX: x + 150,
       clientY: y + 50,
@@ -616,6 +617,7 @@ describe("Carousel", function () {
     const y = instance.Panzoom.$content.getClientRects()[0].top;
 
     triggerEvent(instance.Panzoom.$container, "mousedown", {
+      button: 0,
       buttons: 1,
       clientX: x + 150,
       clientY: y + 50,