Merge pull request #33 from e2b-dev/add-more-cua-methods

Add mouse_press and mouse_release for additional controls

Author: mishushakov

README.md

@@ -176,6 +176,8 @@ desktop.middle_click()
 desktop.scroll(10) # Scroll by the amount. Positive for up, negative for down.
 desktop.mouse_move(100, 200) # Move to x, y coordinates
 desktop.drag((100, 100), (200, 200)) # Drag using the mouse
+desktop.mouse_press("left") # Press the mouse button
+desktop.mouse_release("left") # Release the mouse button
 ```
 
 **JavaScript**
@@ -192,6 +194,8 @@ await desktop.middleClick()
 await desktop.scroll(10) // Scroll by the amount. Positive for up, negative for down.
 await desktop.moveMouse(100, 200) // Move to x, y coordinates
 await desktop.drag([100, 100], [200, 200]) // Drag using the mouse
+await desktop.mousePress("left") // Press the mouse button
+await desktop.mouseRelease("left") // Release the mouse button
 ```
 
 ### Keyboard control

packages/js-sdk/README.md

@@ -106,6 +106,8 @@ await desktop.middleClick()
 await desktop.scroll(10) // Scroll by the amount. Positive for up, negative for down.
 await desktop.moveMouse(100, 200) // Move to x, y coordinates
 await desktop.drag([100, 100], [200, 200]) // Drag using the mouse
+await desktop.mousePress("left") // Press the mouse button
+await desktop.mouseRelease("left") // Release the mouse button
 ```
 
 ### Keyboard control

packages/js-sdk/src/sandbox.ts

@@ -13,6 +13,12 @@ interface ScreenSize {
   height: number;
 }
 
+const MOUSE_BUTTONS = {
+  left: 1,
+  right: 3,
+  middle: 2
+}
+
 /**
  * Configuration options for the Sandbox environment.
  * @interface SandboxOpts
@@ -280,6 +286,20 @@ export class Sandbox extends SandboxBase {
     );
   }
 
+  /**
+   * Press the mouse button.
+   */
+  async mousePress(button: 'left' | 'right' | 'middle' = 'left'): Promise<void> {
+    await this.commands.run(`xdotool mousedown ${MOUSE_BUTTONS[button]}`, { envs: { DISPLAY: this.display } });
+  }
+
+  /**
+   * Release the mouse button.
+   */
+  async mouseRelease(button: 'left' | 'right' | 'middle' = 'left'): Promise<void> {
+    await this.commands.run(`xdotool mouseup ${MOUSE_BUTTONS[button]}`, { envs: { DISPLAY: this.display } });
+  }
+
   /**
    * Get the current cursor position.
    * @returns A object with the x and y coordinates
@@ -387,10 +407,10 @@ export class Sandbox extends SandboxBase {
    * @param to - The ending position.
    */
   async drag([x1, y1]: [number, number], [x2, y2]: [number, number]): Promise<void> {
-    await this.commands.run(
-      `xdotool mousemove ${x1} ${y1} && xdotool mousedown 1 && xdotool mousemove ${x2} ${y2} && xdotool mouseup 1`,
-      { envs: { DISPLAY: this.display } }
-    );
+    await this.moveMouse(x1, y1);
+    await this.mousePress();
+    await this.moveMouse(x2, y2);
+    await this.mouseRelease();
   }
 
   /**

packages/python-sdk/README.md

@@ -103,6 +103,8 @@ desktop.middle_click()
 desktop.scroll(10) # Scroll by the amount. Positive for up, negative for down.
 desktop.mouse_move(100, 200) # Move to x, y coordinates
 desktop.drag((100, 100), (200, 200)) # Drag using the mouse
+desktop.mouse_press("left") # Press the mouse button
+desktop.mouse_release("left") # Release the mouse button
 ```
 
 ### Keyboard control

packages/python-sdk/e2b_desktop/main.py

@@ -6,6 +6,11 @@
 
 from e2b import Sandbox as SandboxBase, CommandHandle, CommandResult, TimeoutException, CommandExitException
 
+MOUSE_BUTTONS = {
+    "left": 1,
+    "right": 3,
+    "middle": 2
+}
 
 class _VNCServer:
     def __init__(self, desktop: "Sandbox") -> None:
@@ -288,6 +293,18 @@ def move_mouse(self, x: int, y: int):
         """
         self.commands.run(f"xdotool mousemove --sync {x} {y}", envs={"DISPLAY": self._display})
 
+    def mouse_press(self, button: Literal["left", "right", "middle"] = "left"):
+        """
+        Press the mouse button.
+        """
+        self.commands.run(f"xdotool mousedown {MOUSE_BUTTONS[button]}", envs={"DISPLAY": self._display})
+
+    def mouse_release(self, button: Literal["left", "right", "middle"] = "left"):
+        """
+        Release the mouse button.
+        """
+        self.commands.run(f"xdotool mouseup {MOUSE_BUTTONS[button]}", envs={"DISPLAY": self._display})
+        
     def get_cursor_position(self) -> tuple[int, int]:
         """
         Get the current cursor position.
@@ -365,7 +382,10 @@ def drag(self, fr: tuple[int, int], to: tuple[int, int]):
         :param from: The starting position.
         :param to: The ending position.
         """
-        self.commands.run(f"xdotool mousemove {fr[0]} {fr[1]} && xdotool mousedown 1 && xdotool mousemove {to[0]} {to[1]} && xdotool mouseup 1", envs={"DISPLAY": self._display})
+        self.move_mouse(fr[0], fr[1])
+        self.mouse_press()
+        self.move_mouse(to[0], to[1])
+        self.mouse_release()
 
     def wait(self, ms: int):
         """