Merge pull request #30 from e2b-dev/add-cua-methods

Add CUA methods

Author: mishushakov

.changeset/nice-clubs-scream.md

@@ -0,0 +1,6 @@
+---
+"@e2b/desktop-python": minor
+"@e2b/desktop": minor
+---
+
+openai cua support

README.md

@@ -175,6 +175,7 @@ desktop.right_click()
 desktop.middle_click()
 desktop.scroll(10) # Scroll by the amount. Positive for up, negative for down.
 desktop.mouse_move(100, 200) # Move to x, y coordinates
+desktop.drag((100, 100), (200, 200)) # Drag using the mouse
 ```
 
 **JavaScript**
@@ -190,6 +191,7 @@ await desktop.rightClick()
 await desktop.middleClick()
 await desktop.scroll(10) // Scroll by the amount. Positive for up, negative for down.
 await desktop.moveMouse(100, 200) // Move to x, y coordinates
+await desktop.drag([100, 100], [200, 200]) // Drag using the mouse
 ```
 
 ### Keyboard control
@@ -208,7 +210,7 @@ desktop.write("Fast typing!", chunk_size=50, delay_in_ms=25)  # Faster typing
 desktop.press("enter")
 desktop.press("space")
 desktop.press("backspace")
-desktop.press("ctrl+c")
+desktop.press(["ctrl", "c"]) # Key combination
 ```
 
 **JavaScript**
@@ -226,7 +228,7 @@ await desktop.write('Fast typing!', { chunkSize: 50, delayInMs: 25 }) // Faster
 await desktop.press('enter')
 await desktop.press('space')
 await desktop.press('backspace')
-await desktop.press('ctrl+c') // Copy
+await desktop.press(['ctrl', 'c']) // Key combination
 ```
 
 ### Screenshot
@@ -305,6 +307,26 @@ const out = await desktop.commands.run('ls -la /home/user')
 console.log(out)
 ```
 
+### Wait
+
+**Python**
+
+```python
+from e2b_desktop import Sandbox
+desktop = Sandbox()
+
+desktop.wait(1000) # Wait for 1 second
+``` 
+
+**JavaScript**
+
+```javascript
+import { Sandbox } from '@e2b/desktop'
+
+const desktop = await Sandbox.create()
+await desktop.wait(1000) // Wait for 1 second
+```
+
 ## Under the hood
 
 The desktop-like environment is based on Linux and [Xfce](https://www.xfce.org/) at the moment. We chose Xfce because it's a fast and lightweight environment that's also popular and actively supported. However, this Sandbox template is fully customizable and you can create your own desktop environment.

packages/js-sdk/README.md

@@ -105,6 +105,7 @@ await desktop.rightClick()
 await desktop.middleClick()
 await desktop.scroll(10) // Scroll by the amount. Positive for up, negative for down.
 await desktop.moveMouse(100, 200) // Move to x, y coordinates
+await desktop.drag([100, 100], [200, 200]) // Drag using the mouse
 ```
 
 ### Keyboard control
@@ -122,7 +123,7 @@ await desktop.write('Fast typing!', { chunkSize: 50, delayInMs: 25 }) // Faster
 await desktop.press('enter')
 await desktop.press('space')
 await desktop.press('backspace')
-await desktop.press('ctrl+c') // Copy
+await desktop.press(['ctrl', 'c']) // Key combination
 ```
 
 ### Screenshot
@@ -160,6 +161,15 @@ const out = await desktop.commands.run('ls -la /home/user')
 console.log(out)
 ```
 
+### Wait
+
+```javascript
+import { Sandbox } from '@e2b/desktop'
+
+const desktop = await Sandbox.create()
+await desktop.wait(1000) // Wait for 1 second
+```
+
 ## Under the hood
 
 The desktop-like environment is based on Linux and [Xfce](https://www.xfce.org/) at the moment. We chose Xfce because it's a fast and lightweight environment that's also popular and actively supported. However, this Sandbox template is fully customizable and you can create your own desktop environment.

packages/js-sdk/src/sandbox.ts

@@ -373,14 +373,34 @@ export class Sandbox extends SandboxBase {
 
   /**
    * Press a key.
-   * @param key - The key to press (e.g. "enter", "space", "backspace", etc.).
+   * @param key - The key to press (e.g. "enter", "space", "backspace", etc.). Can be a single key or an array of keys.
    */
-  async press(key: string): Promise<void> {
+  async press(key: string | string[]): Promise<void> {
     await this.commands.run(
-      `xdotool key ${key}`, { envs: { DISPLAY: this.display } }
+      `xdotool key ${Array.isArray(key) ? key.join('+') : key}`, { envs: { DISPLAY: this.display } }
     );
   }
 
+  /**
+   * Drag the mouse from the given position to the given position.
+   * @param from - The starting position.
+   * @param to - The ending position.
+   */
+  async drag([x1, y1]: [number, number], [x2, y2]: [number, number]): Promise<void> {
+    await this.commands.run(
+      `xdotool mousemove ${x1} ${y1} && xdotool mousedown 1 && xdotool mousemove ${x2} ${y2} && xdotool mouseup 1`,
+      { envs: { DISPLAY: this.display } }
+    );
+  }
+
+  /**
+   * Wait for the given amount of time.
+   * @param ms - The amount of time to wait in milliseconds.
+   */
+  async wait(ms: number): Promise<void> {
+    await this.commands.run(`sleep ${ms / 1000}`, { envs: { DISPLAY: this.display } });
+  }
+
   /**
    * Open a file or a URL in the default application.
    * @param fileOrUrl - The file or URL to open.

packages/python-sdk/README.md

@@ -102,6 +102,7 @@ desktop.right_click()
 desktop.middle_click()
 desktop.scroll(10) # Scroll by the amount. Positive for up, negative for down.
 desktop.mouse_move(100, 200) # Move to x, y coordinates
+desktop.drag((100, 100), (200, 200)) # Drag using the mouse
 ```
 
 ### Keyboard control
@@ -118,7 +119,7 @@ desktop.write("Fast typing!", chunk_size=50, delay_in_ms=25)  # Faster typing
 desktop.press("enter")
 desktop.press("space")
 desktop.press("backspace")
-desktop.press("ctrl+c")
+desktop.press(["ctrl", "c"]) # Key combination
 ```
 
 ### Screenshot
@@ -156,6 +157,15 @@ out = desktop.commands.run("ls -la /home/user")
 print(out)
 ```
 
+### Wait
+
+```python
+from e2b_desktop import Sandbox
+desktop = Sandbox()
+
+desktop.wait(1000) # Wait for 1 second
+```
+
 ## Under the hood
 
 The desktop-like environment is based on Linux and [Xfce](https://www.xfce.org/) at the moment. We chose Xfce because it's a fast and lightweight environment that's also popular and actively supported. However, this Sandbox template is fully customizable and you can create your own desktop environment.

packages/python-sdk/e2b_desktop/main.py

@@ -347,14 +347,34 @@ def break_into_chunks(text: str, n: int):
                 f"xdotool type --delay {delay_in_ms} {quote_string(chunk)}", envs={"DISPLAY": self._display}
             )
 
-    def press(self, key: str):
+    def press(self, key: str | list[str]):
         """
         Press a key.
 
         :param key: The key to press (e.g. "enter", "space", "backspace", etc.).
         """
+        if isinstance(key, list):
+            key = "+".join(key)
+
         self.commands.run(f"xdotool key {key}", envs={"DISPLAY": self._display})
 
+    def drag(self, fr: tuple[int, int], to: tuple[int, int]):
+        """
+        Drag the mouse from the given position to the given position.
+
+        :param from: The starting position.
+        :param to: The ending position.
+        """
+        self.commands.run(f"xdotool mousemove {fr[0]} {fr[1]} && xdotool mousedown 1 && xdotool mousemove {to[0]} {to[1]} && xdotool mouseup 1", envs={"DISPLAY": self._display})
+
+    def wait(self, ms: int):
+        """
+        Wait for the given amount of time.
+
+        :param ms: The amount of time to wait in milliseconds.
+        """
+        self.commands.run(f"sleep {ms / 1000}", envs={"DISPLAY": self._display})
+
     def open(self, file_or_url: str):
         """
         Open a file or a URL in the default application.