from typing import Literal import playwright.async_api async def highlight_by_box_async( page: playwright.async_api.Page, box: dict, color: Literal["blue", "red"] = "blue" ): """Highlights the target element based on its bounding box attributes.""" assert color in ("blue", "red") if box: left, top, width, height = box["x"], box["y"], box["width"], box["height"] await page.evaluate( f"""\ const overlay = document.createElement('div'); document.body.appendChild(overlay); overlay.setAttribute('style', ` all: initial; position: fixed; border: 2px solid transparent; /* Start with transparent border */ borderRadius: 10px; /* Add rounded corners */ boxShadow: 0 0 0px {color}; /* Initial boxShadow with 0px spread */ left: {left - 2}px; /* Adjust left position to accommodate initial shadow spread */ top: {top - 2}px; /* Adjust top position likewise */ width: {width}px; height: {height}px; z-index: 2147483646; /* Maximum value - 1 */ pointerEvents: none; /* Ensure the overlay does not interfere with user interaction */ `); // Animate the boxShadow to create a "wave" effect let spread = 0; // Initial spread radius of the boxShadow const waveInterval = setInterval(() => {{ spread += 10; // Increase the spread radius to simulate the wave moving outward overlay.style.boxShadow = `0 0 40px ${{spread}}px {color}`; // Update boxShadow to new spread radius overlay.style.opacity = 1 - spread / 38; // Gradually decrease opacity to fade out the wave if (spread >= 38) {{ // Assuming 76px ~ 2cm spread radius clearInterval(waveInterval); // Stop the animation once the spread radius reaches 2cm document.body.removeChild(overlay); // Remove the overlay from the document }} }}, 200); // Adjust the interval as needed to control the speed of the wave animation """ ) # Wait a bit to let users see the highlight await page.wait_for_timeout(1000) # Adjust delay as needed async def smooth_move_visual_cursor_to_async( page: playwright.async_api.Page, x: float, y: float, speed: float = 400 ): """ Smoothly moves the visual cursor to a specific point, with constant movement speed. Args: x: target location X coordinate (in viewport pixels) y: target location Y coordinate (in viewport pixels) speed: cursor speed (in pixels per second) """ movement_time = await page.evaluate( """\ ([targetX, targetY, speed]) => { // create cursor if needed if (!("browsergym_visual_cursor" in window)) { if (window.trustedTypes && window.trustedTypes.createPolicy) { window.trustedTypes.createPolicy('default', { createHTML: (string, sink) => string }); } let cursor = document.createElement('div'); cursor.setAttribute('id', 'browsergym-visual-cursor'); cursor.innerHTML = ` `; cursor.setAttribute('style', ` all: initial; position: fixed; opacity: 0.7; /* Slightly transparent */ z-index: 2147483647; /* Maximum value */ pointer-events: none; /* Ensures the SVG doesn't interfere with page interactions */ `); // Calculate center position within the viewport const centerX = window.innerWidth / 2; const centerY = window.innerHeight / 2; cursor.style.left = `${centerX}px`; cursor.style.top = `${centerY}px`; // save cursor element window.browsergym_visual_cursor = cursor; window.browsergym_visual_cursor_n_owners = 0; } // recover cursor let cursor = window.browsergym_visual_cursor; // attach cursor to document document.body.appendChild(cursor); window.browsergym_visual_cursor_n_owners += 1; x = parseFloat(cursor.style.left); y = parseFloat(cursor.style.top); dx = targetX - x; dy = targetY - y; dist = Math.hypot(dx, dy); movement_time = (dist / speed) * 1000; // seconds to milliseconds still_wait_time = 1000; // Adjust steps based on distance to keep movement speed consistent // 1 step per 10 pixels of distance, adjust as needed steps = Math.max(1, Math.trunc(dist / 10)); step_dx = dx / steps; step_dy = dy / steps; step_dist = dist / steps; step_wait_time = Math.max(10, movement_time / steps); let step = 0; let time_still = 0; const cursorInterval = setInterval(() => { // move cursor if (step < steps) { x += step_dx; y += step_dy; cursor.style.left = `${x}px`; cursor.style.top = `${y}px`; } // still cursor (wait a bit) else if (time_still < still_wait_time) { time_still += step_wait_time; } // stop and detach cursor else { clearInterval(cursorInterval); window.browsergym_visual_cursor_n_owners -= 1; if (window.browsergym_visual_cursor_n_owners <= 0) { document.body.removeChild(cursor); } } step += 1; }, step_wait_time); return movement_time; }""", [x, y, speed], ) await page.wait_for_timeout(movement_time) async def check_for_overlay_async( page: playwright.async_api.Page, bid: str, element: playwright.async_api.ElementHandle, box: dict ): if not element: return False visibility = await element.get_attribute("browsergym_visibility_ratio") if visibility is not None: return float(visibility) >= 0.5 """Checks if a given element is the topmost element at its center position by default. If check_corners is True, it checks if any of the corners is visible.""" if box: # corners points_to_check = [ (box["x"], box["y"]), (box["x"] + box["width"], box["y"]), (box["x"], box["y"] + box["height"]), (box["x"] + box["width"], box["y"] + box["height"]), ] for x, y in points_to_check: # Execute JavaScript to find the topmost element at the point. top_element = page.evaluate( f"""() => {{ const el = document.elementFromPoint({x}, {y}); return el ? el.outerHTML : ''; }}""" ) # Check if the topmost element is the element we're interested in. if top_element and bid in top_element: return True return False async def add_demo_mode_effects_async( page: playwright.async_api.Page, elem: playwright.async_api.ElementHandle, bid: str, demo_mode: Literal["off", "default", "all_blue", "only_visible_elements"], move_cursor: bool = True, highlight_box: bool = True, ): if demo_mode == "off": return """Adds visual effects to the target element""" box = await elem.bounding_box() # box = extract_bounds_cdp(page, bid) if box: center_x, center_y = box["x"] + box["width"] / 2, box["y"] + box["height"] / 2 is_top_element = await check_for_overlay_async(page, bid, elem, box) if demo_mode == "only_visible_elements": if not is_top_element: return else: color = "blue" elif demo_mode == "default": if is_top_element: color = "blue" else: color = "red" elif demo_mode == "all_blue": color = "blue" if move_cursor: await smooth_move_visual_cursor_to_async(page, center_x, center_y) if highlight_box: await highlight_by_box_async(page, box, color=color)