scripts/pip-interaction-smoke.mjs - maka - Git at Google

 #!/usr/bin/env node
 // Real-window smoke for the picture-in-picture mirror's interaction.
 //
 // The unit tests drive a fake window, so they prove the physics and the state
 // machine and nothing about Electron. This runs the real thing: a real parent
 // window, a real child panel, the real preload and renderer, and real input
 // events delivered to the page. Everything it asserts is something that was
 // either measured on this machine or read out of Codex's binary.
 //
 // Run: npm --workspace @maka/desktop run build:main && npm --workspace @maka/desktop run build:overlay
 //      && npx electron scripts/pip-interaction-smoke.mjs
 import { app, BrowserWindow, screen } from 'electron';
 import { fileURLToPath } from 'node:url';
 import { dirname, join } from 'node:path';

 const here = dirname(fileURLToPath(import.meta.url));
 const dist = join(here, '..', 'apps', 'desktop', 'dist');
 const { createComputerUsePipController } = await import(
   join(dist, 'main', 'computer-use', 'pip-window.js')
 );

 const PNG_1X1 =
   'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';

 let failures = 0;
 const check = (label, pass, detail) => {
   if (!pass) failures += 1;
   console.log(`[${pass ? 'PASS' : 'FAIL'}] ${label}${detail ? ` — ${detail}` : ''}`);
 };
 const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

 /**
  * Poll until the mirror stops moving, so assertions never race the spring.
  *
  * Requires several identical samples rather than two: the settle is
  * overdamped, so its last stretch moves less than a point per frame and two
  * consecutive reads can round to the same value while it is still travelling.
  * Two was enough to report the mirror had arrived a point short of the anchor.
  */
 async function settled(win, timeoutMs = 4000) {
   let last = null;
   let same = 0;
   const deadline = Date.now() + timeoutMs;
   while (Date.now() < deadline) {
     const bounds = win.getBounds();
     if (last && bounds.x === last.x && bounds.y === last.y) {
       if (++same >= 6) return bounds;
     } else {
       same = 0;
     }
     last = bounds;
     await sleep(60);
   }
   return win.getBounds();
 }

 app.whenReady().then(async () => {
   const parent = new BrowserWindow({ x: 120, y: 120, width: 1000, height: 700, show: true });
   await parent.loadURL('data:text/html,<body style="background:%23202028"></body>');
   await sleep(300);

   // The pointer the drag reads. Driven from here so the gesture is exact
   // rather than dependent on whatever the real mouse happens to be doing.
   let pointer = { x: 0, y: 0 };
   let pip = null;
   const created = [];

   pip = createComputerUsePipController({
     resolveAnchorRect: () => parent.getBounds(),
     subscribeAnchorChanges: (cb) => {
       parent.on('move', cb);
       parent.on('resize', cb);
       return () => {
         parent.off('move', cb);
         parent.off('resize', cb);
       };
     },
     resolveParentWindow: () => parent,
     cursorPoint: () => pointer,
     workAreaFor: (rect) => screen.getDisplayMatching(rect).workArea,
     preloadPath: join(dist, 'overlay', 'pip-preload.cjs'),
     htmlPath: join(dist, 'overlay', 'pip.html'),
   });

   pip.present({
     sessionId: 'smoke',
     base64: PNG_1X1,
     mimeType: 'image/png',
     widthPx: 1000,
     heightPx: 800,
   });
   await sleep(900);

   const mirror = BrowserWindow.getAllWindows().find((w) => w !== parent);
   if (!mirror) {
     console.log('the mirror never opened; stopping.');
     app.exit(2);
     return;
   }
   created.push(mirror);

   // ── the two properties the child-window fix bought ────────────────────────
   check('the mirror does not float above other apps', mirror.isAlwaysOnTop() === false);
   // `parent.isFocused()` is about the whole app being frontmost, which a
   // headless-ish run cannot promise. What the mirror owes is narrower: it must
   // never be the focused window itself.
   check('the mirror never took focus', mirror.isFocused() === false);
   check('the mirror is the app window’s child', mirror.getParentWindow() === parent);

   const seated = mirror.getBounds();
   const app0 = parent.getBounds();
   check(
     'it is seated on the app window’s bottom-right, inset 24pt',
     seated.x === app0.x + app0.width - seated.width - 24 &&
       seated.y === app0.y + app0.height - seated.height - 24,
     `${JSON.stringify(seated)} against ${JSON.stringify(app0)}`,
   );

   // ── a pure move is carried by the window server, not by us ────────────────
   parent.setBounds({ ...app0, x: app0.x + 160 });
   await sleep(400);
   check(
     'moving the app carries the mirror with it',
     mirror.getBounds().x === seated.x + 160,
     `${seated.x} → ${mirror.getBounds().x}`,
   );

   // ── click-through until pointed at ────────────────────────────────────────
   const contents = mirror.webContents;
   const inMirror = (dx, dy) => ({ x: dx, y: dy });
   // Hover is decided from the real pointer on the main side, so the test moves
   // the pointer rather than injecting a page event.
   const mid = mirror.getBounds();
   pointer = { x: mid.x + 40, y: mid.y + 40 };
   await sleep(300);
   const hovered = await contents.executeJavaScript(
     "document.getElementById('controls').dataset.visible",
   );
   check('the controls appear on hover', hovered === '1', `data-visible=${hovered}`);

   // ── throw it at the opposite corner ───────────────────────────────────────
   const before = mirror.getBounds();
   // Press in the middle of the tile, clear of both the controls (top-right,
   // 20x20 inset 8) and the resize grip (20x20 inset 8, on whichever corner is
   // opposite the anchor). This used to press at (20, 20), which is inside the
   // grip's 8..28 box: the grip stops the event propagating, so no drag ever
   // started, the moves ran as a resize instead, and the five checks after it
   // failed on the cascade. This half of the script had never completed.
   const GRAB = { x: Math.round(before.width / 2), y: Math.round(before.height / 2) };
   pointer = { x: before.x + GRAB.x, y: before.y + GRAB.y };
   contents.sendInputEvent({
     type: 'mouseDown',
     ...inMirror(GRAB.x, GRAB.y),
     button: 'left',
     clickCount: 1,
   });
   await sleep(60);
   const appNow = parent.getBounds();
   // Three samples so the tracker sees real motion, ending fast enough that the
   // release reads as a throw rather than a place.
   for (const [fx, fy] of [
     [0.7, 0.7],
     [0.4, 0.4],
     [0.22, 0.18],
   ]) {
     pointer = { x: appNow.x + appNow.width * fx, y: appNow.y + appNow.height * fy };
     contents.sendInputEvent({ type: 'mouseMove', ...inMirror(GRAB.x, GRAB.y) });
     await sleep(30);
   }
   contents.sendInputEvent({
     type: 'mouseUp',
     ...inMirror(GRAB.x, GRAB.y),
     button: 'left',
     clickCount: 1,
   });

   const landed = await settled(mirror);
   check(
     'thrown up and left, it settles on the top-left anchor',
     landed.x === appNow.x + 24 && landed.y === appNow.y + 24,
     `${JSON.stringify(landed)} against app ${JSON.stringify(appNow)}`,
   );
   check(
     'and the controller agrees which corner that is',
     pip.currentAlignment() === 'top-left',
     pip.currentAlignment(),
   );

   // ── a resize returns it to the corner the user chose ──────────────────────
   parent.setBounds({ ...appNow, width: appNow.width - 200, height: appNow.height - 120 });
   await sleep(500);
   const afterResize = mirror.getBounds();
   const appResized = parent.getBounds();
   check(
     'a resize keeps it on the chosen corner, not the default one',
     afterResize.x === appResized.x + 24 && afterResize.y === appResized.y + 24,
     JSON.stringify(afterResize),
   );

   // ── the grip resizes it, away from whichever corner it rests on ──────────
   const beforeGrip = mirror.getBounds();
   const gripAt = await contents.executeJavaScript(
     "(() => { const r = document.getElementById('resize').getBoundingClientRect(); return { x: Math.round(r.x + r.width / 2), y: Math.round(r.y + r.height / 2) }; })()",
   );
   // The tile is anchored top-left, so its top-left corner is the one held
   // still and the grip has to be on the bottom-right — the corner that moves.
   // Pinning the grip to the tile's top-left, which is what the CSS did before
   // main started sending the alignment down, put the handle on the corner the
   // gesture holds: the pointer left it on the first pixel of the drag.
   check(
     'the grip sits on the corner opposite the anchor',
     gripAt.x > beforeGrip.width / 2 && gripAt.y > beforeGrip.height / 2,
     `grip at ${JSON.stringify(gripAt)} in a ${beforeGrip.width}x${beforeGrip.height} tile ` +
       `anchored ${pip.currentAlignment()}`,
   );
   pointer = { x: beforeGrip.x + gripAt.x, y: beforeGrip.y + gripAt.y };
   contents.sendInputEvent({ type: 'mouseMove', ...gripAt });
   await sleep(150);
   contents.sendInputEvent({ type: 'mouseDown', ...gripAt, button: 'left', clickCount: 1 });
   await sleep(60);
   // Anchored top-left, so the grip is below the anchor and dragging down grows.
   for (const dy of [30, 60, 90]) {
     pointer = { x: pointer.x, y: beforeGrip.y + gripAt.y + dy };
     contents.sendInputEvent({ type: 'mouseMove', ...gripAt });
     await sleep(60);
   }
   contents.sendInputEvent({ type: 'mouseUp', ...gripAt, button: 'left', clickCount: 1 });
   await sleep(300);
   const afterGrip = mirror.getBounds();
   const grew =
     Math.max(afterGrip.width, afterGrip.height) > Math.max(beforeGrip.width, beforeGrip.height);
   check(
     'dragging the grip away from the anchor grows the mirror',
     grew,
     `${beforeGrip.width}x${beforeGrip.height} → ${afterGrip.width}x${afterGrip.height}`,
   );
   check(
     'and it stays inside Codex’s [100, 400] clamp',
     Math.max(afterGrip.width, afterGrip.height) <= 400 &&
       Math.min(afterGrip.width, afterGrip.height) >= 1,
     `${afterGrip.width}x${afterGrip.height}`,
   );
   check(
     'and it is still on the corner it was resting on',
     afterGrip.x === appResized.x + 24 && afterGrip.y === appResized.y + 24,
     JSON.stringify(afterGrip),
   );

   // ── hide dismisses it ─────────────────────────────────────────────────────
   //
   // The controls are click-through until the *main-side* pointer says the
   // pointer is on the mirror — that is the whole point of the hover poll — so
   // the fixture has to move its pointer as well as send the page event. It
   // did not, and the grip drag happened to leave the pointer one point past
   // the tile's bottom edge, so the window was still ignoring mouse events and
   // the controls still had `pointer-events: none` when the click arrived.
   const tile = mirror.getBounds();
   pointer = { x: tile.x + tile.width / 2, y: tile.y + tile.height / 2 };
   await sleep(200);
   const visibleNow = await contents.executeJavaScript(
     "document.getElementById('controls').dataset.visible",
   );
   check('the controls are on before the click', visibleNow === '1', `data-visible=${visibleNow}`);
   const hideAt = await contents.executeJavaScript(
     '(() => { const b = document.querySelector(\'[data-control="hide"]\'); const r = b.getBoundingClientRect(); return { x: Math.round(r.x + r.width / 2), y: Math.round(r.y + r.height / 2) }; })()',
   );
   pointer = { x: tile.x + hideAt.x, y: tile.y + hideAt.y };
   contents.sendInputEvent({ type: 'mouseMove', ...hideAt });
   await sleep(120);
   contents.sendInputEvent({ type: 'mouseDown', ...hideAt, button: 'left', clickCount: 1 });
   contents.sendInputEvent({ type: 'mouseUp', ...hideAt, button: 'left', clickCount: 1 });
   await sleep(400);
   check('the hide control dismisses the mirror', pip.isVisible() === false);

   pip.destroyAll();
   for (const w of created) if (!w.isDestroyed()) w.destroy();
   parent.destroy();
   console.log(failures === 0 ? '\nPIP INTERACTION OK' : `\n${failures} check(s) failed`);
   app.exit(failures === 0 ? 0 : 1);
 });
	#!/usr/bin/env node
	// Real-window smoke for the picture-in-picture mirror's interaction.
	//
	// The unit tests drive a fake window, so they prove the physics and the state
	// machine and nothing about Electron. This runs the real thing: a real parent
	// window, a real child panel, the real preload and renderer, and real input
	// events delivered to the page. Everything it asserts is something that was
	// either measured on this machine or read out of Codex's binary.
	//
	// Run: npm --workspace @maka/desktop run build:main && npm --workspace @maka/desktop run build:overlay
	// && npx electron scripts/pip-interaction-smoke.mjs
	import { app, BrowserWindow, screen } from 'electron';
	import { fileURLToPath } from 'node:url';
	import { dirname, join } from 'node:path';

	const here = dirname(fileURLToPath(import.meta.url));
	const dist = join(here, '..', 'apps', 'desktop', 'dist');
	const { createComputerUsePipController } = await import(
	join(dist, 'main', 'computer-use', 'pip-window.js')
	);

	const PNG_1X1 =
	'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==';

	let failures = 0;
	const check = (label, pass, detail) => {
	if (!pass) failures += 1;
	console.log(`[${pass ? 'PASS' : 'FAIL'}] ${label}${detail ? ` — ${detail}` : ''}`);
	};
	const sleep = (ms) => new Promise((r) => setTimeout(r, ms));

	/**
	* Poll until the mirror stops moving, so assertions never race the spring.
	*
	* Requires several identical samples rather than two: the settle is
	* overdamped, so its last stretch moves less than a point per frame and two
	* consecutive reads can round to the same value while it is still travelling.
	* Two was enough to report the mirror had arrived a point short of the anchor.
	*/
	async function settled(win, timeoutMs = 4000) {
	let last = null;
	let same = 0;
	const deadline = Date.now() + timeoutMs;
	while (Date.now() < deadline) {
	const bounds = win.getBounds();
	if (last && bounds.x === last.x && bounds.y === last.y) {
	if (++same >= 6) return bounds;
	} else {
	same = 0;
	}
	last = bounds;
	await sleep(60);
	}
	return win.getBounds();
	}

	app.whenReady().then(async () => {
	const parent = new BrowserWindow({ x: 120, y: 120, width: 1000, height: 700, show: true });
	await parent.loadURL('data:text/html,<body style="background:%23202028"></body>');
	await sleep(300);

	// The pointer the drag reads. Driven from here so the gesture is exact
	// rather than dependent on whatever the real mouse happens to be doing.
	let pointer = { x: 0, y: 0 };
	let pip = null;
	const created = [];

	pip = createComputerUsePipController({
	resolveAnchorRect: () => parent.getBounds(),
	subscribeAnchorChanges: (cb) => {
	parent.on('move', cb);
	parent.on('resize', cb);
	return () => {
	parent.off('move', cb);
	parent.off('resize', cb);
	};
	},
	resolveParentWindow: () => parent,
	cursorPoint: () => pointer,
	workAreaFor: (rect) => screen.getDisplayMatching(rect).workArea,
	preloadPath: join(dist, 'overlay', 'pip-preload.cjs'),
	htmlPath: join(dist, 'overlay', 'pip.html'),
	});

	pip.present({
	sessionId: 'smoke',
	base64: PNG_1X1,
	mimeType: 'image/png',
	widthPx: 1000,
	heightPx: 800,
	});
	await sleep(900);

	const mirror = BrowserWindow.getAllWindows().find((w) => w !== parent);
	if (!mirror) {
	console.log('the mirror never opened; stopping.');
	app.exit(2);
	return;
	}
	created.push(mirror);

	// ── the two properties the child-window fix bought ────────────────────────
	check('the mirror does not float above other apps', mirror.isAlwaysOnTop() === false);
	// `parent.isFocused()` is about the whole app being frontmost, which a
	// headless-ish run cannot promise. What the mirror owes is narrower: it must
	// never be the focused window itself.
	check('the mirror never took focus', mirror.isFocused() === false);
	check('the mirror is the app window’s child', mirror.getParentWindow() === parent);

	const seated = mirror.getBounds();
	const app0 = parent.getBounds();
	check(
	'it is seated on the app window’s bottom-right, inset 24pt',
	seated.x === app0.x + app0.width - seated.width - 24 &&
	seated.y === app0.y + app0.height - seated.height - 24,
	`${JSON.stringify(seated)} against ${JSON.stringify(app0)}`,
	);

	// ── a pure move is carried by the window server, not by us ────────────────
	parent.setBounds({ ...app0, x: app0.x + 160 });
	await sleep(400);
	check(
	'moving the app carries the mirror with it',
	mirror.getBounds().x === seated.x + 160,
	`${seated.x} → ${mirror.getBounds().x}`,
	);

	// ── click-through until pointed at ────────────────────────────────────────
	const contents = mirror.webContents;
	const inMirror = (dx, dy) => ({ x: dx, y: dy });
	// Hover is decided from the real pointer on the main side, so the test moves
	// the pointer rather than injecting a page event.
	const mid = mirror.getBounds();
	pointer = { x: mid.x + 40, y: mid.y + 40 };
	await sleep(300);
	const hovered = await contents.executeJavaScript(
	"document.getElementById('controls').dataset.visible",
	);
	check('the controls appear on hover', hovered === '1', `data-visible=${hovered}`);

	// ── throw it at the opposite corner ───────────────────────────────────────
	const before = mirror.getBounds();
	// Press in the middle of the tile, clear of both the controls (top-right,
	// 20x20 inset 8) and the resize grip (20x20 inset 8, on whichever corner is
	// opposite the anchor). This used to press at (20, 20), which is inside the
	// grip's 8..28 box: the grip stops the event propagating, so no drag ever
	// started, the moves ran as a resize instead, and the five checks after it
	// failed on the cascade. This half of the script had never completed.
	const GRAB = { x: Math.round(before.width / 2), y: Math.round(before.height / 2) };
	pointer = { x: before.x + GRAB.x, y: before.y + GRAB.y };
	contents.sendInputEvent({
	type: 'mouseDown',
	...inMirror(GRAB.x, GRAB.y),
	button: 'left',
	clickCount: 1,
	});
	await sleep(60);
	const appNow = parent.getBounds();
	// Three samples so the tracker sees real motion, ending fast enough that the
	// release reads as a throw rather than a place.
	for (const [fx, fy] of [
	[0.7, 0.7],
	[0.4, 0.4],
	[0.22, 0.18],
	]) {
	pointer = { x: appNow.x + appNow.width * fx, y: appNow.y + appNow.height * fy };
	contents.sendInputEvent({ type: 'mouseMove', ...inMirror(GRAB.x, GRAB.y) });
	await sleep(30);
	}
	contents.sendInputEvent({
	type: 'mouseUp',
	...inMirror(GRAB.x, GRAB.y),
	button: 'left',
	clickCount: 1,
	});

	const landed = await settled(mirror);
	check(
	'thrown up and left, it settles on the top-left anchor',
	landed.x === appNow.x + 24 && landed.y === appNow.y + 24,
	`${JSON.stringify(landed)} against app ${JSON.stringify(appNow)}`,
	);
	check(
	'and the controller agrees which corner that is',
	pip.currentAlignment() === 'top-left',
	pip.currentAlignment(),
	);

	// ── a resize returns it to the corner the user chose ──────────────────────
	parent.setBounds({ ...appNow, width: appNow.width - 200, height: appNow.height - 120 });
	await sleep(500);
	const afterResize = mirror.getBounds();
	const appResized = parent.getBounds();
	check(
	'a resize keeps it on the chosen corner, not the default one',
	afterResize.x === appResized.x + 24 && afterResize.y === appResized.y + 24,
	JSON.stringify(afterResize),
	);

	// ── the grip resizes it, away from whichever corner it rests on ──────────
	const beforeGrip = mirror.getBounds();
	const gripAt = await contents.executeJavaScript(
	"(() => { const r = document.getElementById('resize').getBoundingClientRect(); return { x: Math.round(r.x + r.width / 2), y: Math.round(r.y + r.height / 2) }; })()",
	);
	// The tile is anchored top-left, so its top-left corner is the one held
	// still and the grip has to be on the bottom-right — the corner that moves.
	// Pinning the grip to the tile's top-left, which is what the CSS did before
	// main started sending the alignment down, put the handle on the corner the
	// gesture holds: the pointer left it on the first pixel of the drag.
	check(
	'the grip sits on the corner opposite the anchor',
	gripAt.x > beforeGrip.width / 2 && gripAt.y > beforeGrip.height / 2,
	`grip at ${JSON.stringify(gripAt)} in a ${beforeGrip.width}x${beforeGrip.height} tile ` +
	`anchored ${pip.currentAlignment()}`,
	);
	pointer = { x: beforeGrip.x + gripAt.x, y: beforeGrip.y + gripAt.y };
	contents.sendInputEvent({ type: 'mouseMove', ...gripAt });
	await sleep(150);
	contents.sendInputEvent({ type: 'mouseDown', ...gripAt, button: 'left', clickCount: 1 });
	await sleep(60);
	// Anchored top-left, so the grip is below the anchor and dragging down grows.
	for (const dy of [30, 60, 90]) {
	pointer = { x: pointer.x, y: beforeGrip.y + gripAt.y + dy };
	contents.sendInputEvent({ type: 'mouseMove', ...gripAt });
	await sleep(60);
	}
	contents.sendInputEvent({ type: 'mouseUp', ...gripAt, button: 'left', clickCount: 1 });
	await sleep(300);
	const afterGrip = mirror.getBounds();
	const grew =
	Math.max(afterGrip.width, afterGrip.height) > Math.max(beforeGrip.width, beforeGrip.height);
	check(
	'dragging the grip away from the anchor grows the mirror',
	grew,
	`${beforeGrip.width}x${beforeGrip.height} → ${afterGrip.width}x${afterGrip.height}`,
	);
	check(
	'and it stays inside Codex’s [100, 400] clamp',
	Math.max(afterGrip.width, afterGrip.height) <= 400 &&
	Math.min(afterGrip.width, afterGrip.height) >= 1,
	`${afterGrip.width}x${afterGrip.height}`,
	);
	check(
	'and it is still on the corner it was resting on',
	afterGrip.x === appResized.x + 24 && afterGrip.y === appResized.y + 24,
	JSON.stringify(afterGrip),
	);

	// ── hide dismisses it ─────────────────────────────────────────────────────
	//
	// The controls are click-through until the main-side pointer says the
	// pointer is on the mirror — that is the whole point of the hover poll — so
	// the fixture has to move its pointer as well as send the page event. It
	// did not, and the grip drag happened to leave the pointer one point past
	// the tile's bottom edge, so the window was still ignoring mouse events and
	// the controls still had `pointer-events: none` when the click arrived.
	const tile = mirror.getBounds();
	pointer = { x: tile.x + tile.width / 2, y: tile.y + tile.height / 2 };
	await sleep(200);
	const visibleNow = await contents.executeJavaScript(
	"document.getElementById('controls').dataset.visible",
	);
	check('the controls are on before the click', visibleNow === '1', `data-visible=${visibleNow}`);
	const hideAt = await contents.executeJavaScript(
	'(() => { const b = document.querySelector(\'[data-control="hide"]\'); const r = b.getBoundingClientRect(); return { x: Math.round(r.x + r.width / 2), y: Math.round(r.y + r.height / 2) }; })()',
	);
	pointer = { x: tile.x + hideAt.x, y: tile.y + hideAt.y };
	contents.sendInputEvent({ type: 'mouseMove', ...hideAt });
	await sleep(120);
	contents.sendInputEvent({ type: 'mouseDown', ...hideAt, button: 'left', clickCount: 1 });
	contents.sendInputEvent({ type: 'mouseUp', ...hideAt, button: 'left', clickCount: 1 });
	await sleep(400);
	check('the hide control dismisses the mirror', pip.isVisible() === false);

	pip.destroyAll();
	for (const w of created) if (!w.isDestroyed()) w.destroy();
	parent.destroy();
	console.log(failures === 0 ? '\nPIP INTERACTION OK' : `\n${failures} check(s) failed`);
	app.exit(failures === 0 ? 0 : 1);
	});