open-design/e2e/ui/app-design-files.test.ts
Patrick A 32fd5286b5
chore(e2e): improve test framework quality (#2305)
* chore(e2e): improve test framework quality

- Add lib/timeouts.ts with CI-scaled short/medium/long/xlong constants
- Add lib/playwright/mock-factory.ts to centralise standard localStorage,
  /api/agents, and /api/app-config mock setup; migrate critical-smoke and
  workspace-keyboard-flows to use applyStandardMocks()
- Delete empty lib/shared.ts placeholder
- Replace waitFor({ state: 'detached' }).catch(() => {}) with
  waitFor({ state: 'hidden' }) in all UI tests; 'hidden' resolves
  immediately when the element was never in the DOM, eliminating the
  silent error-swallowing catch
- Remove redundant .catch(() => false) from all isVisible() call sites
  since isVisible() never throws in Playwright
- Convert .waitFor().then(() => true).catch(() => false) guards in
  openDesignFile() to explicit try/catch blocks for clarity
- Simplify sendPrompt() in app.test.ts: replace the 3-attempt manual
  retry loop with a single fill + pressSequentially fallback; the core
  workaround for contenteditable unreliability is preserved but the
  loop structure is gone

* fix(e2e): guard routeMockAgents to GET only

routeMockAgents was intercepting all HTTP methods and returning the mock
fixture, silently swallowing any agent mutation requests. Mirror the
GET-only guard from routeAppConfig so writes fall through to the daemon.

* fix(e2e): address code review findings

- sendPrompt() in app.test.ts, workspace-keyboard-flows.test.ts,
  app-restoration.test.ts: drop fill() (unreliable on contenteditable,
  inputValue() always returns '' for them) and go straight to
  pressSequentially(), which types key-by-key and is authoritative
- Import T from timeouts.ts in app.test.ts and use T.short for the
  input/button waits, making the timeouts module non-dead

* fix(e2e): resolve adversarial review findings

- Revert sendPrompt to fill(): chat-composer-input is a textarea, not
  contenteditable; fill() is atomic and ~60x faster than pressSequentially
- Use T.medium in all waitForLoadingToClear calls: CI workers scale this
  to 20s automatically via the CI env var, eliminating cold-runner flakes
- Add T import to 6 files that needed it for T.medium
- Fix openDesignFile try/catch scope in app-manual-edit: previously the
  catch block only caught waitFor but click/expect errors were also swallowed;
  now only waitFor is inside try, real interaction failures propagate
- Fix regex escaping: .replace('.', '\\.') -> .replace(/\./g, '\\.') in
  app-manual-edit and app-design-files to handle multi-dot filenames
- Migrate entry-chrome-flows.test.ts to applyStandardMocks: it had the
  identical 3-call setup pattern as the factory but was not migrated
- Add GET method guard to project-management-flows app-config route handler,
  matching the pattern used by every other route handler in the suite
- Remove no-op 'as const' from timeouts.ts: Math.ceil returns number,
  not a literal, so the assertion had no effect
- Update e2e/AGENTS.md: remove deleted lib/shared.ts entry, document
  lib/timeouts.ts and lib/playwright/mock-factory.ts

* fix(e2e): scope openDesignFile try/catch to waitFor only

Move click and expect(preview).toBeVisible() outside the catch block so
that a regression in either open path (tab-click or file-list fallback)
fails loudly instead of being silently absorbed. The try now wraps only
the fileTabButton.waitFor existence probe; the subsequent click and final
assertion are unconditional.

---------

Co-authored-by: Patrick A <186436799+eefynet@users.noreply.github.com>
Co-authored-by: Patrick A <259201958+eefynet@users.noreply.github.com>
2026-05-23 00:24:32 +08:00

466 lines
16 KiB
TypeScript

import { expect, test } from '@playwright/test';
import type { Locator, Page, Request, Response } from '@playwright/test';
import { automatedUiScenarios } from '@/playwright/resources';
import type { UiScenario } from '@/playwright/resources';
import { T } from '@/timeouts';
const STORAGE_KEY = 'open-design:config';
test.describe.configure({ timeout: 30_000 });
test.beforeEach(async ({ page }) => {
await page.addInitScript((key) => {
window.localStorage.setItem(
key,
JSON.stringify({
mode: 'daemon',
apiKey: '',
baseUrl: 'https://api.anthropic.com',
model: 'claude-sonnet-4-5',
agentId: 'mock',
skillId: null,
designSystemId: null,
onboardingCompleted: true,
agentModels: {},
privacyDecisionAt: 1,
telemetry: { metrics: false, content: false, artifactManifest: false },
}),
);
}, STORAGE_KEY);
await page.route('**/api/app-config', async (route) => {
if (route.request().method() !== 'GET') {
await route.continue();
return;
}
await route.fulfill({
json: {
config: {
onboardingCompleted: true,
agentId: 'mock',
skillId: null,
designSystemId: null,
agentModels: {},
privacyDecisionAt: 1,
telemetry: { metrics: false, content: false, artifactManifest: false },
},
},
});
});
});
const designFileFlows = new Set([
'design-files-upload',
'design-files-delete',
'design-files-tab-persistence',
'uploaded-image-renders-in-preview',
'python-source-preview',
]);
for (const entry of automatedUiScenarios().filter((scenario) => designFileFlows.has(scenario.flow ?? ''))) {
test(`${entry.id}: ${entry.title}`, async ({ page }) => {
await page.route('**/api/agents', async (route) => {
await route.fulfill({
json: {
agents: [
{
id: 'mock',
name: 'Mock Agent',
bin: 'mock-agent',
available: true,
version: 'test',
models: [{ id: 'default', label: 'Default' }],
},
],
},
});
});
await gotoEntryHome(page);
await createProject(page, entry);
await expectWorkspaceReady(page);
if (entry.flow === 'design-files-upload') {
await runDesignFilesUploadFlow(page);
return;
}
if (entry.flow === 'design-files-delete') {
await runDesignFilesDeleteFlow(page);
return;
}
if (entry.flow === 'design-files-tab-persistence') {
await runDesignFilesTabPersistenceFlow(page);
return;
}
if (entry.flow === 'uploaded-image-renders-in-preview') {
await runUploadedImageRendersInPreviewFlow(page, entry);
return;
}
if (entry.flow === 'python-source-preview') {
await runPythonSourcePreviewFlow(page, entry);
}
});
}
async function createProject(page: Page, entry: UiScenario) {
await createProjectNameOnly(page, entry);
await page.getByTestId('create-project').click();
}
async function createProjectNameOnly(page: Page, entry: UiScenario) {
await openNewProjectModal(page);
if (entry.create.tab) {
await page.getByTestId(`new-project-tab-${entry.create.tab}`).click();
}
await page.getByTestId('new-project-name').fill(entry.create.projectName);
}
async function gotoEntryHome(page: Page) {
await page.goto('/', { waitUntil: 'domcontentloaded' });
await waitForLoadingToClear(page);
const privacyDialog = page.getByRole('dialog').filter({ hasText: 'Help us improve Open Design' });
if (await privacyDialog.isVisible()) {
await privacyDialog.getByRole('button', { name: /not now/i }).click();
await expect(privacyDialog).toHaveCount(0);
}
await expect(page.getByTestId('home-hero')).toBeVisible();
await expect(page.getByTestId('home-hero-input')).toBeVisible();
}
async function openNewProjectModal(page: Page) {
await page.getByTestId('entry-nav-new-project').click();
await expect(page.getByTestId('new-project-modal')).toBeVisible();
await expect(page.getByTestId('new-project-panel')).toBeVisible();
}
async function expectWorkspaceReady(page: Page) {
await waitForLoadingToClear(page);
await expect(page).toHaveURL(/\/projects\//);
await expect(page.getByTestId('chat-composer')).toBeVisible();
await expect(page.getByTestId('chat-composer-input')).toBeVisible();
await expect(page.getByTestId('file-workspace')).toBeVisible();
}
async function getCurrentProjectContext(page: Page): Promise<{ projectId: string; conversationId: string }> {
const current = new URL(page.url());
const [, projects, projectId, maybeConversations, conversationId] = current.pathname.split('/');
if (projects !== 'projects' || !projectId) {
throw new Error(`unexpected project route: ${current.pathname}`);
}
if (maybeConversations === 'conversations' && conversationId) {
return { projectId, conversationId };
}
const response = await page.request.get(`/api/projects/${projectId}/conversations`);
expect(response.ok()).toBeTruthy();
const { conversations } = (await response.json()) as {
conversations: Array<{ id: string; updatedAt: number }>;
};
const active = [...conversations].sort((a, b) => b.updatedAt - a.updatedAt)[0];
if (!active) throw new Error(`no conversations found for project ${projectId}`);
return { projectId, conversationId: active.id };
}
async function seedProjectFile(
page: Page,
projectId: string,
name: string,
content: string,
encoding?: 'base64',
artifactManifest?: Record<string, unknown>,
) {
const response = await page.request.post(
`/api/projects/${projectId}/files`,
{
data: {
name,
content,
...(encoding ? { encoding } : {}),
...(artifactManifest ? { artifactManifest } : {}),
},
timeout: 15_000,
},
);
expect(response.ok()).toBeTruthy();
}
async function seedHtmlArtifact(page: Page, projectId: string, fileName: string, content: string) {
const resp = await page.request.post(
`/api/projects/${projectId}/files`,
{
data: {
name: fileName,
content,
artifactManifest: {
version: 1,
kind: 'html',
title: fileName,
entry: fileName,
renderer: 'html',
exports: ['html'],
},
},
timeout: 15_000,
},
);
expect(resp.ok()).toBeTruthy();
}
async function listProjectFilesFromApi(
page: Page,
projectId: string,
): Promise<Array<{ name: string; kind: string }>> {
const response = await page.request.get(`/api/projects/${projectId}/files`);
expect(response.ok()).toBeTruthy();
const { files } = (await response.json()) as { files: Array<{ name: string; kind: string }> };
return files;
}
async function expectProjectFileToContain(
page: Page,
projectId: string,
fileName: string,
expected: string,
) {
await expect
.poll(async () => {
const response = await page.request.get(`/api/projects/${projectId}/files/${fileName}`);
if (!response.ok()) return '';
return response.text();
}, { timeout: 15_000 })
.toContain(expected);
}
async function expectScenarioFiles(
page: Page,
entry: UiScenario,
projectId: string,
) {
if (!entry.expectedFiles?.length) return;
const files = await listProjectFilesFromApi(page, projectId);
for (const expectedFile of entry.expectedFiles) {
const actual = files.find((file) => file.name === expectedFile.name);
expect(actual, `missing expected file ${expectedFile.name}`).toBeDefined();
if (expectedFile.kind) {
expect(actual?.kind).toBe(expectedFile.kind);
}
if (expectedFile.previewText) {
await expectProjectFileToContain(page, projectId, expectedFile.name, expectedFile.previewText);
}
}
}
async function expectScenarioPreviewText(page: Page, entry: UiScenario) {
if (!entry.expectedPreviewText) return;
const frame = page.frameLocator('[data-testid="artifact-preview-frame"]');
await expect(frame.getByText(entry.expectedPreviewText, { exact: false })).toBeVisible();
}
async function expectScenarioProjectState(
page: Page,
entry: UiScenario,
projectId: string,
) {
await expectScenarioFiles(page, entry, projectId);
await expectScenarioPreviewText(page, entry);
}
async function expectProjectFilesToIncludeSuffixes(
page: Page,
projectId: string,
suffixes: string[],
) {
await expect
.poll(async () => {
const names = (await listProjectFilesFromApi(page, projectId)).map((file) => file.name);
return suffixes.every((suffix) => names.some((name) => name.endsWith(suffix)));
})
.toBe(true);
}
async function openDesignFile(page: Page, fileName: string) {
const preview = page.getByTestId('artifact-preview-frame');
if (await preview.isVisible()) return;
const fileTab = page.getByRole('tab', { name: new RegExp(fileName.replace(/\./g, '\\.'), 'i') });
if (await fileTab.isVisible()) {
await fileTab.click();
return;
}
await page.getByRole('button', { name: new RegExp(fileName.replace(/\./g, '\\.')) }).click();
await page.getByTestId('design-file-preview').getByRole('button', { name: 'Open' }).click();
}
async function waitForLoadingToClear(page: Page) {
await page.getByText('Loading Open Design…').waitFor({ state: 'hidden', timeout: T.medium });
}
async function runUploadedImageRendersInPreviewFlow(page: Page, entry: UiScenario) {
const { projectId } = await getCurrentProjectContext(page);
const pngBase64 = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO5W6McAAAAASUVORK5CYII=';
await seedProjectFile(page, projectId, 'brand.png', pngBase64, 'base64');
await seedHtmlArtifact(
page,
projectId,
'image-preview.html',
'<!doctype html><html><body><main><h1>Image Preview</h1><img alt="Brand logo" src="brand.png"></main></body></html>',
);
await page.reload();
await openDesignFile(page, 'image-preview.html');
const image = page.frameLocator('[data-testid="artifact-preview-frame"]').getByRole('img', { name: 'Brand logo' });
await expect(image).toBeVisible();
await expect
.poll(async () => image.evaluate((img: HTMLImageElement) => img.complete && img.naturalWidth > 0))
.toBe(true);
await expectScenarioProjectState(page, entry, projectId);
}
async function runPythonSourcePreviewFlow(page: Page, entry: UiScenario) {
const { projectId } = await getCurrentProjectContext(page);
await seedProjectFile(page, projectId, 'app.py', 'def greet():\n return "hello from python"\n');
await page.reload();
await openDesignFile(page, 'app.py');
await expect(page.locator('.code-viewer')).toContainText('def greet');
await expect(page.locator('.code-viewer')).toContainText('hello from python');
await expectScenarioFiles(page, entry, projectId);
}
async function runDesignFilesUploadFlow(page: Page) {
const { projectId } = await getCurrentProjectContext(page);
await page.getByTestId('design-files-upload-input').setInputFiles({
name: 'moodboard.png',
mimeType: 'image/png',
buffer: Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO5W6McAAAAASUVORK5CYII=',
'base64',
),
});
await expect(page.getByRole('tab', { name: /moodboard\.png/i })).toBeVisible();
await page.getByTestId('design-files-tab').click();
const fileRow = page.locator('[data-testid^="design-file-row-"]', {
hasText: 'moodboard.png',
});
await expect(fileRow).toBeVisible();
const nameBtn = fileRow.getByRole('button').first();
await nameBtn.click();
const preview = page.getByTestId('design-file-preview');
await expect(preview).toBeVisible();
await expect(preview.getByText(/moodboard\.png/i)).toBeVisible();
await nameBtn.dblclick();
await expect(page.getByRole('tab', { name: /moodboard\.png/i })).toBeVisible();
await expectProjectFilesToIncludeSuffixes(page, projectId, ['moodboard.png']);
}
async function runDesignFilesDeleteFlow(page: Page) {
const { projectId } = await getCurrentProjectContext(page);
page.on('dialog', async (dialog) => {
await dialog.accept();
});
const pngBytes = Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO5W6McAAAAASUVORK5CYII=',
'base64',
);
await page.getByTestId('design-files-upload-input').setInputFiles({
name: 'keep-me.png',
mimeType: 'image/png',
buffer: pngBytes,
});
await expect(page.getByRole('tab', { name: /keep-me\.png/i })).toBeVisible();
await page.getByTestId('design-files-upload-input').setInputFiles({
name: 'trash-me.png',
mimeType: 'image/png',
buffer: pngBytes,
});
await expect(page.getByRole('tab', { name: /trash-me\.png/i })).toBeVisible();
await page.getByTestId('design-files-tab').click();
const fileRow = page.locator('[data-testid^="design-file-row-"]', {
hasText: 'trash-me.png',
});
await expect(fileRow).toBeVisible();
await fileRow.hover();
await fileRow.locator('[data-testid^="design-file-menu-"]').click();
await expect(page.getByTestId('design-file-menu-popover')).toBeVisible();
await page.locator('[data-testid^="design-file-delete-"]').click();
await expect(fileRow).toHaveCount(0);
await expect(page.getByRole('tab', { name: /trash-me\.png/i })).toHaveCount(0);
await expect(page.getByTestId('design-files-tab')).toHaveAttribute('aria-selected', 'true');
await expect(page.getByRole('tab', { name: /keep-me\.png/i })).toBeVisible();
await expect
.poll(async () => {
const names = (await listProjectFilesFromApi(page, projectId)).map((file) => file.name);
return (
names.length === 1 &&
names.some((name) => name.endsWith('keep-me.png')) &&
names.every((name) => !name.endsWith('trash-me.png'))
);
})
.toBe(true);
}
async function runDesignFilesTabPersistenceFlow(page: Page) {
const { projectId } = await getCurrentProjectContext(page);
const pngBytes = Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO5W6McAAAAASUVORK5CYII=',
'base64',
);
await page.getByTestId('design-files-upload-input').setInputFiles({
name: 'first-tab.png',
mimeType: 'image/png',
buffer: pngBytes,
});
await expect(page.getByRole('tab', { name: /first-tab\.png/i })).toBeVisible();
await page.getByTestId('design-files-upload-input').setInputFiles({
name: 'second-tab.png',
mimeType: 'image/png',
buffer: pngBytes,
});
const firstTab = page.getByRole('tab', { name: /first-tab\.png/i });
const secondTab = page.getByRole('tab', { name: /second-tab\.png/i });
await expect(firstTab).toBeVisible();
await expect(secondTab).toBeVisible();
await firstTab.click();
await expect(firstTab).toHaveAttribute('aria-selected', 'true');
await expect(secondTab).toHaveAttribute('aria-selected', 'false');
await page.reload();
const restoredFirstTab = page.getByRole('tab', { name: /first-tab\.png/i });
await expect(restoredFirstTab).toBeVisible();
await expect(restoredFirstTab).toHaveAttribute('aria-selected', 'true');
// The refreshed workspace restores the active file tab, while other project files
// remain available from the Design Files list until the user reopens them.
await page.getByTestId('design-files-tab').click();
const secondFileRow = page.locator('[data-testid^="design-file-row-"]', {
hasText: 'second-tab.png',
});
await expect(secondFileRow).toBeVisible();
await secondFileRow.getByRole('button').first().click();
await page.getByTestId('design-file-preview').getByRole('button', { name: 'Open' }).click();
const restoredSecondTab = page.getByRole('tab', { name: /second-tab\.png/i });
await expect(restoredSecondTab).toBeVisible();
await expect(restoredSecondTab).toHaveAttribute('aria-selected', 'true');
await expect(restoredFirstTab).toHaveAttribute('aria-selected', 'false');
await expectProjectFilesToIncludeSuffixes(page, projectId, ['first-tab.png', 'second-tab.png']);
}
function homeDesignCard(page: Page, name: string): Locator {
return page.locator('.design-card', {
has: page.locator('.design-card-name', { hasText: name }),
});
}