vndangkhoa/open-design
1
0
Fork 0
mirror of https://github.com/nexu-io/open-design.git synced 2026-06-01 03:14:35 +07:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

test: harden e2e smoke and release reports (#1140)

Browse source 
* test: harden e2e inspect specs

* test: wire e2e release reports

* chore: bump packaged beta base to 0.6.1

* test: run release smoke vitest directly

* test: add suite-owned tools-dev lifecycle

* ci: harden stable release packaging

* fix(release,e2e): gate stable signing on verify and harden suite cleanup

- restore `needs: [metadata, verify]` on the stable release `build_mac`,
  `build_mac_intel`, `build_win`, and `build_linux` jobs so Apple
  signing/notarization and Windows release builds cannot run before
  pnpm guard, typecheck, and layout checks complete on the metadata commit.
- in `runToolsDevSuite`, drop the `started` flag and always attempt
  `stopToolsDevWeb` in `finally`; record stop errors in diagnostics, and
  when the test body succeeded, escalate the stop failure to the suite
  result and rethrow — so orphan daemon/web processes from an interrupted
  `startToolsDevWeb` or a broken shutdown can no longer pass silently.

Addresses PR #1140 review feedback from lefarcen and mrcfps.
This commit is contained in:
PerishFire 2026-05-11 13:11:16 +08:00 committed by GitHub
parent 1dc0224599
commit 976edaf38e
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
32 changed files with 2501 additions and 332 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.github/scripts/release/github/stable-notes.sh vendored
View file

@ -17,12 +17,13 @@ cat > "$notes_file" <<EOF
- R2 metadata: $public_origin/$RELEASE_CHANNEL/latest/metadata.json
- E2E report: $public_origin/$RELEASE_CHANNEL/versions/$RELEASE_VERSION/report.zip
- mac signed/notarized: $RELEASE_SIGNED
- mac x64 signed/notarized: ${MAC_INTEL_SIGNED:-false}
- windows signed: false
- branch: $BRANCH_NAME
- commit: $GITHUB_SHA
See [CHANGELOG.md](https://github.com/${GITHUB_REPOSITORY}/blob/$VERSION_TAG/CHANGELOG.md) for the full release notes.
This stable release ships mac arm64 DMG/update ZIP, Windows x64 NSIS installer assets, checksums, updater feed files, and a zipped packaged e2e spec report. Linux AppImage packaging remains optional through the stable Linux lane.
This stable release ships mac arm64/x64 DMG and ZIP assets, Windows x64 NSIS installer assets, checksums, updater feed files, and a zipped packaged e2e spec report. Linux AppImage packaging remains optional through the stable Linux lane.
EOF
echo "notes_file=$notes_file" >> "$GITHUB_OUTPUT"

2
.github/scripts/release/r2/publish.sh vendored
View file

@ -266,7 +266,7 @@ if (enabled("ENABLE_MAC_INTEL")) {
arch: "x64",
enabled: true,
feed: null,
signed: false,
signed: env.MAC_INTEL_SIGNED === "true",
artifacts: {
dmg: fileEntry("mac-intel", env.MAC_INTEL_DMG, "application/x-apple-diskimage"),
zip: fileEntry("mac-intel", env.MAC_INTEL_ZIP, "application/zip"),

2
.github/scripts/release/r2/summary.sh vendored
View file

@ -48,7 +48,7 @@ const platforms = {
},
macIntel: {
enabled: enabled("ENABLE_MAC_INTEL"),
signed: false,
signed: env.MAC_INTEL_SIGNED === "true",
},
};

9
.github/scripts/release/r2/verify.sh vendored
View file

@ -48,6 +48,7 @@ downloaded_metadata="$RUNNER_TEMP/metadata.json"
curl -fsSL "$R2_METADATA_URL?run=${GITHUB_RUN_ID:-local}" -o "$downloaded_metadata"
DOWNLOADED_METADATA="$downloaded_metadata" \
EXPECTED_CHANNEL="$RELEASE_CHANNEL" \
EXPECTED_MAC_INTEL_SIGNED="${MAC_INTEL_SIGNED:-}" \
EXPECTED_NIGHTLY_NUMBER="${NIGHTLY_NUMBER:-}" \
EXPECTED_RELEASE_VERSION="$RELEASE_VERSION" \
node --input-type=module <<'NODE'
@ -73,6 +74,12 @@ if (metadata.channel === "beta") {
}
}
}
if (process.env.EXPECTED_MAC_INTEL_SIGNED !== "") {
const expected = process.env.EXPECTED_MAC_INTEL_SIGNED === "true";
if (metadata.platforms?.macIntel?.signed !== expected) {
throw new Error("unexpected metadata platforms.macIntel.signed: " + metadata.platforms?.macIntel?.signed);
}
}
NODE
if [ "$ENABLE_MAC" = "true" ]; then
@ -98,6 +105,7 @@ if [ "$ENABLE_MAC" = "true" ]; then
fi
require_report_file "mac/manifest.json"
require_report_file "mac/screenshots/open-design-mac-smoke.png"
require_report_file "mac/suite-result.json"
require_report_file "mac/tools-pack.json"
require_report_file "mac/tools-pack.log"
require_report_file "mac/vitest.log"
@ -117,6 +125,7 @@ if [ "$ENABLE_WIN" = "true" ]; then
curl -fsSI "$R2_WIN_INSTALLER_URL" >/dev/null
require_report_file "win/manifest.json"
require_report_file "win/screenshots/open-design-win-smoke.png"
require_report_file "win/suite-result.json"
require_report_file "win/tools-pack.json"
require_report_file "win/vitest.log"
fi

46
.github/workflows/release-beta.yml vendored
View file

@ -249,28 +249,15 @@ jobs:
- name: Smoke beta mac packaged runtime
working-directory: e2e
env:
OD_PACKAGED_E2E_BUILD_JSON_PATH: ${{ runner.temp }}/mac-tools-pack-build.json
OD_PACKAGED_E2E_BUILD_LOG_PATH: ${{ runner.temp }}/mac-tools-pack-build.log
OD_PACKAGED_E2E_MAC: "1"
OD_PACKAGED_E2E_NAMESPACE: release-beta
OD_PACKAGED_E2E_SCREENSHOT_PATH: ${{ runner.temp }}/release-report/mac/screenshots/open-design-mac-smoke.png
OD_PACKAGED_E2E_REPORT_DIR: ${{ runner.temp }}/release-report/mac
OD_PACKAGED_E2E_TOOLS_PACK_DIR: ${{ runner.temp }}/tools-pack
run: |
set -euo pipefail
report_dir="$RUNNER_TEMP/release-report/mac"
mkdir -p "$report_dir/screenshots"
cat > "$report_dir/manifest.json" <<EOF
{
"platform": "mac",
"spec": "specs/mac.spec.ts",
"namespace": "release-beta",
"screenshot": "screenshots/open-design-mac-smoke.png",
"githubRunId": "$GITHUB_RUN_ID",
"githubRunAttempt": "$GITHUB_RUN_ATTEMPT",
"commit": "$GITHUB_SHA"
}
EOF
cp "$RUNNER_TEMP/mac-tools-pack-build.json" "$report_dir/tools-pack.json"
cp "$RUNNER_TEMP/mac-tools-pack-build.log" "$report_dir/tools-pack.log"
pnpm test specs/mac.spec.ts 2>&1 | tee "$report_dir/vitest.log"
pnpm exec tsx scripts/release-smoke.ts mac specs/mac.spec.ts
- name: Upload mac e2e spec report
if: ${{ always() }}
@ -302,7 +289,7 @@ jobs:
name: Build beta mac x64
needs: metadata
if: ${{ inputs.enable_mac_intel }}
runs-on: macos-13
runs-on: macos-15-intel
steps:
- name: Checkout
uses: actions/checkout@v6.0.2
@ -468,29 +455,16 @@ jobs:
- name: Smoke beta windows packaged runtime
working-directory: e2e
env:
OD_PACKAGED_E2E_BUILD_JSON_PATH: ${{ runner.temp }}/windows-tools-pack-build.json
OD_PACKAGED_E2E_WIN: "1"
OD_PACKAGED_E2E_NAMESPACE: release-beta-win
OD_PACKAGED_E2E_REPORT_DIR: ${{ runner.temp }}/release-report/win
OD_PACKAGED_E2E_TOOLS_PACK_DIR: ${{ runner.temp }}/tools-pack
run: |
$ErrorActionPreference = "Stop"
$reportDir = Join-Path $env:RUNNER_TEMP "release-report/win"
$screenshotDir = Join-Path $reportDir "screenshots"
New-Item -ItemType Directory -Force -Path $screenshotDir | Out-Null
$env:OD_PACKAGED_E2E_SCREENSHOT_PATH = Join-Path $screenshotDir "open-design-win-smoke.png"
@{
platform = "win"
spec = "specs/win.spec.ts"
namespace = "release-beta-win"
screenshot = "screenshots/open-design-win-smoke.png"
githubRunId = $env:GITHUB_RUN_ID
githubRunAttempt = $env:GITHUB_RUN_ATTEMPT
commit = $env:GITHUB_SHA
} | ConvertTo-Json | Set-Content -Path (Join-Path $reportDir "manifest.json")
Copy-Item -Force -Path (Join-Path $env:RUNNER_TEMP "windows-tools-pack-build.json") -Destination (Join-Path $reportDir "tools-pack.json")
pnpm test specs/win.spec.ts 2>&1 | Tee-Object -FilePath (Join-Path $reportDir "vitest.log")
$testExitCode = $LASTEXITCODE
if ($testExitCode -ne 0) {
exit $testExitCode
pnpm exec tsx scripts/release-smoke.ts win specs/win.spec.ts
if ($LASTEXITCODE -ne 0) {
exit $LASTEXITCODE
}
- name: Upload windows e2e spec report

204
.github/workflows/release-stable.yml vendored
View file

@ -171,9 +171,6 @@ jobs:
fi
done
- name: Apply release package version
run: npm pkg set "version=${{ needs.metadata.outputs.stable_version }}" --prefix apps/packaged
- name: Prepare Apple signing certificate
env:
APPLE_SIGNING_CERTIFICATE_BASE64: ${{ secrets.APPLE_SIGNING_CERTIFICATE_BASE64 }}
@ -206,7 +203,8 @@ jobs:
--dir "$tools_pack_dir"
--namespace release-stable
--portable
--mac-compression maximum
--app-version "${{ needs.metadata.outputs.stable_version }}"
--mac-compression normal
--to all
--json
--signed
@ -297,30 +295,17 @@ jobs:
- name: Smoke release mac packaged runtime
working-directory: e2e
env:
OD_PACKAGED_E2E_BUILD_JSON_PATH: ${{ runner.temp }}/mac-tools-pack-build.json
OD_PACKAGED_E2E_BUILD_LOG_PATH: ${{ runner.temp }}/mac-tools-pack-build.log
OD_PACKAGED_E2E_MAC: "1"
OD_PACKAGED_E2E_NAMESPACE: release-stable
OD_PACKAGED_E2E_SCREENSHOT_PATH: ${{ runner.temp }}/release-report/mac/screenshots/open-design-mac-smoke.png
OD_PACKAGED_E2E_RELEASE_CHANNEL: ${{ needs.metadata.outputs.channel }}
OD_PACKAGED_E2E_RELEASE_VERSION: ${{ needs.metadata.outputs.release_version }}
OD_PACKAGED_E2E_REPORT_DIR: ${{ runner.temp }}/release-report/mac
OD_PACKAGED_E2E_TOOLS_PACK_DIR: ${{ runner.temp }}/tools-pack
run: |
set -euo pipefail
report_dir="$RUNNER_TEMP/release-report/mac"
mkdir -p "$report_dir/screenshots"
cat > "$report_dir/manifest.json" <<EOF
{
"channel": "${{ needs.metadata.outputs.channel }}",
"platform": "mac",
"releaseVersion": "${{ needs.metadata.outputs.release_version }}",
"spec": "specs/mac.spec.ts",
"namespace": "release-stable",
"screenshot": "screenshots/open-design-mac-smoke.png",
"githubRunId": "$GITHUB_RUN_ID",
"githubRunAttempt": "$GITHUB_RUN_ATTEMPT",
"commit": "$GITHUB_SHA"
}
EOF
cp "$RUNNER_TEMP/mac-tools-pack-build.json" "$report_dir/tools-pack.json"
cp "$RUNNER_TEMP/mac-tools-pack-build.log" "$report_dir/tools-pack.log"
pnpm test specs/mac.spec.ts 2>&1 | tee "$report_dir/vitest.log"
pnpm exec tsx scripts/release-smoke.ts mac specs/mac.spec.ts
- name: Upload mac e2e spec report
if: ${{ always() }}
@ -349,7 +334,7 @@ jobs:
build_mac_intel:
name: Build release mac intel x64
needs: [metadata, verify]
runs-on: macos-13
runs-on: macos-15-intel
env:
GH_TOKEN: ${{ github.token }}
steps:
@ -371,19 +356,37 @@ jobs:
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Apply release package version
run: npm pkg set "version=${{ needs.metadata.outputs.stable_version }}" --prefix apps/packaged
- name: Prepare Apple signing certificate
env:
APPLE_SIGNING_CERTIFICATE_BASE64: ${{ secrets.APPLE_SIGNING_CERTIFICATE_BASE64 }}
APPLE_SIGNING_CERTIFICATE_PASSWORD: ${{ secrets.APPLE_SIGNING_CERTIFICATE_PASSWORD }}
run: |
set -euo pipefail
cert_path="$RUNNER_TEMP/open-design-signing.p12"
if ! printf '%s' "$APPLE_SIGNING_CERTIFICATE_BASE64" | base64 --decode > "$cert_path" 2>/dev/null; then
printf '%s' "$APPLE_SIGNING_CERTIFICATE_BASE64" | base64 -D > "$cert_path"
fi
{
echo "CSC_LINK=$cert_path"
echo "CSC_KEY_PASSWORD=$APPLE_SIGNING_CERTIFICATE_PASSWORD"
} >> "$GITHUB_ENV"
- name: Build release mac intel artifacts
env:
APPLE_ID: ${{ secrets.APPLE_ID }}
APPLE_APP_SPECIFIC_PASSWORD: ${{ secrets.APPLE_APP_SPECIFIC_PASSWORD }}
APPLE_TEAM_ID: ${{ secrets.APPLE_TEAM_ID }}
run: |
set -euo pipefail
pnpm exec tools-pack mac build \
--dir "$RUNNER_TEMP/tools-pack" \
--namespace release-stable-intel \
--portable \
--mac-compression maximum \
--app-version "${{ needs.metadata.outputs.stable_version }}" \
--mac-compression normal \
--to all \
--json
--json \
--signed
- name: Prepare mac intel release assets
id: assets
@ -420,6 +423,32 @@ jobs:
uses: actions/setup-node@v6
with:
node-version: 24
cache: pnpm
cache-dependency-path: pnpm-lock.yaml
- name: Compute Windows tools-pack cache key
id: win_tools_pack_cache_key
shell: pwsh
env:
WIN_TOOLS_PACK_ORIGIN_KEY: ${{ hashFiles('package.json', 'pnpm-lock.yaml', 'pnpm-workspace.yaml', 'apps/daemon/**', 'apps/web/**', 'apps/desktop/**', 'apps/packaged/**', 'packages/contracts/**', 'packages/sidecar-proto/**', 'packages/sidecar/**', 'packages/platform/**', 'tools/pack/bin/**', 'tools/pack/package.json', 'tools/pack/resources/**', 'tools/pack/src/**', 'tools/pack/tsconfig.json', 'assets/community-pets/**', 'assets/frames/**', 'craft/**', 'design-systems/**', 'prompt-templates/**', 'skills/**', '.github/workflows/release-stable.yml', '.github/scripts/release/cache/win.ps1') }}
run: |
if ([string]::IsNullOrWhiteSpace($env:WIN_TOOLS_PACK_ORIGIN_KEY)) {
throw "Windows tools-pack cache origin key is empty"
}
$prefix = "tools-pack-win-v7-stable-$env:RUNNER_OS-"
"origin=$env:WIN_TOOLS_PACK_ORIGIN_KEY" | Out-File -FilePath $env:GITHUB_OUTPUT -Append
"prefix=$prefix" | Out-File -FilePath $env:GITHUB_OUTPUT -Append
"key=$prefix$env:WIN_TOOLS_PACK_ORIGIN_KEY" | Out-File -FilePath $env:GITHUB_OUTPUT -Append
- name: Restore Windows tools-pack cache
id: win_tools_pack_cache_restore
uses: actions/cache/restore@v5
continue-on-error: true
with:
path: ${{ runner.temp }}/tools-pack-cache
key: ${{ steps.win_tools_pack_cache_key.outputs.key }}
restore-keys: |
${{ steps.win_tools_pack_cache_key.outputs.prefix }}
- name: Install dependencies
run: pnpm install --frozen-lockfile
@ -432,54 +461,77 @@ jobs:
}
choco install nsis -y --no-progress
- name: Apply release package version
run: npm pkg set "version=${{ needs.metadata.outputs.stable_version }}" --prefix apps/packaged
- name: Build release windows artifacts
id: win_tools_pack_build
shell: pwsh
run: |
$ErrorActionPreference = "Stop"
$toolsPackDir = "${{ runner.temp }}/tools-pack"
$cacheDir = "${{ runner.temp }}/tools-pack-cache"
$buildJsonPath = Join-Path $env:RUNNER_TEMP "windows-tools-pack-build.json"
$buildOutput = pnpm exec tools-pack win build `
--dir "${{ runner.temp }}/tools-pack" `
--namespace release-stable-win `
--portable `
--to nsis `
--json
if ($LASTEXITCODE -ne 0) {
throw "Windows tools-pack build exited with code $LASTEXITCODE"
$buildArgs = @(
"exec", "tools-pack", "win", "build",
"--dir", $toolsPackDir,
"--cache-dir", $cacheDir,
"--namespace", "release-stable-win",
"--portable",
"--app-version", "${{ needs.metadata.outputs.stable_version }}",
"--to", "nsis",
"--json"
)
"cache_failed=false" | Out-File -FilePath $env:GITHUB_OUTPUT -Append
try {
$buildOutput = pnpm @buildArgs
if ($LASTEXITCODE -ne 0) {
throw "Windows tools-pack cached build exited with code $LASTEXITCODE"
}
} catch {
Write-Warning "Windows tools-pack cached build failed; removing restored cache and retrying without cache."
"cache_failed=true" | Out-File -FilePath $env:GITHUB_OUTPUT -Append
Remove-Item -Recurse -Force -ErrorAction SilentlyContinue $cacheDir
$buildOutput = pnpm exec tools-pack win build `
--dir $toolsPackDir `
--namespace release-stable-win `
--portable `
--app-version "${{ needs.metadata.outputs.stable_version }}" `
--to nsis `
--json
if ($LASTEXITCODE -ne 0) {
throw "Windows tools-pack uncached fallback build exited with code $LASTEXITCODE"
}
}
$buildOutput | Set-Content -Path $buildJsonPath
$buildOutput
- name: Delete failed Windows tools-pack cache
if: ${{ steps.win_tools_pack_build.outputs.cache_failed == 'true' && steps.win_tools_pack_cache_restore.outputs.cache-matched-key != '' }}
shell: pwsh
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
run: |
$matchedKey = "${{ steps.win_tools_pack_cache_restore.outputs.cache-matched-key }}"
$caches = @(gh cache list --key $matchedKey --limit 100 --json id,key,ref | ConvertFrom-Json | Where-Object { $_.key -eq $matchedKey })
foreach ($cache in $caches) {
gh cache delete $cache.id
}
"deletedFailedCacheKey=$matchedKey count=$($caches.Count)"
- name: Smoke release windows packaged runtime
working-directory: e2e
env:
OD_PACKAGED_E2E_BUILD_JSON_PATH: ${{ runner.temp }}/windows-tools-pack-build.json
OD_PACKAGED_E2E_WIN: "1"
OD_PACKAGED_E2E_NAMESPACE: release-stable-win
OD_PACKAGED_E2E_RELEASE_CHANNEL: ${{ needs.metadata.outputs.channel }}
OD_PACKAGED_E2E_RELEASE_VERSION: ${{ needs.metadata.outputs.release_version }}
OD_PACKAGED_E2E_REPORT_DIR: ${{ runner.temp }}/release-report/win
OD_PACKAGED_E2E_TOOLS_PACK_DIR: ${{ runner.temp }}/tools-pack
run: |
$ErrorActionPreference = "Stop"
$reportDir = Join-Path $env:RUNNER_TEMP "release-report/win"
$screenshotDir = Join-Path $reportDir "screenshots"
New-Item -ItemType Directory -Force -Path $screenshotDir | Out-Null
$env:OD_PACKAGED_E2E_SCREENSHOT_PATH = Join-Path $screenshotDir "open-design-win-smoke.png"
@{
channel = "${{ needs.metadata.outputs.channel }}"
platform = "win"
releaseVersion = "${{ needs.metadata.outputs.release_version }}"
spec = "specs/win.spec.ts"
namespace = "release-stable-win"
screenshot = "screenshots/open-design-win-smoke.png"
githubRunId = $env:GITHUB_RUN_ID
githubRunAttempt = $env:GITHUB_RUN_ATTEMPT
commit = $env:GITHUB_SHA
} | ConvertTo-Json | Set-Content -Path (Join-Path $reportDir "manifest.json")
Copy-Item -Force -Path (Join-Path $env:RUNNER_TEMP "windows-tools-pack-build.json") -Destination (Join-Path $reportDir "tools-pack.json")
pnpm test specs/win.spec.ts 2>&1 | Tee-Object -FilePath (Join-Path $reportDir "vitest.log")
$testExitCode = $LASTEXITCODE
if ($testExitCode -ne 0) {
exit $testExitCode
pnpm exec tsx scripts/release-smoke.ts win specs/win.spec.ts
if ($LASTEXITCODE -ne 0) {
exit $LASTEXITCODE
}
- name: Upload windows e2e spec report
@ -490,6 +542,35 @@ jobs:
path: ${{ runner.temp }}/release-report/win
if-no-files-found: warn
- name: Prune Windows tools-pack cache
shell: pwsh
continue-on-error: true
run: ./.github/scripts/release/cache/win.ps1
- name: Save Windows tools-pack cache
if: ${{ success() && (steps.win_tools_pack_cache_restore.outputs.cache-hit != 'true' || steps.win_tools_pack_build.outputs.cache_failed == 'true') }}
uses: actions/cache/save@v5
continue-on-error: true
with:
path: ${{ runner.temp }}/tools-pack-cache
key: ${{ steps.win_tools_pack_cache_key.outputs.key }}
- name: Retain recent Windows tools-pack caches
if: ${{ success() }}
shell: pwsh
continue-on-error: true
env:
GH_TOKEN: ${{ github.token }}
run: |
$prefix = "${{ steps.win_tools_pack_cache_key.outputs.prefix }}"
$keep = 3
$caches = @(gh cache list --key $prefix --sort created_at --order desc --limit 100 --json id,key,createdAt | ConvertFrom-Json)
$stale = @($caches | Select-Object -Skip $keep)
foreach ($cache in $stale) {
gh cache delete $cache.id
}
"actionsCachePrefix=$prefix kept=$([Math]::Min($caches.Count, $keep)) deleted=$($stale.Count)"
- name: Prepare windows release assets
shell: pwsh
env:
@ -535,9 +616,6 @@ jobs:
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Apply release package version
run: npm pkg set "version=${{ needs.metadata.outputs.stable_version }}" --prefix apps/packaged
# `--containerized` builds the AppImage inside the electronuserland/builder
# Docker image (glibc 2.27 baseline) so the resulting binary runs on older
# distros than ubuntu-latest's glibc 2.39. Docker is preinstalled on the
@ -549,6 +627,7 @@ jobs:
--dir "$RUNNER_TEMP/tools-pack" \
--namespace release-stable-linux \
--portable \
--app-version "${{ needs.metadata.outputs.stable_version }}" \
--to appimage \
--containerized \
--json
@ -602,6 +681,7 @@ jobs:
ENABLE_MAC_INTEL: "true"
ENABLE_WIN: "true"
GITHUB_RELEASE_ENABLED: ${{ needs.metadata.outputs.github_release_enabled }}
MAC_INTEL_SIGNED: "true"
NIGHTLY_NUMBER: ${{ needs.metadata.outputs.nightly_number }}
RELEASE_CHANNEL: ${{ needs.metadata.outputs.channel }}
RELEASE_NAME: ${{ needs.metadata.outputs.release_name }}

2
apps/packaged/package.json
View file

@ -1,6 +1,6 @@
{
"name": "@open-design/packaged",
"version": "0.6.0",
"version": "0.6.1",
"private": true,
"type": "module",
"main": "./dist/index.mjs",

25
e2e/AGENTS.md
View file

@ -4,19 +4,33 @@ Follow the root `AGENTS.md` first. This package owns user-level end-to-end smoke
## Directory layout
- `specs/`: highest-ROI end-to-end smoke tests suitable for PR or release gating. Keep this layer small and expand it only for regressions that justify always-on signal.
- `tests/`: broader user-level end-to-end coverage, including Vitest checks that intentionally span app/package/resource boundaries. Add feature-depth scenarios here instead of bloating `specs/`.
- `specs/`: highest-ROI, long-running core business capability regressions suitable for PR or release gating. Each spec should describe one nearly orthogonal product capability chain, such as main dialog generation, Pet, Orbit, or packaged runtime. Keep this layer small and expand it only when a core capability deserves always-on signal.
- `tests/`: broader user-level end-to-end coverage and local hotspot checks that intentionally span app/package/resource boundaries. Prefer adding tests here when a repeated or high-risk local capability naturally falls out of a core spec. Do not build a speculative coverage matrix before the core spec needs it.
- `ui/`: flat Playwright UI automation test files only. Keep helpers, resources, and non-Playwright harnesses out of this directory.
- `resources/`: declarative resources for e2e suites, such as Playwright UI scenario lists.
- `lib/fake-agents.ts`: shared fake local agent CLI harness used by UI and pure-inspect daemon specs.
- `lib/shared.ts`: tiny cross-suite shared helpers only.
- `lib/vitest/`: Vitest-specific helpers.
- `lib/vitest/`: Vitest-specific atomic helpers only. Helpers describe actions such as namespace lifecycle, mock servers, HTTP calls, tools-dev commands, inspect, logs, and reports; they should not hide core business scenario decisions.
- `lib/vitest/report.ts`: the report boundary. Specs save curated output through `report.save(<relpath>, <blob>)` or `report.json(<relpath>, value)`; release workflows should consume only the final report path, not its internal file layout.
- `createSmokeSuite(...).with.*`: suite-owned lifecycle composition. Prefer this shape for namespace-bound resources such as `suite.with.toolsDev(...)` so specs keep business workflow code in the foreground.
- `lib/playwright/`: Playwright-specific fixtures, resource accessors, route helpers, and UI actions.
- `scripts/playwright.ts`: Playwright auxiliary subcommands such as artifact cleanup; it must not wrap `playwright test`.
## Spec and test model
- Start from `specs/`: define orthogonal long-form core capabilities first, then let supporting `tests/` and `lib/` grow from those chains.
- `specs/` should read as business/system workflows, for example `dialog/main.spec.ts`, `orbit/run.spec.ts`, or `pet/main.spec.ts`.
- `tests/` should pin reusable local hotspots, such as `tools-dev/inspect.test.ts`, provider mocks, report lifecycle, artifact file shape, or namespace cleanup.
- High-confidence infrastructure checks may be added to `tests/` before a full core spec exists, but most tests should be extracted only after a spec proves the local hotspot matters.
- Treat `tests/` as maintainable support material, not permanent coverage inventory. Merge, split, shrink, or delete tests as product capabilities evolve.
- Keep new non-UI e2e smoke chains pure inspect by default. Do not use Playwright for these chains; use daemon/web APIs, sidecar IPC, tools-dev/tools-pack inspect, logs, reports, and screenshots when available.
- External service dependencies must use temporary server-level mocks. Do not rely on real API keys, real provider accounts, or UI-level route patching for core e2e smoke.
- Every atomic suite must run in an isolated namespace. Successful suites should keep only curated reports and high-value artifacts, then clean process/runtime scratch. Failed suites should preserve runtime scratch, logs, mock requests, screenshots, and report pointers for diagnosis.
## Naming and tools
- `specs/` files must be `*.spec.ts`.
- `tests/` files must be `*.test.ts`.
- `specs/` files must be `*.spec.ts`; `tests/` files must be `*.test.ts`.
- Prefer directory hierarchy over long file names. Basenames should normally be three words or fewer, such as `main.spec.ts`, `run.spec.ts`, `inspect.test.ts`, or `report.test.ts`.
- `ui/` files must be flat `*.test.ts` Playwright tests. Do not add subdirectories, TSX, Vitest, jsdom, Testing Library, or React harness tests under `ui/`.
- E2E Vitest tests use Node APIs; do not add JSX/TSX, jsdom, or browser-component tests under `specs/` or `tests/`.
- Web component/runtime tests belong in `apps/web/tests/`, not `e2e/ui/`.
@ -29,6 +43,7 @@ Run commands from this directory:
```bash
pnpm test specs/mac.spec.ts
pnpm test tests/tools-dev/inspect.test.ts
pnpm test specs
pnpm test tests
pnpm typecheck

300
e2e/lib/fake-agents.ts Normal file
View file

@ -0,0 +1,300 @@
import { chmod, mkdir, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
export type FakeAgentId =
| 'claude'
| 'codex'
| 'copilot'
| 'cursor-agent'
| 'deepseek'
| 'gemini'
| 'opencode'
| 'qoder'
| 'qwen';
export type FakeAgentRuntime = {
agentId: FakeAgentId;
bin: string;
envKey: string;
env: Record<string, string>;
};
export type FakeAgentRuntimeOptions = {
root?: string;
runtimeIds?: FakeAgentId[];
};
const AGENT_BIN_NAMES: Record<FakeAgentId, string> = {
claude: 'claude-e2e.js',
codex: 'codex-e2e.js',
copilot: 'copilot-e2e.js',
'cursor-agent': 'cursor-agent-e2e.js',
deepseek: 'deepseek-e2e.js',
gemini: 'gemini-e2e.js',
opencode: 'opencode-e2e.js',
qoder: 'qodercli-e2e.js',
qwen: 'qwen-e2e.js',
};
const AGENT_BIN_ENV_KEYS: Record<FakeAgentId, string> = {
claude: 'CLAUDE_BIN',
codex: 'CODEX_BIN',
copilot: 'COPILOT_BIN',
'cursor-agent': 'CURSOR_AGENT_BIN',
deepseek: 'DEEPSEEK_BIN',
gemini: 'GEMINI_BIN',
opencode: 'OPENCODE_BIN',
qoder: 'QODER_BIN',
qwen: 'QWEN_BIN',
};
export const FAKE_AGENT_RUNTIME_IDS: FakeAgentId[] = [
'claude',
'gemini',
'opencode',
'cursor-agent',
'qwen',
'qoder',
'copilot',
];
export async function createFakeAgentRuntimes(
runtimeIds?: FakeAgentId[],
): Promise<Record<FakeAgentId, FakeAgentRuntime>>;
export async function createFakeAgentRuntimes(
options?: FakeAgentRuntimeOptions,
): Promise<Record<FakeAgentId, FakeAgentRuntime>>;
export async function createFakeAgentRuntimes(
input: FakeAgentId[] | FakeAgentRuntimeOptions = {},
): Promise<Record<FakeAgentId, FakeAgentRuntime>> {
const runtimeIds = Array.isArray(input)
? input
: (input.runtimeIds ?? ['codex', ...FAKE_AGENT_RUNTIME_IDS]);
const root = Array.isArray(input)
? path.join(tmpdir(), `open-design-fake-agents-${process.pid}`)
: (input.root ?? path.join(tmpdir(), `open-design-fake-agents-${process.pid}`));
await mkdir(root, { recursive: true });
const runtimes = {} as Record<FakeAgentId, FakeAgentRuntime>;
for (const agentId of runtimeIds) {
const script = path.join(root, AGENT_BIN_NAMES[agentId]);
const bin = process.platform === 'win32'
? script.replace(/\.js$/i, '.cmd')
: script;
await writeFile(script, renderFakeAgentScript(agentId), 'utf8');
if (process.platform === 'win32') {
await writeFile(bin, '@echo off\r\nnode "%~dp0%~n0.js" %*\r\n', 'utf8');
} else {
await chmod(bin, 0o755);
}
const envKey = AGENT_BIN_ENV_KEYS[agentId];
runtimes[agentId] = { agentId, bin, envKey, env: { [envKey]: bin } };
}
return runtimes;
}
function renderFakeAgentScript(agentId: FakeAgentId): string {
return `#!/usr/bin/env node
const agentId = ${JSON.stringify(agentId)};
const args = process.argv.slice(2);
if (args.includes('--version')) {
process.stdout.write(agentId + '-e2e 0.0.0\\n');
process.exitCode = 0;
} else if (agentId === 'claude' && args[0] === '-p' && args.includes('--help')) {
process.stdout.write('--add-dir --include-partial-messages\\n');
process.exitCode = 0;
} else if ((agentId === 'opencode' || agentId === 'cursor-agent') && args[0] === 'models') {
process.stdout.write('fake/default\\n');
process.exitCode = 0;
} else {
let prompt = '';
let emitted = false;
process.stdin.setEncoding('utf8');
process.stdin.resume();
process.stdin.on('data', (chunk) => { prompt += chunk; });
process.stdin.on('end', () => {
void emitRun(prompt).catch(failUnhandled);
});
if (process.stdin.isTTY || agentId === 'deepseek') {
prompt = args.join(' ');
void emitRun(prompt).catch(failUnhandled);
}
async function emitRun(promptText) {
if (emitted) return;
emitted = true;
if (promptText.includes('Return an intentional daemon smoke failure')) {
emitFailure();
return;
}
const isChunked = promptText.includes('Create a chunked deterministic smoke artifact');
const isFollowUp = promptText.includes('Create a follow-up deterministic smoke artifact');
const isDefaultSmoke = promptText.includes('Create a deterministic smoke artifact');
const isOrbit = promptText.includes("Create today's Orbit daily digest as a Live Artifact.");
if (isOrbit) {
await emitOrbitRun();
return;
}
const isRuntime = promptText.match(/Fake runtime smoke for ([a-z0-9-]+)/i);
const runtimeId = isRuntime ? isRuntime[1] : agentId;
const heading = isChunked ? 'Chunked Daemon Smoke' : isFollowUp ? 'Follow-up Daemon Smoke' : isDefaultSmoke ? 'Real Daemon Smoke' : 'Fake Agent Runtime ' + runtimeId;
const identifier = isChunked ? 'chunked-daemon-smoke' : isFollowUp ? 'follow-up-daemon-smoke' : isDefaultSmoke ? 'real-daemon-smoke' : 'fake-agent-runtime-' + runtimeId;
const text = isChunked ? 'Chunked through the daemon run path.' : isFollowUp ? 'Generated after an earlier daemon turn.' : isDefaultSmoke ? 'Generated through the daemon run path.' : 'Generated through fake ' + runtimeId + ' runtime.';
const html = '<!doctype html><html><body><main><h1>' + heading + '</h1><p>' + text + '</p></main></body></html>';
const artifact = '<artifact identifier="' + identifier + '" type="text/html" title="' + heading + '">' + html + '</artifact>';
emitSuccess(artifact, isChunked);
process.exitCode = 0;
}
function writeJson(value) {
process.stdout.write(JSON.stringify(value) + '\\n');
}
function emitSuccess(artifact, isChunked) {
const first = artifact.slice(0, Math.ceil(artifact.length / 2));
const second = artifact.slice(Math.ceil(artifact.length / 2));
switch (agentId) {
case 'codex':
writeJson({ type: 'thread.started' });
writeJson({ type: 'turn.started' });
if (isChunked) {
writeJson({ type: 'item.completed', item: { type: 'agent_message', text: first } });
writeJson({ type: 'item.completed', item: { type: 'agent_message', text: second } });
} else {
writeJson({ type: 'item.completed', item: { type: 'agent_message', text: artifact } });
}
writeJson({ type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 1 } });
return;
case 'claude':
writeJson({ type: 'system', subtype: 'init', model: 'fake-claude', session_id: 'fake-session' });
writeJson({ type: 'assistant', message: { id: 'msg-1', content: [{ type: 'text', text: artifact }] } });
writeJson({ type: 'result', usage: { input_tokens: 1, output_tokens: 1 }, total_cost_usd: 0, duration_ms: 1, stop_reason: 'end_turn' });
return;
case 'gemini':
writeJson({ type: 'init', session_id: 'fake-gemini', model: 'fake-gemini' });
writeJson({ type: 'message', role: 'assistant', content: artifact, delta: true });
writeJson({ type: 'result', status: 'success', stats: { input_tokens: 1, output_tokens: 1, cached: 0, duration_ms: 1 } });
return;
case 'opencode':
writeJson({ type: 'step_start', sessionID: 'fake-opencode', part: { type: 'step-start' } });
writeJson({ type: 'text', sessionID: 'fake-opencode', part: { type: 'text', text: artifact } });
writeJson({ type: 'step_finish', sessionID: 'fake-opencode', part: { type: 'step-finish', tokens: { input: 1, output: 1 }, cost: 0 } });
return;
case 'cursor-agent':
writeJson({ type: 'system', subtype: 'init', model: 'fake-cursor' });
writeJson({ type: 'assistant', timestamp_ms: 1, message: { role: 'assistant', content: [{ type: 'text', text: artifact }] } });
writeJson({ type: 'result', duration_ms: 1, usage: { inputTokens: 1, outputTokens: 1, cacheReadTokens: 0, cacheWriteTokens: 0 } });
return;
case 'qoder':
writeJson({ type: 'system', subtype: 'init', qodercli_version: '0.0.0', model: 'fake-qoder', session_id: 'fake-qoder' });
writeJson({ type: 'assistant', message: { content: [{ type: 'text', text: artifact }] }, session_id: 'fake-qoder' });
writeJson({ type: 'result', subtype: 'success', duration_ms: 1, is_error: false, stop_reason: 'end_turn', total_cost_usd: 0, usage: { input_tokens: 1, output_tokens: 1 } });
return;
case 'copilot':
writeJson({ type: 'session.tools_updated', data: { model: 'fake-copilot' } });
writeJson({ type: 'assistant.turn_start', data: {} });
writeJson({ type: 'assistant.message_delta', data: { deltaContent: artifact } });
writeJson({ type: 'result', success: true, exitCode: 0, usage: { input_tokens: 1, output_tokens: 1, sessionDurationMs: 1 } });
return;
case 'qwen':
case 'deepseek':
process.stdout.write(artifact + '\\n');
return;
default:
process.stdout.write(artifact + '\\n');
}
}
async function emitOrbitRun() {
const artifact = await createOrbitLiveArtifact();
const text = 'Orbit fake digest registered live artifact ' + artifact.id + ' for project ' + artifact.projectId + '.';
emitSuccess(text, false);
process.exitCode = 0;
}
async function createOrbitLiveArtifact() {
const baseUrl = process.env.OD_DAEMON_URL;
const token = process.env.OD_TOOL_TOKEN;
if (!baseUrl || !token) {
throw new Error('Orbit fake run requires OD_DAEMON_URL and OD_TOOL_TOKEN');
}
const url = new URL('/api/tools/live-artifacts/create', baseUrl);
const payload = {
input: {
title: 'Orbit Daily Digest',
slug: 'orbit-daily-digest',
preview: { type: 'html', entry: 'index.html' },
document: {
format: 'html_template_v1',
templatePath: 'template.html',
generatedPreviewPath: 'index.html',
dataPath: 'data.json',
dataJson: {
headline: 'Orbit daily digest',
takeaway1: 'Fake connector activity was summarized through the daemon Orbit path.',
takeaway2: 'The live artifact tool token was accepted.',
takeaway3: 'The digest can be opened and previewed from the Orbit project.',
checked: 'fake activity feed and fake task updates',
},
},
},
templateHtml: '<!doctype html><html><body><main><h1>{{data.headline}}</h1><ul><li>{{data.takeaway1}}</li><li>{{data.takeaway2}}</li><li>{{data.takeaway3}}</li></ul><p>{{data.checked}}</p></main></body></html>',
provenanceJson: {
generatedAt: new Date().toISOString(),
generatedBy: 'agent',
sources: [{ label: 'Fake Orbit e2e data', type: 'derived' }],
},
};
const response = await fetch(url, {
method: 'POST',
headers: {
authorization: 'Bearer ' + token,
'content-type': 'application/json',
},
body: JSON.stringify(payload),
});
const text = await response.text();
let body = {};
try {
body = text ? JSON.parse(text) : {};
} catch {
body = { raw: text };
}
if (!response.ok || !body.artifact) {
throw new Error('Orbit live artifact create failed: HTTP ' + response.status + ' ' + text.slice(0, 500));
}
return body.artifact;
}
function failUnhandled(error) {
process.stderr.write((error && error.stack ? error.stack : String(error)) + '\\n');
process.exitCode = 1;
}
function emitFailure() {
switch (agentId) {
case 'codex':
writeJson({ type: 'thread.started' });
writeJson({ type: 'turn.started' });
writeJson({ type: 'turn.failed', error: { message: 'intentional fake codex failure' } });
process.exitCode = 0;
return;
case 'opencode':
writeJson({ type: 'error', error: { data: { message: 'intentional fake opencode failure' } } });
process.exitCode = 0;
return;
case 'qoder':
writeJson({ type: 'assistant', message: { content: [] }, error: { message: 'intentional fake qoder failure' } });
process.exitCode = 0;
return;
default:
process.stderr.write('intentional fake ' + agentId + ' failure\\n');
process.exitCode = 1;
}
}
}
`;
}

222
e2e/lib/playwright/fake-agents.ts
View file

@ -1,213 +1,9 @@
import { chmod, mkdir, writeFile } from 'node:fs/promises';
import { tmpdir } from 'node:os';
import path from 'node:path';
export type FakeAgentId =
| 'claude'
| 'codex'
| 'copilot'
| 'cursor-agent'
| 'deepseek'
| 'gemini'
| 'opencode'
| 'qoder'
| 'qwen';
type FakeAgentRuntime = {
agentId: FakeAgentId;
bin: string;
envKey: string;
env: Record<string, string>;
};
const AGENT_BIN_NAMES: Record<FakeAgentId, string> = {
claude: 'claude-e2e.js',
codex: 'codex-e2e.js',
copilot: 'copilot-e2e.js',
'cursor-agent': 'cursor-agent-e2e.js',
deepseek: 'deepseek-e2e.js',
gemini: 'gemini-e2e.js',
opencode: 'opencode-e2e.js',
qoder: 'qodercli-e2e.js',
qwen: 'qwen-e2e.js',
};
const AGENT_BIN_ENV_KEYS: Record<FakeAgentId, string> = {
claude: 'CLAUDE_BIN',
codex: 'CODEX_BIN',
copilot: 'COPILOT_BIN',
'cursor-agent': 'CURSOR_AGENT_BIN',
deepseek: 'DEEPSEEK_BIN',
gemini: 'GEMINI_BIN',
opencode: 'OPENCODE_BIN',
qoder: 'QODER_BIN',
qwen: 'QWEN_BIN',
};
export const FAKE_AGENT_RUNTIME_IDS: FakeAgentId[] = [
'claude',
'gemini',
'opencode',
'cursor-agent',
'qwen',
'qoder',
'copilot',
];
export async function createFakeAgentRuntimes(
runtimeIds: FakeAgentId[] = ['codex', ...FAKE_AGENT_RUNTIME_IDS],
): Promise<Record<FakeAgentId, FakeAgentRuntime>> {
const root = path.join(tmpdir(), `open-design-playwright-fake-agents-${process.pid}`);
await mkdir(root, { recursive: true });
const runtimes = {} as Record<FakeAgentId, FakeAgentRuntime>;
for (const agentId of runtimeIds) {
const script = path.join(root, AGENT_BIN_NAMES[agentId]);
const bin = process.platform === 'win32'
? script.replace(/\.js$/i, '.cmd')
: script;
await writeFile(script, renderFakeAgentScript(agentId), 'utf8');
if (process.platform === 'win32') {
await writeFile(bin, '@echo off\r\nnode "%~dp0%~n0.js" %*\r\n', 'utf8');
} else {
await chmod(bin, 0o755);
}
const envKey = AGENT_BIN_ENV_KEYS[agentId];
runtimes[agentId] = { agentId, bin, envKey, env: { [envKey]: bin } };
}
return runtimes;
}
function renderFakeAgentScript(agentId: FakeAgentId): string {
return `#!/usr/bin/env node
const agentId = ${JSON.stringify(agentId)};
const args = process.argv.slice(2);
if (args.includes('--version')) {
process.stdout.write(agentId + '-e2e 0.0.0\\n');
process.exitCode = 0;
} else if (agentId === 'claude' && args[0] === '-p' && args.includes('--help')) {
process.stdout.write('--add-dir --include-partial-messages\\n');
process.exitCode = 0;
} else if ((agentId === 'opencode' || agentId === 'cursor-agent') && args[0] === 'models') {
process.stdout.write('fake/default\\n');
process.exitCode = 0;
} else {
let prompt = '';
let emitted = false;
process.stdin.setEncoding('utf8');
process.stdin.resume();
process.stdin.on('data', (chunk) => { prompt += chunk; });
process.stdin.on('end', () => {
emitRun(prompt);
});
if (process.stdin.isTTY || agentId === 'deepseek') {
prompt = args.join(' ');
emitRun(prompt);
}
function emitRun(promptText) {
if (emitted) return;
emitted = true;
if (promptText.includes('Return an intentional daemon smoke failure')) {
emitFailure();
return;
}
const isChunked = promptText.includes('Create a chunked deterministic smoke artifact');
const isFollowUp = promptText.includes('Create a follow-up deterministic smoke artifact');
const isDefaultSmoke = promptText.includes('Create a deterministic smoke artifact');
const isRuntime = promptText.match(/Fake runtime smoke for ([a-z0-9-]+)/i);
const runtimeId = isRuntime ? isRuntime[1] : agentId;
const heading = isChunked ? 'Chunked Daemon Smoke' : isFollowUp ? 'Follow-up Daemon Smoke' : isDefaultSmoke ? 'Real Daemon Smoke' : 'Fake Agent Runtime ' + runtimeId;
const identifier = isChunked ? 'chunked-daemon-smoke' : isFollowUp ? 'follow-up-daemon-smoke' : isDefaultSmoke ? 'real-daemon-smoke' : 'fake-agent-runtime-' + runtimeId;
const text = isChunked ? 'Chunked through the daemon run path.' : isFollowUp ? 'Generated after an earlier daemon turn.' : isDefaultSmoke ? 'Generated through the daemon run path.' : 'Generated through fake ' + runtimeId + ' runtime.';
const html = '<!doctype html><html><body><main><h1>' + heading + '</h1><p>' + text + '</p></main></body></html>';
const artifact = '<artifact identifier="' + identifier + '" type="text/html" title="' + heading + '">' + html + '</artifact>';
emitSuccess(artifact, isChunked);
process.exitCode = 0;
}
function writeJson(value) {
process.stdout.write(JSON.stringify(value) + '\\n');
}
function emitSuccess(artifact, isChunked) {
const first = artifact.slice(0, Math.ceil(artifact.length / 2));
const second = artifact.slice(Math.ceil(artifact.length / 2));
switch (agentId) {
case 'codex':
writeJson({ type: 'thread.started' });
writeJson({ type: 'turn.started' });
if (isChunked) {
writeJson({ type: 'item.completed', item: { type: 'agent_message', text: first } });
writeJson({ type: 'item.completed', item: { type: 'agent_message', text: second } });
} else {
writeJson({ type: 'item.completed', item: { type: 'agent_message', text: artifact } });
}
writeJson({ type: 'turn.completed', usage: { input_tokens: 1, output_tokens: 1 } });
return;
case 'claude':
writeJson({ type: 'system', subtype: 'init', model: 'fake-claude', session_id: 'fake-session' });
writeJson({ type: 'assistant', message: { id: 'msg-1', content: [{ type: 'text', text: artifact }] } });
writeJson({ type: 'result', usage: { input_tokens: 1, output_tokens: 1 }, total_cost_usd: 0, duration_ms: 1, stop_reason: 'end_turn' });
return;
case 'gemini':
writeJson({ type: 'init', session_id: 'fake-gemini', model: 'fake-gemini' });
writeJson({ type: 'message', role: 'assistant', content: artifact, delta: true });
writeJson({ type: 'result', status: 'success', stats: { input_tokens: 1, output_tokens: 1, cached: 0, duration_ms: 1 } });
return;
case 'opencode':
writeJson({ type: 'step_start', sessionID: 'fake-opencode', part: { type: 'step-start' } });
writeJson({ type: 'text', sessionID: 'fake-opencode', part: { type: 'text', text: artifact } });
writeJson({ type: 'step_finish', sessionID: 'fake-opencode', part: { type: 'step-finish', tokens: { input: 1, output: 1 }, cost: 0 } });
return;
case 'cursor-agent':
writeJson({ type: 'system', subtype: 'init', model: 'fake-cursor' });
writeJson({ type: 'assistant', timestamp_ms: 1, message: { role: 'assistant', content: [{ type: 'text', text: artifact }] } });
writeJson({ type: 'result', duration_ms: 1, usage: { inputTokens: 1, outputTokens: 1, cacheReadTokens: 0, cacheWriteTokens: 0 } });
return;
case 'qoder':
writeJson({ type: 'system', subtype: 'init', qodercli_version: '0.0.0', model: 'fake-qoder', session_id: 'fake-qoder' });
writeJson({ type: 'assistant', message: { content: [{ type: 'text', text: artifact }] }, session_id: 'fake-qoder' });
writeJson({ type: 'result', subtype: 'success', duration_ms: 1, is_error: false, stop_reason: 'end_turn', total_cost_usd: 0, usage: { input_tokens: 1, output_tokens: 1 } });
return;
case 'copilot':
writeJson({ type: 'session.tools_updated', data: { model: 'fake-copilot' } });
writeJson({ type: 'assistant.turn_start', data: {} });
writeJson({ type: 'assistant.message_delta', data: { deltaContent: artifact } });
writeJson({ type: 'result', success: true, exitCode: 0, usage: { input_tokens: 1, output_tokens: 1, sessionDurationMs: 1 } });
return;
case 'qwen':
case 'deepseek':
process.stdout.write(artifact + '\\n');
return;
default:
process.stdout.write(artifact + '\\n');
}
}
function emitFailure() {
switch (agentId) {
case 'codex':
writeJson({ type: 'thread.started' });
writeJson({ type: 'turn.started' });
writeJson({ type: 'turn.failed', error: { message: 'intentional fake codex failure' } });
process.exitCode = 0;
return;
case 'opencode':
writeJson({ type: 'error', error: { data: { message: 'intentional fake opencode failure' } } });
process.exitCode = 0;
return;
case 'qoder':
writeJson({ type: 'assistant', message: { content: [] }, error: { message: 'intentional fake qoder failure' } });
process.exitCode = 0;
return;
default:
process.stderr.write('intentional fake ' + agentId + ' failure\\n');
process.exitCode = 1;
}
}
}
`;
}
export {
createFakeAgentRuntimes,
FAKE_AGENT_RUNTIME_IDS,
} from '../fake-agents.ts';
export type {
FakeAgentId,
FakeAgentRuntime,
FakeAgentRuntimeOptions,
} from '../fake-agents.ts';

176
e2e/lib/vitest/artifacts.ts Normal file
View file

@ -0,0 +1,176 @@
import { requestJson } from './http.ts';
export type ExtractedArtifact = {
artifactType: string;
html: string;
identifier: string;
rawText: string;
title: string;
};
export type ProjectFile = {
artifactManifest?: ArtifactManifest;
kind: string;
name: string;
size: number;
};
export type ArtifactManifest = {
createdAt?: string;
designSystemId?: string | null;
entry: string;
exports: string[];
kind: string;
metadata?: Record<string, unknown>;
renderer: string;
sourceSkillId?: string;
status: string;
title: string;
updatedAt?: string;
version: number;
};
export function extractArtifactFromRunEvents(events: string): ExtractedArtifact {
const rawText = collectRunText(events);
const artifact = parseArtifactTag(rawText);
if (artifact == null) {
throw new Error(`run events did not include a complete artifact block`);
}
return { ...artifact, rawText };
}
export async function persistExtractedArtifact(
baseUrl: string,
projectId: string,
artifact: ExtractedArtifact,
options: { designSystemId?: string | null; sourceSkillId?: string | null } = {},
): Promise<ProjectFile> {
const fileName = artifactFileNameFor(artifact);
const response = await requestJson<{ file: ProjectFile }>(
baseUrl,
`/api/projects/${encodeURIComponent(projectId)}/files`,
{
body: {
artifactManifest: createHtmlArtifactManifest({
entry: fileName,
metadata: {
artifactType: artifact.artifactType,
identifier: artifact.identifier,
inferred: false,
},
title: artifact.title || artifact.identifier || fileName,
...(options.designSystemId !== undefined ? { designSystemId: options.designSystemId } : {}),
...(options.sourceSkillId != null ? { sourceSkillId: options.sourceSkillId } : {}),
}),
content: artifact.html,
name: fileName,
},
},
);
return response.file;
}
function collectRunText(events: string): string {
let text = '';
for (const frame of events.split(/\n\n+/)) {
const parsed = parseSseFrame(frame);
if (parsed == null) continue;
if (parsed.event === 'stdout' && typeof parsed.data?.chunk === 'string') {
text += parsed.data.chunk;
continue;
}
if (
parsed.event === 'agent' &&
parsed.data?.type === 'text_delta' &&
typeof parsed.data.delta === 'string'
) {
text += parsed.data.delta;
}
}
return text;
}
function parseSseFrame(frame: string): { event: string; data: Record<string, unknown> } | null {
const lines = frame.split(/\r?\n/);
const event = lines
.find((line) => line.startsWith('event:'))
?.slice('event:'.length)
.trim();
const dataLines = lines
.filter((line) => line.startsWith('data:'))
.map((line) => line.slice('data:'.length).trimStart());
if (!event || dataLines.length === 0) return null;
try {
const data = JSON.parse(dataLines.join('\n')) as unknown;
if (!data || typeof data !== 'object' || Array.isArray(data)) return null;
return { event, data: data as Record<string, unknown> };
} catch {
return null;
}
}
function parseArtifactTag(rawText: string): Omit<ExtractedArtifact, 'rawText'> | null {
const match = rawText.match(/<artifact\s+([^>]*)>([\s\S]*?)<\/artifact>/);
if (!match) return null;
const attrs = parseAttrs(match[1] ?? '');
return {
artifactType: attrs.type ?? '',
html: match[2] ?? '',
identifier: attrs.identifier ?? '',
title: attrs.title ?? '',
};
}
function parseAttrs(raw: string): Record<string, string> {
const attrs: Record<string, string> = {};
const pattern = /(\w+)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
let match: RegExpExecArray | null = pattern.exec(raw);
while (match !== null) {
const key = match[1];
if (key) attrs[key] = match[2] ?? match[3] ?? '';
match = pattern.exec(raw);
}
return attrs;
}
function artifactFileNameFor(artifact: ExtractedArtifact): string {
const baseName = (artifact.identifier || artifact.title || 'artifact')
.toLowerCase()
.replace(/[^a-z0-9_-]+/g, '-')
.replace(/^-+|-+$/g, '')
.slice(0, 60) || 'artifact';
return `${baseName}${artifactExtensionFor(artifact)}`;
}
function artifactExtensionFor(artifact: ExtractedArtifact): '.html' | '.jsx' | '.tsx' {
const type = artifact.artifactType.toLowerCase();
const identifier = artifact.identifier.toLowerCase();
if (type.includes('tsx') || identifier.endsWith('.tsx')) return '.tsx';
if (type.includes('jsx') || type.includes('react') || identifier.endsWith('.jsx')) return '.jsx';
return '.html';
}
function createHtmlArtifactManifest(input: {
designSystemId?: string | null;
entry: string;
metadata?: Record<string, unknown>;
sourceSkillId?: string;
title: string;
}): ArtifactManifest {
const now = new Date().toISOString();
const manifest: ArtifactManifest = {
entry: input.entry,
exports: ['html', 'pdf', 'zip'],
kind: 'html',
renderer: 'html',
status: 'complete',
title: input.title,
createdAt: now,
updatedAt: now,
version: 1,
};
if (input.metadata !== undefined) manifest.metadata = input.metadata;
if (input.designSystemId !== undefined) manifest.designSystemId = input.designSystemId;
if (input.sourceSkillId !== undefined) manifest.sourceSkillId = input.sourceSkillId;
return manifest;
}

34
e2e/lib/vitest/http.ts Normal file
View file

@ -0,0 +1,34 @@
export type JsonRequestOptions = {
body?: unknown;
headers?: Record<string, string>;
method?: string;
};
export async function requestJson<T>(baseUrl: string, path: string, options: JsonRequestOptions = {}): Promise<T> {
const response = await fetch(new URL(path, ensureTrailingSlash(baseUrl)), {
headers: {
...(options.body === undefined ? {} : { 'content-type': 'application/json' }),
...options.headers,
},
method: options.method ?? (options.body === undefined ? 'GET' : 'POST'),
...(options.body === undefined ? {} : { body: JSON.stringify(options.body) }),
});
const text = await response.text();
if (!response.ok) {
throw new Error(`HTTP ${response.status} ${path}: ${text.slice(0, 500)}`);
}
return (text ? JSON.parse(text) : null) as T;
}
export async function requestText(baseUrl: string, path: string): Promise<string> {
const response = await fetch(new URL(path, ensureTrailingSlash(baseUrl)));
const text = await response.text();
if (!response.ok) {
throw new Error(`HTTP ${response.status} ${path}: ${text.slice(0, 500)}`);
}
return text;
}
function ensureTrailingSlash(value: string): string {
return value.endsWith('/') ? value : `${value}/`;
}

34
e2e/lib/vitest/live-artifacts.ts Normal file
View file

@ -0,0 +1,34 @@
import { requestJson, requestText } from './http.ts';
export type LiveArtifactSummary = {
createdAt: string;
createdByRunId?: string;
id: string;
projectId: string;
refreshStatus: string;
slug: string;
status: string;
title: string;
};
export async function listLiveArtifacts(
baseUrl: string,
projectId: string,
): Promise<LiveArtifactSummary[]> {
const response = await requestJson<{ artifacts: LiveArtifactSummary[] }>(
baseUrl,
`/api/live-artifacts?projectId=${encodeURIComponent(projectId)}`,
);
return response.artifacts;
}
export async function readLiveArtifactPreview(
baseUrl: string,
projectId: string,
artifactId: string,
): Promise<string> {
return await requestText(
baseUrl,
`/api/live-artifacts/${encodeURIComponent(artifactId)}/preview?projectId=${encodeURIComponent(projectId)}`,
);
}

42
e2e/lib/vitest/messages.ts Normal file
View file

@ -0,0 +1,42 @@
import { requestJson } from './http.ts';
export type E2eChatMessage = {
agentId?: string | null;
agentName?: string;
content: string;
createdAt?: number;
endedAt?: number;
events?: unknown[];
id: string;
producedFiles?: unknown[];
role: 'assistant' | 'user';
runId?: string;
runStatus?: 'queued' | 'running' | 'succeeded' | 'failed' | 'canceled';
startedAt?: number;
telemetryFinalized?: boolean;
};
export async function saveMessage(
baseUrl: string,
projectId: string,
conversationId: string,
message: E2eChatMessage,
): Promise<E2eChatMessage> {
return await requestJson<E2eChatMessage>(
baseUrl,
`/api/projects/${encodeURIComponent(projectId)}/conversations/${encodeURIComponent(conversationId)}/messages/${encodeURIComponent(message.id)}`,
{ body: message, method: 'PUT' },
);
}
export async function listMessages(
baseUrl: string,
projectId: string,
conversationId: string,
): Promise<E2eChatMessage[]> {
const response = await requestJson<{ messages: E2eChatMessage[] }>(
baseUrl,
`/api/projects/${encodeURIComponent(projectId)}/conversations/${encodeURIComponent(conversationId)}/messages`,
);
return response.messages;
}

121
e2e/lib/vitest/mock-openai.ts Normal file
View file

@ -0,0 +1,121 @@
import { createServer, type IncomingMessage, type Server, type ServerResponse } from 'node:http';
export type MockOpenAiRequest = {
body: unknown;
headers: Record<string, string | string[] | undefined>;
method: string;
path: string;
receivedAt: string;
};
export type MockOpenAiServer = {
baseUrl: string;
close: () => Promise<void>;
requests: () => MockOpenAiRequest[];
};
export type MockOpenAiServerOptions = {
model: string;
reply?: string;
};
export async function createMockOpenAiServer(options: MockOpenAiServerOptions): Promise<MockOpenAiServer> {
const requests: MockOpenAiRequest[] = [];
const reply = options.reply ?? 'ok';
const server = createServer(async (req, res) => {
const path = req.url ?? '/';
const body = await readJsonBody(req);
requests.push({
body,
headers: redactHeaders(req.headers),
method: req.method ?? 'GET',
path,
receivedAt: new Date().toISOString(),
});
if (req.method === 'GET' && path === '/v1/models') {
return sendJson(res, 200, {
data: [{ id: options.model, object: 'model' }],
object: 'list',
});
}
if (req.method === 'POST' && path === '/v1/chat/completions') {
return sendJson(res, 200, {
choices: [
{
finish_reason: 'stop',
index: 0,
message: { content: reply, role: 'assistant' },
},
],
id: 'chatcmpl-e2e-smoke',
model: options.model,
object: 'chat.completion',
});
}
return sendJson(res, 404, {
error: {
message: `unexpected mock OpenAI path: ${req.method ?? 'GET'} ${path}`,
type: 'not_found',
},
});
});
await new Promise<void>((resolveListen, rejectListen) => {
server.once('error', rejectListen);
server.listen(0, '127.0.0.1', () => resolveListen());
});
const address = server.address();
if (address == null || typeof address === 'string') {
await closeServer(server);
throw new Error('mock OpenAI server did not receive a TCP port');
}
return {
baseUrl: `http://127.0.0.1:${address.port}/v1`,
close: () => closeServer(server),
requests: () => requests.slice(),
};
}
async function readJsonBody(req: IncomingMessage): Promise<unknown> {
const chunks: Buffer[] = [];
for await (const chunk of req) {
chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
}
const raw = Buffer.concat(chunks).toString('utf8');
if (!raw) return null;
try {
return JSON.parse(raw);
} catch {
return raw;
}
}
function sendJson(res: ServerResponse, status: number, value: unknown): void {
res.statusCode = status;
res.setHeader('content-type', 'application/json');
res.end(JSON.stringify(value));
}
function redactHeaders(headers: IncomingMessage['headers']): Record<string, string | string[] | undefined> {
return Object.fromEntries(
Object.entries(headers).map(([key, value]) => [
key,
key.toLowerCase() === 'authorization' || key.toLowerCase() === 'x-api-key' || key.toLowerCase() === 'api-key'
? '[REDACTED]'
: value,
]),
);
}
async function closeServer(server: Server): Promise<void> {
if (!server.listening) return;
await new Promise<void>((resolveClose, rejectClose) => {
server.close((error) => (error == null ? resolveClose() : rejectClose(error)));
});
}

66
e2e/lib/vitest/orbit.ts Normal file
View file

@ -0,0 +1,66 @@
import { requestJson } from './http.ts';
export type OrbitRunStart = {
agentRunId: string;
projectId: string;
};
export type OrbitSummary = {
agentRunId?: string | null;
artifactId?: string | null;
artifactProjectId?: string | null;
completedAt: string;
connectorsChecked: number;
connectorsFailed: number;
connectorsSkipped: number;
connectorsSucceeded: number;
id?: string;
markdown: string;
templateSkillId?: string | null;
trigger?: 'manual' | 'scheduled';
};
export type OrbitStatus = {
config?: {
enabled?: boolean;
templateSkillId?: string | null;
time?: string;
};
lastRun?: OrbitSummary | null;
lastRunsByTemplate?: Record<string, OrbitSummary>;
nextRunAt?: string | null;
running?: boolean;
};
export async function readOrbitStatus(baseUrl: string): Promise<OrbitStatus> {
return await requestJson<OrbitStatus>(baseUrl, '/api/orbit/status');
}
export async function startOrbitRun(baseUrl: string): Promise<OrbitRunStart> {
return await requestJson<OrbitRunStart>(baseUrl, '/api/orbit/run', { body: {} });
}
export async function waitForOrbitSummary(
baseUrl: string,
agentRunId: string,
options: { intervalMs?: number; timeoutMs?: number } = {},
): Promise<OrbitStatus> {
const timeoutMs = options.timeoutMs ?? 30_000;
const intervalMs = options.intervalMs ?? 250;
const startedAt = Date.now();
let last: OrbitStatus | null = null;
while (Date.now() - startedAt < timeoutMs) {
last = await readOrbitStatus(baseUrl);
if (last.running === false && last.lastRun?.agentRunId === agentRunId) {
return last;
}
await delay(intervalMs);
}
throw new Error(`Orbit summary for ${agentRunId} was not finalized within ${timeoutMs}ms; last=${JSON.stringify(last)}`);
}
function delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}

37
e2e/lib/vitest/packaged-report.ts Normal file
View file

@ -0,0 +1,37 @@
import { readFile } from 'node:fs/promises';
import { isAbsolute, join, resolve } from 'node:path';
import { createReport, type E2eReport } from './report.ts';
import { e2eWorkspaceRoot } from './smoke-suite.ts';
export type PackagedReportPlatform = 'mac' | 'win';
export type PackagedSmokeReport = {
report: E2eReport;
saveScreenshot: (path: string) => Promise<void>;
saveSummary: (value: unknown) => Promise<void>;
screenshotRelpath: string;
};
export async function createPackagedSmokeReport(platform: PackagedReportPlatform): Promise<PackagedSmokeReport> {
const root = resolveFromWorkspace(
process.env.OD_PACKAGED_E2E_REPORT_DIR ?? join('.tmp', 'e2e-release-report', platform),
);
const report = await createReport(root);
const screenshotRelpath = `screenshots/open-design-${platform}-smoke.png`;
return {
report,
saveScreenshot: async (path) => {
await report.save(screenshotRelpath, await readFile(path));
},
saveSummary: async (value) => {
await report.json('summary.json', value);
},
screenshotRelpath,
};
}
function resolveFromWorkspace(path: string): string {
return isAbsolute(path) ? path : resolve(e2eWorkspaceRoot(), path);
}

87
e2e/lib/vitest/pets.ts Normal file
View file

@ -0,0 +1,87 @@
import { mkdir, writeFile } from 'node:fs/promises';
import { join } from 'node:path';
import type { SmokeSuite } from './smoke-suite.ts';
import { requestJson } from './http.ts';
export type CodexPetSummary = {
bundled?: boolean;
description: string;
displayName: string;
hatchedAt: number;
id: string;
spritesheetExt: string;
spritesheetUrl: string;
};
export type CodexPetsResponse = {
pets: CodexPetSummary[];
rootDir: string;
};
export type CodexPetFixture = {
description: string;
displayName: string;
id: string;
png: Buffer;
};
const ONE_PIXEL_PNG = Buffer.from(
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mP8/x8AAwMCAO+/p9sAAAAASUVORK5CYII=',
'base64',
);
export async function writeCodexPetFixture(
suite: SmokeSuite,
input: {
description: string;
displayName: string;
id: string;
},
): Promise<CodexPetFixture> {
const petDir = join(suite.codexHomeDir, 'pets', input.id);
await mkdir(petDir, { recursive: true });
await writeFile(
join(petDir, 'pet.json'),
`${JSON.stringify({
description: input.description,
displayName: input.displayName,
id: input.id,
spritesheetPath: 'spritesheet.png',
}, null, 2)}\n`,
'utf8',
);
await writeFile(join(petDir, 'spritesheet.png'), ONE_PIXEL_PNG);
return { ...input, png: ONE_PIXEL_PNG };
}
export async function listCodexPets(baseUrl: string): Promise<CodexPetsResponse> {
return await requestJson<CodexPetsResponse>(baseUrl, '/api/codex-pets');
}
export async function readCodexPetSpritesheet(
baseUrl: string,
petId: string,
): Promise<{
body: Buffer;
cacheControl: string | null;
contentType: string | null;
origin: string | null;
status: number;
}> {
const response = await fetch(
new URL(`/api/codex-pets/${encodeURIComponent(petId)}/spritesheet`, ensureTrailingSlash(baseUrl)),
{ headers: { origin: 'null' } },
);
return {
body: Buffer.from(await response.arrayBuffer()),
cacheControl: response.headers.get('cache-control'),
contentType: response.headers.get('content-type'),
origin: response.headers.get('access-control-allow-origin'),
status: response.status,
};
}
function ensureTrailingSlash(value: string): string {
return value.endsWith('/') ? value : `${value}/`;
}

58
e2e/lib/vitest/report.ts Normal file
View file

@ -0,0 +1,58 @@
import { mkdir, writeFile } from 'node:fs/promises';
import { dirname, join, posix } from 'node:path';
export type ReportBlob = ArrayBuffer | Blob | string | Uint8Array;
export type ReportEntry = {
bytes: number;
path: string;
relpath: string;
};
export type E2eReport = {
root: string;
json: (relpath: string, value: unknown) => Promise<ReportEntry>;
save: (relpath: string, blob: ReportBlob) => Promise<ReportEntry>;
};
export async function createReport(root: string): Promise<E2eReport> {
await mkdir(root, { recursive: true });
async function save(relpath: string, blob: ReportBlob): Promise<ReportEntry> {
const safeRelpath = assertRelativeReportPath(relpath);
const outputPath = join(root, safeRelpath);
const content = await normalizeBlob(blob);
await mkdir(dirname(outputPath), { recursive: true });
await writeFile(outputPath, content);
return {
bytes: typeof content === 'string' ? Buffer.byteLength(content) : content.byteLength,
path: outputPath,
relpath: safeRelpath,
};
}
return {
root,
json: (relpath, value) => save(relpath, `${JSON.stringify(value, null, 2)}\n`),
save,
};
}
export function assertRelativeReportPath(relpath: string): string {
const unixName = relpath.replace(/\\/g, '/');
if (unixName.includes('\0') || unixName.startsWith('/') || /^[A-Za-z]:\//.test(unixName)) {
throw new Error(`report path must be relative: ${relpath}`);
}
const normalized = posix.normalize(unixName);
if (normalized === '.' || normalized === '..' || normalized.startsWith('../')) {
throw new Error(`report path must not escape report root: ${relpath}`);
}
return normalized;
}
async function normalizeBlob(blob: ReportBlob): Promise<Buffer | string> {
if (typeof blob === 'string') return blob;
if (blob instanceof Uint8Array) return Buffer.from(blob);
if (blob instanceof ArrayBuffer) return Buffer.from(blob);
return Buffer.from(await blob.arrayBuffer());
}

74
e2e/lib/vitest/runs.ts Normal file
View file

@ -0,0 +1,74 @@
import { requestJson, requestText } from './http.ts';
export type ChatRunStatus = 'queued' | 'running' | 'succeeded' | 'failed' | 'canceled';
export type ChatRunCreateBody = {
agentId: string;
assistantMessageId: string;
clientRequestId: string;
conversationId: string;
designSystemId?: string | null;
message: string;
model?: string | null;
projectId: string;
reasoning?: string | null;
skillId?: string | null;
};
export type ChatRunStatusBody = {
agentId: string | null;
assistantMessageId: string | null;
conversationId: string | null;
createdAt: number;
exitCode?: number | null;
id: string;
projectId: string | null;
signal?: string | null;
status: ChatRunStatus;
updatedAt: number;
};
export async function startRun(
baseUrl: string,
body: ChatRunCreateBody,
): Promise<{ runId: string }> {
return await requestJson<{ runId: string }>(baseUrl, '/api/runs', { body });
}
export async function readRun(
baseUrl: string,
runId: string,
): Promise<ChatRunStatusBody> {
return await requestJson<ChatRunStatusBody>(baseUrl, `/api/runs/${encodeURIComponent(runId)}`);
}
export async function readRunEvents(baseUrl: string, runId: string): Promise<string> {
return await requestText(baseUrl, `/api/runs/${encodeURIComponent(runId)}/events`);
}
export async function waitForRunStatus(
baseUrl: string,
runId: string,
expected: ChatRunStatus,
options: { intervalMs?: number; timeoutMs?: number } = {},
): Promise<ChatRunStatusBody> {
const timeoutMs = options.timeoutMs ?? 20_000;
const intervalMs = options.intervalMs ?? 250;
const startedAt = Date.now();
let last: ChatRunStatusBody | null = null;
while (Date.now() - startedAt < timeoutMs) {
last = await readRun(baseUrl, runId);
if (last.status === expected) return last;
if (last.status === 'failed' || last.status === 'canceled') {
throw new Error(`run ${runId} reached ${last.status}, expected ${expected}`);
}
await delay(intervalMs);
}
throw new Error(`run ${runId} did not reach ${expected} within ${timeoutMs}ms; last=${last?.status ?? 'unknown'}`);
}
function delay(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}

239
e2e/lib/vitest/smoke-suite.ts Normal file
View file

@ -0,0 +1,239 @@
import { mkdir, rm, writeFile } from 'node:fs/promises';
import { dirname, join } from 'node:path';
import { fileURLToPath } from 'node:url';
import { expect } from 'vitest';
import { assertRelativeReportPath, createReport, type E2eReport } from './report.ts';
import type {
ToolsDevCheckResult,
ToolsDevLogResult,
ToolsDevRuntime,
ToolsDevStartResult,
ToolsDevStatusResult,
} from './tools-dev.ts';
export type SmokeSuite = {
codexHomeDir: string;
dataDir: string;
namespace: string;
report: E2eReport;
root: string;
scratchDir: string;
toolsDevRoot: string;
with: SmokeSuiteWith;
writeScratchJson: (name: string, value: unknown) => Promise<string>;
finalize: (result: SmokeSuiteFinalizeInput) => Promise<string>;
};
export type SmokeSuiteFinalizeInput = {
diagnostics?: unknown;
error?: unknown;
success: boolean;
};
export type SmokeSuiteWith = {
toolsDev: (
run: (context: ToolsDevSuiteContext) => Promise<void>,
options?: ToolsDevSuiteOptions,
) => Promise<string>;
};
export type ToolsDevSuiteContext = {
check: () => Promise<ToolsDevCheckResult>;
logs: () => Promise<Record<string, ToolsDevLogResult>>;
runtime: ToolsDevRuntime;
start: ToolsDevStartResult;
status: ToolsDevStatusResult;
webUrl: string;
};
export type ToolsDevSuiteOptions = {
onFailure?: (input: {
context: ToolsDevSuiteContext | null;
error: unknown;
suite: SmokeSuite;
}) => Promise<void>;
skipFatalLogCheck?: boolean;
};
const e2eRoot = dirname(dirname(dirname(fileURLToPath(import.meta.url))));
const workspaceRoot = dirname(e2eRoot);
export function e2eWorkspaceRoot(): string {
return workspaceRoot;
}
export async function createSmokeSuite(name: string): Promise<SmokeSuite> {
const namespace = `e2e-${sanitizeSegment(name)}-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`;
const root = join(workspaceRoot, '.tmp', 'e2e', namespace);
const reportDir = join(root, 'report');
const scratchDir = join(root, 'scratch');
const codexHomeDir = join(scratchDir, 'codex-home');
const toolsDevRoot = join(scratchDir, 'tools-dev');
const dataDir = join(scratchDir, 'data');
await mkdir(reportDir, { recursive: true });
await mkdir(scratchDir, { recursive: true });
const report = await createReport(reportDir);
async function writeJson(baseDir: string, name: string, value: unknown): Promise<string> {
const safeName = assertRelativeReportPath(name);
const outputPath = join(baseDir, safeName);
await mkdir(dirname(outputPath), { recursive: true });
await writeFile(outputPath, `${JSON.stringify(value, null, 2)}\n`, 'utf8');
return outputPath;
}
const suite: SmokeSuite = {
codexHomeDir,
dataDir,
namespace,
report,
root,
scratchDir,
toolsDevRoot,
with: {
toolsDev: (run, options) => runToolsDevSuite(suite, run, options),
},
writeScratchJson: (name, value) => writeJson(scratchDir, name, value),
async finalize(result) {
await report.json('suite-result.json', {
namespace,
reportPath: report.root,
root,
status: result.success ? 'success' : 'failed',
timestamp: new Date().toISOString(),
});
if (result.success) {
await rm(scratchDir, { force: true, recursive: true });
return report.root;
}
await report.json('failure/preserved-site.json', {
diagnostics: result.diagnostics ?? null,
error: formatUnknown(result.error),
preservedScratchDir: scratchDir,
});
return report.root;
},
};
return suite;
}
async function runToolsDevSuite(
suite: SmokeSuite,
run: (context: ToolsDevSuiteContext) => Promise<void>,
options: ToolsDevSuiteOptions = {},
): Promise<string> {
const toolsDev = await import('./tools-dev.ts');
const runtime = await toolsDev.allocateToolsDevRuntime();
let context: ToolsDevSuiteContext | null = null;
let diagnostics: unknown = null;
let caughtError: unknown = null;
let success = false;
try {
const start = await toolsDev.startToolsDevWeb(suite, runtime);
const webUrl = assertRuntimeUrl(start.web?.status.url, 'web');
const status = await toolsDev.inspectToolsDevStatus(suite);
assertToolsDevStatus(suite, status);
context = {
check: () => toolsDev.inspectToolsDevCheck(suite),
logs: () => toolsDev.readToolsDevLogs(suite),
runtime,
start,
status,
webUrl,
};
await run(context);
if (options.skipFatalLogCheck !== true) {
assertNoFatalLogs(await context.logs());
}
success = true;
} catch (error) {
caughtError = error;
diagnostics = await toolsDev.inspectToolsDevCheck(suite).catch((diagnosticError: unknown) => ({
error: diagnosticError instanceof Error ? diagnosticError.message : String(diagnosticError),
}));
await options.onFailure?.({ context, error, suite }).catch((failureHookError: unknown) => {
diagnostics = {
diagnostics,
failureHookError: failureHookError instanceof Error ? failureHookError.message : String(failureHookError),
};
});
throw error;
} finally {
// startToolsDevWeb may have spawned namespace processes even if it threw before
// resolving, so cleanup must run unconditionally — otherwise orphans poison the
// next smoke run on a shared CI runner.
let stopError: unknown = null;
try {
await toolsDev.stopToolsDevWeb(suite);
} catch (error) {
stopError = error;
}
if (stopError != null) {
diagnostics = {
diagnostics,
stopError: stopError instanceof Error ? stopError.message : String(stopError),
};
// If the test body already failed, the catch block rethrew it; treat the stop
// failure as a side effect. If the body succeeded, the stop failure is the
// test failure — silent leaks are worse than a noisy assertion.
if (caughtError == null) {
success = false;
caughtError = stopError;
}
}
await suite.finalize({ diagnostics, error: caughtError, success });
if (stopError != null && caughtError === stopError) {
throw stopError;
}
}
return suite.report.root;
}
function assertRuntimeUrl(value: string | null | undefined, app: string): string {
if (typeof value !== 'string' || !value.startsWith('http://')) {
throw new Error(`${app} runtime did not expose an http URL: ${String(value)}`);
}
return value;
}
function assertToolsDevStatus(suite: SmokeSuite, status: ToolsDevStatusResult): void {
expect(status.namespace).toBe(suite.namespace);
expect(status.apps?.daemon?.state).toBe('running');
expect(status.apps?.web?.state).toBe('running');
}
function assertNoFatalLogs(logs: Record<string, { lines: string[] }>): void {
const combined = Object.values(logs)
.flatMap((entry) => entry.lines)
.join('\n');
expect(combined).not.toMatch(/ERR_UNSUPPORTED_NODE_MODULES_TYPE_STRIPPING/);
expect(combined).not.toMatch(/standalone Next\.js server exited/i);
expect(combined).not.toMatch(/packaged runtime failed/i);
expect(combined).not.toMatch(/Agent completed without producing any output/i);
}
function sanitizeSegment(value: string): string {
const safe = value.replace(/[^A-Za-z0-9._-]+/g, '-').replace(/^-+|-+$/g, '');
return safe || 'suite';
}
function formatUnknown(value: unknown): string | null {
if (value == null) return null;
if (value instanceof Error) {
return value.stack ?? value.message;
}
if (typeof value === 'string') return value;
try {
return JSON.stringify(value);
} catch {
return String(value);
}
}

182
e2e/lib/vitest/tools-dev.ts Normal file
View file

@ -0,0 +1,182 @@
import { execFile } from 'node:child_process';
import { createServer } from 'node:net';
import { promisify } from 'node:util';
import { e2eWorkspaceRoot, type SmokeSuite } from './smoke-suite.ts';
const execFileAsync = promisify(execFile);
export type ToolsDevAppStatus = {
pid?: number;
state?: string;
title?: string | null;
updatedAt?: string;
url?: string | null;
windowVisible?: boolean;
};
export type ToolsDevStartResult = {
daemon?: {
app: 'daemon';
created: boolean;
logPath: string;
pid?: number;
status: ToolsDevAppStatus;
};
web?: {
app: 'web';
created: boolean;
logPath: string;
pid?: number;
status: ToolsDevAppStatus;
};
};
export type ToolsDevStatusResult = {
apps?: Record<string, ToolsDevAppStatus | null>;
namespace?: string;
status?: string;
};
export type ToolsDevLogResult = {
lines: string[];
logPath: string;
};
export type ToolsDevCheckResult = {
apps?: Record<string, ToolsDevAppStatus | null>;
diagnostics?: unknown;
logs?: Record<string, ToolsDevLogResult>;
namespace?: string;
};
export type ToolsDevRuntime = {
daemonPort: number;
webPort: number;
};
export async function allocateToolsDevRuntime(): Promise<ToolsDevRuntime> {
const [daemonPort, webPort] = await Promise.all([findFreePort(), findFreePort()]);
if (daemonPort === webPort) return await allocateToolsDevRuntime();
return { daemonPort, webPort };
}
export async function startToolsDevWeb(suite: SmokeSuite, runtime: ToolsDevRuntime): Promise<ToolsDevStartResult> {
return await runToolsDevJson<ToolsDevStartResult>(
suite,
[
'start',
'web',
'--namespace',
suite.namespace,
'--tools-dev-root',
suite.toolsDevRoot,
'--daemon-port',
String(runtime.daemonPort),
'--web-port',
String(runtime.webPort),
'--json',
],
);
}
export async function stopToolsDevWeb(suite: SmokeSuite): Promise<unknown> {
return await runToolsDevJson<unknown>(
suite,
[
'stop',
'web',
'--namespace',
suite.namespace,
'--tools-dev-root',
suite.toolsDevRoot,
'--json',
],
);
}
export async function inspectToolsDevStatus(suite: SmokeSuite): Promise<ToolsDevStatusResult> {
return await runToolsDevJson<ToolsDevStatusResult>(
suite,
[
'status',
'--namespace',
suite.namespace,
'--tools-dev-root',
suite.toolsDevRoot,
'--json',
],
);
}
export async function inspectToolsDevCheck(suite: SmokeSuite): Promise<ToolsDevCheckResult> {
return await runToolsDevJson<ToolsDevCheckResult>(
suite,
[
'check',
'--namespace',
suite.namespace,
'--tools-dev-root',
suite.toolsDevRoot,
'--json',
],
);
}
export async function readToolsDevLogs(suite: SmokeSuite): Promise<Record<string, ToolsDevLogResult>> {
return await runToolsDevJson<Record<string, ToolsDevLogResult>>(
suite,
[
'logs',
'--namespace',
suite.namespace,
'--tools-dev-root',
suite.toolsDevRoot,
'--json',
],
);
}
async function runToolsDevJson<T>(suite: SmokeSuite, args: string[]): Promise<T> {
const { stdout } = await execFileAsync('pnpm', ['tools-dev', ...args], {
cwd: e2eWorkspaceRoot(),
env: {
...process.env,
CODEX_HOME: suite.codexHomeDir,
OD_DATA_DIR: suite.dataDir,
OD_MEDIA_CONFIG_DIR: suite.dataDir,
},
maxBuffer: 20 * 1024 * 1024,
});
return parseJsonOutput<T>(stdout);
}
function parseJsonOutput<T>(stdout: string): T {
const trimmed = stdout.trim();
if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
return JSON.parse(trimmed) as T;
}
const objectStart = stdout.lastIndexOf('\n{');
const arrayStart = stdout.lastIndexOf('\n[');
const jsonStart = Math.max(objectStart, arrayStart);
if (jsonStart < 0) {
throw new Error(`Expected JSON output from tools-dev, got: ${stdout}`);
}
return JSON.parse(stdout.slice(jsonStart + 1)) as T;
}
async function findFreePort(): Promise<number> {
const server = createServer();
await new Promise<void>((resolveListen, rejectListen) => {
server.once('error', rejectListen);
server.listen(0, '127.0.0.1', () => resolveListen());
});
const address = server.address();
await new Promise<void>((resolveClose, rejectClose) => {
server.close((error) => (error == null ? resolveClose() : rejectClose(error)));
});
if (address == null || typeof address === 'string') {
throw new Error('failed to allocate a local TCP port');
}
return address.port;
}

143
e2e/scripts/release-smoke.ts Normal file
View file

@ -0,0 +1,143 @@
import { spawn } from 'node:child_process';
import { existsSync } from 'node:fs';
import { readFile } from 'node:fs/promises';
import { dirname, isAbsolute, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { createReport } from '../lib/vitest/report.ts';
type Platform = 'mac' | 'win';
const e2eRoot = dirname(dirname(fileURLToPath(import.meta.url)));
const workspaceRoot = dirname(e2eRoot);
async function main(): Promise<void> {
const platform = parsePlatform(process.argv[2]);
const spec = process.argv[3] ?? defaultSpec(platform);
const namespace = process.env.OD_PACKAGED_E2E_NAMESPACE ?? defaultNamespace(platform);
const reportRoot = resolveFromWorkspace(
process.env.OD_PACKAGED_E2E_REPORT_DIR ?? join('.tmp', 'release-report', platform),
);
const report = await createReport(reportRoot);
process.env.OD_PACKAGED_E2E_REPORT_DIR = report.root;
await report.json('manifest.json', {
...(process.env.OD_PACKAGED_E2E_RELEASE_CHANNEL == null
? {}
: { channel: process.env.OD_PACKAGED_E2E_RELEASE_CHANNEL }),
...(process.env.OD_PACKAGED_E2E_RELEASE_VERSION == null
? {}
: { releaseVersion: process.env.OD_PACKAGED_E2E_RELEASE_VERSION }),
commit: process.env.GITHUB_SHA ?? null,
generatedAt: new Date().toISOString(),
githubRunAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null,
githubRunId: process.env.GITHUB_RUN_ID ?? null,
namespace,
platform,
reportPath: report.root,
screenshot: `screenshots/open-design-${platform}-smoke.png`,
spec,
});
await saveRequiredSource(report, 'tools-pack.json', process.env.OD_PACKAGED_E2E_BUILD_JSON_PATH);
await saveOptionalSource(report, 'tools-pack.log', process.env.OD_PACKAGED_E2E_BUILD_LOG_PATH);
const startedAt = Date.now();
const result = await runVitest(spec).catch((error: unknown) => ({
exitCode: 1,
log: formatUnknown(error),
}));
await report.save('vitest.log', result.log);
await report.json('suite-result.json', {
durationMs: Date.now() - startedAt,
exitCode: result.exitCode,
namespace,
platform,
reportPath: report.root,
spec,
status: result.exitCode === 0 ? 'success' : 'failed',
timestamp: new Date().toISOString(),
});
if (result.exitCode !== 0) {
process.exitCode = result.exitCode;
}
}
async function saveRequiredSource(
report: Awaited<ReturnType<typeof createReport>>,
relpath: string,
sourcePath: string | undefined,
): Promise<void> {
if (sourcePath == null || sourcePath === '') {
throw new Error(`missing source path for ${relpath}`);
}
const resolved = resolveFromWorkspace(sourcePath);
if (!existsSync(resolved)) {
throw new Error(`source file for ${relpath} does not exist: ${resolved}`);
}
await report.save(relpath, await readFile(resolved));
}
async function saveOptionalSource(
report: Awaited<ReturnType<typeof createReport>>,
relpath: string,
sourcePath: string | undefined,
): Promise<void> {
if (sourcePath == null || sourcePath === '') return;
const resolved = resolveFromWorkspace(sourcePath);
if (!existsSync(resolved)) return;
await report.save(relpath, await readFile(resolved));
}
async function runVitest(spec: string): Promise<{ exitCode: number; log: string }> {
const chunks: string[] = [];
const child = spawn(process.execPath, [join(e2eRoot, 'node_modules', 'vitest', 'vitest.mjs'), 'run', '-c', 'vitest.config.ts', spec], {
cwd: e2eRoot,
env: process.env,
stdio: ['ignore', 'pipe', 'pipe'],
});
child.stdout.on('data', (chunk: Buffer) => {
chunks.push(chunk.toString('utf8'));
process.stdout.write(chunk);
});
child.stderr.on('data', (chunk: Buffer) => {
chunks.push(chunk.toString('utf8'));
process.stderr.write(chunk);
});
const exitCode = await new Promise<number>((resolveExit, rejectExit) => {
child.once('error', rejectExit);
child.once('exit', (code) => resolveExit(code ?? 1));
});
return { exitCode, log: chunks.join('') };
}
function parsePlatform(value: string | undefined): Platform {
if (value === 'mac' || value === 'win') return value;
throw new Error('usage: tsx scripts/release-smoke.ts <mac|win> [spec]');
}
function defaultSpec(platform: Platform): string {
return platform === 'mac' ? 'specs/mac.spec.ts' : 'specs/win.spec.ts';
}
function defaultNamespace(platform: Platform): string {
return platform === 'mac' ? 'release-beta' : 'release-beta-win';
}
function resolveFromWorkspace(path: string): string {
return isAbsolute(path) ? path : resolve(workspaceRoot, path);
}
function formatUnknown(value: unknown): string {
if (value instanceof Error) return value.stack ?? `${value.name}: ${value.message}`;
try {
return JSON.stringify(value, null, 2);
} catch {
return String(value);
}
}
await main();

206
e2e/specs/dialog/main.spec.ts Normal file
View file

@ -0,0 +1,206 @@
// @vitest-environment node
import { randomUUID } from 'node:crypto';
import { join } from 'node:path';
import { describe, expect, test } from 'vitest';
import { createFakeAgentRuntimes } from '@/fake-agents';
import {
extractArtifactFromRunEvents,
persistExtractedArtifact,
type ProjectFile,
} from '@/vitest/artifacts';
import { requestJson, requestText } from '@/vitest/http';
import { listMessages, saveMessage } from '@/vitest/messages';
import { startRun, readRunEvents, waitForRunStatus } from '@/vitest/runs';
import { createSmokeSuite } from '@/vitest/smoke-suite';
const GENERATED_FILE = 'real-daemon-smoke.html';
const GENERATED_HEADING = 'Real Daemon Smoke';
const PROMPT = 'Create a deterministic smoke artifact';
type ProjectResponse = {
conversationId: string;
project: {
id: string;
metadata?: {
kind?: string;
};
name: string;
};
};
type ProjectFilesResponse = {
files: ProjectFile[];
};
describe('dialog main spec', () => {
test('creates a project, runs the configured agent, streams events, and persists the generated artifact', async () => {
const suite = await createSmokeSuite('dialog-main');
await suite.with.toolsDev(async ({ runtime, status, webUrl }) => {
const fakeAgents = await createFakeAgentRuntimes({
root: join(suite.scratchDir, 'fake-agents'),
runtimeIds: ['codex'],
});
await requestJson<{ config: Record<string, unknown> }>(webUrl, '/api/app-config', {
body: {
agentCliEnv: { codex: fakeAgents.codex.env },
agentId: 'codex',
agentModels: { codex: { model: 'default', reasoning: 'default' } },
designSystemId: null,
onboardingCompleted: true,
skillId: null,
telemetry: { artifactManifest: true, content: false, metrics: false },
},
method: 'PUT',
});
const project = await requestJson<ProjectResponse>(webUrl, '/api/projects', {
body: {
designSystemId: null,
id: randomUUID(),
metadata: { kind: 'prototype' },
name: 'Dialog main smoke project',
pendingPrompt: null,
skillId: null,
},
});
expect(project.conversationId).toEqual(expect.any(String));
expect(project.project.metadata?.kind).toBe('prototype');
const requestId = `dialog-main-${Date.now()}`;
const now = Date.now();
const userMessageId = `user-${requestId}`;
const assistantMessageId = `assistant-${requestId}`;
await saveMessage(webUrl, project.project.id, project.conversationId, {
content: PROMPT,
createdAt: now,
id: userMessageId,
role: 'user',
});
await saveMessage(webUrl, project.project.id, project.conversationId, {
agentId: 'codex',
agentName: 'Codex',
content: '',
createdAt: now,
events: [],
id: assistantMessageId,
role: 'assistant',
runStatus: 'running',
startedAt: now,
});
const run = await startRun(webUrl, {
agentId: 'codex',
assistantMessageId,
clientRequestId: requestId,
conversationId: project.conversationId,
designSystemId: null,
message: PROMPT,
model: 'default',
projectId: project.project.id,
reasoning: 'default',
skillId: null,
});
const finalRun = await waitForRunStatus(webUrl, run.runId, 'succeeded', { timeoutMs: 30_000 });
expect(finalRun.projectId).toBe(project.project.id);
expect(finalRun.conversationId).toBe(project.conversationId);
expect(finalRun.agentId).toBe('codex');
const events = await readRunEvents(webUrl, run.runId);
expect(events).toContain('real-daemon-smoke');
expect(events).toContain('"type":"usage"');
expect(events).toContain('"status":"succeeded"');
const artifact = extractArtifactFromRunEvents(events);
expect(artifact.identifier).toBe('real-daemon-smoke');
expect(artifact.title).toBe(GENERATED_HEADING);
expect(artifact.html).toContain(GENERATED_HEADING);
const persistedArtifact = await persistExtractedArtifact(
webUrl,
project.project.id,
artifact,
{ designSystemId: null, sourceSkillId: null },
);
expect(persistedArtifact.name).toBe(GENERATED_FILE);
expect(persistedArtifact.kind).toBe('html');
const files = await requestJson<ProjectFilesResponse>(
webUrl,
`/api/projects/${encodeURIComponent(project.project.id)}/files`,
);
const generated = files.files.find((file) => file.name === GENERATED_FILE);
expect(generated?.kind).toBe('html');
expect(generated?.artifactManifest?.title).toBe(GENERATED_HEADING);
expect(generated?.artifactManifest?.renderer).toBe('html');
const rawHtml = await requestText(
webUrl,
`/api/projects/${encodeURIComponent(project.project.id)}/raw/${GENERATED_FILE}`,
);
expect(rawHtml).toContain(GENERATED_HEADING);
expect(rawHtml).toContain('Generated through the daemon run path.');
await saveMessage(webUrl, project.project.id, project.conversationId, {
agentId: 'codex',
agentName: 'Codex',
content: artifact.rawText,
createdAt: now,
endedAt: Date.now(),
events: [],
id: assistantMessageId,
producedFiles: [persistedArtifact],
role: 'assistant',
runId: finalRun.id,
runStatus: 'succeeded',
startedAt: now,
telemetryFinalized: true,
});
const messages = await listMessages(
webUrl,
project.project.id,
project.conversationId,
);
const assistantMessage = messages.find((message) => message.id === finalRun.assistantMessageId);
expect(assistantMessage?.runStatus).toBe('succeeded');
expect(assistantMessage?.producedFiles).toEqual(
expect.arrayContaining([
expect.objectContaining({
artifactManifest: expect.objectContaining({ title: GENERATED_HEADING }),
name: GENERATED_FILE,
}),
]),
);
await suite.report.json('summary.json', {
artifact: {
manifest: generated?.artifactManifest,
name: generated?.name,
size: generated?.size,
},
conversationId: project.conversationId,
files: files.files.map((file) => ({
artifactManifest: file.artifactManifest,
kind: file.kind,
name: file.name,
size: file.size,
})),
namespace: suite.namespace,
project: project.project,
run: finalRun,
runtime: {
daemonPort: runtime.daemonPort,
webPort: runtime.webPort,
webUrl,
},
status,
});
});
}, 180_000);
});

47
e2e/specs/mac.spec.ts
View file

@ -1,13 +1,14 @@
// @vitest-environment node
import { execFile } from 'node:child_process';
import { access, stat } from 'node:fs/promises';
import { access, mkdir, stat } from 'node:fs/promises';
import { dirname, isAbsolute, join, resolve, sep } from 'node:path';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
import { afterAll, beforeAll, describe, expect, test } from 'vitest';
import { createPackagedSmokeReport } from '@/vitest/packaged-report';
import { createDesktopHarness, STORAGE_KEY, waitFor } from '../lib/desktop/desktop-test-helpers.ts';
const execFileAsync = promisify(execFile);
@ -16,9 +17,7 @@ const workspaceRoot = dirname(e2eRoot);
const toolsPackDir = resolveFromWorkspace(process.env.OD_PACKAGED_E2E_TOOLS_PACK_DIR ?? '.tmp/tools-pack');
const namespace = process.env.OD_PACKAGED_E2E_NAMESPACE ?? 'release-beta';
const pnpmCommand = process.env.OD_E2E_PNPM_COMMAND ?? 'pnpm';
const screenshotPath = resolveFromWorkspace(
process.env.OD_PACKAGED_E2E_SCREENSHOT_PATH ?? join(toolsPackDir, 'screenshots', `${namespace}.png`),
);
const screenshotPath = join(toolsPackDir, 'screenshots', `${namespace}.png`);
const outputNamespaceRoot = join(toolsPackDir, 'out', 'mac', 'namespaces', namespace);
const runtimeNamespaceRoot = join(toolsPackDir, 'runtime', 'mac', 'namespaces', namespace);
@ -110,6 +109,7 @@ macDescribe('packaged mac runtime smoke', () => {
let started = false;
test('installs, starts, inspects, stops, and uninstalls the built mac artifact', async () => {
const report = await createPackagedSmokeReport('mac');
let passed = false;
try {
const install = await runToolsPackJson<MacInstallResult>('install');
@ -147,11 +147,14 @@ macDescribe('packaged mac runtime smoke', () => {
expect(value.health.ok).toBe(true);
expect(value.health.version).toEqual(expect.any(String));
await mkdir(dirname(screenshotPath), { recursive: true });
const screenshot = await runToolsPackJson<MacInspectResult>('inspect', ['--path', screenshotPath]);
expect(screenshot.screenshot?.path).toBe(screenshotPath);
expect(await fileSizeBytes(screenshotPath)).toBeGreaterThan(0);
await report.saveScreenshot(screenshotPath);
assertLogPathsAndContent(await runToolsPackJson<LogsResult>('logs'));
const logs = await runToolsPackJson<LogsResult>('logs');
assertLogPathsAndContent(logs);
const stop = await runToolsPackJson<MacStopResult>('stop');
started = false;
@ -165,6 +168,28 @@ macDescribe('packaged mac runtime smoke', () => {
expect(uninstall.installedAppPath).toBe(install.installedAppPath);
expect(uninstall.removed).toBe(true);
expect(await pathExists(install.installedAppPath)).toBe(false);
await report.saveSummary({
health: value,
install: {
detached: install.detached,
dmgPath: install.dmgPath,
installedAppPath: install.installedAppPath,
mountPoint: install.mountPoint,
},
logs: summarizeLogs(logs),
namespace,
screenshot: report.screenshotRelpath,
start: {
appPath: start.appPath,
executablePath: start.executablePath,
logPath: start.logPath,
pid: start.pid,
source: start.source,
status: start.status,
},
stop,
uninstall,
});
passed = true;
} finally {
if (!passed) {
@ -529,6 +554,18 @@ function assertLogPathsAndContent(result: LogsResult): void {
expect(combined).not.toMatch(/packaged runtime failed/i);
}
function summarizeLogs(result: LogsResult): Record<string, { lineCount: number; logPath: string }> {
return Object.fromEntries(
Object.entries(result.logs).map(([app, entry]) => [
app,
{
lineCount: entry.lines.length,
logPath: entry.logPath,
},
]),
);
}
async function printPackagedLogs(): Promise<void> {
const result = await runToolsPackJson<LogsResult>('logs');
for (const [app, entry] of Object.entries(result.logs)) {

110
e2e/specs/orbit/run.spec.ts Normal file
View file

@ -0,0 +1,110 @@
// @vitest-environment node
import { join } from 'node:path';
import { describe, expect, test } from 'vitest';
import { createFakeAgentRuntimes } from '@/fake-agents';
import { listLiveArtifacts, readLiveArtifactPreview } from '@/vitest/live-artifacts';
import { readOrbitStatus, startOrbitRun, waitForOrbitSummary } from '@/vitest/orbit';
import { readRun, waitForRunStatus } from '@/vitest/runs';
import { createSmokeSuite } from '@/vitest/smoke-suite';
import { requestJson } from '@/vitest/http';
type ProjectResponse = {
project: {
id: string;
metadata?: {
kind?: string;
trigger?: string;
};
name: string;
skillId?: string | null;
};
};
describe('orbit run spec', () => {
test('starts a manual Orbit run and publishes a live artifact through the agent tool path', async () => {
const suite = await createSmokeSuite('orbit-run');
await suite.with.toolsDev(async ({ runtime, status, webUrl }) => {
const fakeAgents = await createFakeAgentRuntimes({
root: join(suite.scratchDir, 'fake-agents'),
runtimeIds: ['codex'],
});
await requestJson<{ config: Record<string, unknown> }>(webUrl, '/api/app-config', {
body: {
agentCliEnv: { codex: fakeAgents.codex.env },
agentId: 'codex',
agentModels: { codex: { model: 'default', reasoning: 'default' } },
designSystemId: null,
onboardingCompleted: true,
orbit: { enabled: false, templateSkillId: null, time: '08:00' },
skillId: null,
telemetry: { artifactManifest: true, content: false, metrics: false },
},
method: 'PUT',
});
const before = await readOrbitStatus(webUrl);
expect(before.running).toBe(false);
expect(before.config?.enabled).toBe(false);
expect(before.config?.templateSkillId).toBe(null);
const orbit = await startOrbitRun(webUrl);
expect(orbit.projectId).toMatch(/^orbit-/);
expect(orbit.agentRunId).toEqual(expect.any(String));
const runningStatus = await readRun(webUrl, orbit.agentRunId);
expect(runningStatus.projectId).toBe(orbit.projectId);
expect(runningStatus.agentId).toBe('codex');
const finalRun = await waitForRunStatus(webUrl, orbit.agentRunId, 'succeeded', { timeoutMs: 30_000 });
expect(finalRun.status).toBe('succeeded');
const finalOrbit = await waitForOrbitSummary(webUrl, orbit.agentRunId, { timeoutMs: 30_000 });
const summary = finalOrbit.lastRun;
expect(summary?.agentRunId).toBe(orbit.agentRunId);
expect(summary?.artifactProjectId).toBe(orbit.projectId);
expect(summary?.artifactId).toEqual(expect.any(String));
expect(summary?.connectorsChecked).toBe(1);
expect(summary?.connectorsSucceeded).toBe(1);
expect(summary?.connectorsFailed).toBe(0);
expect(summary?.markdown).toContain('Orbit Agent');
const project = await requestJson<ProjectResponse>(webUrl, `/api/projects/${encodeURIComponent(orbit.projectId)}`);
expect(project.project.skillId).toBe('live-artifact');
expect(project.project.metadata?.kind).toBe('orbit');
expect(project.project.metadata?.trigger).toBe('manual');
const artifacts = await listLiveArtifacts(webUrl, orbit.projectId);
const artifact = artifacts.find((candidate) => candidate.id === summary?.artifactId);
expect(artifact).toEqual(expect.objectContaining({
createdByRunId: orbit.agentRunId,
projectId: orbit.projectId,
refreshStatus: 'idle',
status: 'active',
title: 'Orbit Daily Digest',
}));
const preview = await readLiveArtifactPreview(webUrl, orbit.projectId, artifact?.id ?? '');
expect(preview).toContain('Orbit daily digest');
expect(preview).toContain('Fake connector activity');
await suite.report.json('summary.json', {
artifact,
namespace: suite.namespace,
orbit: finalOrbit,
project: project.project,
run: finalRun,
runtime: {
daemonPort: runtime.daemonPort,
webPort: runtime.webPort,
webUrl,
},
status,
});
});
}, 180_000);
});

80
e2e/specs/pet/main.spec.ts Normal file
View file

@ -0,0 +1,80 @@
// @vitest-environment node
import { join } from 'node:path';
import { describe, expect, test } from 'vitest';
import {
listCodexPets,
readCodexPetSpritesheet,
writeCodexPetFixture,
} from '@/vitest/pets';
import { createSmokeSuite } from '@/vitest/smoke-suite';
const USER_PET_ID = 'qa-inspect-pet';
const USER_PET_NAME = 'QA Inspect Pet';
const USER_PET_DESCRIPTION = 'Seeded by the Pet pure inspect spec.';
describe('pet main spec', () => {
test('serves deterministic Codex pet registry entries and spritesheets', async () => {
const suite = await createSmokeSuite('pet-main');
await suite.with.toolsDev(async ({ runtime, status, webUrl }) => {
const fixture = await writeCodexPetFixture(suite, {
description: USER_PET_DESCRIPTION,
displayName: USER_PET_NAME,
id: USER_PET_ID,
});
const registry = await listCodexPets(webUrl);
expect(registry.rootDir).toBe(join(suite.codexHomeDir, 'pets'));
const userPet = registry.pets.find((pet) => pet.id === USER_PET_ID);
expect(userPet).toEqual(expect.objectContaining({
bundled: false,
description: USER_PET_DESCRIPTION,
displayName: USER_PET_NAME,
id: USER_PET_ID,
spritesheetExt: 'png',
spritesheetUrl: `/api/codex-pets/${USER_PET_ID}/spritesheet`,
}));
const bundledPet = registry.pets.find((pet) => pet.id === 'clippit');
expect(bundledPet).toEqual(expect.objectContaining({
bundled: true,
id: 'clippit',
spritesheetExt: 'webp',
}));
const userSheet = await readCodexPetSpritesheet(webUrl, USER_PET_ID);
expect(userSheet.status).toBe(200);
expect(userSheet.contentType).toMatch(/^image\/png\b/);
expect(userSheet.cacheControl).toBe('no-store');
expect(userSheet.origin).toBe('null');
expect(userSheet.body.equals(fixture.png)).toBe(true);
const bundledSheet = await readCodexPetSpritesheet(webUrl, 'clippit');
expect(bundledSheet.status).toBe(200);
expect(bundledSheet.contentType).toMatch(/^image\/webp\b/);
expect(bundledSheet.body.byteLength).toBeGreaterThan(0);
const escapedSheet = await readCodexPetSpritesheet(webUrl, '../../etc/passwd');
expect(escapedSheet.status).toBe(404);
await suite.report.json('summary.json', {
namespace: suite.namespace,
registry: {
bundledCount: registry.pets.filter((pet) => pet.bundled).length,
rootDir: registry.rootDir,
userPet,
},
runtime: {
daemonPort: runtime.daemonPort,
webPort: runtime.webPort,
webUrl,
},
status,
});
});
}, 180_000);
});

50
e2e/specs/win.spec.ts
View file

@ -1,13 +1,15 @@
// @vitest-environment node
import { execFile } from 'node:child_process';
import { readFile, stat } from 'node:fs/promises';
import { mkdir, readFile, stat } from 'node:fs/promises';
import { basename, dirname, isAbsolute, join, resolve, sep } from 'node:path';
import { fileURLToPath } from 'node:url';
import { promisify } from 'node:util';
import { describe, expect, test } from 'vitest';
import { createPackagedSmokeReport } from '@/vitest/packaged-report';
const execFileAsync = promisify(execFile);
const e2eRoot = dirname(dirname(fileURLToPath(import.meta.url)));
const workspaceRoot = dirname(e2eRoot);
@ -19,9 +21,7 @@ const installIdentity = resolveInstallIdentity(namespace);
const outputNamespaceRoot = join(toolsPackDir, 'out', 'win', 'namespaces', namespace);
const runtimeNamespaceRoot = join(toolsPackDir, 'runtime', 'win', 'namespaces', namespace);
const screenshotPath = resolveFromWorkspace(
process.env.OD_PACKAGED_E2E_SCREENSHOT_PATH ?? join(toolsPackDir, 'screenshots', `${namespace}.png`),
);
const screenshotPath = join(toolsPackDir, 'screenshots', `${namespace}.png`);
const healthExpression = "fetch('/api/health').then(async response => ({ health: await response.json(), href: location.href, status: response.status, title: document.title }))";
type DesktopStatus = {
@ -133,6 +133,7 @@ winDescribe('packaged windows runtime smoke', () => {
let started = false;
test('installs, starts, inspects with eval and screenshot, stops, and uninstalls the built windows artifact', async () => {
const report = await createPackagedSmokeReport('win');
let passed = false;
const timings: SmokeTiming[] = [];
try {
@ -190,11 +191,13 @@ winDescribe('packaged windows runtime smoke', () => {
expect(value.health.ok).toBe(true);
expect(value.health.version).toEqual(expect.any(String));
await mkdir(dirname(screenshotPath), { recursive: true });
const screenshot = await measureSmokeStep(timings, 'inspect screenshot', async () =>
runToolsPackJson<WinInspectResult>('inspect', ['--path', screenshotPath]),
);
expect(screenshot.screenshot?.path).toBe(screenshotPath);
expect(await fileSizeBytes(screenshotPath)).toBeGreaterThan(0);
await report.saveScreenshot(screenshotPath);
const logs = await measureSmokeStep(timings, 'logs', async () => runToolsPackJson<LogsResult>('logs'));
assertLogPathsAndContent(logs);
@ -217,6 +220,33 @@ winDescribe('packaged windows runtime smoke', () => {
expect(uninstall.residueObservation?.uninstallerExists).toBe(false);
expect(uninstall.residueObservation?.startMenuShortcutExists).toBe(false);
expect(uninstall.residueObservation?.userDesktopShortcutExists).toBe(false);
await report.saveSummary({
health: value,
install: {
desktopShortcutExists: install.desktopShortcutExists,
installDir: install.installDir,
installPayload: install.installPayload,
installerPath: install.installerPath,
registryEntryCount: install.registryEntries.length,
startMenuShortcutExists: install.startMenuShortcutExists,
timingPath: install.timingPath,
uninstallerPath: install.uninstallerPath,
},
installTiming,
logs: summarizeLogs(logs),
namespace,
screenshot: report.screenshotRelpath,
start: {
executablePath: start.executablePath,
logPath: start.logPath,
pid: start.pid,
source: start.source,
status: start.status,
},
stop,
timings,
uninstall,
});
passed = true;
} finally {
if (!passed) {
@ -343,6 +373,18 @@ function assertLogPathsAndContent(result: LogsResult): void {
expect(combined).not.toMatch(/standalone Next\.js server exited/i);
}
function summarizeLogs(result: LogsResult): Record<string, { lineCount: number; logPath: string }> {
return Object.fromEntries(
Object.entries(result.logs).map(([app, entry]) => [
app,
{
lineCount: entry.lines.length,
logPath: entry.logPath,
},
]),
);
}
async function printPackagedLogs(): Promise<void> {
const result = await runToolsPackJson<LogsResult>('logs');
for (const [app, entry] of Object.entries(result.logs)) {

46
e2e/tests/report/lifecycle.test.ts Normal file
View file

@ -0,0 +1,46 @@
// @vitest-environment node
import { mkdtemp, readFile, rm } from 'node:fs/promises';
import { join } from 'node:path';
import { tmpdir } from 'node:os';
import { afterEach, describe, expect, test } from 'vitest';
import { createReport } from '@/vitest/report';
const roots: string[] = [];
describe('report lifecycle', () => {
afterEach(async () => {
await Promise.all(roots.splice(0).map((root) => rm(root, { force: true, recursive: true })));
});
test('saves binary and JSON entries under the report root', async () => {
const root = await makeRoot();
const report = await createReport(root);
const binary = await report.save('evidence/sample.bin', Buffer.from([1, 2, 3]));
const json = await report.json('summary.json', { ok: true });
expect(binary.relpath).toBe('evidence/sample.bin');
expect(binary.path).toBe(join(root, 'evidence', 'sample.bin'));
expect(binary.bytes).toBe(3);
expect(await readFile(binary.path)).toEqual(Buffer.from([1, 2, 3]));
expect(json.path).toBe(join(root, 'summary.json'));
expect(JSON.parse(await readFile(json.path, 'utf8'))).toEqual({ ok: true });
});
test('rejects absolute paths and parent traversal', async () => {
const report = await createReport(await makeRoot());
await expect(report.save('/tmp/out.txt', 'x')).rejects.toThrow(/relative/);
await expect(report.save('../out.txt', 'x')).rejects.toThrow(/escape/);
await expect(report.save('nested/../../out.txt', 'x')).rejects.toThrow(/escape/);
});
});
async function makeRoot(): Promise<string> {
const root = await mkdtemp(join(tmpdir(), 'open-design-e2e-report-'));
roots.push(root);
return root;
}

164
e2e/tests/tools-dev/inspect.test.ts Normal file
View file

@ -0,0 +1,164 @@
// @vitest-environment node
import { randomUUID } from 'node:crypto';
import { describe, expect, test } from 'vitest';
import { requestJson, requestText } from '@/vitest/http';
import { createMockOpenAiServer } from '@/vitest/mock-openai';
import { createSmokeSuite } from '@/vitest/smoke-suite';
type HealthResponse = {
ok?: unknown;
service?: unknown;
version?: unknown;
};
type ProjectResponse = {
project: {
id: string;
metadata?: {
kind?: string;
};
name: string;
};
};
type ProjectFilesResponse = {
files: Array<{
artifactManifest?: {
kind?: string;
renderer?: string;
status?: string;
title?: string;
};
kind?: string;
name: string;
size: number;
}>;
};
type ProviderConnectionResponse = {
kind: string;
latencyMs: number;
model?: string;
ok: boolean;
sample?: string;
status?: number;
};
describe('tools-dev pure inspect smoke', () => {
test('runs a namespace-isolated web/daemon runtime with a mocked OpenAI provider and artifact file', async () => {
const suite = await createSmokeSuite('tools-dev-provider');
const mock = await createMockOpenAiServer({ model: 'e2e-smoke-model' });
try {
await suite.with.toolsDev(
async ({ runtime, status, webUrl }) => {
const health = await requestJson<HealthResponse>(webUrl, '/api/health');
expect(health.ok).toBe(true);
expect(health.version).toEqual(expect.any(String));
const config = await requestJson<{ config: Record<string, unknown> }>(webUrl, '/api/app-config', {
body: {
agentId: null,
agentModels: {},
onboardingCompleted: true,
privacyDecisionAt: Date.now(),
telemetry: { artifactManifest: true, content: false, metrics: false },
},
method: 'PUT',
});
expect(config.config.onboardingCompleted).toBe(true);
const project = await requestJson<ProjectResponse>(webUrl, '/api/projects', {
body: {
id: randomUUID(),
name: 'Pure inspect smoke project',
metadata: { kind: 'prototype' },
pendingPrompt: 'Create a deterministic inspect smoke artifact',
},
});
expect(project.project.id).toEqual(expect.any(String));
expect(project.project.metadata?.kind).toBe('prototype');
const file = await requestJson<{ file: { name: string; size: number } }>(
webUrl,
`/api/projects/${encodeURIComponent(project.project.id)}/files`,
{
body: {
artifactManifest: {
entry: 'index.html',
exports: ['html'],
kind: 'html',
renderer: 'html',
status: 'complete',
title: 'Pure Inspect Smoke Artifact',
version: 1,
},
content: '<!doctype html><html><body><main data-e2e="pure-inspect-smoke">ok</main></body></html>',
name: 'index.html',
},
},
);
expect(file.file.name).toBe('index.html');
expect(file.file.size).toBeGreaterThan(0);
const files = await requestJson<ProjectFilesResponse>(webUrl, `/api/projects/${encodeURIComponent(project.project.id)}/files`);
const indexFile = files.files.find((entry) => entry.name === 'index.html');
expect(indexFile?.artifactManifest?.title).toBe('Pure Inspect Smoke Artifact');
expect(indexFile?.artifactManifest?.renderer).toBe('html');
const rawHtml = await requestText(webUrl, `/api/projects/${encodeURIComponent(project.project.id)}/raw/index.html`);
expect(rawHtml).toContain('data-e2e="pure-inspect-smoke"');
const connection = await requestJson<ProviderConnectionResponse>(webUrl, '/api/test/connection', {
body: {
apiKey: 'sk-e2e-placeholder',
baseUrl: mock.baseUrl,
mode: 'provider',
model: 'e2e-smoke-model',
protocol: 'openai',
},
});
expect(connection.ok).toBe(true);
expect(connection.kind).toBe('success');
expect(connection.sample).toBe('ok');
expect(mock.requests().map((request) => request.path)).toEqual(['/v1/models', '/v1/chat/completions']);
await suite.report.json('summary.json', {
connection,
files: files.files.map((entry) => ({
artifactManifest: entry.artifactManifest,
kind: entry.kind,
name: entry.name,
size: entry.size,
})),
health,
mockRequests: mock.requests().map((request) => ({
body: request.body,
method: request.method,
path: request.path,
receivedAt: request.receivedAt,
})),
namespace: suite.namespace,
project: project.project,
runtime: {
daemonPort: runtime.daemonPort,
webPort: runtime.webPort,
webUrl,
},
status,
});
},
{
onFailure: async () => {
await suite.writeScratchJson('failure/mock-requests.json', mock.requests());
},
},
);
} finally {
await mock.close();
}
}, 180_000);
});

10
scripts/guard.ts
View file

@ -2,6 +2,10 @@ import { readFile, readdir } from "node:fs/promises";
import path from "node:path";
const repoRoot = path.resolve(import.meta.dirname, "..");
const allowedE2eScripts = new Set([
"e2e/scripts/playwright.ts",
"e2e/scripts/release-smoke.ts",
]);
type GuardCheck = {
name: string;
@ -304,13 +308,13 @@ async function checkE2eLayout(): Promise<boolean> {
}
if (repositoryPath.startsWith("e2e/scripts/")) {
if (repositoryPath !== "e2e/scripts/playwright.ts") {
violations.push(`${repositoryPath} -> e2e scripts currently allow only scripts/playwright.ts`);
if (!allowedE2eScripts.has(repositoryPath)) {
violations.push(`${repositoryPath} -> e2e scripts must be an approved package-owned entrypoint`);
}
continue;
}
violations.push(`${repositoryPath} -> e2e source files must live in specs/, tests/, ui/, resources/, lib/, or scripts/playwright.ts`);
violations.push(`${repositoryPath} -> e2e source files must live in specs/, tests/, ui/, resources/, lib/, or approved scripts`);
}
if (violations.length > 0) {

12
scripts/release-stable.ts
View file

@ -301,6 +301,18 @@ async function validateStableNightlyMetadata(options: {
requireVersionedUrlField(macZip, "url", expectedVersionUrl, `${sourceName}.platforms.mac.artifacts.zip`);
requireVersionedUrlField(macZip, "sha256Url", expectedVersionUrl, `${sourceName}.platforms.mac.artifacts.zip`);
const macIntel = requireObjectField(platforms, "macIntel", `${sourceName}.platforms`);
expectBooleanField(macIntel, "enabled", true, `${sourceName}.platforms.macIntel`);
expectStringField(macIntel, "arch", "x64", `${sourceName}.platforms.macIntel`);
expectBooleanField(macIntel, "signed", true, `${sourceName}.platforms.macIntel`);
const macIntelArtifacts = requireObjectField(macIntel, "artifacts", `${sourceName}.platforms.macIntel`);
const macIntelDmg = requireObjectField(macIntelArtifacts, "dmg", `${sourceName}.platforms.macIntel.artifacts`);
requireVersionedUrlField(macIntelDmg, "url", expectedVersionUrl, `${sourceName}.platforms.macIntel.artifacts.dmg`);
requireVersionedUrlField(macIntelDmg, "sha256Url", expectedVersionUrl, `${sourceName}.platforms.macIntel.artifacts.dmg`);
const macIntelZip = requireObjectField(macIntelArtifacts, "zip", `${sourceName}.platforms.macIntel.artifacts`);
requireVersionedUrlField(macIntelZip, "url", expectedVersionUrl, `${sourceName}.platforms.macIntel.artifacts.zip`);
requireVersionedUrlField(macIntelZip, "sha256Url", expectedVersionUrl, `${sourceName}.platforms.macIntel.artifacts.zip`);
const win = requireObjectField(platforms, "win", `${sourceName}.platforms`);
expectBooleanField(win, "enabled", true, `${sourceName}.platforms.win`);
expectStringField(win, "arch", "x64", `${sourceName}.platforms.win`);