mirror of
https://github.com/ZSeven-W/openpencil.git
synced 2026-05-31 19:04:29 +07:00
feat(ai): detect sibling overlaps in snapshot_layout and guide fixes
snapshot_layout now emits an `overlaps` array listing sibling pairs whose rendered bounds intersect, so text-only agents can diagnose stacking bugs without a screenshot. When the shared parent has `layout: "none"` the reason string points at the real cause (absolute x/y stacking) instead of letting models hedge with height/padding tweaks. Handler prompt adds a matching diagnosis workflow so agents fix the parent layout rather than resizing the overlapping children.
This commit is contained in:
parent
0154873fe9
commit
572f71a1a8
3 changed files with 75 additions and 21 deletions
|
|
@ -85,6 +85,16 @@ WORKFLOW:
|
|||
3. When inserting, use "after" parameter with a sibling ID to place the new node in the correct position.
|
||||
4. After each operation, write 1-2 sentences summarizing what changed.
|
||||
|
||||
DIAGNOSING OVERLAP / STACKING BUGS — read this before "fixing" any visual overlap:
|
||||
- When snapshot_layout.overlaps is non-empty, two or more siblings share screen area. Do NOT blindly enlarge heights, shrink fonts, or tweak padding — those are surface patches.
|
||||
- Inspect the overlapping nodes' shared PARENT via batch_get. Look at its \`layout\` field:
|
||||
• \`layout: "none"\` (or missing) → children positioned via absolute x/y. OpenPencil's renderer has a known bug where absolute-positioned children stack vertically instead of honoring x/y. This is almost always the true root cause.
|
||||
• \`layout: "vertical"\` with gap=0 and children using textGrowth:"fit_content" → text can visually touch; bump \`gap\` or add padding on the children.
|
||||
- Preferred fix for \`layout: "none"\` parents that contain stacked content (badges, titles, rows):
|
||||
update_node(parent, { layout: "vertical", gap: 8, alignItems: "flex-start" })
|
||||
and strip the children's absolute x/y (the flex engine positions them).
|
||||
- For a circle/ring with centered content: NEVER use \`layout: "none"\`. Use a frame with cornerRadius = width/2, layout:"horizontal", alignItems:"center", justifyContent:"center", children:[ the text/icon ].
|
||||
|
||||
INSERT_NODE GUIDE — always include complete node data with children:
|
||||
- Button example: {"type":"frame","name":"My Button","width":"fill_container","height":50,"cornerRadius":8,"fill":[{"type":"solid","color":"#1877F2"}],"layout":"horizontal","gap":8,"alignItems":"center","justifyContent":"center","children":[{"type":"icon_font","name":"Icon","iconName":"facebook","width":20,"height":20,"fill":[{"type":"solid","color":"#FFFFFF"}]},{"type":"text","name":"Label","text":"Continue with Facebook","fontSize":15,"fontWeight":600,"fill":[{"type":"solid","color":"#FFFFFF"}]}]}
|
||||
- Text example: {"type":"text","name":"Title","text":"Hello","fontSize":24,"fontWeight":700,"fill":[{"type":"solid","color":"#1A1A2E"}]}
|
||||
|
|
|
|||
|
|
@ -394,11 +394,11 @@ export class AgentToolExecutor {
|
|||
|
||||
const { getNodeBounds } = await import('@/stores/document-tree-utils');
|
||||
|
||||
const buildLayout = (
|
||||
nodes: typeof children,
|
||||
maxDepth: number,
|
||||
depth = 0,
|
||||
): {
|
||||
// Overlap accumulator — flat list of sibling pairs whose rendered bounds
|
||||
// intersect. Gives the agent a text-based "screenshot equivalent" so it can
|
||||
// spot layout bugs (notably `layout:"none"` parents stacking children at
|
||||
// the same y) without needing a vision-capable model.
|
||||
type LayoutEntry = {
|
||||
id: string;
|
||||
name?: string;
|
||||
type: string;
|
||||
|
|
@ -407,21 +407,44 @@ export class AgentToolExecutor {
|
|||
width: number;
|
||||
height: number;
|
||||
children?: unknown[];
|
||||
}[] =>
|
||||
nodes.map((node) => {
|
||||
};
|
||||
const overlaps: Array<{ parentId: string | null; a: string; b: string; reason: string }> = [];
|
||||
const OVERLAP_EPS = 4; // ignore sub-pixel touches
|
||||
|
||||
const detectSiblingOverlaps = (
|
||||
entries: LayoutEntry[],
|
||||
parentId: string | null,
|
||||
parentLayout: string | undefined,
|
||||
) => {
|
||||
for (let i = 0; i < entries.length; i++) {
|
||||
for (let j = i + 1; j < entries.length; j++) {
|
||||
const a = entries[i];
|
||||
const b = entries[j];
|
||||
if (a.width <= 0 || a.height <= 0 || b.width <= 0 || b.height <= 0) continue;
|
||||
const xOverlap = Math.min(a.x + a.width, b.x + b.width) - Math.max(a.x, b.x);
|
||||
const yOverlap = Math.min(a.y + a.height, b.y + b.height) - Math.max(a.y, b.y);
|
||||
if (xOverlap > OVERLAP_EPS && yOverlap > OVERLAP_EPS) {
|
||||
const reason = !parentLayout || parentLayout === 'none'
|
||||
? 'parent has layout:"none" — absolute x/y can stack children at the same position; switch parent to layout:"vertical"|"horizontal" with gap'
|
||||
: `siblings overlap by ~${Math.round(xOverlap)}x${Math.round(yOverlap)} px — check gap/padding on parent`;
|
||||
overlaps.push({ parentId, a: a.id, b: b.id, reason });
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const buildLayout = (
|
||||
nodes: typeof children,
|
||||
maxDepth: number,
|
||||
parentId: string | null,
|
||||
parentLayout: string | undefined,
|
||||
depth = 0,
|
||||
): LayoutEntry[] => {
|
||||
const entries = nodes.map((node) => {
|
||||
// Use layout-computed bounds from SkiaEngine, fall back to stored values
|
||||
const computed = renderNodeMap.get(node.id);
|
||||
const b = computed ?? getNodeBounds(node, allChildren);
|
||||
const entry: {
|
||||
id: string;
|
||||
name?: string;
|
||||
type: string;
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
children?: unknown[];
|
||||
} = {
|
||||
const entry: LayoutEntry = {
|
||||
id: node.id,
|
||||
name: node.name,
|
||||
type: node.type,
|
||||
|
|
@ -431,12 +454,25 @@ export class AgentToolExecutor {
|
|||
height: Math.round(b.h),
|
||||
};
|
||||
if ('children' in node && node.children?.length && depth < maxDepth) {
|
||||
entry.children = buildLayout(node.children, maxDepth, depth + 1);
|
||||
const childLayout = (node as { layout?: string }).layout;
|
||||
entry.children = buildLayout(
|
||||
node.children,
|
||||
maxDepth,
|
||||
node.id,
|
||||
childLayout,
|
||||
depth + 1,
|
||||
);
|
||||
}
|
||||
return entry;
|
||||
});
|
||||
detectSiblingOverlaps(entries, parentId, parentLayout);
|
||||
return entries;
|
||||
};
|
||||
|
||||
return { success: true, data: buildLayout(children, 3) };
|
||||
const tree = buildLayout(children, 3, null, undefined);
|
||||
return overlaps.length > 0
|
||||
? { success: true, data: { tree, overlaps } }
|
||||
: { success: true, data: tree };
|
||||
}
|
||||
|
||||
private async handleFindEmptySpace(args: {
|
||||
|
|
|
|||
|
|
@ -95,7 +95,11 @@ export function getCrudToolDefs(): ToolDef[] {
|
|||
{
|
||||
name: 'snapshot_layout',
|
||||
description:
|
||||
'Get a compact layout snapshot of the current page showing node positions and sizes',
|
||||
'Get a compact layout snapshot of the current page showing node positions and sizes. ' +
|
||||
'When sibling nodes visually overlap, the result includes an `overlaps` array (parentId, a, b, reason) — ' +
|
||||
'use it as a text-only screenshot-replacement to diagnose visual bugs like stacked badges or overlapping text. ' +
|
||||
'If an overlap reason mentions `layout:"none"`, fix the PARENT frame (set layout to "vertical" or "horizontal" with a gap); ' +
|
||||
'do not just resize the overlapping children.',
|
||||
level: TOOL_AUTH_MAP.snapshot_layout,
|
||||
parameters: {
|
||||
type: 'object',
|
||||
|
|
@ -219,7 +223,11 @@ export function getDesignToolDefs(): ToolDef[] {
|
|||
{
|
||||
name: 'snapshot_layout',
|
||||
description:
|
||||
'Get a compact layout snapshot of the current page showing node positions and sizes',
|
||||
'Get a compact layout snapshot of the current page showing node positions and sizes. ' +
|
||||
'When sibling nodes visually overlap, the result includes an `overlaps` array (parentId, a, b, reason) — ' +
|
||||
'use it as a text-only screenshot-replacement to diagnose visual bugs like stacked badges or overlapping text. ' +
|
||||
'If an overlap reason mentions `layout:"none"`, fix the PARENT frame (set layout to "vertical" or "horizontal" with a gap); ' +
|
||||
'do not just resize the overlapping children.',
|
||||
level: TOOL_AUTH_MAP.snapshot_layout,
|
||||
parameters: {
|
||||
type: 'object',
|
||||
|
|
|
|||
Loading…
Reference in a new issue