Skip to content

Commit 33da068

Browse files
yuyutaotaoyuyutaotao
committed
feat: include htmlTagName in page description
1 parent b3250db commit 33da068

File tree

10 files changed

+236
-103
lines changed

10 files changed

+236
-103
lines changed

‎packages/midscene/src/ai-model/prompt/util.ts

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,7 @@ function cropFieldInformation(
294294
const elementInfosDescription: Array<Record<string, any>> = elementsInfo.map(
295295
(item) => {
296296
const { id, attributes = {}, rect, content } = item;
297+
let htmlTagName = '';
297298
const tailorContent = truncateText(content, truncateTextLength);
298299
const tailorAttributes = Object.keys(attributes).reduce(
299300
(res, currentKey: string) => {
@@ -304,6 +305,10 @@ function cropFieldInformation(
304305
if (!filterNonTextContent) {
305306
res[currentKey] = attributeVal.replace(/\sNode$/, '');
306307
}
308+
} else if (currentKey === 'htmlTagName') {
309+
if (!['<span>', '<p>', '<div>'].includes(attributeVal)) {
310+
htmlTagName = attributeVal;
311+
}
307312
} else {
308313
res[currentKey] = truncateText(attributeVal);
309314
}
@@ -321,6 +326,7 @@ function cropFieldInformation(
321326
...(Object.keys(tailorAttributes).length && !tailorContent
322327
? { attributes: tailorAttributes }
323328
: {}),
329+
...(htmlTagName ? { htmlTagName } : {}),
324330
rect: {
325331
left: rect.left,
326332
top: rect.top,

‎packages/midscene/tests/ai/evaluate/ai-data/inspect/todo.json

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,12 +11,22 @@
1111
}
1212
]
1313
},
14+
{
15+
"prompt": "Input 'Study Rust tomorrow' in the task box input and press the Enter key ",
16+
"multi": false,
17+
"response": [
18+
{
19+
"id": "fbc2d0029b7a9037",
20+
"indexId": 1
21+
}
22+
]
23+
},
1424
{
1525
"prompt": "任务列表中的第二项名称",
1626
"multi": false,
1727
"response": [
1828
{
19-
"id": "14b20d09632977fb",
29+
"id": "dd1b5a5792d39a54",
2030
"indexId": 7
2131
}
2232
]

‎packages/midscene/tests/ai/evaluate/test-data/todo-input-with-value/element-snapshot.json

Lines changed: 68 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@
77
"nodeType": "TEXT Node",
88
"locator": "",
99
"attributes": {
10-
"nodeType": "TEXT Node"
10+
"nodeType": "TEXT Node",
11+
"htmlTagName": "h1"
1112
},
1213
"center": [
1314
640,
@@ -65,7 +66,8 @@
6566
"nodeType": "TEXT Node",
6667
"locator": "",
6768
"attributes": {
68-
"nodeType": "TEXT Node"
69+
"nodeType": "TEXT Node",
70+
"htmlTagName": "label"
6971
},
7072
"center": [
7173
960,
@@ -144,27 +146,29 @@
144146
"screenHeight": 720
145147
},
146148
{
147-
"id": "c1fb39ce9a84bef9",
149+
"id": "6baafaffbd3eb3ec",
150+
"nodePath": "0-0-1-1-0-0-1",
151+
"nodeHashId": "6baafaffbd3eb3ec",
152+
"locator": "[_midscene_retrieve_task_id='6baafaffbd3eb3ec']",
153+
"nodeType": "FORM_ITEM Node",
148154
"indexId": 5,
149-
"nodePath": "0-0-1-1-0-0-1-0",
150-
"nodeHashId": "c1fb39ce9a84bef9",
151-
"nodeType": "TEXT Node",
152-
"locator": "",
153155
"attributes": {
154-
"nodeType": "TEXT Node"
156+
"data-testid": "todo-item-label",
157+
"htmlTagName": "<label>",
158+
"nodeType": "FORM_ITEM Node"
155159
},
156-
"center": [
157-
497,
158-
225
159-
],
160160
"content": "Learn Python",
161161
"rect": {
162-
"left": 425,
163-
"top": 211,
164-
"width": 143,
165-
"height": 28,
162+
"left": 365,
163+
"top": 196,
164+
"width": 550,
165+
"height": 59,
166166
"zoom": 1
167167
},
168+
"center": [
169+
640,
170+
226
171+
],
168172
"zoom": 1,
169173
"screenWidth": 1280,
170174
"screenHeight": 720
@@ -200,27 +204,29 @@
200204
"screenHeight": 720
201205
},
202206
{
203-
"id": "14b20d09632977fb",
207+
"id": "dd1b5a5792d39a54",
208+
"nodePath": "0-0-1-1-1-0-1",
209+
"nodeHashId": "dd1b5a5792d39a54",
210+
"locator": "[_midscene_retrieve_task_id='dd1b5a5792d39a54']",
211+
"nodeType": "FORM_ITEM Node",
204212
"indexId": 7,
205-
"nodePath": "0-0-1-1-1-0-1-0",
206-
"nodeHashId": "14b20d09632977fb",
207-
"nodeType": "TEXT Node",
208-
"locator": "",
209213
"attributes": {
210-
"nodeType": "TEXT Node"
214+
"data-testid": "todo-item-label",
215+
"htmlTagName": "<label>",
216+
"nodeType": "FORM_ITEM Node"
211217
},
212-
"center": [
213-
484,
214-
285
215-
],
216218
"content": "Learn Rust",
217219
"rect": {
218-
"left": 425,
219-
"top": 271,
220-
"width": 117,
221-
"height": 28,
220+
"left": 365,
221+
"top": 256,
222+
"width": 550,
223+
"height": 59,
222224
"zoom": 1
223225
},
226+
"center": [
227+
640,
228+
286
229+
],
224230
"zoom": 1,
225231
"screenWidth": 1280,
226232
"screenHeight": 720
@@ -235,6 +241,7 @@
235241
"attributes": {
236242
"class": ".destroy",
237243
"data-testid": "todo-item-button",
244+
"htmlTagName": "button",
238245
"nodeType": "BUTTON Node"
239246
},
240247
"content": "×",
@@ -284,27 +291,29 @@
284291
"screenHeight": 720
285292
},
286293
{
287-
"id": "3277d35f7c939507",
294+
"id": "93a83d3ed5ab442d",
295+
"nodePath": "0-0-1-1-2-0-1",
296+
"nodeHashId": "93a83d3ed5ab442d",
297+
"locator": "[_midscene_retrieve_task_id='93a83d3ed5ab442d']",
298+
"nodeType": "FORM_ITEM Node",
288299
"indexId": 10,
289-
"nodePath": "0-0-1-1-2-0-1-0",
290-
"nodeHashId": "3277d35f7c939507",
291-
"nodeType": "TEXT Node",
292-
"locator": "",
293300
"attributes": {
294-
"nodeType": "TEXT Node"
301+
"data-testid": "todo-item-label",
302+
"htmlTagName": "<label>",
303+
"nodeType": "FORM_ITEM Node"
295304
},
296-
"center": [
297-
470,
298-
345
299-
],
300305
"content": "Learn AI",
301306
"rect": {
302-
"left": 425,
303-
"top": 331,
304-
"width": 89,
305-
"height": 28,
307+
"left": 365,
308+
"top": 316,
309+
"width": 550,
310+
"height": 59,
306311
"zoom": 1
307312
},
313+
"center": [
314+
640,
315+
346
316+
],
308317
"zoom": 1,
309318
"screenWidth": 1280,
310319
"screenHeight": 720
@@ -317,7 +326,8 @@
317326
"nodeType": "TEXT Node",
318327
"locator": "",
319328
"attributes": {
320-
"nodeType": "TEXT Node"
329+
"nodeType": "TEXT Node",
330+
"htmlTagName": "a"
321331
},
322332
"center": [
323333
565,
@@ -343,7 +353,8 @@
343353
"nodeType": "TEXT Node",
344354
"locator": "",
345355
"attributes": {
346-
"nodeType": "TEXT Node"
356+
"nodeType": "TEXT Node",
357+
"htmlTagName": "a"
347358
},
348359
"center": [
349360
613,
@@ -369,7 +380,8 @@
369380
"nodeType": "TEXT Node",
370381
"locator": "",
371382
"attributes": {
372-
"nodeType": "TEXT Node"
383+
"nodeType": "TEXT Node",
384+
"htmlTagName": "a"
373385
},
374386
"center": [
375387
688,
@@ -396,6 +408,7 @@
396408
"locator": "[_midscene_retrieve_task_id='69e78b5a71b57768']",
397409
"attributes": {
398410
"class": ".clear-completed",
411+
"htmlTagName": "button",
399412
"nodeType": "BUTTON Node"
400413
},
401414
"content": "Clear completed",
@@ -422,7 +435,8 @@
422435
"nodeType": "TEXT Node",
423436
"locator": "",
424437
"attributes": {
425-
"nodeType": "TEXT Node"
438+
"nodeType": "TEXT Node",
439+
"htmlTagName": "p"
426440
},
427441
"center": [
428442
640,
@@ -448,7 +462,8 @@
448462
"nodeType": "TEXT Node",
449463
"locator": "",
450464
"attributes": {
451-
"nodeType": "TEXT Node"
465+
"nodeType": "TEXT Node",
466+
"htmlTagName": "p"
452467
},
453468
"center": [
454469
640,
@@ -474,7 +489,8 @@
474489
"nodeType": "TEXT Node",
475490
"locator": "",
476491
"attributes": {
477-
"nodeType": "TEXT Node"
492+
"nodeType": "TEXT Node",
493+
"htmlTagName": "p"
478494
},
479495
"center": [
480496
616,
@@ -500,7 +516,8 @@
500516
"nodeType": "TEXT Node",
501517
"locator": "",
502518
"attributes": {
503-
"nodeType": "TEXT Node"
519+
"nodeType": "TEXT Node",
520+
"htmlTagName": "a"
504521
},
505522
"center": [
506523
656,

‎packages/midscene/tests/ai/evaluate/test-data/todo-input-with-value/output.png

-267 Bytes
Loading

‎packages/midscene/tests/ai/evaluate/test-data/todo-input-with-value/output_without_text.png

208 Bytes
Loading

0 commit comments

Comments
 (0)