diff --git a/Include/internal/pycore_stackref.h b/Include/internal/pycore_stackref.h index 329045b5faaa22..a3af30df4fda07 100644 --- a/Include/internal/pycore_stackref.h +++ b/Include/internal/pycore_stackref.h @@ -615,6 +615,25 @@ PyStackRef_FromPyObjectBorrow(PyObject *obj) return (_PyStackRef){ .bits = (uintptr_t)obj | Py_TAG_REFCNT}; } +/* Create a stackref from a pre-tagged operand (tag bits already set). + Used by _LOAD_CONST_INLINE_BORROW variants where the operand is + tagged at trace creation time to avoid tagging on every execution. */ +static inline _PyStackRef +_PyStackRef_FromPreTagged(uintptr_t tagged) +{ + assert(tagged & Py_TAG_REFCNT); + return (_PyStackRef){ .bits = tagged }; +} +#define PyStackRef_FromPreTagged(ptr) _PyStackRef_FromPreTagged((uintptr_t)(ptr)) + +/* Tag a PyObject pointer as a borrowed operand for BORROW variants. */ +#define PyStackRef_TagBorrow(ptr) \ + ((uintptr_t)(ptr) | Py_TAG_REFCNT) + +/* Strip tag bits from a pre-tagged operand to recover the PyObject pointer. */ +#define PyStackRef_UntagBorrow(tagged) \ + ((PyObject *)((uintptr_t)(tagged) & ~Py_TAG_BITS)) + /* WARNING: This macro evaluates its argument more than once */ #ifdef _WIN32 #define PyStackRef_DUP(REF) \ diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 6b96f9bc78e8fe..5505bc136cc092 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -1093,327 +1093,322 @@ extern "C" { #define _LOAD_CONST_INLINE_r12 1305 #define _LOAD_CONST_INLINE_r23 1306 #define _LOAD_CONST_INLINE_BORROW_r01 1307 -#define _LOAD_CONST_INLINE_BORROW_r12 1308 -#define _LOAD_CONST_INLINE_BORROW_r23 1309 -#define _LOAD_DEREF_r01 1310 -#define _LOAD_FAST_r01 1311 -#define _LOAD_FAST_r12 1312 -#define _LOAD_FAST_r23 1313 -#define _LOAD_FAST_0_r01 1314 -#define _LOAD_FAST_0_r12 1315 -#define _LOAD_FAST_0_r23 1316 -#define _LOAD_FAST_1_r01 1317 -#define _LOAD_FAST_1_r12 1318 -#define _LOAD_FAST_1_r23 1319 -#define _LOAD_FAST_2_r01 1320 -#define _LOAD_FAST_2_r12 1321 -#define _LOAD_FAST_2_r23 1322 -#define _LOAD_FAST_3_r01 1323 -#define _LOAD_FAST_3_r12 1324 -#define _LOAD_FAST_3_r23 1325 -#define _LOAD_FAST_4_r01 1326 -#define _LOAD_FAST_4_r12 1327 -#define _LOAD_FAST_4_r23 1328 -#define _LOAD_FAST_5_r01 1329 -#define _LOAD_FAST_5_r12 1330 -#define _LOAD_FAST_5_r23 1331 -#define _LOAD_FAST_6_r01 1332 -#define _LOAD_FAST_6_r12 1333 -#define _LOAD_FAST_6_r23 1334 -#define _LOAD_FAST_7_r01 1335 -#define _LOAD_FAST_7_r12 1336 -#define _LOAD_FAST_7_r23 1337 -#define _LOAD_FAST_AND_CLEAR_r01 1338 -#define _LOAD_FAST_AND_CLEAR_r12 1339 -#define _LOAD_FAST_AND_CLEAR_r23 1340 -#define _LOAD_FAST_BORROW_r01 1341 -#define _LOAD_FAST_BORROW_r12 1342 -#define _LOAD_FAST_BORROW_r23 1343 -#define _LOAD_FAST_BORROW_0_r01 1344 -#define _LOAD_FAST_BORROW_0_r12 1345 -#define _LOAD_FAST_BORROW_0_r23 1346 -#define _LOAD_FAST_BORROW_1_r01 1347 -#define _LOAD_FAST_BORROW_1_r12 1348 -#define _LOAD_FAST_BORROW_1_r23 1349 -#define _LOAD_FAST_BORROW_2_r01 1350 -#define _LOAD_FAST_BORROW_2_r12 1351 -#define _LOAD_FAST_BORROW_2_r23 1352 -#define _LOAD_FAST_BORROW_3_r01 1353 -#define _LOAD_FAST_BORROW_3_r12 1354 -#define _LOAD_FAST_BORROW_3_r23 1355 -#define _LOAD_FAST_BORROW_4_r01 1356 -#define _LOAD_FAST_BORROW_4_r12 1357 -#define _LOAD_FAST_BORROW_4_r23 1358 -#define _LOAD_FAST_BORROW_5_r01 1359 -#define _LOAD_FAST_BORROW_5_r12 1360 -#define _LOAD_FAST_BORROW_5_r23 1361 -#define _LOAD_FAST_BORROW_6_r01 1362 -#define _LOAD_FAST_BORROW_6_r12 1363 -#define _LOAD_FAST_BORROW_6_r23 1364 -#define _LOAD_FAST_BORROW_7_r01 1365 -#define _LOAD_FAST_BORROW_7_r12 1366 -#define _LOAD_FAST_BORROW_7_r23 1367 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1368 -#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1369 -#define _LOAD_FAST_CHECK_r01 1370 -#define _LOAD_FAST_CHECK_r12 1371 -#define _LOAD_FAST_CHECK_r23 1372 -#define _LOAD_FAST_LOAD_FAST_r02 1373 -#define _LOAD_FAST_LOAD_FAST_r13 1374 -#define _LOAD_FROM_DICT_OR_DEREF_r11 1375 -#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1376 -#define _LOAD_GLOBAL_r00 1377 -#define _LOAD_GLOBAL_BUILTINS_r01 1378 -#define _LOAD_GLOBAL_MODULE_r01 1379 -#define _LOAD_LOCALS_r01 1380 -#define _LOAD_LOCALS_r12 1381 -#define _LOAD_LOCALS_r23 1382 -#define _LOAD_NAME_r01 1383 -#define _LOAD_SMALL_INT_r01 1384 -#define _LOAD_SMALL_INT_r12 1385 -#define _LOAD_SMALL_INT_r23 1386 -#define _LOAD_SMALL_INT_0_r01 1387 -#define _LOAD_SMALL_INT_0_r12 1388 -#define _LOAD_SMALL_INT_0_r23 1389 -#define _LOAD_SMALL_INT_1_r01 1390 -#define _LOAD_SMALL_INT_1_r12 1391 -#define _LOAD_SMALL_INT_1_r23 1392 -#define _LOAD_SMALL_INT_2_r01 1393 -#define _LOAD_SMALL_INT_2_r12 1394 -#define _LOAD_SMALL_INT_2_r23 1395 -#define _LOAD_SMALL_INT_3_r01 1396 -#define _LOAD_SMALL_INT_3_r12 1397 -#define _LOAD_SMALL_INT_3_r23 1398 -#define _LOAD_SPECIAL_r00 1399 -#define _LOAD_SUPER_ATTR_ATTR_r31 1400 -#define _LOAD_SUPER_ATTR_METHOD_r32 1401 -#define _LOCK_OBJECT_r01 1402 -#define _LOCK_OBJECT_r11 1403 -#define _LOCK_OBJECT_r22 1404 -#define _LOCK_OBJECT_r33 1405 -#define _MAKE_CALLARGS_A_TUPLE_r33 1406 -#define _MAKE_CELL_r00 1407 -#define _MAKE_FUNCTION_r12 1408 -#define _MAKE_HEAP_SAFE_r01 1409 -#define _MAKE_HEAP_SAFE_r11 1410 -#define _MAKE_HEAP_SAFE_r22 1411 -#define _MAKE_HEAP_SAFE_r33 1412 -#define _MAKE_WARM_r00 1413 -#define _MAKE_WARM_r11 1414 -#define _MAKE_WARM_r22 1415 -#define _MAKE_WARM_r33 1416 -#define _MAP_ADD_r20 1417 -#define _MATCH_CLASS_r33 1418 -#define _MATCH_KEYS_r23 1419 -#define _MATCH_MAPPING_r02 1420 -#define _MATCH_MAPPING_r12 1421 -#define _MATCH_MAPPING_r23 1422 -#define _MATCH_SEQUENCE_r02 1423 -#define _MATCH_SEQUENCE_r12 1424 -#define _MATCH_SEQUENCE_r23 1425 -#define _MAYBE_EXPAND_METHOD_r00 1426 -#define _MAYBE_EXPAND_METHOD_KW_r11 1427 -#define _MONITOR_CALL_r00 1428 -#define _MONITOR_CALL_KW_r11 1429 -#define _MONITOR_JUMP_BACKWARD_r00 1430 -#define _MONITOR_JUMP_BACKWARD_r11 1431 -#define _MONITOR_JUMP_BACKWARD_r22 1432 -#define _MONITOR_JUMP_BACKWARD_r33 1433 -#define _MONITOR_RESUME_r00 1434 -#define _NOP_r00 1435 -#define _NOP_r11 1436 -#define _NOP_r22 1437 -#define _NOP_r33 1438 -#define _POP_EXCEPT_r10 1439 -#define _POP_ITER_r20 1440 -#define _POP_JUMP_IF_FALSE_r00 1441 -#define _POP_JUMP_IF_FALSE_r10 1442 -#define _POP_JUMP_IF_FALSE_r21 1443 -#define _POP_JUMP_IF_FALSE_r32 1444 -#define _POP_JUMP_IF_TRUE_r00 1445 -#define _POP_JUMP_IF_TRUE_r10 1446 -#define _POP_JUMP_IF_TRUE_r21 1447 -#define _POP_JUMP_IF_TRUE_r32 1448 -#define _POP_TOP_r10 1449 -#define _POP_TOP_FLOAT_r00 1450 -#define _POP_TOP_FLOAT_r10 1451 -#define _POP_TOP_FLOAT_r21 1452 -#define _POP_TOP_FLOAT_r32 1453 -#define _POP_TOP_INT_r00 1454 -#define _POP_TOP_INT_r10 1455 -#define _POP_TOP_INT_r21 1456 -#define _POP_TOP_INT_r32 1457 -#define _POP_TOP_NOP_r00 1458 -#define _POP_TOP_NOP_r10 1459 -#define _POP_TOP_NOP_r21 1460 -#define _POP_TOP_NOP_r32 1461 -#define _POP_TOP_OPARG_r00 1462 -#define _POP_TOP_UNICODE_r00 1463 -#define _POP_TOP_UNICODE_r10 1464 -#define _POP_TOP_UNICODE_r21 1465 -#define _POP_TOP_UNICODE_r32 1466 -#define _PUSH_EXC_INFO_r02 1467 -#define _PUSH_EXC_INFO_r12 1468 -#define _PUSH_EXC_INFO_r23 1469 -#define _PUSH_FRAME_r10 1470 -#define _PUSH_NULL_r01 1471 -#define _PUSH_NULL_r12 1472 -#define _PUSH_NULL_r23 1473 -#define _PUSH_NULL_CONDITIONAL_r00 1474 -#define _PUSH_TAGGED_ZERO_r01 1475 -#define _PUSH_TAGGED_ZERO_r12 1476 -#define _PUSH_TAGGED_ZERO_r23 1477 -#define _PY_FRAME_EX_r31 1478 -#define _PY_FRAME_GENERAL_r01 1479 -#define _PY_FRAME_KW_r11 1480 -#define _REPLACE_WITH_TRUE_r02 1481 -#define _REPLACE_WITH_TRUE_r12 1482 -#define _REPLACE_WITH_TRUE_r23 1483 -#define _RESUME_CHECK_r00 1484 -#define _RESUME_CHECK_r11 1485 -#define _RESUME_CHECK_r22 1486 -#define _RESUME_CHECK_r33 1487 -#define _RETURN_GENERATOR_r01 1488 -#define _RETURN_VALUE_r11 1489 -#define _SAVE_RETURN_OFFSET_r00 1490 -#define _SAVE_RETURN_OFFSET_r11 1491 -#define _SAVE_RETURN_OFFSET_r22 1492 -#define _SAVE_RETURN_OFFSET_r33 1493 -#define _SEND_r33 1494 -#define _SEND_GEN_FRAME_r33 1495 -#define _SETUP_ANNOTATIONS_r00 1496 -#define _SET_ADD_r10 1497 -#define _SET_FUNCTION_ATTRIBUTE_r01 1498 -#define _SET_FUNCTION_ATTRIBUTE_r11 1499 -#define _SET_FUNCTION_ATTRIBUTE_r21 1500 -#define _SET_FUNCTION_ATTRIBUTE_r32 1501 -#define _SET_IP_r00 1502 -#define _SET_IP_r11 1503 -#define _SET_IP_r22 1504 -#define _SET_IP_r33 1505 -#define _SET_UPDATE_r11 1506 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 1507 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 1508 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 1509 -#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1510 -#define _SPILL_OR_RELOAD_r01 1511 -#define _SPILL_OR_RELOAD_r02 1512 -#define _SPILL_OR_RELOAD_r03 1513 -#define _SPILL_OR_RELOAD_r10 1514 -#define _SPILL_OR_RELOAD_r12 1515 -#define _SPILL_OR_RELOAD_r13 1516 -#define _SPILL_OR_RELOAD_r20 1517 -#define _SPILL_OR_RELOAD_r21 1518 -#define _SPILL_OR_RELOAD_r23 1519 -#define _SPILL_OR_RELOAD_r30 1520 -#define _SPILL_OR_RELOAD_r31 1521 -#define _SPILL_OR_RELOAD_r32 1522 -#define _START_EXECUTOR_r00 1523 -#define _STORE_ATTR_r20 1524 -#define _STORE_ATTR_INSTANCE_VALUE_r21 1525 -#define _STORE_ATTR_SLOT_r21 1526 -#define _STORE_ATTR_WITH_HINT_r21 1527 -#define _STORE_DEREF_r10 1528 -#define _STORE_FAST_LOAD_FAST_r11 1529 -#define _STORE_FAST_STORE_FAST_r20 1530 -#define _STORE_GLOBAL_r10 1531 -#define _STORE_NAME_r10 1532 -#define _STORE_SLICE_r30 1533 -#define _STORE_SUBSCR_r30 1534 -#define _STORE_SUBSCR_DICT_r31 1535 -#define _STORE_SUBSCR_DICT_KNOWN_HASH_r31 1536 -#define _STORE_SUBSCR_LIST_INT_r32 1537 -#define _SWAP_r11 1538 -#define _SWAP_2_r02 1539 -#define _SWAP_2_r12 1540 -#define _SWAP_2_r22 1541 -#define _SWAP_2_r33 1542 -#define _SWAP_3_r03 1543 -#define _SWAP_3_r13 1544 -#define _SWAP_3_r23 1545 -#define _SWAP_3_r33 1546 -#define _SWAP_FAST_r01 1547 -#define _SWAP_FAST_r11 1548 -#define _SWAP_FAST_r22 1549 -#define _SWAP_FAST_r33 1550 -#define _SWAP_FAST_0_r01 1551 -#define _SWAP_FAST_0_r11 1552 -#define _SWAP_FAST_0_r22 1553 -#define _SWAP_FAST_0_r33 1554 -#define _SWAP_FAST_1_r01 1555 -#define _SWAP_FAST_1_r11 1556 -#define _SWAP_FAST_1_r22 1557 -#define _SWAP_FAST_1_r33 1558 -#define _SWAP_FAST_2_r01 1559 -#define _SWAP_FAST_2_r11 1560 -#define _SWAP_FAST_2_r22 1561 -#define _SWAP_FAST_2_r33 1562 -#define _SWAP_FAST_3_r01 1563 -#define _SWAP_FAST_3_r11 1564 -#define _SWAP_FAST_3_r22 1565 -#define _SWAP_FAST_3_r33 1566 -#define _SWAP_FAST_4_r01 1567 -#define _SWAP_FAST_4_r11 1568 -#define _SWAP_FAST_4_r22 1569 -#define _SWAP_FAST_4_r33 1570 -#define _SWAP_FAST_5_r01 1571 -#define _SWAP_FAST_5_r11 1572 -#define _SWAP_FAST_5_r22 1573 -#define _SWAP_FAST_5_r33 1574 -#define _SWAP_FAST_6_r01 1575 -#define _SWAP_FAST_6_r11 1576 -#define _SWAP_FAST_6_r22 1577 -#define _SWAP_FAST_6_r33 1578 -#define _SWAP_FAST_7_r01 1579 -#define _SWAP_FAST_7_r11 1580 -#define _SWAP_FAST_7_r22 1581 -#define _SWAP_FAST_7_r33 1582 -#define _TIER2_RESUME_CHECK_r00 1583 -#define _TIER2_RESUME_CHECK_r11 1584 -#define _TIER2_RESUME_CHECK_r22 1585 -#define _TIER2_RESUME_CHECK_r33 1586 -#define _TO_BOOL_r11 1587 -#define _TO_BOOL_BOOL_r01 1588 -#define _TO_BOOL_BOOL_r11 1589 -#define _TO_BOOL_BOOL_r22 1590 -#define _TO_BOOL_BOOL_r33 1591 -#define _TO_BOOL_INT_r02 1592 -#define _TO_BOOL_INT_r12 1593 -#define _TO_BOOL_INT_r23 1594 -#define _TO_BOOL_LIST_r02 1595 -#define _TO_BOOL_LIST_r12 1596 -#define _TO_BOOL_LIST_r23 1597 -#define _TO_BOOL_NONE_r01 1598 -#define _TO_BOOL_NONE_r11 1599 -#define _TO_BOOL_NONE_r22 1600 -#define _TO_BOOL_NONE_r33 1601 -#define _TO_BOOL_STR_r02 1602 -#define _TO_BOOL_STR_r12 1603 -#define _TO_BOOL_STR_r23 1604 -#define _TRACE_RECORD_r00 1605 -#define _UNARY_INVERT_r12 1606 -#define _UNARY_NEGATIVE_r12 1607 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r02 1608 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r12 1609 -#define _UNARY_NEGATIVE_FLOAT_INPLACE_r23 1610 -#define _UNARY_NOT_r01 1611 -#define _UNARY_NOT_r11 1612 -#define _UNARY_NOT_r22 1613 -#define _UNARY_NOT_r33 1614 -#define _UNPACK_EX_r10 1615 -#define _UNPACK_SEQUENCE_r10 1616 -#define _UNPACK_SEQUENCE_LIST_r10 1617 -#define _UNPACK_SEQUENCE_TUPLE_r10 1618 -#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1619 -#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 1620 -#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 1621 -#define _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 1622 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 1623 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 1624 -#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 1625 -#define _WITH_EXCEPT_START_r33 1626 -#define _YIELD_VALUE_r11 1627 -#define MAX_UOP_REGS_ID 1627 +#define _LOAD_DEREF_r01 1308 +#define _LOAD_FAST_r01 1309 +#define _LOAD_FAST_r12 1310 +#define _LOAD_FAST_r23 1311 +#define _LOAD_FAST_0_r01 1312 +#define _LOAD_FAST_0_r12 1313 +#define _LOAD_FAST_0_r23 1314 +#define _LOAD_FAST_1_r01 1315 +#define _LOAD_FAST_1_r12 1316 +#define _LOAD_FAST_1_r23 1317 +#define _LOAD_FAST_2_r01 1318 +#define _LOAD_FAST_2_r12 1319 +#define _LOAD_FAST_2_r23 1320 +#define _LOAD_FAST_3_r01 1321 +#define _LOAD_FAST_3_r12 1322 +#define _LOAD_FAST_3_r23 1323 +#define _LOAD_FAST_4_r01 1324 +#define _LOAD_FAST_4_r12 1325 +#define _LOAD_FAST_4_r23 1326 +#define _LOAD_FAST_5_r01 1327 +#define _LOAD_FAST_5_r12 1328 +#define _LOAD_FAST_5_r23 1329 +#define _LOAD_FAST_6_r01 1330 +#define _LOAD_FAST_6_r12 1331 +#define _LOAD_FAST_6_r23 1332 +#define _LOAD_FAST_7_r01 1333 +#define _LOAD_FAST_7_r12 1334 +#define _LOAD_FAST_7_r23 1335 +#define _LOAD_FAST_AND_CLEAR_r01 1336 +#define _LOAD_FAST_AND_CLEAR_r12 1337 +#define _LOAD_FAST_AND_CLEAR_r23 1338 +#define _LOAD_FAST_BORROW_r01 1339 +#define _LOAD_FAST_BORROW_r12 1340 +#define _LOAD_FAST_BORROW_r23 1341 +#define _LOAD_FAST_BORROW_0_r01 1342 +#define _LOAD_FAST_BORROW_0_r12 1343 +#define _LOAD_FAST_BORROW_0_r23 1344 +#define _LOAD_FAST_BORROW_1_r01 1345 +#define _LOAD_FAST_BORROW_1_r12 1346 +#define _LOAD_FAST_BORROW_1_r23 1347 +#define _LOAD_FAST_BORROW_2_r01 1348 +#define _LOAD_FAST_BORROW_2_r12 1349 +#define _LOAD_FAST_BORROW_2_r23 1350 +#define _LOAD_FAST_BORROW_3_r01 1351 +#define _LOAD_FAST_BORROW_3_r12 1352 +#define _LOAD_FAST_BORROW_3_r23 1353 +#define _LOAD_FAST_BORROW_4_r01 1354 +#define _LOAD_FAST_BORROW_4_r12 1355 +#define _LOAD_FAST_BORROW_4_r23 1356 +#define _LOAD_FAST_BORROW_5_r01 1357 +#define _LOAD_FAST_BORROW_5_r12 1358 +#define _LOAD_FAST_BORROW_5_r23 1359 +#define _LOAD_FAST_BORROW_6_r01 1360 +#define _LOAD_FAST_BORROW_6_r12 1361 +#define _LOAD_FAST_BORROW_6_r23 1362 +#define _LOAD_FAST_BORROW_7_r01 1363 +#define _LOAD_FAST_BORROW_7_r12 1364 +#define _LOAD_FAST_BORROW_7_r23 1365 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r02 1366 +#define _LOAD_FAST_BORROW_LOAD_FAST_BORROW_r13 1367 +#define _LOAD_FAST_CHECK_r01 1368 +#define _LOAD_FAST_CHECK_r12 1369 +#define _LOAD_FAST_CHECK_r23 1370 +#define _LOAD_FAST_LOAD_FAST_r02 1371 +#define _LOAD_FAST_LOAD_FAST_r13 1372 +#define _LOAD_FROM_DICT_OR_DEREF_r11 1373 +#define _LOAD_FROM_DICT_OR_GLOBALS_r11 1374 +#define _LOAD_GLOBAL_r00 1375 +#define _LOAD_GLOBAL_BUILTINS_r01 1376 +#define _LOAD_GLOBAL_MODULE_r01 1377 +#define _LOAD_LOCALS_r01 1378 +#define _LOAD_LOCALS_r12 1379 +#define _LOAD_LOCALS_r23 1380 +#define _LOAD_NAME_r01 1381 +#define _LOAD_SMALL_INT_r01 1382 +#define _LOAD_SMALL_INT_r12 1383 +#define _LOAD_SMALL_INT_r23 1384 +#define _LOAD_SMALL_INT_0_r01 1385 +#define _LOAD_SMALL_INT_0_r12 1386 +#define _LOAD_SMALL_INT_0_r23 1387 +#define _LOAD_SMALL_INT_1_r01 1388 +#define _LOAD_SMALL_INT_1_r12 1389 +#define _LOAD_SMALL_INT_1_r23 1390 +#define _LOAD_SMALL_INT_2_r01 1391 +#define _LOAD_SMALL_INT_2_r12 1392 +#define _LOAD_SMALL_INT_2_r23 1393 +#define _LOAD_SMALL_INT_3_r01 1394 +#define _LOAD_SMALL_INT_3_r12 1395 +#define _LOAD_SMALL_INT_3_r23 1396 +#define _LOAD_SPECIAL_r00 1397 +#define _LOAD_SUPER_ATTR_ATTR_r31 1398 +#define _LOAD_SUPER_ATTR_METHOD_r32 1399 +#define _LOCK_OBJECT_r01 1400 +#define _LOCK_OBJECT_r11 1401 +#define _LOCK_OBJECT_r22 1402 +#define _LOCK_OBJECT_r33 1403 +#define _MAKE_CALLARGS_A_TUPLE_r33 1404 +#define _MAKE_CELL_r00 1405 +#define _MAKE_FUNCTION_r12 1406 +#define _MAKE_HEAP_SAFE_r01 1407 +#define _MAKE_HEAP_SAFE_r11 1408 +#define _MAKE_HEAP_SAFE_r22 1409 +#define _MAKE_HEAP_SAFE_r33 1410 +#define _MAKE_WARM_r00 1411 +#define _MAKE_WARM_r11 1412 +#define _MAKE_WARM_r22 1413 +#define _MAKE_WARM_r33 1414 +#define _MAP_ADD_r20 1415 +#define _MATCH_CLASS_r33 1416 +#define _MATCH_KEYS_r23 1417 +#define _MATCH_MAPPING_r02 1418 +#define _MATCH_MAPPING_r12 1419 +#define _MATCH_MAPPING_r23 1420 +#define _MATCH_SEQUENCE_r02 1421 +#define _MATCH_SEQUENCE_r12 1422 +#define _MATCH_SEQUENCE_r23 1423 +#define _MAYBE_EXPAND_METHOD_r00 1424 +#define _MAYBE_EXPAND_METHOD_KW_r11 1425 +#define _MONITOR_CALL_r00 1426 +#define _MONITOR_CALL_KW_r11 1427 +#define _MONITOR_JUMP_BACKWARD_r00 1428 +#define _MONITOR_JUMP_BACKWARD_r11 1429 +#define _MONITOR_JUMP_BACKWARD_r22 1430 +#define _MONITOR_JUMP_BACKWARD_r33 1431 +#define _MONITOR_RESUME_r00 1432 +#define _NOP_r00 1433 +#define _NOP_r11 1434 +#define _NOP_r22 1435 +#define _NOP_r33 1436 +#define _POP_EXCEPT_r10 1437 +#define _POP_ITER_r20 1438 +#define _POP_JUMP_IF_FALSE_r00 1439 +#define _POP_JUMP_IF_FALSE_r10 1440 +#define _POP_JUMP_IF_FALSE_r21 1441 +#define _POP_JUMP_IF_FALSE_r32 1442 +#define _POP_JUMP_IF_TRUE_r00 1443 +#define _POP_JUMP_IF_TRUE_r10 1444 +#define _POP_JUMP_IF_TRUE_r21 1445 +#define _POP_JUMP_IF_TRUE_r32 1446 +#define _POP_TOP_r10 1447 +#define _POP_TOP_FLOAT_r00 1448 +#define _POP_TOP_FLOAT_r10 1449 +#define _POP_TOP_FLOAT_r21 1450 +#define _POP_TOP_FLOAT_r32 1451 +#define _POP_TOP_INT_r00 1452 +#define _POP_TOP_INT_r10 1453 +#define _POP_TOP_INT_r21 1454 +#define _POP_TOP_INT_r32 1455 +#define _POP_TOP_NOP_r00 1456 +#define _POP_TOP_NOP_r10 1457 +#define _POP_TOP_NOP_r21 1458 +#define _POP_TOP_NOP_r32 1459 +#define _POP_TOP_OPARG_r00 1460 +#define _POP_TOP_UNICODE_r00 1461 +#define _POP_TOP_UNICODE_r10 1462 +#define _POP_TOP_UNICODE_r21 1463 +#define _POP_TOP_UNICODE_r32 1464 +#define _PUSH_EXC_INFO_r02 1465 +#define _PUSH_EXC_INFO_r12 1466 +#define _PUSH_EXC_INFO_r23 1467 +#define _PUSH_FRAME_r10 1468 +#define _PUSH_NULL_r01 1469 +#define _PUSH_NULL_r12 1470 +#define _PUSH_NULL_r23 1471 +#define _PUSH_NULL_CONDITIONAL_r00 1472 +#define _PUSH_TAGGED_ZERO_r01 1473 +#define _PUSH_TAGGED_ZERO_r12 1474 +#define _PUSH_TAGGED_ZERO_r23 1475 +#define _PY_FRAME_EX_r31 1476 +#define _PY_FRAME_GENERAL_r01 1477 +#define _PY_FRAME_KW_r11 1478 +#define _REPLACE_WITH_TRUE_r02 1479 +#define _REPLACE_WITH_TRUE_r12 1480 +#define _REPLACE_WITH_TRUE_r23 1481 +#define _RESUME_CHECK_r00 1482 +#define _RESUME_CHECK_r11 1483 +#define _RESUME_CHECK_r22 1484 +#define _RESUME_CHECK_r33 1485 +#define _RETURN_GENERATOR_r01 1486 +#define _RETURN_VALUE_r11 1487 +#define _SAVE_RETURN_OFFSET_r00 1488 +#define _SAVE_RETURN_OFFSET_r11 1489 +#define _SAVE_RETURN_OFFSET_r22 1490 +#define _SAVE_RETURN_OFFSET_r33 1491 +#define _SEND_r33 1492 +#define _SEND_GEN_FRAME_r33 1493 +#define _SETUP_ANNOTATIONS_r00 1494 +#define _SET_ADD_r10 1495 +#define _SET_FUNCTION_ATTRIBUTE_r01 1496 +#define _SET_FUNCTION_ATTRIBUTE_r11 1497 +#define _SET_FUNCTION_ATTRIBUTE_r21 1498 +#define _SET_FUNCTION_ATTRIBUTE_r32 1499 +#define _SET_IP_r00 1500 +#define _SET_IP_r11 1501 +#define _SET_IP_r22 1502 +#define _SET_IP_r33 1503 +#define _SET_UPDATE_r11 1504 +#define _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 1505 +#define _SPILL_OR_RELOAD_r01 1506 +#define _SPILL_OR_RELOAD_r02 1507 +#define _SPILL_OR_RELOAD_r03 1508 +#define _SPILL_OR_RELOAD_r10 1509 +#define _SPILL_OR_RELOAD_r12 1510 +#define _SPILL_OR_RELOAD_r13 1511 +#define _SPILL_OR_RELOAD_r20 1512 +#define _SPILL_OR_RELOAD_r21 1513 +#define _SPILL_OR_RELOAD_r23 1514 +#define _SPILL_OR_RELOAD_r30 1515 +#define _SPILL_OR_RELOAD_r31 1516 +#define _SPILL_OR_RELOAD_r32 1517 +#define _START_EXECUTOR_r00 1518 +#define _STORE_ATTR_r20 1519 +#define _STORE_ATTR_INSTANCE_VALUE_r21 1520 +#define _STORE_ATTR_SLOT_r21 1521 +#define _STORE_ATTR_WITH_HINT_r21 1522 +#define _STORE_DEREF_r10 1523 +#define _STORE_FAST_LOAD_FAST_r11 1524 +#define _STORE_FAST_STORE_FAST_r20 1525 +#define _STORE_GLOBAL_r10 1526 +#define _STORE_NAME_r10 1527 +#define _STORE_SLICE_r30 1528 +#define _STORE_SUBSCR_r30 1529 +#define _STORE_SUBSCR_DICT_r31 1530 +#define _STORE_SUBSCR_DICT_KNOWN_HASH_r31 1531 +#define _STORE_SUBSCR_LIST_INT_r32 1532 +#define _SWAP_r11 1533 +#define _SWAP_2_r02 1534 +#define _SWAP_2_r12 1535 +#define _SWAP_2_r22 1536 +#define _SWAP_2_r33 1537 +#define _SWAP_3_r03 1538 +#define _SWAP_3_r13 1539 +#define _SWAP_3_r23 1540 +#define _SWAP_3_r33 1541 +#define _SWAP_FAST_r01 1542 +#define _SWAP_FAST_r11 1543 +#define _SWAP_FAST_r22 1544 +#define _SWAP_FAST_r33 1545 +#define _SWAP_FAST_0_r01 1546 +#define _SWAP_FAST_0_r11 1547 +#define _SWAP_FAST_0_r22 1548 +#define _SWAP_FAST_0_r33 1549 +#define _SWAP_FAST_1_r01 1550 +#define _SWAP_FAST_1_r11 1551 +#define _SWAP_FAST_1_r22 1552 +#define _SWAP_FAST_1_r33 1553 +#define _SWAP_FAST_2_r01 1554 +#define _SWAP_FAST_2_r11 1555 +#define _SWAP_FAST_2_r22 1556 +#define _SWAP_FAST_2_r33 1557 +#define _SWAP_FAST_3_r01 1558 +#define _SWAP_FAST_3_r11 1559 +#define _SWAP_FAST_3_r22 1560 +#define _SWAP_FAST_3_r33 1561 +#define _SWAP_FAST_4_r01 1562 +#define _SWAP_FAST_4_r11 1563 +#define _SWAP_FAST_4_r22 1564 +#define _SWAP_FAST_4_r33 1565 +#define _SWAP_FAST_5_r01 1566 +#define _SWAP_FAST_5_r11 1567 +#define _SWAP_FAST_5_r22 1568 +#define _SWAP_FAST_5_r33 1569 +#define _SWAP_FAST_6_r01 1570 +#define _SWAP_FAST_6_r11 1571 +#define _SWAP_FAST_6_r22 1572 +#define _SWAP_FAST_6_r33 1573 +#define _SWAP_FAST_7_r01 1574 +#define _SWAP_FAST_7_r11 1575 +#define _SWAP_FAST_7_r22 1576 +#define _SWAP_FAST_7_r33 1577 +#define _TIER2_RESUME_CHECK_r00 1578 +#define _TIER2_RESUME_CHECK_r11 1579 +#define _TIER2_RESUME_CHECK_r22 1580 +#define _TIER2_RESUME_CHECK_r33 1581 +#define _TO_BOOL_r11 1582 +#define _TO_BOOL_BOOL_r01 1583 +#define _TO_BOOL_BOOL_r11 1584 +#define _TO_BOOL_BOOL_r22 1585 +#define _TO_BOOL_BOOL_r33 1586 +#define _TO_BOOL_INT_r02 1587 +#define _TO_BOOL_INT_r12 1588 +#define _TO_BOOL_INT_r23 1589 +#define _TO_BOOL_LIST_r02 1590 +#define _TO_BOOL_LIST_r12 1591 +#define _TO_BOOL_LIST_r23 1592 +#define _TO_BOOL_NONE_r01 1593 +#define _TO_BOOL_NONE_r11 1594 +#define _TO_BOOL_NONE_r22 1595 +#define _TO_BOOL_NONE_r33 1596 +#define _TO_BOOL_STR_r02 1597 +#define _TO_BOOL_STR_r12 1598 +#define _TO_BOOL_STR_r23 1599 +#define _TRACE_RECORD_r00 1600 +#define _UNARY_INVERT_r12 1601 +#define _UNARY_NEGATIVE_r12 1602 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r02 1603 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r12 1604 +#define _UNARY_NEGATIVE_FLOAT_INPLACE_r23 1605 +#define _UNARY_NOT_r01 1606 +#define _UNARY_NOT_r11 1607 +#define _UNARY_NOT_r22 1608 +#define _UNARY_NOT_r33 1609 +#define _UNPACK_EX_r10 1610 +#define _UNPACK_SEQUENCE_r10 1611 +#define _UNPACK_SEQUENCE_LIST_r10 1612 +#define _UNPACK_SEQUENCE_TUPLE_r10 1613 +#define _UNPACK_SEQUENCE_TWO_TUPLE_r12 1614 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r03 1615 +#define _UNPACK_SEQUENCE_UNIQUE_THREE_TUPLE_r13 1616 +#define _UNPACK_SEQUENCE_UNIQUE_TUPLE_r10 1617 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r02 1618 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r12 1619 +#define _UNPACK_SEQUENCE_UNIQUE_TWO_TUPLE_r23 1620 +#define _WITH_EXCEPT_START_r33 1621 +#define _YIELD_VALUE_r11 1622 +#define MAX_UOP_REGS_ID 1622 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 8465fd4345e9a5..e9d1a96170d009 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -396,8 +396,8 @@ const uint32_t _PyUop_Flags[MAX_UOP_ID+1] = { [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY] = HAS_DEOPT_FLAG, [_LOAD_CONST_INLINE] = HAS_PURE_FLAG, - [_LOAD_CONST_INLINE_BORROW] = HAS_PURE_FLAG, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = 0, + [_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, + [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = HAS_ESCAPES_FLAG, [_START_EXECUTOR] = HAS_DEOPT_FLAG, [_MAKE_WARM] = 0, [_FATAL_ERROR] = 0, @@ -3692,20 +3692,20 @@ const _PyUopCachingInfo _PyUop_Caching[MAX_UOP_ID+1] = { }, }, [_LOAD_CONST_INLINE_BORROW] = { - .best = { 0, 1, 2, 2 }, + .best = { 0, 0, 0, 0 }, .entries = { { 1, 0, _LOAD_CONST_INLINE_BORROW_r01 }, - { 2, 1, _LOAD_CONST_INLINE_BORROW_r12 }, - { 3, 2, _LOAD_CONST_INLINE_BORROW_r23 }, + { -1, -1, -1 }, + { -1, -1, -1 }, { -1, -1, -1 }, }, }, [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = { - .best = { 0, 1, 2, 3 }, + .best = { 3, 3, 3, 3 }, .entries = { - { 3, 0, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03 }, - { 3, 1, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13 }, - { 3, 2, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23 }, + { -1, -1, -1 }, + { -1, -1, -1 }, + { -1, -1, -1 }, { 3, 3, _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33 }, }, }, @@ -4703,11 +4703,6 @@ const uint16_t _PyUop_Uncached[MAX_UOP_REGS_ID+1] = { [_LOAD_CONST_INLINE_r12] = _LOAD_CONST_INLINE, [_LOAD_CONST_INLINE_r23] = _LOAD_CONST_INLINE, [_LOAD_CONST_INLINE_BORROW_r01] = _LOAD_CONST_INLINE_BORROW, - [_LOAD_CONST_INLINE_BORROW_r12] = _LOAD_CONST_INLINE_BORROW, - [_LOAD_CONST_INLINE_BORROW_r23] = _LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33] = _SHUFFLE_3_LOAD_CONST_INLINE_BORROW, [_START_EXECUTOR_r00] = _START_EXECUTOR, [_MAKE_WARM_r00] = _MAKE_WARM, @@ -5668,8 +5663,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_LOAD_CONST_INLINE_r23] = "_LOAD_CONST_INLINE_r23", [_LOAD_CONST_INLINE_BORROW] = "_LOAD_CONST_INLINE_BORROW", [_LOAD_CONST_INLINE_BORROW_r01] = "_LOAD_CONST_INLINE_BORROW_r01", - [_LOAD_CONST_INLINE_BORROW_r12] = "_LOAD_CONST_INLINE_BORROW_r12", - [_LOAD_CONST_INLINE_BORROW_r23] = "_LOAD_CONST_INLINE_BORROW_r23", [_LOAD_DEREF] = "_LOAD_DEREF", [_LOAD_DEREF_r01] = "_LOAD_DEREF_r01", [_LOAD_FAST] = "_LOAD_FAST", @@ -5934,9 +5927,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_REGS_ID+1] = { [_SET_UPDATE] = "_SET_UPDATE", [_SET_UPDATE_r11] = "_SET_UPDATE_r11", [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13", - [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23", [_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33] = "_SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33", [_SPILL_OR_RELOAD] = "_SPILL_OR_RELOAD", [_SPILL_OR_RELOAD_r01] = "_SPILL_OR_RELOAD_r01", diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-04-04-19-35-32.gh-issue-145742.SU9RYL.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-04-19-35-32.gh-issue-145742.SU9RYL.rst new file mode 100644 index 00000000000000..0909ffe44bd331 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-04-04-19-35-32.gh-issue-145742.SU9RYL.rst @@ -0,0 +1,2 @@ +Optimize _LOAD_CONST_INLINE_BORROW by pre-tagging operands at trace +creation. Patch by Donghee Na. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 7de889b93b71a7..acec4b6303893e 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -6104,11 +6104,11 @@ dummy_func( } tier2 pure op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = PyStackRef_FromPyObjectBorrow(ptr); + value = PyStackRef_FromPreTagged(ptr); } tier2 op(_SHUFFLE_3_LOAD_CONST_INLINE_BORROW, (ptr/4, callable, null, arg -- res, a, c)) { - res = PyStackRef_FromPyObjectBorrow(ptr); + res = PyStackRef_FromPreTagged(ptr); a = arg; c = callable; INPUTS_DEAD(); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 93d39bee1b9ff6..397f0850115cfe 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -22300,118 +22300,17 @@ assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); _PyStackRef value; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - value = PyStackRef_FromPyObjectBorrow(ptr); + _PyFrame_SetStackPointer(frame, stack_pointer); + value = PyStackRef_FromPreTagged(ptr); + stack_pointer = _PyFrame_GetStackPointer(frame); _tos_cache0 = value; + _tos_cache1 = PyStackRef_ZERO_BITS; + _tos_cache2 = PyStackRef_ZERO_BITS; SET_CURRENT_CACHED_VALUES(1); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } - case _LOAD_CONST_INLINE_BORROW_r12: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - value = PyStackRef_FromPyObjectBorrow(ptr); - _tos_cache1 = value; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _LOAD_CONST_INLINE_BORROW_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef value; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - value = PyStackRef_FromPyObjectBorrow(ptr); - _tos_cache2 = value; - _tos_cache1 = _stack_item_1; - _tos_cache0 = _stack_item_0; - SET_CURRENT_CACHED_VALUES(3); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r03: { - CHECK_CURRENT_CACHED_VALUES(0); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; - arg = stack_pointer[-1]; - callable = stack_pointer[-3]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -3; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r13: { - CHECK_CURRENT_CACHED_VALUES(1); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; - _PyStackRef _stack_item_0 = _tos_cache0; - arg = _stack_item_0; - callable = stack_pointer[-2]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -2; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r23: { - CHECK_CURRENT_CACHED_VALUES(2); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - _PyStackRef arg; - _PyStackRef callable; - _PyStackRef res; - _PyStackRef a; - _PyStackRef c; - _PyStackRef _stack_item_0 = _tos_cache0; - _PyStackRef _stack_item_1 = _tos_cache1; - arg = _stack_item_1; - callable = stack_pointer[-1]; - PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); - a = arg; - c = callable; - _tos_cache2 = c; - _tos_cache1 = a; - _tos_cache0 = res; - SET_CURRENT_CACHED_VALUES(3); - stack_pointer += -1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); - break; - } - case _SHUFFLE_3_LOAD_CONST_INLINE_BORROW_r33: { CHECK_CURRENT_CACHED_VALUES(3); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); @@ -22426,13 +22325,22 @@ arg = _stack_item_2; callable = _stack_item_0; PyObject *ptr = (PyObject *)CURRENT_OPERAND0_64(); - res = PyStackRef_FromPyObjectBorrow(ptr); + stack_pointer[0] = callable; + stack_pointer[1] = _stack_item_1; + stack_pointer[2] = arg; + stack_pointer += 3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + res = PyStackRef_FromPreTagged(ptr); + stack_pointer = _PyFrame_GetStackPointer(frame); a = arg; c = callable; _tos_cache2 = c; _tos_cache1 = a; _tos_cache0 = res; SET_CURRENT_CACHED_VALUES(3); + stack_pointer += -3; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); assert(WITHIN_STACK_BOUNDS_IGNORING_CACHE()); break; } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 095bcfc639bc65..60496247216893 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -177,12 +177,13 @@ convert_global_to_const(_PyUOpInstruction *inst, PyObject *obj) if (res == NULL) { return NULL; } - if (_Py_IsImmortal(res)) { + bool borrow = _Py_IsImmortal(res); + if (borrow) { inst->opcode = _LOAD_CONST_INLINE_BORROW; } else { inst->opcode = _LOAD_CONST_INLINE; } - inst->operand0 = (uint64_t)res; + inst->operand0 = borrow ? PyStackRef_TagBorrow(res) : (uint64_t)res; return res; } @@ -233,7 +234,13 @@ add_op(JitOptContext *ctx, _PyUOpInstruction *this_instr, out->format = this_instr->format; out->oparg = (oparg); out->target = this_instr->target; - out->operand0 = (operand0); + if (opcode == _LOAD_CONST_INLINE_BORROW || + opcode == _SHUFFLE_3_LOAD_CONST_INLINE_BORROW) { + out->operand0 = PyStackRef_TagBorrow((PyObject *)operand0); + } + else { + out->operand0 = (operand0); + } out->operand1 = this_instr->operand1; ctx->out_buffer.next++; } diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 6e4882143fbe50..31712a011f0708 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -890,7 +890,7 @@ dummy_func(void) { } op(_LOAD_CONST_INLINE_BORROW, (ptr/4 -- value)) { - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); } op(_POP_TOP_OPARG, (args[oparg] --)) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index c3c889e9de9a7e..fc72d66f8aa6af 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -5489,7 +5489,7 @@ case _LOAD_CONST_INLINE_BORROW: { JitOptRef value; PyObject *ptr = (PyObject *)this_instr->operand0; - value = PyJitRef_Borrow(sym_new_const(ctx, ptr)); + value = PyJitRef_Borrow(sym_new_const(ctx, PyStackRef_UntagBorrow(ptr))); CHECK_STACK_BOUNDS(1); stack_pointer[0] = value; stack_pointer += 1; diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 414ca18be4654c..898b98ff3c82ab 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -766,7 +766,7 @@ def escaping_call_in_simple_stmt(stmt: SimpleStmt, result: dict[SimpleStmt, Esca continue #if not tkn.text.startswith(("Py", "_Py", "monitor")): # continue - if tkn.text.startswith(("sym_", "optimize_", "PyJitRef")): + if tkn.text.startswith(("sym_", "optimize_", "PyJitRef", "PyStackRef_Tag", "PyStackRef_Untag")): # Optimize functions continue if tkn.text.endswith("Check"):