1313. 1313 after.is_empty() || after.starts_with(|c: char| !c.is_ascii_alphanumeric());
1314. 1314 if word_start && word_end && before.trim_end().ends_with([',', ';', ':', '.', '\n']) {
1315. 1315 return true;
1316. 1316 }
1317. 1317 }
1318. 1318 false
1319. 1319 }
1320. 1320}
1321. 1321
1322. 1322fn starts_with_words(tokens: &[String], phrase: &str) -> Option<usize> {
1323. 1323 let needle: Vec<&str> = phrase.split(' ').collect();
1324. 1324 (tokens.len() >= needle.len() && needle.iter().zip(tokens).all(|(a, b)| a == b))
1325. 1325 .then_some(needle.len())
1326. 1326}
1327. 1327
1328. 1328/// Drop `count` leading words from `text`, keeping the original spelling of
1329. 1329/// what remains.
1330. 1330fn drop_leading_words(text: &str, count: usize) -> &str {
1331. 1331 let mut seen = 0usize;
1332. 1332 let mut in_word = false;
1333. 1333 for (i, c) in text.char_indices() {
1334. 1334 let is_word = c.is_ascii_alphanumeric();
1335. 1335 if is_word && !in_word {
1336. 1336 if seen == count {
1337. 1337 return &text[i..];
1338. 1338 }
1339. 1339 seen += 1;
1340. 1340 }
1341. 1341 in_word = is_word;
1342. 1342 }
1343. 1343 ""
1344. 1344}
1345. 1345
1346. 1346/// Read one clause: strip what leads it, then decide its kind.
1347. 1347pub(crate) fn read_clause(
1348. 1348 text: &str,
1349. 1349 boundary: crate::strand::Boundary,
1350. 1350 question_mark: bool,
1351. 1351) -> ClauseRead {
1352. 1352 let mut rest = text.trim();
1353. 1353 let mut greeting = 0.0f64;
1354. 1354 let mut stripped = Vec::new();
1355. 1355 let mut acknowledged = false;
1356. 1356 // Greetings, acknowledgements, polite openers and sequencing words, in
1357. 1357 // any order, until none applies.
1358. 1358 loop {
1359. 1359 let tokens = words(rest);
1360. 1360 let Some(first) = tokens.first() else {
1361. 1361 break;
1362. 1362 };
1363. 1363 if let Some((phrase, weight)) = SOCIAL_PHRASES
1364. 1364 .iter()
1365. 1365 .find(|(phrase, _)| starts_with_words(&tokens, phrase).is_some())
1366. 1366 .filter(|(phrase, _)| starts_with_words(&tokens, phrase) != Some(tokens.len()))
1367. 1367 {
1368. 1368 // A social phrase leading a longer clause is a greeting.
1369. 1369 if let Some(n) = starts_with_words(&tokens, phrase) {
1370. 1370 greeting = greeting.max(*weight);
1371. 1371 stripped.push(format!("greeting `{phrase}`"));
1372. 1372 rest = drop_leading_words(rest, n);
1373. 1373 continue;
1374. 1374 }
1375. 1375 }
1376. 1376 if tokens.len() > 1
1377. 1377 && let Some((word, weight)) = SOCIAL_WORDS.iter().find(|(word, _)| word == first)
1378. 1378 {
1379. 1379 greeting = greeting.max(*weight);
1380. 1380 stripped.push(format!("greeting `{word}`"));
1381. 1381 rest = drop_leading_words(rest, 1);
1382. 1382 continue;
1383. 1383 }
1384. 1384 if tokens.len() > 1 && ACKNOWLEDGEMENTS.contains(&first.as_str()) {
1385. 1385 acknowledged = true;
1386. 1386 stripped.push(format!("acknowledgement `{first}`"));
1387. 1387 rest = drop_leading_words(rest, 1);
1388. 1388 continue;
1389. 1389 }
1390. 1390 if let Some(n) = PREAMBLES
1391. 1391 .iter()
1392. 1392 .filter_map(|preamble| starts_with_words(&tokens, preamble))
1393. 1393 .max()
1394. 1394 .filter(|n| *n < tokens.len())
1395. 1395 {
1396. 1396 stripped.push(format!("opener `{}`", tokens[..n].join(" ")));
1397. 1397 rest = drop_leading_words(rest, n);
1398. 1398 continue;
1399. 1399 }
1400. 1400 if tokens.len() > 1 && LEAD_FILLERS.contains(&first.as_str()) {
1401. 1401 rest = drop_leading_words(rest, 1);
1402. 1402 continue;
1403. 1403 }
1404. 1404 break;
1405. 1405 }
1406. 1406 let rest = rest.trim_start_matches(|c: char| !c.is_ascii_alphanumeric() && c.is_ascii());
1407. 1407 let tokens = Tokens::new(rest);
1408. 1408 let mut consumed = vec![false; tokens.len()];
1409. 1409 let mut social_phrase = false;
1410. 1410 for (phrase, _) in SOCIAL_PHRASES {
1411. 1411 let words: Vec<&str> = split_phrase(phrase);
1412. 1412 if let Some(at) = tokens.phrase_at(&words) {
1413. 1413 social_phrase = true;
1414. 1414 for flag in consumed.iter_mut().skip(at).take(words.len()) {
1415. 1415 *flag = true;
1416. 1416 }
1417. 1417 }
1418. 1418 }
1419. 1419 let lone_social = tokens.len() == 1
1420. 1420 && SOCIAL_WORDS
1421. 1421 .iter()
1422. 1422 .any(|(word, _)| tokens.get(0) == Some(*word));
1423. 1423 // A wish is a request, and the words that say so are not verbs of
1424. 1424 // their own: the `what` of "what I need is…" asks nothing.
1425. 1425 let desire = DESIRE_PREFIXES
1426. 1426 .iter()
1427. 1427 .filter_map(|prefix| starts_with_words(&tokens.words, prefix))
1428. 1428 .max();
1429. 1429 if let Some(length) = desire {
1430. 1430 for flag in consumed.iter_mut().take(length) {
1431. 1431 *flag = true;
1432. 1432 }
1433. 1433 }
1434. 1434 let mut read = ClauseRead {
1435. 1435 text: rest.to_string(),
1436. 1436 boundary,
1437. 1437 kind: ClauseKind::Statement,
1438. 1438 tokens,
1439. 1439 consumed,
1440. 1440 greeting,
1441. 1441 stripped,
1442. 1442 question_mark,
1443. 1443 };
1444. 1444 read.kind = classify(
1445. 1445 &read,
1446. 1446 social_phrase,
1447. 1447 lone_social,
1448. 1448 acknowledged,
1449. 1449 desire.is_some(),
1450. 1450 );
1451. 1451 read
1452. 1452}
1453. 1453
1454. 1454fn classify(
1455. 1455 read: &ClauseRead,
1456. 1456 social_phrase: bool,
1457. 1457 lone_social: bool,
1458. 1458 acknowledged: bool,
1459. 1459 desire: bool,
1460. 1460) -> ClauseKind {
1461. 1461 let tokens = &read.tokens;
1462. 1462 if tokens.len() == 0 {
1463. 1463 return if read.greeting > 0.0 || acknowledged {
1464. 1464 ClauseKind::Social
1465. 1465 } else {
1466. 1466 ClauseKind::Statement
1467. 1467 };
1468. 1468 }
1469. 1469 if lone_social || (social_phrase && read.consumed.iter().all(|c| *c)) {
1470. 1470 return ClauseKind::Social;
1471. 1471 }
1472. 1472 if desire {
1473. 1473 return ClauseKind::Request;
1474. 1474 }
1475. 1475 let first = tokens.get(0).unwrap_or("");
1476. 1476 let second = tokens.get(1).unwrap_or("");
1477. 1477 let asks = WH_WORDS.contains(&first)
1478. 1478 || (first == "do" && PERSONS.contains(&second))
1479. 1479 || (first != "do" && AUX_WORDS.contains(&first) && QUESTION_SUBJECTS.contains(&second))
1480. 1480 || read.question_mark;
1481. 1481 if asks && !(social_phrase && read.consumed.first().copied().unwrap_or(false)) {
1482. 1482 return ClauseKind::Question;
1483. 1483 }
1484. 1484 if social_phrase && read.consumed.first().copied().unwrap_or(false) {
1485. 1485 return ClauseKind::Social;
1486. 1486 }
1487. 1487 // A verb heading a later sub-clause ("the build is broken, fix it")
1488. 1488 // makes the clause an instruction whatever its first words say.
1489. 1489 if (1..tokens.len()).any(|i| read.is_head(i) && is_lexicon_verb(tokens, i)) {
1490. 1490 return ClauseKind::Imperative;
1491. 1491 }
1492. 1492 // "Build fails on CI", "the deploy failed", "move occurs because…": a
1493. 1493 // subject, then its verb. The first word is the subject even when it
1494. 1494 // could be a verb somewhere else.
1495. 1495 if STATEMENT_VERBS.contains(&second) || first.chars().any(|c| c.is_ascii_digit()) {
1496. 1496 return ClauseKind::Statement;
1497. 1497 }
1498. 1498 if read.is_head(0) && is_lexicon_verb(tokens, 0) {
1499. 1499 return ClauseKind::Imperative;
1500. 1500 }
1501. 1501 if SUBJECT_STARTERS.contains(&first) {
1502. 1502 return ClauseKind::Statement;
1503. 1503 }
1504. 1504 ClauseKind::Request
1505. 1505}
1506. 1506
1507. 1507// ------------------------------------------------------------ material ---
1508. 1508
1509. 1509/// A request with its pasted material set aside.
1510. 1510#[derive(Debug, Clone, Default, PartialEq)]
1511. 1511pub(crate) struct Prepared {
1512. 1512 /// The text that may vote.
1513. 1513 pub instruction: String,
1514. 1514 /// Fenced code blocks removed.
1515. 1515 pub fenced_blocks: usize,
1516. 1516 /// Lines of unfenced pasted material removed.
1517. 1517 pub pasted_lines: usize,
1518. 1518}
1519. 1519
1520. 1520fn is_list_item(line: &str) -> bool {
1521. 1521 let trimmed = line.trim_start();
1522. 1522 if trimmed.starts_with("- ") || trimmed.starts_with("* ") || trimmed.starts_with("+ ") {
1523. 1523 return true;
1524. 1524 }
1525. 1525 let digits = trimmed.chars().take_while(char::is_ascii_digit).count();
1526. 1526 digits > 0
1527. 1527 && digits <= 3
1528. 1528 && (trimmed[digits..].starts_with(". ") || trimmed[digits..].starts_with(") "))
1529. 1529}
1530. 1530
1531. 1531/// Whether a line reads as something a person wrote to the agent, rather
1532. 1532/// than something pasted for it to look at.
1533. 1533fn reads_as_request(line: &str) -> bool {
1534. 1534 let trimmed = line.trim();
1535. 1535 if trimmed.is_empty() {
1536. 1536 return true;
1537. 1537 }
1538. 1538 // Indented lines are code, stack frames and tracebacks.
1539. 1539 if line.starts_with('\t') || line.starts_with(" ") {
1540. 1540 return false;
1541. 1541 }
1542. 1542 if is_list_item(line) {
1543. 1543 return true;
1544. 1544 }
1545. 1545 let tokens = Tokens::new(trimmed);
1546. 1546 if let Some(first) = tokens.get(0)
1547. 1547 && (is_lexicon_verb(&tokens, 0)
1548. 1548 || WH_WORDS.contains(&first)
1549. 1549 || AUX_WORDS.contains(&first)
1550. 1550 || SOCIAL_WORDS.iter().any(|(word, _)| *word == first)
1551. 1551 || PREAMBLES
1552. 1552 .iter()
1553. 1553 .any(|preamble| preamble.split(' ').next() == Some(first)))
1554. 1554 {
1555. 1555 return true;
1556. 1556 }
1557. 1557 let visible: Vec<char> = trimmed.chars().filter(|c| !c.is_whitespace()).collect();
1558. 1558 if visible.is_empty() {
1559. 1559 return true;
1560. 1560 }
1561. 1561 let prose = visible
1562. 1562 .iter()
1563. 1563 .filter(|c| {
1564. 1564 c.is_alphabetic()
1565. 1565 || matches!(
1566. 1566 c,
1567. 1567 ',' | '.' | '\'' | '?' | '!' | ':' | ';' | '-' | '"' | '(' | ')'
1568. 1568 )
1569. 1569 })
1570. 1570 .count();
1571. 1571 let letters = visible.iter().filter(|c| c.is_alphabetic()).count();
1572. 1572 let prose_like = prose as f64 / visible.len() as f64 >= 0.9
1573. 1573 && letters as f64 / visible.len() as f64 >= 0.7
1574. 1574 && tokens.len() >= 2;
1575. 1575 let ends_like_prose = trimmed.ends_with(['.', '?', '!', ':']);
1576. 1576 prose_like || (ends_like_prose && letters as f64 / visible.len() as f64 >= 0.6)
1577. 1577}
1578. 1578
1579. 1579/// Set pasted material aside: fenced blocks, and runs of two or more lines
1580. 1580/// that do not read as a request (or one very long one). What remains is the
1581. 1581/// request itself.
1582. 1582pub(crate) fn prepare(raw: &str) -> Prepared {
1583. 1583 fn flush<'a>(run: &mut Vec<&'a str>, kept: &mut Vec<&'a str>, prepared: &mut Prepared) {
1584. 1584 let material = run.len() >= 2 || run.iter().any(|line| line.len() >= 160);
1585. 1585 if material {
1586. 1586 prepared.pasted_lines += run.len();
1587. 1587 } else {
1588. 1588 kept.append(run);
1589. 1589 }
1590. 1590 run.clear();
1591. 1591 }
1592. 1592 let text = clean_request_text(raw);
1593. 1593 let mut prepared = Prepared::default();
1594. 1594 let mut kept: Vec<&str> = Vec::new();
1595. 1595 let mut run: Vec<&str> = Vec::new();
1596. 1596 let mut in_fence = false;
1597. 1597 for line in text.lines() {
1598. 1598 let trimmed = line.trim_start();
1599. 1599 if trimmed.starts_with("```") || trimmed.starts_with("~~~") {
1600. 1600 if !in_fence {
1601. 1601 flush(&mut run, &mut kept, &mut prepared);
1602. 1602 prepared.fenced_blocks += 1;
1603. 1603 }
1604. 1604 in_fence = !in_fence;
1605. 1605 continue;
1606. 1606 }
1607. 1607 if in_fence {
1608. 1608 continue;
1609. 1609 }
1610. 1610 if reads_as_request(line) {
1611. 1611 flush(&mut run, &mut kept, &mut prepared);
1612. 1612 kept.push(line);
1613. 1613 } else {
1614. 1614 run.push(line);
1615. 1615 }
1616. 1616 }
1617. 1617 flush(&mut run, &mut kept, &mut prepared);
1618. 1618 prepared.instruction = kept.join("\n");
1619. 1619 prepared
1620. 1620}
1621. 1621
1622. 1622/// Strip prompt scaffolding and runner control blocks before extracting
1623. 1623/// intent. Delegates to [`crate::control`], which owns the tag vocabulary.
1624. 1624pub(crate) fn clean_request_text(raw: &str) -> String {
1625. 1625 let mut text = crate::control::strip_control_blocks(raw);
1626. 1626 while let Some(start) = text.find("[Scheduled-run context:") {
1627. 1627 if let Some(end_offset) = text[start..].find(']') {
1628. 1628 let end = start + end_offset + 1;
1629. 1629 text.replace_range(start..end, "");
1630. 1630 } else {
1631. 1631 text.truncate(start);
1632. 1632 break;
1633. 1633 }
1634. 1634 }
1635. 1635 text.trim().to_string()
1636. 1636}
1637. 1637
1638. 1638// ------------------------------------------------------------- digest ---
1639. 1639
1640. 1640/// A digest of every table tier 1 reads, so a lexicon change that forgets
1641. 1641/// to bump [`crate::RESOLVER_VERSION`] fails a test rather than silently
1642. 1642/// invalidating every ledger row that claims `reproducible: true`.
1643. 1643///
1644. 1644/// Scoring constants are deliberately part of it too: a changed weight is
1645. 1645/// as much a new resolver as a new word.
1646. 1646pub fn lexicon_digest() -> String {
1647. 1647 use sha2::{Digest, Sha256};
1648. 1648 let mut out = String::new();
1649. 1649 for (word, act, weight) in ACT_VERBS {
1650. 1650 out.push_str(&format!("act:{word}:{}:{weight}\n", act.as_str()));
1651. 1651 }
1652. 1652 for (word, weight) in SOCIAL_WORDS {
1653. 1653 out.push_str(&format!("social:{word}:{weight}\n"));
1654. 1654 }
1655. 1655 for (phrase, weight) in SOCIAL_PHRASES {
1656. 1656 out.push_str(&format!("social-phrase:{phrase}:{weight}\n"));
1657. 1657 }
1658. 1658 let lists: [(&str, &[&str]); 15] = [
1659. 1659 ("ack", ACKNOWLEDGEMENTS),
1660. 1660 ("preamble", PREAMBLES),
1661. 1661 ("filler", LEAD_FILLERS),
1662. 1662 ("desire", DESIRE_PREFIXES),
1663. 1663 ("requester", REQUESTER),
1664. 1664 ("statement-verb", STATEMENT_VERBS),
1665. 1665 ("aux", AUX_WORDS),
1666. 1666 ("wh", WH_WORDS),
1667. 1667 ("question-subject", QUESTION_SUBJECTS),
1668. 1668 ("person", PERSONS),
1669. 1669 ("subject", SUBJECT_STARTERS),
1670. 1670 ("checkable", CHECKABLE_WORDS),
1671. 1671 ("local", LOCAL_NOUNS),
1672. 1672 ("time", TIME_WORDS),
1673. 1673 ("head-predecessor", HEAD_PREDECESSORS),
1674. 1674 ];
1675. 1675 for (name, list) in lists {
1676. 1676 for word in list {
1677. 1677 out.push_str(&format!("{name}:{word}\n"));
1678. 1678 }
1679. 1679 }
1680. 1680 for (word, stakes, weight) in STAKES_WORDS {
1681. 1681 out.push_str(&format!("stakes:{word}:{}:{weight}\n", stakes.as_str()));
1682. 1682 }
1683. 1683 for (word, evidence, weight) in EVIDENCE_WORDS {
1684. 1684 out.push_str(&format!("evidence:{word}:{}:{weight}\n", evidence.as_str()));
1685. 1685 }
1686. 1686 for word in EXACT_EVIDENCE_WORDS {
1687. 1687 out.push_str(&format!("evidence-exact:{word}\n"));
1688. 1688 }
1689. 1689 for (phrase, weight) in ASSURANCE_PHRASES {
1690. 1690 out.push_str(&format!("assurance:{phrase}:{weight}\n"));
1691. 1691 }
1692. 1692 for (phrase, weight) in RECENCY_PHRASES {
1693. 1693 out.push_str(&format!("recency:{phrase}:{weight}\n"));
1694. 1694 }
1695. 1695 for (phrase, horizon, weight) in HORIZON_PHRASES {
1696. 1696 out.push_str(&format!("horizon:{phrase}:{}:{weight}\n", horizon.as_str()));
1697. 1697 }
1698. 1698 for word in RESIDE_SUBJECTS {
1699. 1699 out.push_str(&format!("live-reside-subject:{word}\n"));
1700. 1700 }
1701. 1701 for word in RESIDE_PREPOSITIONS {
1702. 1702 out.push_str(&format!("live-reside-preposition:{word}\n"));
1703. 1703 }
1704. 1704 for word in LIVE_COPULAS {
1705. 1705 out.push_str(&format!("live-copula:{word}\n"));
1706. 1706 }
1707. 1707 for word in DEICTIC_WORDS {
1708. 1708 out.push_str(&format!("deictic:{word}\n"));
1709. 1709 }
1710. 1710 for word in BARE_PRONOUNS {
1711. 1711 out.push_str(&format!("bare:{word}\n"));
1712. 1712 }
1713. 1713 for word in RECURRENCE_ADJECTIVES {
1714. 1714 out.push_str(&format!("recurrence-adjective:{word}\n"));
1715. 1715 }
1716. 1716 for word in DETERMINERS {
1717. 1717 out.push_str(&format!("determiner:{word}\n"));
1718. 1718 }
1719. 1719 out.push_str(&format!("floor:{}\n", Votes::<Act>::ESCALATION_FLOOR));
1720. 1720 out.push_str(&format!("imperative-bonus:{IMPERATIVE_BONUS}\n"));
1721. 1721 crate::strand::segmentation_fingerprint(&mut out);
1722. 1722 format!("{:x}", Sha256::digest(out.as_bytes()))
1723. 1723}
1724. 1724
1725. 1725// --------------------------------------------------------- extraction ---
1726. 1726
1727. 1727/// Tier 1 over a whole request read as one part. A pure function of
1728. 1728/// `request`; the resolver reads requests part by part through
1729. 1729/// [`extract_part`].
1730. 1730pub fn extract(request: &Request<'_>) -> Extraction {
1731. 1731 let prepared = prepare(request.text);
1732. 1732 let clauses: Vec<ClauseRead> = crate::strand::segment(&prepared.instruction)
1733. 1733 .into_iter()
1734. 1734 .map(|clause| read_clause(&clause.text, clause.boundary, clause.question))
1735. 1735 .collect();
1736. 1736 let all: Vec<&ClauseRead> = clauses.iter().collect();
1737. 1737 extract_part(
1738. 1738 request,
1739. 1739 &all,
1740. 1740 &prepared,
1741. 1741 list_items(&prepared.instruction),
1742. 1742 false,
1743. 1743 )
1744. 1744}
1745. 1745
1746. 1746/// How many enumerated items the request holds.
1747. 1747pub(crate) fn list_items(instruction: &str) -> usize {
1748. 1748 instruction
1749. 1749 .lines()
1750. 1750 .filter(|line| is_list_item(line))
1751. 1751 .count()
1752. 1752}
1753. 1753
1754. 1754/// Tier 1 over one part of a request: the clauses that make it up, and the
1755. 1755/// pasted material and enumeration of the request it belongs to.
1756. 1756pub(crate) fn extract_part(
1757. 1757 request: &Request<'_>,
1758. 1758 clauses: &[&ClauseRead],
1759. 1759 prepared: &Prepared,
1760. 1760 enumerated: usize,
1761. 1761 earlier_parts: bool,
1762. 1762) -> Extraction {
1763. 1763 let mut out = Extraction {
1764. 1764 attendance: request
1765. 1765 .attendance_override
1766. 1766 .unwrap_or_else(|| request.surface.implied_attendance()),
1767. 1767 ..Extraction::default()
1768. 1768 };
1769. 1769 out.signals.push(Signal::new(
1770. 1770 SignalKind::Surface,
1771. 1771 format!("surface:{}", request.surface.as_str()),
1772. 1772 0.0,
1773. 1773 format!(
1774. 1774 "arrived on {} ⇒ {} by default",
1775. 1775 request.surface.as_str(),
1776. 1776 out.attendance.as_str()
1777. 1777 ),
1778. 1778 ));
1779. 1779
1780. 1780 let part_words: Vec<String> = clauses
1781. 1781 .iter()
1782. 1782 .flat_map(|clause| clause.tokens.words.iter().cloned())
1783. 1783 .collect();
1784. 1784 let checkable = part_words
1785. 1785 .iter()
1786. 1786 .any(|word| CHECKABLE_WORDS.contains(&word.as_str()));
1787. 1787 let mut asks = false;
1788. 1788 let mut work = false;
1789. 1789
1790. 1790 for clause in clauses {
1791. 1791 for stripped in &clause.stripped {
1792. 1792 out.signals.push(Signal::new(
1793. 1793 SignalKind::Lexical,
1794. 1794 "stripped",
1795. 1795 0.0,
1796. 1796 format!("{stripped} set aside"),
1797. 1797 ));
1798. 1798 }
1799. 1799 if clause.greeting > 0.0 {
1800. 1800 out.act.add(Act::Converse, clause.greeting);
1801. 1801 out.signals.push(Signal::new(
1802. 1802 SignalKind::Lexical,
1803. 1803 "social:greeting",
1804. 1804 clause.greeting,
1805. 1805 "a greeting ⇒ converse".to_string(),
1806. 1806 ));
1807. 1807 }
1808. 1808 work |= clause.kind.is_work();
1809. 1809 read_acts(clause, &mut out);
1810. 1810 if clause.kind == ClauseKind::Question {
1811. 1811 asks = true;
1812. 1812 }
1813. 1813 read_stakes_and_evidence(clause, checkable, &mut out);
1814. 1814 read_recency(clause, &mut out);
1815. 1815 if matches!(clause.kind, ClauseKind::Imperative | ClauseKind::Request) {
1816. 1816 read_horizon(clause, &mut out);
1817. 1817 }
1818. 1818 read_structure(clause, &mut out);
1819. 1819 }
1820. 1820 if asks {
1821. 1821 out.signals.push(Signal::new(
1822. 1822 SignalKind::Structural,
1823. 1823 "question",
1824. 1824 0.0,
1825. 1825 "the part asks a question",
1826. 1826 ));
1827. 1827 }
1828. 1828
1829. 1829 // A greeting and nothing else is one reply.
1830. 1830 if !work
1831. 1831 && out
1832. 1832 .act
1833. 1833 .ranked()
1834. 1834 .iter()
1835. 1835 .all(|(act, _)| *act == Act::Converse)
1836. 1836 && !out.act.is_empty()
1837. 1837 {
1838. 1838 out.horizon.add(Horizon::Immediate, 0.6);
1839. 1839 out.signals.push(Signal::new(
1840. 1840 SignalKind::Structural,
1841. 1841 "social-only",
1842. 1842 0.6,
1843. 1843 "social talk and nothing else ⇒ immediate",
1844. 1844 ));
1845. 1845 }
1846. 1846
1847. 1847 // --- material ---------------------------------------------------------
1848. 1848 if prepared.fenced_blocks > 0 {
1849. 1849 out.act.add(Act::Modify, 0.3);
1850. 1850 out.signals.push(Signal::new(
1851. 1851 SignalKind::Structural,
1852. 1852 "code-fence",
1853. 1853 0.3,
1854. 1854 format!("{} fenced block(s) set aside", prepared.fenced_blocks),
1855. 1855 ));
1856. 1856 }
1857. 1857 if prepared.pasted_lines > 0 {
1858. 1858 out.input_modalities.push(Modality::Data);
1859. 1859 out.signals.push(Signal::new(
1860. 1860 SignalKind::Structural,
1861. 1861 "pasted-material",
1862. 1862 0.0,
1863. 1863 format!(
1864. 1864 "{} pasted line(s) read as material, not instructions",
1865. 1865 prepared.pasted_lines
1866. 1866 ),
1867. 1867 ));
1868. 1868 }
1869. 1869 // Enumerated steps are the honest multi-step marker: a list the user
1870. 1870 // wrote themselves. Multi-step is a session, not a durable obligation.
1871. 1871 if enumerated >= 2 && work {
1872. 1872 out.horizon.add(Horizon::Session, 0.9);
1873. 1873 out.signals.push(Signal::new(
1874. 1874 SignalKind::Structural,
1875. 1875 "enumerated-steps",
1876. 1876 0.9,
1877. 1877 format!("{enumerated} enumerated items ⇒ session"),
1878. 1878 ));
1879. 1879 }
1880. 1880
1881. 1881 // --- deixis -----------------------------------------------------------
1882. 1882 let deictic: Vec<&str> = DEICTIC_WORDS
1883. 1883 .iter()
1884. 1884 .copied()
1885. 1885 .filter(|w| part_words.iter().any(|t| t == w))
1886. 1886 .collect();
1887. 1887 out.deictic = !deictic.is_empty();
1888. 1888 if !deictic.is_empty() {
1889. 1889 let bare = clauses.iter().any(|clause| {
1890. 1890 clause.kind.is_work()
1891. 1891 && clause
1892. 1892 .tokens
1893. 1893 .words
1894. 1894 .last()
1895. 1895 .is_some_and(|last| BARE_PRONOUNS.contains(&last.as_str()))
1896. 1896 });
1897. 1897 // Pointing at something is only ambiguous when there is nothing to
1898. 1898 // point at: the first part of a first message with no attachment.
1899. 1899 // Mid-conversation, after an earlier part of the same message ("explain
1900. 1900 // the parser, then refactor it"), or with an attached file, it is
1901. 1901 // ordinary and clear.
1902. 1902 if request.history.turn_index == 0 && request.attachments.is_empty() && !earlier_parts {
1903. 1903 if bare {
1904. 1904 out.clarity.add(Clarity::Ambiguous, 0.7);
1905. 1905 out.signals.push(Signal::new(
1906. 1906 SignalKind::Deictic,
1907. 1907 "unresolved-reference",
1908. 1908 0.7,
1909. 1909 format!("`{}` with no prior context", deictic.join("`, `")),
1910. 1910 ));
1911. 1911 }
1912. 1912 } else {
1913. 1913 out.clarity.add(Clarity::Clear, 0.4);
1914. 1914 out.signals.push(Signal::new(
1915. 1915 SignalKind::Deictic,
1916. 1916 "resolved-reference",
1917. 1917 0.4,
1918. 1918 format!("`{}` resolvable from context", deictic.join("`, `")),
1919. 1919 ));
1920. 1920 }
1921. 1921 }
1922. 1922
1923. 1923 // --- workspace --------------------------------------------------------
1924. 1924 if request.workspace.is_repo {
1925. 1925 out.domains_from_environment.push("engineering".into());
1926. 1926 out.signals.push(Signal::new(
1927. 1927 SignalKind::Workspace,
1928. 1928 "git-repo",
1929. 1929 0.3,
1930. 1930 "workspace is a git repository",
1931. 1931 ));
1932. 1932 }
1933. 1933 if request.workspace.has_uncommitted_changes {
1934. 1934 out.stakes_from_environment.add(Stakes::Reversible, 0.5);
1935. 1935 out.signals.push(Signal::new(
1936. 1936 SignalKind::Workspace,
1937. 1937 "dirty-tree",
1938. 1938 0.5,
1939. 1939 "uncommitted changes present",
1940. 1940 ));
1941. 1941 }
1942. 1942
1943. 1943 // --- session ----------------------------------------------------------
1944. 1944 if let Some(previous) = request.history.previous_act {
1945. 1945 // Continuity is a real prior, but a weak one: people change subject.
1946. 1946 out.act.add(previous, 0.25);
1947. 1947 out.signals.push(Signal::new(
1948. 1948 SignalKind::Session,
1949. 1949 format!("continues:{}", previous.as_str()),
1950. 1950 0.25,
1951. 1951 format!("previous turn read as {}", previous.as_str()),
1952. 1952 ));
1953. 1953 }
1954. 1954
1955. 1955 // --- attachments ------------------------------------------------------
1956. 1956 for attachment in request.attachments {
1957. 1957 out.input_modalities.push(attachment.modality);
1958. 1958 out.signals.push(Signal::new(
1959. 1959 SignalKind::Attachment,
1960. 1960 format!("input:{}", attachment.modality.as_str()),
1961. 1961 1.0,
1962. 1962 format!(
1963. 1963 "attached {} `{}`",
1964. 1964 attachment.modality.as_str(),
1965. 1965 attachment.name
1966. 1966 ),
1967. 1967 ));
1968. 1968 if attachment.modality != Modality::Text {
1969. 1969 out.act.add(Act::Analyze, 0.4);
1970. 1970 }
1971. 1971 }
1972. 1972 if !out.input_modalities.contains(&Modality::Text) {
1973. 1973 out.input_modalities.push(Modality::Text);
1974. 1974 }
1975. 1975 out.output_modalities.push(Modality::Text);
1976. 1976
1977. 1977 out.input_modalities.sort();
1978. 1978 out.input_modalities.dedup();
1979. 1979 out.output_modalities.sort();
1980. 1980 out.output_modalities.dedup();
1981. 1981 out.domains.sort();
1982. 1982 out.domains.dedup();
1983. 1983 out
1984. 1984}
1985. 1985
1986. 1986fn read_acts(clause: &ClauseRead, out: &mut Extraction) {
1987. 1987 let tokens = &clause.tokens;
1988. 1988 for (target, act, weight) in VERB_TARGETS.iter() {
1989. 1989 let Some(position) = tokens.position(target, &clause.consumed) else {
1990. 1990 continue;
1991. 1991 };
1992. 1992 let head = clause.is_head(position);
1993. 1993 let weight = match clause.kind {
1994. 1994 // A statement's first word is its subject, and its other nouns
1995. 1995 // are not verbs; only a verb heading a later sub-clause ("it
1996. 1996 // crashes, fix it") asks for anything.
1997. 1997 ClauseKind::Statement if !head || position == 0 => continue,
1998. 1998 ClauseKind::Social if !head => continue,
1999. 1999 // In a question the head asks; other verbs are what it asks
2000. 2000 // about ("did the email send?") and count for half.
2001. 2001 ClauseKind::Question if !head => weight * 0.5,
2002. 2002 _ if head => weight * IMPERATIVE_BONUS,
2003. 2003 _ => *weight,
2004. 2004 };
2005. 2005 // "Alert me", "send me the report": delivering the result to the
2006. 2006 // person asking is an answer, not an operation on someone else.
2007. 2007 let to_requester = *act == Act::Operate
2008. 2008 && tokens
2009. 2009 .get(position + 1)
2010. 2010 .is_some_and(|next| REQUESTER.contains(&next));
2011. 2011 let (act, weight) = if to_requester {
2012. 2012 (Act::Answer, weight * 0.5)
2013. 2013 } else {
2014. 2014 (*act, weight)
2015. 2015 };
2016. 2016 out.act.add(act, weight);
2017. 2017 out.signals.push(Signal::new(
2018. 2018 SignalKind::Lexical,
2019. 2019 format!("verb:{}", target.word),
2020. 2020 weight,
2021. 2021 if to_requester {
2022. 2022 format!("`{} me` delivers to the requester ⇒ answer", target.word)
2023. 2023 } else if position == 0 {
2024. 2024 format!("`{}` (leading verb) ⇒ {}", target.word, act.as_str())
2025. 2025 } else if head {
2026. 2026 format!("`{}` (clause head verb) ⇒ {}", target.word, act.as_str())
2027. 2027 } else {
2028. 2028 format!("`{}` ⇒ {}", target.word, act.as_str())
2029. 2029 },
2030. 2030 ));
2031. 2031 }
2032. 2032 if clause.kind == ClauseKind::Question {
2033. 2033 let first = tokens.get(0).unwrap_or("");
2034. 2034 if !WH_WORDS.contains(&first) {
2035. 2035 let weight = if AUX_WORDS.contains(&first) {
2036. 2036 0.8 * IMPERATIVE_BONUS
2037. 2037 } else {
2038. 2038 0.8
2039. 2039 };
2040. 2040 out.act.add(Act::Answer, weight);
2041. 2041 out.signals.push(Signal::new(
2042. 2042 SignalKind::Lexical,
2043. 2043 "question:head",
2044. 2044 weight,
2045. 2045 if AUX_WORDS.contains(&first) {
2046. 2046 format!("`{first}` opens a question ⇒ answer")
2047. 2047 } else {
2048. 2048 "a question mark ⇒ answer".to_string()
2049. 2049 },
2050. 2050 ));
2051. 2051 }
2052. 2052 }
2053. 2053 for (phrase, weight) in SOCIAL_PHRASES {
2054. 2054 if tokens.phrase_at(&split_phrase(phrase)).is_some() {
2055. 2055 out.act.add(Act::Converse, *weight);
2056. 2056 out.signals.push(Signal::new(
2057. 2057 SignalKind::Lexical,
2058. 2058 format!("social:{}", phrase.replace(' ', "-")),
2059. 2059 *weight,
2060. 2060 format!("`{phrase}` ⇒ converse"),
2061. 2061 ));
2062. 2062 }
2063. 2063 }
2064. 2064 if clause.kind == ClauseKind::Social
2065. 2065 && let Some(first) = tokens.get(0)
2066. 2066 && let Some((word, weight)) = SOCIAL_WORDS.iter().find(|(word, _)| *word == first)
2067. 2067 {
2068. 2068 out.act.add(Act::Converse, *weight);
2069. 2069 out.signals.push(Signal::new(
2070. 2070 SignalKind::Lexical,
2071. 2071 format!("social:{word}"),
2072. 2072 *weight,
2073. 2073 format!("`{word}` ⇒ converse"),
2074. 2074 ));
2075. 2075 }
2076. 2076}
2077. 2077
2078. 2078fn read_stakes_and_evidence(clause: &ClauseRead, checkable: bool, out: &mut Extraction) {
2079. 2079 let tokens = &clause.tokens;
2080. 2080 for (phrase, stakes, weight) in STAKES_WORDS {
2081. 2081 let words = split_phrase(phrase);
2082. 2082 let hit = if words.len() > 1 {
2083. 2083 tokens.phrase_at(&words).is_some()
2084. 2084 } else {
2085. 2085 tokens.position(&Target::new(phrase), &[]).is_some()
2086. 2086 };
2087. 2087 if hit {
2088. 2088 out.stakes_from_words.add(*stakes, *weight);
2089. 2089 out.signals.push(Signal::new(
2090. 2090 SignalKind::Lexical,
2091. 2091 format!("stakes:{}", phrase.replace(' ', "-")),
2092. 2092 *weight,
2093. 2093 format!("`{phrase}` ⇒ {}", stakes.as_str()),
2094. 2094 ));
2095. 2095 }
2096. 2096 }
2097. 2097 for (phrase, evidence, weight) in EVIDENCE_WORDS {
2098. 2098 let words = split_phrase(phrase);
2099. 2099 let hit = if words.len() > 1 {
2100. 2100 tokens.phrase_at(&words).is_some()
2101. 2101 } else if EXACT_EVIDENCE_WORDS.contains(phrase) {
2102. 2102 tokens.words.iter().any(|t| t == phrase)
2103. 2103 } else {
2104. 2104 tokens.position(&Target::new(phrase), &[]).is_some()
2105. 2105 };
2106. 2106 if hit {
2107. 2107 out.evidence.add(*evidence, *weight);
2108. 2108 out.signals.push(Signal::new(
2109. 2109 SignalKind::Lexical,
2110. 2110 format!("evidence:{}", phrase.replace(' ', "-")),
2111. 2111 *weight,
2112. 2112 format!("`{phrase}` ⇒ {}", evidence.as_str()),
2113. 2113 ));
2114. 2114 }
2115. 2115 }
2116. 2116 for (phrase, weight) in ASSURANCE_PHRASES {
2117. 2117 let words = split_phrase(phrase);
2118. 2118 let hit = if words.len() > 1 {
2119. 2119 tokens.phrase_at(&words).is_some()
2120. 2120 } else {
2121. 2121 tokens.position(&Target::new(phrase), &[]).is_some()
2122. 2122 };
2123. 2123 if hit && checkable {
2124. 2124 out.evidence.add(Evidence::Verified, *weight);
2125. 2125 out.signals.push(Signal::new(
2126. 2126 SignalKind::Lexical,
2127. 2127 format!("evidence:{}", phrase.replace(' ', "-")),
2128. 2128 *weight,
2129. 2129 format!("`{phrase}` something checkable ⇒ verified"),
2130. 2130 ));
2131. 2131 }
2132. 2132 }
2133. 2133}
2134. 2134
2135. 2135fn read_recency(clause: &ClauseRead, out: &mut Extraction) {
2136. 2136 let tokens = &clause.tokens;
2137. 2137 let only_live_is_current =
2138. 2138 live_means_current(&tokens.words) && tokens.words.iter().any(|word| word == LIVE_WORD);
2139. 2139 if TIME_WORDS
2140. 2140 .iter()
2141. 2141 .any(|word| tokens.words.iter().any(|t| t == word))
2142. 2142 && !only_live_is_current
2143. 2143 {
2144. 2144 return;
2145. 2145 }
2146. 2146 for (phrase, weight) in RECENCY_PHRASES {
2147. 2147 let words = split_phrase(phrase);
2148. 2148 let Some(at) = tokens.phrase_at(&words) else {
2149. 2149 continue;
2150. 2150 };
2151. 2151 // "We live in the city" is not a question about time.
2152. 2152 if *phrase == LIVE_WORD && !live_means_current(&tokens.words) {
2153. 2153 continue;
2154. 2154 }
2155. 2155 let next = tokens.get(at + words.len()).unwrap_or("");
2156. 2156 // "today's": the possessive splits into `today` `s`; look past it.
2157. 2157 let next = if next == "s" {
2158. 2158 tokens.get(at + words.len() + 1).unwrap_or("")
2159. 2159 } else {
2160. 2160 next
2161. 2161 };
2162. 2162 // The thing that is current: the noun after the word ("the current
2163. 2163 // directory"), just before it ("is my branch up to date"), or what a
2164. 2164 // wh-question asks about ("what tasks are scheduled right now").
2165. 2165 let before = (at.saturating_sub(2)..at).filter_map(|i| tokens.get(i));
2166. 2166 let asked_about = tokens
2167. 2167 .get(0)
2168. 2168 .filter(|first| matches!(*first, "what" | "which"))
2169. 2169 .and_then(|_| tokens.get(1));
2170. 2170 if LOCAL_NOUNS.contains(&next)
2171. 2171 || before.into_iter().any(|word| LOCAL_NOUNS.contains(&word))
2172. 2172 || asked_about.is_some_and(|word| LOCAL_NOUNS.contains(&word))
2173. 2173 {
2174. 2174 continue;
2175. 2175 }
2176. 2176 if out.recency.as_ref().is_none_or(|(_, best)| *weight > *best) {
2177. 2177 out.recency = Some((phrase.to_string(), *weight));
2178. 2178 }
2179. 2179 }
2180. 2180 if let Some((phrase, weight)) = out.recency.clone()
2181. 2181 && !out
2182. 2182 .signals
2183. 2183 .iter()
2184. 2184 .any(|s| s.name == format!("recency:{}", phrase.replace(' ', "-")))
2185. 2185 {
2186. 2186 out.signals.push(Signal::new(
2187. 2187 SignalKind::Lexical,
2188. 2188 format!("recency:{}", phrase.replace(' ', "-")),
2189. 2189 weight,
2190. 2190 format!("`{phrase}` ⇒ a current value, if the part asks for a fact"),
2191. 2191 ));
2192. 2192 }
2193. 2193}
2194. 2194
2195. 2195fn read_horizon(clause: &ClauseRead, out: &mut Extraction) {
2196. 2196 let tokens = &clause.tokens;
2197. 2197 for (phrase, horizon, weight) in HORIZON_PHRASES {
2198. 2198 // A recurrence adjective names a thing, not a schedule: "the nightly
2199. 2199 // job", "our daily report". Only the adverb says the work recurs.
2200. 2200 if RECURRENCE_ADJECTIVES.contains(phrase)
2201. 2201 && tokens
2202. 2202 .words
2203. 2203 .windows(2)
2204. 2204 .any(|pair| DETERMINERS.contains(&pair[0].as_str()) && pair[1] == *phrase)
2205. 2205 && !tokens
2206. 2206 .words
2207. 2207 .windows(2)
2208. 2208 .any(|pair| !DETERMINERS.contains(&pair[0].as_str()) && pair[1] == *phrase)
2209. 2209 && tokens.get(0) != Some(phrase)
2210. 2210 {
2211. 2211 continue;
2212. 2212 }
2213. 2213 if tokens.phrase_at(&split_phrase(phrase)).is_some() {
2214. 2214 out.horizon.add(*horizon, *weight);
2215. 2215 out.signals.push(Signal::new(
2216. 2216 SignalKind::Lexical,
2217. 2217 format!("horizon:{}", phrase.replace(' ', "-")),
2218. 2218 *weight,
2219. 2219 format!("`{phrase}` ⇒ {}", horizon.as_str()),
2220. 2220 ));
2221. 2221 }
2222. 2222 }
2223. 2223}
2224. 2224
2225. 2225fn read_structure(clause: &ClauseRead, out: &mut Extraction) {
2226. 2226 let text = &clause.text;
2227. 2227 let lower = text.to_ascii_lowercase();
2228. 2228 if lower.contains("http://") || lower.contains("https://") {
2229. 2229 out.act.add(Act::Analyze, 0.3);
2230. 2230 out.signals.push(Signal::new(
2231. 2231 SignalKind::Structural,
2232. 2232 "url",
2233. 2233 0.3,
2234. 2234 "a URL to fetch",
2235. 2235 ));
2236. 2236 }
2237. 2237 let path_like = text
2238. 2238 .split_whitespace()
2239. 2239 .any(|w| w.contains('/') && w.contains('.') && !w.contains("://"));
2240. 2240 if path_like {
2241. 2241 out.signals.push(Signal::new(
2242. 2242 SignalKind::Structural,
2243. 2243 "path-mention",
2244. 2244 0.4,
2245. 2245 "a file path is named",
2246. 2246 ));
2247. 2247 out.act.add(Act::Modify, 0.2);
2248. 2248 out.clarity.add(Clarity::Clear, 0.5);
2249. 2249 }
2250. 2250}
2251. 2251
2252. 2252#[cfg(test)]
2253. 2253#[allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
2254. 2254mod tests {
2255. 2255 use super::*;
2256. 2256
2257. 2257 fn request<'a>(text: &'a str) -> Request<'a> {
2258. 2258 Request {
2259. 2259 text,
2260. 2260 ..Request::default()
2261. 2261 }
2262. 2262 }
2263. 2263
2264. 2264 #[test]
2265. 2265 fn temporal_deixis_is_recorded_and_local_nouns_are_not() {
2266. 2266 let now = extract(&request("what is the current price of copper"));
2267. 2267 assert_eq!(now.recency.as_ref().map(|r| r.0.as_str()), Some("current"));
2268. 2268 assert!(now.signals.iter().any(|s| s.name.starts_with("recency:")));
2269. 2269 // The evidence standard is left to the request's own words.
2270. 2270 assert_ne!(now.evidence.winner().map(|w| w.0), Some(Evidence::Cited));
2271. 2271
2272. 2272 for local in [
2273. 2273 "what's in the current directory",
2274. 2274 "update the README to reflect the latest changes",
2275. 2275 "fix the live reload bug",
2276. 2276 "what time is it right now",
2277. 2277 "what's today's date",
2278. 2278 ] {
2279. 2279 assert!(extract(&request(local)).recency.is_none(), "{local}");
2280. 2280 }
2281. 2281 assert!(
2282. 2282 extract(&request("explain how copper is refined"))
2283. 2283 .recency
2284. 2284 .is_none()
2285. 2285 );
2286. 2286 }
2287. 2287
2288. 2288 #[test]
2289. 2289 fn live_as_reside_is_neither_current_nor_irreversible() {
2290. 2290 for text in [
2291. 2291 "Help me plan Saturday with the kids. They are 6 and 9, we live in the city, and I would like a couple of options for the afternoon.",
2292. 2292 "deploy the site to wherever I live near",
2293. 2293 "update the doc: my son lives with his grandparents",
2294. 2294 "rewrite the letter, we have lived in Pune for years",
2295. 2295 "fix the budget for living costs",
2296. 2296 "where do you live",
2297. 2297 "change the address, my kids live in the suburbs",
2298. 2298 ] {
2299. 2299 let x = extract(&request(text));
2300. 2300 assert!(x.recency.is_none(), "{text}: {:?}", x.recency);
2301. 2301 assert!(
2302. 2302 !x.signals
2303. 2303 .iter()
2304. 2304 .any(|s| s.name.starts_with("stakes:") && s.name.contains("live")),
2305. 2305 "{text}"
2306. 2306 );
2307. 2307 }
2308. 2308 }
2309. 2309
2310. 2310 /// `live` in the sense of *current* is a recency word — whether it asks
2311. 2311 /// for live data is then the act's question (a request for a fact) — and
2312. 2312 /// "go live" is a launch, which raises stakes for the work that does it.