59
60
/// `｜` (U+FF5C), the bar every DSML marker above is built from.
61
const FULLWIDTH_BAR: char = '｜';
62
+/// Two of them — the spelling [`fold_dsml_bars`] rewrites into one.
63
+const DOUBLED_BAR: &str = "｜｜";
64
/// The name those markers wrap, and the only place [`fold_dsml_bars`] folds.
63
-const DSML_MARKER_NAME: &str = "DSML";
65
+const DSML_TAG_NAME: &str = "DSML";
66
67
const DSML_TOOL_CALLS_OPEN: &str = "<｜DSML｜tool_calls>";
68
const DSML_TOOL_CALLS_CLOSE: &str = "<｜DSML｜/tool_calls>";
461
// The fold runs on the whole buffer, not the chunk: a doubled bar split
462
// across two deltas is only visible once they are joined, and a marker
463
// is never emitted before then because a partial prefix is held back.
464
+ // Both the single- and doubled-bar spellings are partial prefixes
465
+ // (START_MARKERS is single-bar), so a doubled-bar prefix is flushed as
466
+ // text on this push; the fold then rewrites the buffer and the next
467
+ // push re-runs `drain` against canonical markers.
468
if let Some(folded) = fold_dsml_bars(&self.pending) {
469
self.pending = folded;
470
}
685
/// turns out not to be a call comes back with single bars — markup either way.
686
///
687
/// Returns `None` when there is nothing to fold — in particular for any buffer
682
-/// without a `DSML` marker, which is the common case and costs no allocation.
688
+/// without a doubled bar, which is the common case and costs no allocation.
689
fn fold_dsml_bars(text: &str) -> Option<String> {
684
- if !text.contains(DSML_MARKER_NAME) {
690
+ if !text.contains(DSML_TAG_NAME) {
691
+ return None;
692
+ }
693
+ // A DSML tag with only single bars is canonical; only the doubled-bar
694
+ // spelling needs rewriting, so the buffer-walking allocation can be
695
+ // skipped when the only defect isn't present.
696
+ if !text.contains(DOUBLED_BAR) {
697
return None;
698
}
699
let mut folded = String::with_capacity(text.len());
700
let mut changed = false;
701
let mut rest = text;
690
- while let Some(idx) = rest.find(DSML_MARKER_NAME) {
702
+ while let Some(idx) = rest.find(DSML_TAG_NAME) {
703
let (before, from_name) = rest.split_at(idx);
704
// The run of bars closing the tag being opened, i.e. `<｜｜` in
705
// `<｜｜DSML｜｜invoke`. Counted in bars, not bytes: a lone fullwidth bar
710
changed |= opening_run > 1;
711
folded.push(FULLWIDTH_BAR);
712
}
701
- folded.push_str(DSML_MARKER_NAME);
702
- let after = &from_name[DSML_MARKER_NAME.len()..];
713
+ folded.push_str(DSML_TAG_NAME);
714
+ let after = &from_name[DSML_TAG_NAME.len()..];
715
// The run opening the tag being closed, i.e. `｜｜>` in
716
// `<｜DSML｜｜/invoke>`.
717
let closing_run = bar_run_len(after, true);
726
}
727
728
/// Number of fullwidth bars in the run at the end of `text`, or at its start
717
-/// when `leading` is set.
729
+/// when `leading` is set. The caller passes the slice anchored to `DSML`, so
730
+/// the run being counted always sits at the boundary, never in the middle.
731
fn bar_run_len(text: &str, leading: bool) -> usize {
732
let bytes = if leading {
733
text.len() - text.trim_start_matches(FULLWIDTH_BAR).len()