Skip to content

Add text element metadata to types #9235

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
charley-oai merged 6 commits into from
Jan 15, 2026
Merged

Add text element metadata to types #9235

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
622a70a
Add text element metadata to types
charley-oai Jan 14, 2026
328465c
Always serialize text element vectors
charley-oai Jan 14, 2026
cae1ad6
Serialize empty user message metadata
charley-oai Jan 14, 2026
d5bbcdf
Always serialize text element placeholders
charley-oai Jan 14, 2026
527190c
Fix tests
charley-oai Jan 14, 2026
00b75c6
Add some tests
charley-oai Jan 15, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 31 additions & 3 deletions codex-rs/app-server-protocol/src/protocol/thread_history.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,8 @@ impl ThreadHistoryBuilder {
if !payload.message.trim().is_empty() {
content.push(UserInput::Text {
text: payload.message.clone(),
// TODO: Thread text element ranges into thread history. Empty keeps old behavior.
Comment thread
charley-oai marked this conversation as resolved.
text_elements: Vec::new(),
});
}
if let Some(images) = &payload.images {
Expand Down Expand Up @@ -244,6 +246,8 @@ mod tests {
EventMsg::UserMessage(UserMessageEvent {
message: "First turn".into(),
images: Some(vec!["https://example.com/one.png".into()]),
text_elements: Vec::new(),
local_images: Vec::new(),
}),
EventMsg::AgentMessage(AgentMessageEvent {
message: "Hi there".into(),
Expand All @@ -257,6 +261,8 @@ mod tests {
EventMsg::UserMessage(UserMessageEvent {
message: "Second turn".into(),
images: None,
text_elements: Vec::new(),
local_images: Vec::new(),
}),
EventMsg::AgentMessage(AgentMessageEvent {
message: "Reply two".into(),
Expand All @@ -277,6 +283,7 @@ mod tests {
content: vec![
UserInput::Text {
text: "First turn".into(),
text_elements: Vec::new(),
},
UserInput::Image {
url: "https://example.com/one.png".into(),
Expand Down Expand Up @@ -308,7 +315,8 @@ mod tests {
ThreadItem::UserMessage {
id: "item-4".into(),
content: vec![UserInput::Text {
text: "Second turn".into()
text: "Second turn".into(),
text_elements: Vec::new(),
}],
}
);
Expand All @@ -327,6 +335,8 @@ mod tests {
EventMsg::UserMessage(UserMessageEvent {
message: "Turn start".into(),
images: None,
text_elements: Vec::new(),
local_images: Vec::new(),
}),
EventMsg::AgentReasoning(AgentReasoningEvent {
text: "first summary".into(),
Expand Down Expand Up @@ -371,6 +381,8 @@ mod tests {
EventMsg::UserMessage(UserMessageEvent {
message: "Please do the thing".into(),
images: None,
text_elements: Vec::new(),
local_images: Vec::new(),
}),
EventMsg::AgentMessage(AgentMessageEvent {
message: "Working...".into(),
Expand All @@ -381,6 +393,8 @@ mod tests {
EventMsg::UserMessage(UserMessageEvent {
message: "Let's try again".into(),
images: None,
text_elements: Vec::new(),
local_images: Vec::new(),
}),
EventMsg::AgentMessage(AgentMessageEvent {
message: "Second attempt complete.".into(),
Expand All @@ -398,7 +412,8 @@ mod tests {
ThreadItem::UserMessage {
id: "item-1".into(),
content: vec![UserInput::Text {
text: "Please do the thing".into()
text: "Please do the thing".into(),
text_elements: Vec::new(),
}],
}
);
Expand All @@ -418,7 +433,8 @@ mod tests {
ThreadItem::UserMessage {
id: "item-3".into(),
content: vec![UserInput::Text {
text: "Let's try again".into()
text: "Let's try again".into(),
text_elements: Vec::new(),
}],
}
);
Expand All @@ -437,13 +453,17 @@ mod tests {
EventMsg::UserMessage(UserMessageEvent {
message: "First".into(),
images: None,
text_elements: Vec::new(),
local_images: Vec::new(),
}),
EventMsg::AgentMessage(AgentMessageEvent {
message: "A1".into(),
}),
EventMsg::UserMessage(UserMessageEvent {
message: "Second".into(),
images: None,
text_elements: Vec::new(),
local_images: Vec::new(),
}),
EventMsg::AgentMessage(AgentMessageEvent {
message: "A2".into(),
Expand All @@ -452,6 +472,8 @@ mod tests {
EventMsg::UserMessage(UserMessageEvent {
message: "Third".into(),
images: None,
text_elements: Vec::new(),
local_images: Vec::new(),
}),
EventMsg::AgentMessage(AgentMessageEvent {
message: "A3".into(),
Expand All @@ -469,6 +491,7 @@ mod tests {
id: "item-1".into(),
content: vec![UserInput::Text {
text: "First".into(),
text_elements: Vec::new(),
}],
},
ThreadItem::AgentMessage {
Expand All @@ -486,6 +509,7 @@ mod tests {
id: "item-3".into(),
content: vec![UserInput::Text {
text: "Third".into(),
text_elements: Vec::new(),
}],
},
ThreadItem::AgentMessage {
Expand All @@ -504,13 +528,17 @@ mod tests {
EventMsg::UserMessage(UserMessageEvent {
message: "One".into(),
images: None,
text_elements: Vec::new(),
local_images: Vec::new(),
}),
EventMsg::AgentMessage(AgentMessageEvent {
message: "A1".into(),
}),
EventMsg::UserMessage(UserMessageEvent {
message: "Two".into(),
images: None,
text_elements: Vec::new(),
local_images: Vec::new(),
}),
EventMsg::AgentMessage(AgentMessageEvent {
message: "A2".into(),
Expand Down
52 changes: 46 additions & 6 deletions codex-rs/app-server-protocol/src/protocol/v2.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1543,21 +1543,55 @@ pub struct TurnInterruptParams {
pub struct TurnInterruptResponse {}

// User input types
#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub struct ByteRange {
pub start: usize,
pub end: usize,
}

#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(rename_all = "camelCase")]
#[ts(export_to = "v2/")]
pub struct TextElement {
/// Byte range in the parent `text` buffer that this element occupies.
pub byte_range: ByteRange,
/// Optional human-readable placeholder for the element, displayed in the UI.
pub placeholder: Option<String>,
}

#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, JsonSchema, TS)]
#[serde(tag = "type", rename_all = "camelCase")]
#[ts(tag = "type")]
#[ts(export_to = "v2/")]
pub enum UserInput {
Text { text: String },
Image { url: String },
LocalImage { path: PathBuf },
Skill { name: String, path: PathBuf },
Text {
text: String,
/// UI-defined spans within `text` used to render or persist special elements.
#[serde(default)]
text_elements: Vec<TextElement>,
Comment thread
charley-oai marked this conversation as resolved.
},
Image {
url: String,
},
LocalImage {
path: PathBuf,
},
Skill {
name: String,
path: PathBuf,
},
}

impl UserInput {
pub fn into_core(self) -> CoreUserInput {
match self {
UserInput::Text { text } => CoreUserInput::Text { text },
UserInput::Text { text, .. } => CoreUserInput::Text {
text,
// TODO: Thread text element ranges into v2 inputs. Empty keeps old behavior.
text_elements: Vec::new(),
},
UserInput::Image { url } => CoreUserInput::Image { image_url: url },
UserInput::LocalImage { path } => CoreUserInput::LocalImage { path },
UserInput::Skill { name, path } => CoreUserInput::Skill { name, path },
Expand All @@ -1568,7 +1602,11 @@ impl UserInput {
impl From<CoreUserInput> for UserInput {
fn from(value: CoreUserInput) -> Self {
match value {
CoreUserInput::Text { text } => UserInput::Text { text },
CoreUserInput::Text { text, .. } => UserInput::Text {
text,
// TODO: Thread text element ranges from core into v2 inputs.
text_elements: Vec::new(),
},
CoreUserInput::Image { image_url } => UserInput::Image { url: image_url },
CoreUserInput::LocalImage { path } => UserInput::LocalImage { path },
CoreUserInput::Skill { name, path } => UserInput::Skill { name, path },
Expand Down Expand Up @@ -2160,6 +2198,7 @@ mod tests {
content: vec![
CoreUserInput::Text {
text: "hello".to_string(),
text_elements: Vec::new(),
},
CoreUserInput::Image {
image_url: "https://example.com/image.png".to_string(),
Expand All @@ -2181,6 +2220,7 @@ mod tests {
content: vec![
UserInput::Text {
text: "hello".to_string(),
text_elements: Vec::new(),
},
UserInput::Image {
url: "https://example.com/image.png".to_string(),
Expand Down
8 changes: 7 additions & 1 deletion codex-rs/app-server-test-client/src/main.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -256,7 +256,11 @@ fn send_message_v2_with_policies(
println!("< thread/start response: {thread_response:?}");
let mut turn_params = TurnStartParams {
thread_id: thread_response.thread.id.clone(),
input: vec![V2UserInput::Text { text: user_message }],
input: vec![V2UserInput::Text {
text: user_message,
// Plain text conversion has no UI element ranges.
text_elements: Vec::new(),
}],
..Default::default()
};
turn_params.approval_policy = approval_policy;
Expand Down Expand Up @@ -288,6 +292,7 @@ fn send_follow_up_v2(
thread_id: thread_response.thread.id.clone(),
input: vec![V2UserInput::Text {
text: first_message,
text_elements: Vec::new(),
}],
..Default::default()
};
Expand All @@ -299,6 +304,7 @@ fn send_follow_up_v2(
thread_id: thread_response.thread.id.clone(),
input: vec![V2UserInput::Text {
text: follow_up_message,
text_elements: Vec::new(),
}],
..Default::default()
};
Expand Down
13 changes: 11 additions & 2 deletions codex-rs/app-server/src/codex_message_processor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3125,7 +3125,11 @@ impl CodexMessageProcessor {
let mapped_items: Vec<CoreInputItem> = items
.into_iter()
.map(|item| match item {
WireInputItem::Text { text } => CoreInputItem::Text { text },
WireInputItem::Text { text } => CoreInputItem::Text {
text,
// TODO: Thread text element ranges into v1 input handling.
text_elements: Vec::new(),
},
WireInputItem::Image { image_url } => CoreInputItem::Image { image_url },
WireInputItem::LocalImage { path } => CoreInputItem::LocalImage { path },
})
Expand Down Expand Up @@ -3171,7 +3175,11 @@ impl CodexMessageProcessor {
let mapped_items: Vec<CoreInputItem> = items
.into_iter()
.map(|item| match item {
WireInputItem::Text { text } => CoreInputItem::Text { text },
WireInputItem::Text { text } => CoreInputItem::Text {
text,
// TODO: Thread text element ranges into v1 input handling.
text_elements: Vec::new(),
},
WireInputItem::Image { image_url } => CoreInputItem::Image { image_url },
WireInputItem::LocalImage { path } => CoreInputItem::LocalImage { path },
})
Expand Down Expand Up @@ -3333,6 +3341,7 @@ impl CodexMessageProcessor {
id: turn_id.clone(),
content: vec![V2UserInput::Text {
text: display_text.to_string(),
text_elements: Vec::new(),
}],
}]
};
Expand Down
3 changes: 3 additions & 0 deletions codex-rs/app-server/tests/suite/v2/output_schema.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ async fn turn_start_accepts_output_schema_v2() -> Result<()> {
thread_id: thread.id.clone(),
input: vec![V2UserInput::Text {
text: "Hello".to_string(),
text_elements: Vec::new(),
}],
output_schema: Some(output_schema.clone()),
..Default::default()
Expand Down Expand Up @@ -142,6 +143,7 @@ async fn turn_start_output_schema_is_per_turn_v2() -> Result<()> {
thread_id: thread.id.clone(),
input: vec![V2UserInput::Text {
text: "Hello".to_string(),
text_elements: Vec::new(),
}],
output_schema: Some(output_schema.clone()),
..Default::default()
Expand Down Expand Up @@ -183,6 +185,7 @@ async fn turn_start_output_schema_is_per_turn_v2() -> Result<()> {
thread_id: thread.id.clone(),
input: vec![V2UserInput::Text {
text: "Hello again".to_string(),
text_elements: Vec::new(),
}],
output_schema: None,
..Default::default()
Expand Down
3 changes: 2 additions & 1 deletion codex-rs/app-server/tests/suite/v2/thread_fork.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,8 @@ async fn thread_fork_creates_new_thread_and_emits_started() -> Result<()> {
assert_eq!(
content,
&vec![UserInput::Text {
text: preview.to_string()
text: preview.to_string(),
text_elements: Vec::new(),
}]
);
}
Expand Down
3 changes: 2 additions & 1 deletion codex-rs/app-server/tests/suite/v2/thread_resume.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,8 @@ async fn thread_resume_returns_rollout_history() -> Result<()> {
assert_eq!(
content,
&vec![UserInput::Text {
text: preview.to_string()
text: preview.to_string(),
text_elements: Vec::new(),
}]
);
}
Expand Down
8 changes: 6 additions & 2 deletions codex-rs/app-server/tests/suite/v2/thread_rollback.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@ async fn thread_rollback_drops_last_turns_and_persists_to_rollout() -> Result<()
thread_id: thread.id.clone(),
input: vec![V2UserInput::Text {
text: first_text.to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
Expand All @@ -77,6 +78,7 @@ async fn thread_rollback_drops_last_turns_and_persists_to_rollout() -> Result<()
thread_id: thread.id.clone(),
input: vec![V2UserInput::Text {
text: "Second".to_string(),
text_elements: Vec::new(),
}],
..Default::default()
})
Expand Down Expand Up @@ -115,7 +117,8 @@ async fn thread_rollback_drops_last_turns_and_persists_to_rollout() -> Result<()
assert_eq!(
content,
&vec![V2UserInput::Text {
text: first_text.to_string()
text: first_text.to_string(),
text_elements: Vec::new(),
}]
);
}
Expand Down Expand Up @@ -143,7 +146,8 @@ async fn thread_rollback_drops_last_turns_and_persists_to_rollout() -> Result<()
assert_eq!(
content,
&vec![V2UserInput::Text {
text: first_text.to_string()
text: first_text.to_string(),
text_elements: Vec::new(),
}]
);
}
Expand Down
1 change: 1 addition & 0 deletions codex-rs/app-server/tests/suite/v2/turn_interrupt.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ async fn turn_interrupt_aborts_running_turn() -> Result<()> {
thread_id: thread.id.clone(),
input: vec![V2UserInput::Text {
text: "run sleep".to_string(),
text_elements: Vec::new(),
}],
cwd: Some(working_directory.clone()),
..Default::default()
Expand Down
Loading
Loading