Added a way to read what the AI received from your sources (#1100)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
Paul KoudelkaandThorsten Sommer authored and GitHub committed 2026-10-26 10:00:01 +01:00
1 parent e0d8e18f34
commit cbc4ce5413
229 files changed
+12739 -1269

No files matched your search

+37 -6
View File
@@ -1342,6 +1342,13 @@ async fn stream_spreadsheet_as_csv(file_path: &str) -> Result<ChunkStream> {
};
number_of_sheets += 1;
//
// Rows are numbered as the spreadsheet program numbers them, so a passage can be found
// there again. The range of a sheet starts at its first cell with content, which need
// not be in the first row. Row 0 is the fence in front of the first row.
//
let rows_before_range = range.start().map_or(0, |(row, _)| row as usize);
let mut row_idx = 0;
tx.blocking_send(Ok(Chunk::new(
"```csv".to_string(),
@@ -1350,7 +1357,7 @@ async fn stream_spreadsheet_as_csv(file_path: &str) -> Result<ChunkStream> {
row_number: row_idx,
}
))).ok();
for row in range.rows() {
row_idx += 1;
let cells = row.iter().map(|cell| cell.to_string()).collect::<Vec<_>>();
@@ -1367,7 +1374,7 @@ async fn stream_spreadsheet_as_csv(file_path: &str) -> Result<ChunkStream> {
content,
Metadata::Spreadsheet {
sheet_name: sheet_name.clone(),
row_number: row_idx,
row_number: rows_before_range + row_idx,
}
))).is_err() {
return;
@@ -1378,7 +1385,7 @@ async fn stream_spreadsheet_as_csv(file_path: &str) -> Result<ChunkStream> {
"```".to_string(),
Metadata::Spreadsheet {
sheet_name: sheet_name.clone(),
row_number: row_idx,
row_number: rows_before_range + row_idx,
}
))).ok();
}
@@ -1871,9 +1878,14 @@ fn document_metadata_to_markdown(metadata: &DocumentMetadata) -> Option<String>
}
push_presentation_metadata_field(&mut fields, "Created", metadata.created_at.as_deref());
push_presentation_metadata_field(&mut fields, "Modified", metadata.modified_at.as_deref());
for (name, value) in &metadata.custom {
fields.push(format!("Custom {name}: {}", sanitize_presentation_metadata_value(value)));
}
//
// The custom properties of the document are left out. Most of them are internal properties of
// document libraries such as SharePoint, e.g., content type and document ids, which tell the
// AI nothing and would only add noise to the first chunk, where the search finds them. A
// sensitivity label, which does matter, consists of several such properties and needs reading
// of its own.
//
if fields.is_empty() { None } else { Some(format!("<!-- Document Metadata\n{}\n-->", fields.join("\n"))) }
}
@@ -2151,4 +2163,23 @@ mod tests {
assert_eq!(extracted.redacted_count, 0, "nothing should be filtered here: {:?}", extracted.rule_ids);
assert_eq!(extracted.lines, content.lines().collect::<Vec<_>>());
}
/// The metadata comment in front of the first page keeps what tells the AI about the document,
/// but none of its custom properties: content type and document ids of a library such as
/// SharePoint would only add noise to the first chunk.
#[test]
fn the_custom_properties_of_a_document_stay_out_of_its_metadata() {
let metadata = DocumentMetadata {
title: Some("REST API".to_string()),
custom: std::collections::BTreeMap::from([
("ContentTypeId".to_string(), "0x0101002586".to_string()),
("Order".to_string(), "48800".to_string()),
]),
..DocumentMetadata::default()
};
let comment = document_metadata_to_markdown(&metadata).expect("the title makes a comment");
assert!(comment.contains("Title: REST API"), "got: {comment}");
assert!(!comment.contains("Custom") && !comment.contains("48800"), "got: {comment}");
}
}