mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-11-16 03:08:27 +00:00
Added a way to read what the AI received from your sources (#1100)
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
1 parent
e0d8e18f34
commit
cbc4ce5413
229 files changed
+12739
-1269
No files matched your search
@@ -1342,6 +1342,13 @@ async fn stream_spreadsheet_as_csv(file_path: &str) -> Result<ChunkStream> {
|
||||
};
|
||||
|
||||
number_of_sheets += 1;
|
||||
|
||||
//
|
||||
// Rows are numbered as the spreadsheet program numbers them, so a passage can be found
|
||||
// there again. The range of a sheet starts at its first cell with content, which need
|
||||
// not be in the first row. Row 0 is the fence in front of the first row.
|
||||
//
|
||||
let rows_before_range = range.start().map_or(0, |(row, _)| row as usize);
|
||||
let mut row_idx = 0;
|
||||
tx.blocking_send(Ok(Chunk::new(
|
||||
"```csv".to_string(),
|
||||
@@ -1350,7 +1357,7 @@ async fn stream_spreadsheet_as_csv(file_path: &str) -> Result<ChunkStream> {
|
||||
row_number: row_idx,
|
||||
}
|
||||
))).ok();
|
||||
|
||||
|
||||
for row in range.rows() {
|
||||
row_idx += 1;
|
||||
let cells = row.iter().map(|cell| cell.to_string()).collect::<Vec<_>>();
|
||||
@@ -1367,7 +1374,7 @@ async fn stream_spreadsheet_as_csv(file_path: &str) -> Result<ChunkStream> {
|
||||
content,
|
||||
Metadata::Spreadsheet {
|
||||
sheet_name: sheet_name.clone(),
|
||||
row_number: row_idx,
|
||||
row_number: rows_before_range + row_idx,
|
||||
}
|
||||
))).is_err() {
|
||||
return;
|
||||
@@ -1378,7 +1385,7 @@ async fn stream_spreadsheet_as_csv(file_path: &str) -> Result<ChunkStream> {
|
||||
"```".to_string(),
|
||||
Metadata::Spreadsheet {
|
||||
sheet_name: sheet_name.clone(),
|
||||
row_number: row_idx,
|
||||
row_number: rows_before_range + row_idx,
|
||||
}
|
||||
))).ok();
|
||||
}
|
||||
@@ -1871,9 +1878,14 @@ fn document_metadata_to_markdown(metadata: &DocumentMetadata) -> Option<String>
|
||||
}
|
||||
push_presentation_metadata_field(&mut fields, "Created", metadata.created_at.as_deref());
|
||||
push_presentation_metadata_field(&mut fields, "Modified", metadata.modified_at.as_deref());
|
||||
for (name, value) in &metadata.custom {
|
||||
fields.push(format!("Custom {name}: {}", sanitize_presentation_metadata_value(value)));
|
||||
}
|
||||
|
||||
//
|
||||
// The custom properties of the document are left out. Most of them are internal properties of
|
||||
// document libraries such as SharePoint, e.g., content type and document ids, which tell the
|
||||
// AI nothing and would only add noise to the first chunk, where the search finds them. A
|
||||
// sensitivity label, which does matter, consists of several such properties and needs reading
|
||||
// of its own.
|
||||
//
|
||||
if fields.is_empty() { None } else { Some(format!("<!-- Document Metadata\n{}\n-->", fields.join("\n"))) }
|
||||
}
|
||||
|
||||
@@ -2151,4 +2163,23 @@ mod tests {
|
||||
assert_eq!(extracted.redacted_count, 0, "nothing should be filtered here: {:?}", extracted.rule_ids);
|
||||
assert_eq!(extracted.lines, content.lines().collect::<Vec<_>>());
|
||||
}
|
||||
|
||||
/// The metadata comment in front of the first page keeps what tells the AI about the document,
|
||||
/// but none of its custom properties: content type and document ids of a library such as
|
||||
/// SharePoint would only add noise to the first chunk.
|
||||
#[test]
|
||||
fn the_custom_properties_of_a_document_stay_out_of_its_metadata() {
|
||||
let metadata = DocumentMetadata {
|
||||
title: Some("REST API".to_string()),
|
||||
custom: std::collections::BTreeMap::from([
|
||||
("ContentTypeId".to_string(), "0x0101002586".to_string()),
|
||||
("Order".to_string(), "48800".to_string()),
|
||||
]),
|
||||
..DocumentMetadata::default()
|
||||
};
|
||||
|
||||
let comment = document_metadata_to_markdown(&metadata).expect("the title makes a comment");
|
||||
assert!(comment.contains("Title: REST API"), "got: {comment}");
|
||||
assert!(!comment.contains("Custom") && !comment.contains("48800"), "got: {comment}");
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user