Added a way to read what the AI received from your sources (#1100)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

runtime/src/data_restore.rs

@@ -52,10 +52,11 @@ const SETTINGS_FILE_NAME: &str = "settings.json";
 /// (`DataBackupContents`) includes. The runtime refuses a plan which names anything else, so that
 /// whatever goes wrong in the app, a restore never replaces the logs, the downloaded tools, or the
 /// data folder as a whole.
-const REPLACEABLE_ELEMENTS: [(Location, &str); 9] = [
+const REPLACEABLE_ELEMENTS: [(Location, &str); 10] = [
     (Location::ConfigDirectory, SETTINGS_FILE_NAME),
     (Location::DataDirectory, "workspaces/"),
     (Location::DataDirectory, "tempChats/"),
+    (Location::DataDirectory, "databases/provenance/"),
     (Location::DataDirectory, "plugins/"),
     (Location::DataDirectory, "tokenizers/"),
     (Location::DataDirectory, "visualBriefings/"),

runtime/src/file_data.rs

@@ -1342,6 +1342,13 @@ async fn stream_spreadsheet_as_csv(file_path: &str) -> Result<ChunkStream> {
             };
 
             number_of_sheets += 1;
+
+            //
+            // Rows are numbered as the spreadsheet program numbers them, so a passage can be found
+            // there again. The range of a sheet starts at its first cell with content, which need
+            // not be in the first row. Row 0 is the fence in front of the first row.
+            //
+            let rows_before_range = range.start().map_or(0, |(row, _)| row as usize);
             let mut row_idx = 0;
             tx.blocking_send(Ok(Chunk::new(
                 "```csv".to_string(),
@@ -1350,7 +1357,7 @@ async fn stream_spreadsheet_as_csv(file_path: &str) -> Result<ChunkStream> {
                     row_number: row_idx,
                 }
             ))).ok();
-            
+
             for row in range.rows() {
                 row_idx += 1;
                 let cells = row.iter().map(|cell| cell.to_string()).collect::<Vec<_>>();
@@ -1367,7 +1374,7 @@ async fn stream_spreadsheet_as_csv(file_path: &str) -> Result<ChunkStream> {
                     content,
                     Metadata::Spreadsheet {
                         sheet_name: sheet_name.clone(),
-                        row_number: row_idx,
+                        row_number: rows_before_range + row_idx,
                     }
                 ))).is_err() {
                     return;
@@ -1378,7 +1385,7 @@ async fn stream_spreadsheet_as_csv(file_path: &str) -> Result<ChunkStream> {
                 "```".to_string(),
                 Metadata::Spreadsheet {
                     sheet_name: sheet_name.clone(),
-                    row_number: row_idx,
+                    row_number: rows_before_range + row_idx,
                 }
             ))).ok();
         }
@@ -1871,9 +1878,14 @@ fn document_metadata_to_markdown(metadata: &DocumentMetadata) -> Option<String>
     }
     push_presentation_metadata_field(&mut fields, "Created", metadata.created_at.as_deref());
     push_presentation_metadata_field(&mut fields, "Modified", metadata.modified_at.as_deref());
-    for (name, value) in &metadata.custom {
-        fields.push(format!("Custom {name}: {}", sanitize_presentation_metadata_value(value)));
-    }
+
+    //
+    // The custom properties of the document are left out. Most of them are internal properties of
+    // document libraries such as SharePoint, e.g., content type and document ids, which tell the
+    // AI nothing and would only add noise to the first chunk, where the search finds them. A
+    // sensitivity label, which does matter, consists of several such properties and needs reading
+    // of its own.
+    //
     if fields.is_empty() { None } else { Some(format!("<!-- Document Metadata\n{}\n-->", fields.join("\n"))) }
 }
 
@@ -2151,4 +2163,23 @@ mod tests {
         assert_eq!(extracted.redacted_count, 0, "nothing should be filtered here: {:?}", extracted.rule_ids);
         assert_eq!(extracted.lines, content.lines().collect::<Vec<_>>());
     }
+
+    /// The metadata comment in front of the first page keeps what tells the AI about the document,
+    /// but none of its custom properties: content type and document ids of a library such as
+    /// SharePoint would only add noise to the first chunk.
+    #[test]
+    fn the_custom_properties_of_a_document_stay_out_of_its_metadata() {
+        let metadata = DocumentMetadata {
+            title: Some("REST API".to_string()),
+            custom: std::collections::BTreeMap::from([
+                ("ContentTypeId".to_string(), "0x0101002586".to_string()),
+                ("Order".to_string(), "48800".to_string()),
+            ]),
+            ..DocumentMetadata::default()
+        };
+
+        let comment = document_metadata_to_markdown(&metadata).expect("the title makes a comment");
+        assert!(comment.contains("Title: REST API"), "got: {comment}");
+        assert!(!comment.contains("Custom") && !comment.contains("48800"), "got: {comment}");
+    }
 }

runtime/src/qdrant_edge_database.rs

@@ -10,10 +10,10 @@ use qdrant_edge::external::serde_json::{json, Value};
 use qdrant_edge::external::uuid::Uuid;
 use qdrant_edge::{
     Condition, Distance, EdgeConfig, EdgeOptimizersConfig, EdgeShard, EdgeVectorParams,
-    FieldCondition, Filter, HasIdCondition, HnswIndexConfig, Match, MatchValue, NamedQuery,
+    FieldCondition, Filter, HasIdCondition, HnswIndexConfig, IsEmptyCondition, JsonPath, Match, MatchValue, NamedQuery,
     Payload, PayloadOps, PointId, PointInsertOperations, PointOperations, PointStruct, QueryEnum,
-    QueryRequest, ScoredPoint, ScoringQuery, UpdateOperation, ValueVariants, VectorInternal,
-    SetPayloadOp, Vectors, WithPayloadInterface, WithVector,
+    QueryRequest, RetrieveRequest, ScoredPoint, ScoringQuery, UpdateOperation, ValueVariants,
+    VectorInternal, VectorStructInternal, SetPayloadOp, Vectors, WithPayloadInterface, WithVector,
 };
 use serde::{Deserialize, Serialize};
 use tauri::Manager;
@@ -90,7 +90,14 @@ pub struct QdrantEdgeStoragePoint {
     pub relative_path: String,
     pub file_type: String,
     pub page_number: Option<i32>,
+    pub end_page_number: Option<i32>,
     pub chunk_index: i32,
+
+    /// Which indexing of its document wrote the point. A search leaves out every generation of a
+    /// document but the one which can be searched, cf. `QdrantEdgeHiddenChunks`. A point written
+    /// before points had a generation has none in its payload and counts as generation 0.
+    #[serde(default)]
+    pub generation: i32,
     pub text: String,
     pub fingerprint: String,
     pub creation_utc: String,
@@ -125,6 +132,27 @@ pub struct SearchQdrantEdgeEmbeddingRequest {
     /// The only points the search may return. Missing means the whole store. An empty list means
     /// no point at all, so the search finds nothing -- it never falls back to the whole store.
     pub point_ids: Option<Vec<String>>,
+
+    /// The points of documents which the search must not return, e.g., what an indexing attempt
+    /// wrote before the document was complete. Missing means none.
+    #[serde(default)]
+    pub hidden_chunks: Vec<QdrantEdgeHiddenChunks>,
+}
+
+/// The points of one document which a search leaves out.
+#[derive(Deserialize)]
+pub struct QdrantEdgeHiddenChunks {
+    pub parent_file_id: String,
+
+    /// The generation of the document which can be searched, so every other one is left out.
+    /// Missing while the document has no complete generation yet, so every point of it is left out.
+    pub searchable_generation: Option<i32>,
+}
+
+#[derive(Deserialize)]
+pub struct RetrieveQdrantEdgeVectorsRequest {
+    pub store_name: String,
+    pub point_ids: Vec<String>,
 }
 
 #[derive(Deserialize)]
@@ -132,6 +160,11 @@ pub struct DeleteQdrantEdgeEmbeddingByFileRequest {
     pub store_name: String,
     pub file_path: String,
     pub parent_file_id: String,
+
+    /// Points of the document which stay, e.g., the ones an indexing run just wrote. Missing means
+    /// none, so every point of the document goes.
+    #[serde(default)]
+    pub keep_point_ids: Vec<String>,
 }
 
 #[derive(Deserialize)]
@@ -243,6 +276,9 @@ pub struct QdrantEdgeSearchResult {
     pub relative_path: String,
     pub file_type: String,
     pub page_number: Option<i32>,
+
+    // Points stored before the end page was kept have none, which reads as not known:
+    pub end_page_number: Option<i32>,
     pub chunk_index: i32,
     pub text: String,
     pub fingerprint: String,
@@ -251,6 +287,12 @@ pub struct QdrantEdgeSearchResult {
     pub embedded_at_utc: String,
 }
 
+#[derive(Serialize)]
+pub struct QdrantEdgeStoredVector {
+    pub point_id: String,
+    pub vector: Vec<f32>,
+}
+
 #[derive(Clone, Serialize)]
 pub struct QdrantEdgeInfo {
     pub name: String,
@@ -433,17 +475,26 @@ impl QdrantEdgeDatabase {
         Ok(())
     }
 
-    fn search_embedding(&mut self, store_name: &str, vector: Vec<f32>, max_matches: usize, point_ids: Option<Vec<String>>) -> QdrantEdgeResult<Vec<QdrantEdgeSearchResult>> {
+    fn search_embedding(&mut self, store_name: &str, vector: Vec<f32>, max_matches: usize, point_ids: Option<Vec<String>>, hidden_chunks: &[QdrantEdgeHiddenChunks]) -> QdrantEdgeResult<Vec<QdrantEdgeSearchResult>> {
         if max_matches == 0 {
             return Ok(vec![]);
         }
 
-        let filter = match point_ids {
+        let mut filter = match point_ids {
             None => None,
             Some(point_ids) if point_ids.is_empty() => return Ok(vec![]),
             Some(point_ids) => Some(point_id_filter(&point_ids)?),
         };
 
+        if !hidden_chunks.is_empty() {
+            let hidden = hidden_chunks
+                .iter()
+                .map(|document| hidden_chunks_filter(document).map(Condition::Filter))
+                .collect::<QdrantEdgeResult<Vec<_>>>()?;
+
+            filter.get_or_insert_with(Filter::default).must_not = Some(hidden);
+        }
+
         validate_vector_size(vector.len())?;
         let Some(shard) = self.get_existing_store(store_name)? else {
             return Ok(vec![]);
@@ -470,7 +521,49 @@ impl QdrantEdgeDatabase {
             .collect())
     }
 
-    fn delete_embedding_by_file(&mut self, store_name: &str, file_path: &str, parent_file_id: &str) -> QdrantEdgeResult<()> {
+    /// Reads the vectors of the given points, leaving out the points which do not exist.
+    ///
+    /// An indexing run asks for them before it replaces a document: a chunk whose text did not
+    /// change takes over its stored vector instead of going to the embedding provider again.
+    fn retrieve_vectors(&mut self, store_name: &str, point_ids: &[String]) -> QdrantEdgeResult<Vec<QdrantEdgeStoredVector>> {
+        if point_ids.is_empty() {
+            return Ok(vec![]);
+        }
+
+        let point_ids = point_ids
+            .iter()
+            .map(|point_id| to_point_id(point_id))
+            .collect::<QdrantEdgeResult<Vec<_>>>()?;
+
+        let Some(shard) = self.get_existing_store(store_name)? else {
+            return Ok(vec![]);
+        };
+
+        let records = shard.retrieve(RetrieveRequest {
+            point_ids,
+            with_payload: Some(WithPayloadInterface::Bool(false)),
+            with_vector: Some(WithVector::Selector(vec![VECTOR_NAME.to_string()])),
+        })?;
+
+        Ok(records
+            .into_iter()
+            .filter_map(|record| {
+                let Some(VectorStructInternal::Named(mut vectors)) = record.vector else {
+                    return None;
+                };
+
+                match vectors.remove(VECTOR_NAME)? {
+                    VectorInternal::Dense(vector) => Some(QdrantEdgeStoredVector {
+                        point_id: point_id_to_string(record.id),
+                        vector,
+                    }),
+                    _ => None,
+                }
+            })
+            .collect())
+    }
+
+    fn delete_embedding_by_file(&mut self, store_name: &str, file_path: &str, parent_file_id: &str, keep_point_ids: &[String]) -> QdrantEdgeResult<()> {
         let Some(shard) = self.get_existing_store(store_name)? else {
             return Ok(());
         };
@@ -478,11 +571,11 @@ impl QdrantEdgeDatabase {
         // The location may already have changed if a relocation stopped before SQLite caught up.
         if !parent_file_id.is_empty() {
             shard.update(UpdateOperation::PointOperation(
-                PointOperations::DeletePointsByFilter(match_keyword_filter("parent_file_id", parent_file_id)?),
+                PointOperations::DeletePointsByFilter(document_points_filter("parent_file_id", parent_file_id, keep_point_ids)?),
             ))?;
         }
         shard.update(UpdateOperation::PointOperation(
-            PointOperations::DeletePointsByFilter(match_keyword_filter("file_path", file_path)?),
+            PointOperations::DeletePointsByFilter(document_points_filter("file_path", file_path, keep_point_ids)?),
         ))?;
         shard.flush()?;
         Ok(())
@@ -814,13 +907,19 @@ pub async fn insert_qdrant_edge_embedding(Json(request): Json<InsertQdrantEdgeEm
 
 pub async fn search_qdrant_edge_embeddings(Json(request): Json<SearchQdrantEdgeEmbeddingRequest>) -> Json<QdrantEdgeResponse<Vec<QdrantEdgeSearchResult>>> {
     run_qdrant_edge_request(move |database| {
-        database.search_embedding(&request.store_name, request.vector, request.max_matches, request.point_ids)
+        database.search_embedding(&request.store_name, request.vector, request.max_matches, request.point_ids, &request.hidden_chunks)
+    }).await
+}
+
+pub async fn retrieve_qdrant_edge_vectors(Json(request): Json<RetrieveQdrantEdgeVectorsRequest>) -> Json<QdrantEdgeResponse<Vec<QdrantEdgeStoredVector>>> {
+    run_qdrant_edge_request(move |database| {
+        database.retrieve_vectors(&request.store_name, &request.point_ids)
     }).await
 }
 
 pub async fn delete_qdrant_edge_embedding_by_file(Json(request): Json<DeleteQdrantEdgeEmbeddingByFileRequest>) -> Json<QdrantEdgeResponse<()>> {
     run_qdrant_edge_request(move |database| {
-        database.delete_embedding_by_file(&request.store_name, &request.file_path, &request.parent_file_id)
+        database.delete_embedding_by_file(&request.store_name, &request.file_path, &request.parent_file_id, &request.keep_point_ids)
     }).await
 }
 
@@ -1266,7 +1365,9 @@ fn to_qdrant_edge_point(point: QdrantEdgeStoragePoint) -> QdrantEdgeResult<qdran
             "relative_path": point.relative_path,
             "file_type": point.file_type,
             "page_number": point.page_number,
+            "end_page_number": point.end_page_number,
             "chunk_index": point.chunk_index,
+            "generation": point.generation,
             "text": point.text,
             "fingerprint": point.fingerprint,
             "creation_utc": point.creation_utc,
@@ -1292,6 +1393,7 @@ fn to_qdrant_edge_search_result(point: ScoredPoint) -> QdrantEdgeSearchResult {
         relative_path: payload_string(&payload, "relative_path"),
         file_type: payload_string(&payload, "file_type"),
         page_number: payload_i32(&payload, "page_number"),
+        end_page_number: payload_i32(&payload, "end_page_number"),
         chunk_index: payload_i32(&payload, "chunk_index").unwrap_or_default(),
         text: payload_string(&payload, "text"),
         fingerprint: payload_string(&payload, "fingerprint"),
@@ -1347,6 +1449,51 @@ fn match_keyword_filter(field_name: &str, value: &str) -> QdrantEdgeResult<Filte
     })
 }
 
+/// The points of one document, found by one of its payload fields, apart from the points to keep.
+///
+/// An indexing run writes the new points of a document first and removes the old ones last, so a
+/// chunk whose text did not change can still take over its old vector while the run goes on.
+fn document_points_filter(field_name: &str, value: &str, keep_point_ids: &[String]) -> QdrantEdgeResult<Filter> {
+    let mut filter = match_keyword_filter(field_name, value)?;
+    if !keep_point_ids.is_empty() {
+        let keep = keep_point_ids
+            .iter()
+            .map(|point_id| to_point_id(point_id))
+            .collect::<QdrantEdgeResult<HasIdCondition>>()?;
+
+        filter.must_not = Some(vec![Condition::HasId(keep)]);
+    }
+
+    Ok(filter)
+}
+
+/// The points of one document which a search leaves out, cf. `QdrantEdgeHiddenChunks`.
+///
+/// A point without a generation was written before points had one, and is generation 0. So for a
+/// document searched by generation 0, such a point stays searchable, and for any other generation,
+/// it is left out like every other generation.
+fn hidden_chunks_filter(document: &QdrantEdgeHiddenChunks) -> QdrantEdgeResult<Filter> {
+    let mut filter = match_keyword_filter("parent_file_id", &document.parent_file_id)?;
+    if let Some(generation) = document.searchable_generation {
+        let mut searchable = vec![Condition::Field(FieldCondition::new_match(
+            "generation".try_into().map_err(|_| "Invalid payload field name 'generation'.")?,
+            Match::Value(MatchValue {
+                value: ValueVariants::Integer(generation.into()),
+            }),
+        ))];
+
+        if generation == 0 {
+            searchable.push(Condition::IsEmpty(IsEmptyCondition::from(
+                JsonPath::try_from("generation").map_err(|_| "Invalid payload field name 'generation'.")?,
+            )));
+        }
+
+        filter.must_not = Some(searchable);
+    }
+
+    Ok(filter)
+}
+
 /// Restricts a search to the given points. Every id has to be a valid UUID, the same as when the
 /// points were inserted, so a malformed id is an error rather than a point which is silently missed.
 fn point_id_filter(point_ids: &[String]) -> QdrantEdgeResult<Filter> {
@@ -1499,7 +1646,7 @@ mod tests {
         let test_directory = test_directory("no-points");
         let mut database = store_with_one_point_per_axis(&test_directory);
 
-        let found = database.search_embedding(SEARCH_STORE, search_vector(), 10, Some(vec![])).unwrap();
+        let found = database.search_embedding(SEARCH_STORE, search_vector(), 10, Some(vec![]), &[]).unwrap();
 
         // An empty restriction comes from conditions which matched nothing. Searching the whole
         // store instead would answer with exactly what those conditions ruled out.
@@ -1514,11 +1661,11 @@ mod tests {
         let test_directory = test_directory("some-points");
         let mut database = store_with_one_point_per_axis(&test_directory);
 
-        let unrestricted = database.search_embedding(SEARCH_STORE, search_vector(), 10, None).unwrap();
+        let unrestricted = database.search_embedding(SEARCH_STORE, search_vector(), 10, None, &[]).unwrap();
         assert_eq!(found_point_ids(&unrestricted), vec![POINT_X, POINT_Y, POINT_Z]);
 
         let restricted = database
-            .search_embedding(SEARCH_STORE, search_vector(), 10, Some(vec![POINT_Y.to_string(), POINT_Z.to_string()]))
+            .search_embedding(SEARCH_STORE, search_vector(), 10, Some(vec![POINT_Y.to_string(), POINT_Z.to_string()]), &[])
             .unwrap();
         assert_eq!(
             found_point_ids(&restricted),
@@ -1530,6 +1677,76 @@ mod tests {
         fs::remove_dir_all(test_directory).unwrap();
     }
 
+    /// A document read again keeps its searchable generation until the new one is complete, and an
+    /// attempt which failed may stay next to it, so its vectors can be reused. Neither must be
+    /// found twice, nor a document whose first indexing is not complete.
+    #[test]
+    fn a_search_leaves_out_the_hidden_chunks_of_a_document() {
+        let test_directory = test_directory("hidden-chunks");
+        let mut database = QdrantEdgeDatabase::new(test_directory.clone());
+        database.ensure_store_exists(SEARCH_STORE, "Some source", 3).unwrap();
+
+        let mut stored = test_point(POINT_X, vec![1.0, 0.0, 0.0]);
+        stored.parent_file_id = "report".to_string();
+        let mut attempt = test_point(POINT_Y, vec![0.0, 1.0, 0.0]);
+        attempt.parent_file_id = "report".to_string();
+        attempt.generation = 1;
+        let mut other = test_point(POINT_Z, vec![0.0, 0.0, 1.0]);
+        other.parent_file_id = "notes".to_string();
+        database.insert_embedding(SEARCH_STORE, vec![stored, attempt, other]).unwrap();
+
+        let hidden = |parent_file_id: &str, searchable_generation: Option<i32>| QdrantEdgeHiddenChunks {
+            parent_file_id: parent_file_id.to_string(),
+            searchable_generation,
+        };
+
+        let found = database.search_embedding(SEARCH_STORE, search_vector(), 10, None, &[hidden("report", Some(0))]).unwrap();
+        assert_eq!(found_point_ids(&found), vec![POINT_X, POINT_Z], "the attempt is left out, the stored generation is found");
+
+        let found = database.search_embedding(SEARCH_STORE, search_vector(), 10, None, &[hidden("report", Some(1))]).unwrap();
+        assert_eq!(found_point_ids(&found), vec![POINT_Y, POINT_Z], "once the attempt is the searchable generation, the one before is left out");
+
+        let found = database.search_embedding(SEARCH_STORE, search_vector(), 10, None, &[hidden("notes", None)]).unwrap();
+        assert_eq!(found_point_ids(&found), vec![POINT_X, POINT_Y], "a document without a searchable generation is left out entirely");
+
+        let found = database
+            .search_embedding(SEARCH_STORE, search_vector(), 10, Some(vec![POINT_X.to_string(), POINT_Y.to_string()]), &[hidden("report", Some(0))])
+            .unwrap();
+        assert_eq!(found_point_ids(&found), vec![POINT_X], "hidden chunks stay hidden when the search is restricted to some points");
+
+        drop(database);
+        fs::remove_dir_all(test_directory).unwrap();
+    }
+
+    /// Points written before points had a generation are generation 0 without saying so.
+    #[test]
+    fn a_point_without_a_generation_counts_as_generation_0() {
+        let test_directory = test_directory("no-generation");
+        let mut database = QdrantEdgeDatabase::new(test_directory.clone());
+        let mut point = test_point(POINT_X, vec![1.0, 0.0, 0.0]);
+        point.parent_file_id = "report".to_string();
+        let mut point = to_qdrant_edge_point(point).unwrap();
+        point.payload.as_mut().unwrap().0.remove("generation");
+        let (shard, _) = database.get_or_create_store(SEARCH_STORE, 3).unwrap();
+        shard
+            .update(UpdateOperation::PointOperation(PointOperations::UpsertPoints(PointInsertOperations::PointsList(vec![point]))))
+            .unwrap();
+
+        let hidden = |searchable_generation: i32| [QdrantEdgeHiddenChunks {
+            parent_file_id: "report".to_string(),
+            searchable_generation: Some(searchable_generation),
+        }];
+
+        let found = database.search_embedding(SEARCH_STORE, search_vector(), 10, None, &hidden(0)).unwrap();
+        assert_eq!(found_point_ids(&found), vec![POINT_X], "the document is searched by the generation of its old points");
+
+        let found = database.search_embedding(SEARCH_STORE, search_vector(), 10, None, &hidden(1)).unwrap();
+        assert!(found.is_empty(), "once the document is searched by another generation, its old points are left out");
+
+        drop(database);
+        fs::remove_dir_all(test_directory).unwrap();
+    }
+
     #[test]
     fn a_restriction_with_a_malformed_point_id_is_rejected() {
         assert!(point_id_filter(&[POINT_X.to_string()]).is_ok());
@@ -1546,6 +1763,7 @@ mod tests {
         let mut point = test_point(POINT_X, search_vector());
         point.parent_file_id = "original-parent".to_string();
         point.page_number = Some(2);
+        point.end_page_number = Some(3);
         database.insert_embedding(SEARCH_STORE, vec![point]).unwrap();
 
         for _ in 0..2 {
@@ -1561,7 +1779,7 @@ mod tests {
         drop(database);
 
         let mut database = QdrantEdgeDatabase::new(test_directory.clone());
-        let found = database.search_embedding(SEARCH_STORE, search_vector(), 10, None).unwrap();
+        let found = database.search_embedding(SEARCH_STORE, search_vector(), 10, None, &[]).unwrap();
         assert_eq!(found.len(), 1);
         assert_eq!(found[0].point_id, POINT_X);
         assert_eq!(found[0].parent_file_id, "original-parent");
@@ -1572,12 +1790,56 @@ mod tests {
         assert_eq!(found[0].fingerprint, "new-fingerprint");
         assert_eq!(found[0].text, "Some text.");
         assert_eq!(found[0].page_number, Some(2));
+        assert_eq!(found[0].end_page_number, Some(3), "a relocation rewrites where the document is, not which pages a chunk spans");
         assert!((found[0].score - 1.0).abs() < 0.0001);
 
         // SQLite may still know the old path when a relocation was interrupted. Its stable parent
         // id must be enough to remove the vectors if the document changed before the next retry.
-        database.delete_embedding_by_file(SEARCH_STORE, "/tmp/test-data/notes.md", "original-parent").unwrap();
-        assert!(database.search_embedding(SEARCH_STORE, search_vector(), 10, None).unwrap().is_empty());
+        database.delete_embedding_by_file(SEARCH_STORE, "/tmp/test-data/notes.md", "original-parent", &[]).unwrap();
+        assert!(database.search_embedding(SEARCH_STORE, search_vector(), 10, None, &[]).unwrap().is_empty());
+        drop(database);
+        fs::remove_dir_all(test_directory).unwrap();
+    }
+
+    /// An indexing run reads the vectors of a document before it replaces the document, so a chunk
+    /// whose text stays the same keeps its vector instead of being embedded again.
+    #[test]
+    fn retrieving_vectors_returns_the_stored_ones_and_leaves_out_unknown_points() {
+        let test_directory = test_directory("retrieve");
+        let mut database = QdrantEdgeDatabase::new(test_directory.clone());
+        database.insert_embedding(SEARCH_STORE, vec![test_point(POINT_X, vec![1.0, 0.0, 0.0]), test_point(POINT_Y, vec![0.0, 1.0, 0.0])]).unwrap();
+
+        let vectors = database.retrieve_vectors(SEARCH_STORE, &[POINT_Y.to_string(), POINT_Z.to_string()]).unwrap();
+        assert_eq!(vectors.len(), 1, "a point which does not exist is left out");
+        assert_eq!(vectors[0].point_id, POINT_Y);
+        assert_eq!(vectors[0].vector, vec![0.0, 1.0, 0.0], "the vector comes back as it was stored");
+
+        assert!(database.retrieve_vectors("rag_00000000000000000000000000000000", &[POINT_X.to_string()]).unwrap().is_empty(), "a store which does not exist has no vectors to give");
+        assert!(database.retrieve_vectors(SEARCH_STORE, &["not-a-point-id".to_string()]).is_err(), "a malformed id is an error, not a point which is quietly missed");
+
+        drop(database);
+        fs::remove_dir_all(test_directory).unwrap();
+    }
+
+    /// An indexing run writes the new points of a document first and removes the old ones at the
+    /// end, all but the points it just wrote.
+    #[test]
+    fn deleting_the_points_of_a_document_keeps_the_points_named() {
+        let test_directory = test_directory("delete-keep");
+        let mut database = QdrantEdgeDatabase::new(test_directory.clone());
+        database.insert_embedding(SEARCH_STORE, vec![test_point(POINT_X, search_vector()), test_point(POINT_Y, search_vector()), test_point(POINT_Z, search_vector())]).unwrap();
+
+        database.delete_embedding_by_file(SEARCH_STORE, "/tmp/test-data/notes.md", "", &[POINT_X.to_string(), POINT_Z.to_string()]).unwrap();
+
+        let mut remaining = database
+            .retrieve_vectors(SEARCH_STORE, &[POINT_X.to_string(), POINT_Y.to_string(), POINT_Z.to_string()])
+            .unwrap()
+            .into_iter()
+            .map(|stored| stored.point_id)
+            .collect::<Vec<_>>();
+        remaining.sort();
+        assert_eq!(remaining, vec![POINT_X.to_string(), POINT_Z.to_string()], "only the point which is not kept goes");
+
         drop(database);
         fs::remove_dir_all(test_directory).unwrap();
     }
@@ -1601,7 +1863,7 @@ mod tests {
         drop(database);
 
         let mut database = QdrantEdgeDatabase::new(test_directory.clone());
-        let found = database.search_embedding(SEARCH_STORE, search_vector(), point_ids.len(), None).unwrap();
+        let found = database.search_embedding(SEARCH_STORE, search_vector(), point_ids.len(), None, &[]).unwrap();
 
         assert_eq!(closed_wal_segments(&store_path).len(), 1);
         assert_eq!(
@@ -1630,7 +1892,7 @@ mod tests {
         database.insert_embedding(SEARCH_STORE, vec![updated_point]).unwrap();
 
         let found = database
-            .search_embedding(SEARCH_STORE, search_vector(), 1, Some(vec![newest_point_id.clone()]))
+            .search_embedding(SEARCH_STORE, search_vector(), 1, Some(vec![newest_point_id.clone()]), &[])
             .unwrap();
         assert_eq!(found.len(), 1);
         assert_eq!(found[0].text, "Updated text.");
@@ -1654,7 +1916,7 @@ mod tests {
         updated_point.text = "Updated text.".to_string();
         database.insert_embedding(SEARCH_STORE, vec![updated_point]).unwrap();
 
-        let found = database.search_embedding(SEARCH_STORE, search_vector(), point_ids.len(), None).unwrap();
+        let found = database.search_embedding(SEARCH_STORE, search_vector(), point_ids.len(), None, &[]).unwrap();
         assert_eq!(
             found_point_ids(&found).into_iter().collect::<HashSet<_>>(),
             point_ids.iter().map(String::as_str).collect::<HashSet<_>>()
@@ -1719,11 +1981,11 @@ mod tests {
         // The copy lies where the store lies below the data directory, and it is a complete store:
         let staging_path = PathBuf::from(&preparation.staging_path);
         let mut copy = QdrantEdgeDatabase::new(staging_path.join("databases").join("vector_database"));
-        let found = copy.search_embedding(SEARCH_STORE, search_vector(), 3, None).unwrap();
+        let found = copy.search_embedding(SEARCH_STORE, search_vector(), 3, None, &[]).unwrap();
         assert_eq!(found_point_ids(&found), vec![POINT_X, POINT_Y, POINT_Z]);
 
         // The copy unloaded the store, and the next request loads it again:
-        let found = database.search_embedding(SEARCH_STORE, search_vector(), 3, None).unwrap();
+        let found = database.search_embedding(SEARCH_STORE, search_vector(), 3, None, &[]).unwrap();
         assert_eq!(found_point_ids(&found), vec![POINT_X, POINT_Y, POINT_Z]);
 
         drop(copy);
@@ -1773,7 +2035,7 @@ mod tests {
             drop(restored);
 
             let mut restored = QdrantEdgeDatabase::new(restored_base);
-            let found = restored.search_embedding(SEARCH_STORE, vec![1.0, 0.0, 0.0], 10, None).unwrap();
+            let found = restored.search_embedding(SEARCH_STORE, vec![1.0, 0.0, 0.0], 10, None, &[]).unwrap();
             assert_eq!(found.len(), 1);
             assert_eq!(found[0].point_id, POINT_X);
             assert_eq!(found[0].parent_file_id, "original-parent");
@@ -1783,7 +2045,7 @@ mod tests {
             assert_eq!(found[0].embedded_at_utc, "2026-10-01T12:00:00Z");
             assert!((found[0].score - 1.0).abs() < 0.0001);
 
-            let original = database.search_embedding(SEARCH_STORE, vec![1.0, 0.0, 0.0], 10, None).unwrap();
+            let original = database.search_embedding(SEARCH_STORE, vec![1.0, 0.0, 0.0], 10, None, &[]).unwrap();
             assert_eq!(original[0].file_path, old_path, "relocating a restored copy must not change the original store");
             drop(restored);
             drop(database);
@@ -1923,7 +2185,9 @@ mod tests {
             relative_path: "notes.md".to_string(),
             file_type: "md".to_string(),
             page_number: None,
+            end_page_number: None,
             chunk_index: 0,
+            generation: 0,
             text: "Some text.".to_string(),
             fingerprint: String::new(),
             creation_utc: String::new(),

runtime/src/runtime_api.rs

@@ -41,6 +41,7 @@ fn create_router() -> Router {
         .route("/system/qdrant-edge/exists", post(crate::qdrant_edge_database::check_qdrant_edge_store))
         .route("/system/qdrant-edge/insert", post(crate::qdrant_edge_database::insert_qdrant_edge_embedding))
         .route("/system/qdrant-edge/search", post(crate::qdrant_edge_database::search_qdrant_edge_embeddings))
+        .route("/system/qdrant-edge/retrieve", post(crate::qdrant_edge_database::retrieve_qdrant_edge_vectors))
         .route("/system/qdrant-edge/delete-file", post(crate::qdrant_edge_database::delete_qdrant_edge_embedding_by_file))
         .route("/system/qdrant-edge/relocate-file", post(crate::qdrant_edge_database::relocate_qdrant_edge_embedding_file))
         .route("/system/qdrant-edge/optimize", post(crate::qdrant_edge_database::optimize_qdrant_edge_store))