unify how lines in blame results are accessed

* provide `Outcome` with interner and and a list of tokens.
* unify the way tokens are created.

Author: Byron

gitoxide-core/src/repository/blame.rs

@@ -1,4 +1,4 @@
-use std::{ffi::OsStr, path::PathBuf, str::Lines};
+use std::{ffi::OsStr, path::PathBuf};
 
 use anyhow::anyhow;
 use gix::bstr::BStr;
@@ -18,40 +18,31 @@ pub fn blame_file(mut repo: gix::Repository, file: &OsStr, out: impl std::io::Wr
         .into();
     let file_path: &BStr = gix::path::os_str_into_bstr(file)?;
 
-    let blame_entries = gix::blame::file(
+    let outcome = gix::blame::file(
         &repo.objects,
         traverse,
         &mut resource_cache,
         work_dir.clone(),
         file_path,
     )?;
-
-    let absolute_path = work_dir.join(file);
-    let file_content = std::fs::read_to_string(absolute_path)?;
-    let lines = file_content.lines();
-
-    write_blame_entries(out, lines, blame_entries)?;
+    write_blame_entries(out, outcome)?;
 
     Ok(())
 }
 
-fn write_blame_entries(
-    mut out: impl std::io::Write,
-    mut lines: Lines<'_>,
-    blame_entries: Vec<gix::blame::BlameEntry>,
-) -> Result<(), std::io::Error> {
-    for blame_entry in blame_entries {
-        for line_number in blame_entry.range_in_blamed_file {
-            let line = lines.next().unwrap();
-
-            writeln!(
+fn write_blame_entries(mut out: impl std::io::Write, outcome: gix::blame::Outcome) -> Result<(), std::io::Error> {
+    for (entry, lines_in_hunk) in outcome.entries_with_lines() {
+        for ((actual_lno, source_lno), line) in entry
+            .range_in_blamed_file
+            .zip(entry.range_in_original_file)
+            .zip(lines_in_hunk)
+        {
+            write!(
                 out,
-                "{} {} {}",
-                blame_entry.commit_id.to_hex_with_len(8),
-                // `line_number` is 0-based, but we want to show 1-based line numbers (as `git`
-                // does).
-                line_number + 1,
-                line
+                "{short_id} {line_no} {src_line_no} {line}",
+                line_no = actual_lno + 1,
+                src_line_no = source_lno + 1,
+                short_id = entry.commit_id.to_hex_with_len(8),
             )?;
         }
     }

gix-blame/src/file/function.rs

@@ -1,10 +1,10 @@
-use std::{ops::Range, path::PathBuf};
-
+use gix_diff::blob::intern::TokenSource;
 use gix_hash::ObjectId;
 use gix_object::{bstr::BStr, FindExt};
+use std::{ops::Range, path::PathBuf};
 
 use super::{process_changes, Change, Offset, UnblamedHunk};
-use crate::BlameEntry;
+use crate::{BlameEntry, Outcome};
 
 // TODO: do not instantiate anything, get everything passed as argument.
 /// Produce a list of consecutive [`BlameEntry`] instances to indicate in which commits the ranges of the file
@@ -60,28 +60,35 @@ pub fn file<E>(
     // TODO: remove
     worktree_root: PathBuf,
     file_path: &BStr,
-) -> Result<Vec<BlameEntry>, E> {
+) -> Result<Outcome, E> {
     // TODO: `worktree_root` should be removed - read everything from Commit.
     //       Worktree changes should be placed into a temporary commit.
     // TODO: remove this and deduplicate the respective code.
     use gix_object::bstr::ByteSlice;
     let absolute_path = worktree_root.join(gix_path::from_bstr(file_path));
 
-    // TODO  use `imara-diff` to tokenize this just like it will be tokenized when diffing.
-    let number_of_lines = std::fs::read_to_string(absolute_path).unwrap().lines().count();
-
     let mut traverse = traverse.into_iter().peekable();
     let Some(Ok(suspect)) = traverse.peek().map(|res| res.as_ref().map(|item| item.id)) else {
         todo!("return actual error");
     };
 
+    let original_file_blob = std::fs::read(absolute_path).unwrap();
+    let num_lines_in_original = {
+        let mut interner = gix_diff::blob::intern::Interner::new(original_file_blob.len() / 100);
+        tokens_for_diffing(&original_file_blob)
+            .tokenize()
+            .map(|token| interner.intern(token))
+            .count()
+    };
+
     let mut hunks_to_blame = vec![UnblamedHunk::new(
-        0..number_of_lines.try_into().unwrap(),
+        0..num_lines_in_original.try_into().unwrap(),
         suspect,
         Offset::Added(0),
     )];
 
     let mut out = Vec::new();
+    let mut buf = Vec::with_capacity(512);
     'outer: for item in traverse {
         let item = item?;
         let suspect = item.id;
@@ -103,9 +110,8 @@ pub fn file<E>(
             break;
         }
 
-        let mut buffer = Vec::new();
-        let commit_id = odb.find_commit(&suspect, &mut buffer).unwrap().tree();
-        let tree_iter = odb.find_tree_iter(&commit_id, &mut buffer).unwrap();
+        let commit_id = odb.find_commit(&suspect, &mut buf).unwrap().tree();
+        let tree_iter = odb.find_tree_iter(&commit_id, &mut buf).unwrap();
 
         let mut entry_buffer = Vec::new();
         let Some(entry) = tree_iter
@@ -247,7 +253,10 @@ pub fn file<E>(
     // I don’t know yet whether it would make sense to use a data structure instead that preserves
     // order on insertion.
     out.sort_by(|a, b| a.range_in_blamed_file.start.cmp(&b.range_in_blamed_file.start));
-    Ok(coalesce_blame_entries(out))
+    Ok(Outcome {
+        entries: coalesce_blame_entries(out),
+        blob: original_file_blob,
+    })
 }
 
 /// This function merges adjacent blame entries. It merges entries that are adjacent both in the
@@ -416,9 +425,18 @@ fn blob_changes(
         .unwrap();
 
     let outcome = resource_cache.prepare_diff().unwrap();
-    let input = outcome.interned_input();
+    let input = gix_diff::blob::intern::InternedInput::new(
+        tokens_for_diffing(outcome.old.data.as_slice().unwrap_or_default()),
+        tokens_for_diffing(outcome.new.data.as_slice().unwrap_or_default()),
+    );
     let number_of_lines_in_destination = input.after.len();
     let change_recorder = ChangeRecorder::new(number_of_lines_in_destination.try_into().unwrap());
 
     gix_diff::blob::diff(gix_diff::blob::Algorithm::Histogram, &input, change_recorder)
 }
+
+/// Return an iterator over tokens for use in diffing. These usually lines, but iit's important to unify them
+/// so the later access shows the right thing.
+pub(crate) fn tokens_for_diffing(data: &[u8]) -> impl TokenSource<Token = &[u8]> {
+    gix_diff::blob::sources::byte_lines_with_terminator(data)
+}

gix-blame/src/lib.rs

@@ -15,7 +15,7 @@
 #![forbid(unsafe_code)]
 
 mod types;
-pub use types::BlameEntry;
+pub use types::{BlameEntry, Outcome};
 
 mod file;
 pub use file::function::file;

gix-blame/src/types.rs

@@ -3,7 +3,43 @@ use std::{
     ops::{AddAssign, Range, SubAssign},
 };
 
+use crate::file::function::tokens_for_diffing;
 use gix_hash::ObjectId;
+use gix_object::bstr::BString;
+
+/// The outcome of [`file()`](crate::file()).
+pub struct Outcome {
+    /// One entry in sequential order, to associate a hunk in the original file with the commit (and its lines)
+    /// that introduced it.
+    pub entries: Vec<BlameEntry>,
+    /// A buffer with the file content of the *Original File*, ready for tokenization.
+    pub blob: Vec<u8>,
+}
+
+impl Outcome {
+    /// Return an iterator over each entry in [`Self::entries`], along with its lines, line by line.
+    ///
+    /// Note that [`Self::blob`] must be tokenized in exactly the same way as the tokenizer that was used
+    /// to perform the diffs, which is what this method assures.
+    pub fn entries_with_lines(&self) -> impl Iterator<Item = (BlameEntry, Vec<BString>)> + '_ {
+        use gix_diff::blob::intern::TokenSource;
+        let mut interner = gix_diff::blob::intern::Interner::new(self.blob.len() / 100);
+        let lines_as_tokens: Vec<_> = tokens_for_diffing(&self.blob)
+            .tokenize()
+            .map(|token| interner.intern(token))
+            .collect();
+        self.entries.iter().map(move |e| {
+            let Range { start, end } = e.range_in_blamed_file.clone();
+            (
+                e.clone(),
+                lines_as_tokens[start as usize..end as usize]
+                    .iter()
+                    .map(|token| BString::new(interner[*token].into()))
+                    .collect(),
+            )
+        })
+    }
+}
 
 /// Describes the offset of a particular hunk relative to the *Original File*.
 #[derive(Clone, Copy, Debug, PartialEq)]
@@ -68,11 +104,12 @@ impl SubAssign<u32> for Offset {
 /// Both ranges are of the same size, but may use different [starting points](Range::start). Naturally,
 /// they have the same content, which is the reason they are in what is returned by [`file()`](crate::file()).
 // TODO: see if this can be encoded as `start_in_original_file` and `start_in_blamed_file` and a single `len`.
-#[derive(Debug, PartialEq)]
+#[derive(Clone, Debug, PartialEq)]
 pub struct BlameEntry {
     /// The section of tokens in the tokenized version of the *Blamed File* (typically lines).
     pub range_in_blamed_file: Range<u32>,
     /// The section of tokens in the tokenized version of the *Original File* (typically lines).
+    // TODO: figure out why this is basically inverted. Probably that's just it - would make sense with `UnblamedHunk` then.
     pub range_in_original_file: Range<u32>,
     /// The commit that introduced the section into the *Blamed File*.
     pub commit_id: ObjectId,

gix-blame/tests/blame.rs

@@ -199,7 +199,8 @@ macro_rules! mktest {
                 worktree_path,
                 format!("{}.txt", $case).as_str().into(),
             )
-            .unwrap();
+            .unwrap()
+            .entries;
 
             assert_eq!(lines_blamed.len(), $number_of_lines);
 
@@ -259,7 +260,8 @@ fn diff_disparity() {
             worktree_path,
             format!("{case}.txt").as_str().into(),
         )
-        .unwrap();
+        .unwrap()
+        .entries;
 
         assert_eq!(lines_blamed.len(), 5);