add git ref storage for older API versions

.config/nextest.toml

@@ -1,2 +1,6 @@
+# Required for NEXTEST_BIN_EXE_ with hyphens in binary names replaced by
+# underscores.
+nextest-version = "0.9.113"
+
 [profile.ci]
 fail-fast = false

crates/dropshot-api-manager-types/src/validation.rs

@@ -130,42 +130,75 @@ impl ApiSpecFileName {
     }
 
     /// Returns the path of this file relative to the root of the OpenAPI
-    /// documents
+    /// documents.
     pub fn path(&self) -> Utf8PathBuf {
         match &self.kind {
             ApiSpecFileNameKind::Lockstep => {
                 Utf8PathBuf::from_iter([self.basename()])
             }
-            ApiSpecFileNameKind::Versioned { .. } => Utf8PathBuf::from_iter([
-                self.ident.deref().clone(),
-                self.basename(),
-            ]),
+            ApiSpecFileNameKind::Versioned { .. }
+            | ApiSpecFileNameKind::VersionedGitRef { .. } => {
+                Utf8PathBuf::from_iter([
+                    self.ident.deref().clone(),
+                    self.basename(),
+                ])
+            }
         }
     }
 
-    /// Returns the base name of this file path
+    /// Returns the base name of this file path.
     pub fn basename(&self) -> String {
         match &self.kind {
             ApiSpecFileNameKind::Lockstep => format!("{}.json", self.ident),
             ApiSpecFileNameKind::Versioned { version, hash } => {
                 format!("{}-{}-{}.json", self.ident, version, hash)
             }
+            ApiSpecFileNameKind::VersionedGitRef { version, hash } => {
+                format!("{}-{}-{}.json.gitref", self.ident, version, hash)
+            }
         }
     }
 
-    /// For versioned APIs, returns the version part of the filename
+    /// For versioned APIs, returns the version part of the filename.
     pub fn version(&self) -> Option<&semver::Version> {
         match &self.kind {
             ApiSpecFileNameKind::Lockstep => None,
-            ApiSpecFileNameKind::Versioned { version, .. } => Some(version),
+            ApiSpecFileNameKind::Versioned { version, .. }
+            | ApiSpecFileNameKind::VersionedGitRef { version, .. } => {
+                Some(version)
+            }
         }
     }
 
-    /// For versioned APIs, returns the hash part of the filename
+    /// For versioned APIs, returns the hash part of the filename.
     pub fn hash(&self) -> Option<&str> {
         match &self.kind {
             ApiSpecFileNameKind::Lockstep => None,
-            ApiSpecFileNameKind::Versioned { hash, .. } => Some(hash),
+            ApiSpecFileNameKind::Versioned { hash, .. }
+            | ApiSpecFileNameKind::VersionedGitRef { hash, .. } => Some(hash),
+        }
+    }
+
+    /// Returns true if this is a git ref file.
+    pub fn is_git_ref(&self) -> bool {
+        matches!(self.kind, ApiSpecFileNameKind::VersionedGitRef { .. })
+    }
+
+    /// Converts a `VersionedGitRef` to its `Versioned` equivalent.
+    ///
+    /// For non-git ref files, returns a clone of self.
+    pub fn to_json_filename(&self) -> ApiSpecFileName {
+        match &self.kind {
+            ApiSpecFileNameKind::VersionedGitRef { version, hash } => {
+                ApiSpecFileName::new(
+                    self.ident.clone(),
+                    ApiSpecFileNameKind::Versioned {
+                        version: version.clone(),
+                        hash: hash.clone(),
+                    },
+                )
+            }
+            _ => self.clone(),
         }
     }
 }
@@ -182,6 +215,17 @@ pub enum ApiSpecFileNameKind {
         /// The hash of the file contents.
         hash: String,
     },
+    /// The file's path implies a versioned API stored as a git ref.
+    ///
+    /// Instead of storing the full JSON content, a `.gitref` file contains a
+    /// reference in the format `commit:path` that can be used to retrieve the
+    /// content via `git show`.
+    VersionedGitRef {
+        /// The version of the API this document describes.
+        version: semver::Version,
+        /// The hash of the file contents (from the original file).
+        hash: String,
+    },
 }
 
 /// Newtype for API identifiers

crates/dropshot-api-manager/src/apis.rs

@@ -83,6 +83,13 @@ pub struct ManagedApi {
     ///
     /// Default: false (bytewise check is performed for latest version).
     allow_trivial_changes_for_latest: bool,
+
+    /// Per-API override for git ref storage.
+    ///
+    /// - `None`: use the global setting from `ManagedApis`.
+    /// - `Some(true)`: enable git ref storage for this API.
+    /// - `Some(false)`: disable git ref storage for this API.
+    use_git_ref_storage: Option<bool>,
 }
 
 impl fmt::Debug for ManagedApi {
@@ -95,6 +102,7 @@ impl fmt::Debug for ManagedApi {
             api_description: _,
             extra_validation,
             allow_trivial_changes_for_latest,
+            use_git_ref_storage,
         } = self;
 
         f.debug_struct("ManagedApi")
@@ -111,6 +119,7 @@ impl fmt::Debug for ManagedApi {
                 "allow_trivial_changes_for_latest",
                 allow_trivial_changes_for_latest,
             )
+            .field("use_git_ref_storage", use_git_ref_storage)
             .finish()
     }
 }
@@ -132,6 +141,7 @@ impl From<ManagedApiConfig> for ManagedApi {
             api_description,
             extra_validation: None,
             allow_trivial_changes_for_latest: false,
+            use_git_ref_storage: None,
         }
     }
 }
@@ -184,6 +194,30 @@ impl ManagedApi {
         self.allow_trivial_changes_for_latest
     }
 
+    /// Enables git ref storage for this API, overriding the global setting.
+    ///
+    /// When enabled, non-latest blessed API versions are stored as `.gitref`
+    /// files containing a git reference instead of full JSON files.
+    pub fn with_git_ref_storage(mut self) -> Self {
+        self.use_git_ref_storage = Some(true);
+        self
+    }
+
+    /// Disables git ref storage for this API, overriding the global setting.
+    pub fn disable_git_ref_storage(mut self) -> Self {
+        self.use_git_ref_storage = Some(false);
+        self
+    }
+
+    /// Returns the git ref storage setting for this API.
+    ///
+    /// - `None`: use the global setting.
+    /// - `Some(true)`: git ref storage is enabled for this API.
+    /// - `Some(false)`: git ref storage is disabled for this API.
+    pub fn uses_git_ref_storage(&self) -> Option<bool> {
+        self.use_git_ref_storage
+    }
+
     /// Sets extra validation to perform on the OpenAPI document.
     ///
     /// For versioned APIs, extra validation is performed on *all* versions,
@@ -269,16 +303,24 @@ pub struct ManagedApis {
     apis: BTreeMap<ApiIdent, ManagedApi>,
     unknown_apis: BTreeSet<ApiIdent>,
     validation: Option<Box<DynValidationFn>>,
+
+    /// If true, store non-latest blessed API versions as git ref files instead
+    /// of full JSON files. This saves disk space but requires git access to
+    /// read the contents.
+    ///
+    /// The default is false.
+    use_git_ref_storage: bool,
 }
 
 impl fmt::Debug for ManagedApis {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let Self { apis, unknown_apis, validation } = self;
+        let Self { apis, unknown_apis, validation, use_git_ref_storage } = self;
 
         f.debug_struct("ManagedApis")
             .field("apis", apis)
             .field("unknown_apis", unknown_apis)
             .field("validation", &validation.as_ref().map(|_| "..."))
+            .field("use_git_ref_storage", use_git_ref_storage)
             .finish()
     }
 }
@@ -307,6 +349,7 @@ impl ManagedApis {
             apis,
             unknown_apis: BTreeSet::new(),
             validation: None,
+            use_git_ref_storage: false,
         })
     }
 
@@ -343,6 +386,30 @@ impl ManagedApis {
         self.validation.as_deref()
     }
 
+    /// Enables git ref storage for older blessed API versions.
+    ///
+    /// When enabled, non-latest blessed API versions are stored as `.gitref`
+    /// files containing a Git reference instead of full JSON files. This allows
+    /// for Git (including the GitHub web UI) to detect changed OpenAPI
+    /// documents as renames, but Git history is required to be present to read
+    /// older versions.
+    ///
+    /// Individual APIs can override this setting using
+    /// [`ManagedApi::use_git_ref_storage`] or
+    /// [`ManagedApi::disable_git_ref_storage`].
+    pub fn with_git_ref_storage(mut self) -> Self {
+        self.use_git_ref_storage = true;
+        self
+    }
+
+    /// Returns true if git ref storage is enabled for the given API.
+    ///
+    /// This checks the per-API setting first, falling back to the global
+    /// setting if not specified.
+    pub(crate) fn uses_git_ref_storage(&self, api: &ManagedApi) -> bool {
+        api.uses_git_ref_storage().unwrap_or(self.use_git_ref_storage)
+    }
+
     /// Returns the number of APIs managed by this instance.
     pub fn len(&self) -> usize {
         self.apis.len()

crates/dropshot-api-manager/src/cmd/check.rs

@@ -24,10 +24,12 @@ pub(crate) fn check_impl(
 
     eprintln!("{:>HEADER_WIDTH$}", SEPARATOR);
 
-    let (generated, errors) = generated_source.load(apis, &styles)?;
+    let (generated, errors) =
+        generated_source.load(apis, &styles, &env.repo_root)?;
     display_load_problems(&errors, &styles)?;
 
-    let (local_files, errors) = env.local_source.load(apis, &styles)?;
+    let (local_files, errors) =
+        env.local_source.load(apis, &styles, &env.repo_root)?;
     display_load_problems(&errors, &styles)?;
 
     let (blessed, errors) =

crates/dropshot-api-manager/src/cmd/debug.rs

@@ -26,7 +26,8 @@ pub(crate) fn debug_impl(
 
     // Print information about local files.
 
-    let (local_files, errors) = env.local_source.load(apis, &styles)?;
+    let (local_files, errors) =
+        env.local_source.load(apis, &styles, &env.repo_root)?;
     dump_structure(&local_files, &errors);
 
     // Print information about what we found in Git.
@@ -35,7 +36,8 @@ pub(crate) fn debug_impl(
     dump_structure(&blessed, &errors);
 
     // Print information about generated files.
-    let (generated, errors) = generated_source.load(apis, &styles)?;
+    let (generated, errors) =
+        generated_source.load(apis, &styles, &env.repo_root)?;
     dump_structure(&generated, &errors);
 
     // Print result of resolving the differences.

crates/dropshot-api-manager/src/cmd/generate.rs

@@ -43,10 +43,12 @@ pub(crate) fn generate_impl(
         styles.colorize();
     }
 
-    let (generated, errors) = generated_source.load(apis, &styles)?;
+    let (generated, errors) =
+        generated_source.load(apis, &styles, &env.repo_root)?;
     display_load_problems(&errors, &styles)?;
 
-    let (local_files, errors) = env.local_source.load(apis, &styles)?;
+    let (local_files, errors) =
+        env.local_source.load(apis, &styles, &env.repo_root)?;
     display_load_problems(&errors, &styles)?;
 
     let (blessed, errors) =
@@ -159,10 +161,11 @@ pub(crate) fn generate_impl(
         return Ok(GenerateResult::Failures);
     }
 
-    // Finally, check again for any problems.  Since we expect this should have
+    // Finally, check again for any problems. Since we expect this should have
     // fixed everything, be quiet unless we find something amiss.
     let mut nproblems = 0;
-    let (local_files, errors) = env.local_source.load(apis, &styles)?;
+    let (local_files, errors) =
+        env.local_source.load(apis, &styles, &env.repo_root)?;
     eprintln!(
         "{:>HEADER_WIDTH$} all local files",
         "Rechecking".style(styles.success_header),

crates/dropshot-api-manager/src/environment.rs

@@ -199,7 +199,7 @@ pub enum BlessedSource {
 }
 
 impl BlessedSource {
-    /// Load the blessed OpenAPI documents
+    /// Load the blessed OpenAPI documents.
     pub fn load(
         &self,
         repo_root: &Utf8Path,
@@ -215,7 +215,12 @@ impl BlessedSource {
                     local_directory,
                 );
                 let api_files: ApiSpecFilesBuilder<'_, BlessedApiSpecFile> =
-                    walk_local_directory(local_directory, apis, &mut errors)?;
+                    walk_local_directory(
+                        local_directory,
+                        apis,
+                        &mut errors,
+                        repo_root,
+                    )?;
                 Ok((BlessedFiles::from(api_files), errors))
             }
             BlessedSource::GitRevisionMergeBase { revision, directory } => {
@@ -254,11 +259,12 @@ pub enum GeneratedSource {
 }
 
 impl GeneratedSource {
-    /// Load the generated OpenAPI documents (i.e., generating them as needed)
+    /// Load the generated OpenAPI documents (i.e., generating them as needed).
     pub fn load(
         &self,
         apis: &ManagedApis,
         styles: &Styles,
+        repo_root: &Utf8Path,
     ) -> anyhow::Result<(GeneratedFiles, ErrorAccumulator)> {
         let mut errors = ErrorAccumulator::new();
         match self {
@@ -277,8 +283,12 @@ impl GeneratedSource {
                     "Loading".style(styles.success_header),
                     local_directory,
                 );
-                let api_files =
-                    walk_local_directory(local_directory, apis, &mut errors)?;
+                let api_files = walk_local_directory(
+                    local_directory,
+                    apis,
+                    &mut errors,
+                    repo_root,
+                )?;
                 Ok((GeneratedFiles::from(api_files), errors))
             }
         }
@@ -299,11 +309,14 @@ pub enum LocalSource {
 }
 
 impl LocalSource {
-    /// Load the local OpenAPI documents
+    /// Load the local OpenAPI documents.
+    ///
+    /// The `repo_root` parameter is needed to resolve `.gitref` files.
     pub fn load(
         &self,
         apis: &ManagedApis,
         styles: &Styles,
+        repo_root: &Utf8Path,
     ) -> anyhow::Result<(LocalFiles, ErrorAccumulator)> {
         let mut errors = ErrorAccumulator::new();
         match self {
@@ -319,6 +332,7 @@ impl LocalSource {
                         abs_dir,
                         apis,
                         &mut errors,
+                        repo_root,
                     )?,
                     errors,
                 ))