Merge pull request #513 from nix-community/fix-github-reload-efficient

fix: update project maps directly in reload services for GitHub and G…

Author: Mic92

buildbot_nix/buildbot_nix/gitea_projects.py

@@ -181,7 +181,9 @@ def create_reload_builder(self, worker_names: list[str]) -> BuilderConfig:
         """Updates the flake an opens a PR for it."""
         factory = util.BuildFactory()
         factory.addStep(
-            ReloadGiteaProjects(self.config, self.config.project_cache_file),
+            ReloadGiteaProjects(
+                self.config, self.config.project_cache_file, backend=self
+            ),
         )
         factory.addStep(
             CreateGiteaProjectHooks(
@@ -259,6 +261,10 @@ def load_projects(self) -> list[GitProject]:
             for repo in repos
         ]
 
+    def update_projects(self, projects: set[str]) -> None:
+        # Replace atomically to avoid mutating a set read elsewhere
+        self.gitea_projects = projects
+
     def are_projects_cached(self) -> bool:
         return self.config.project_cache_file.exists()
 
@@ -371,10 +377,12 @@ def __init__(
         self,
         config: GiteaConfig,
         project_cache_file: Path,
+        backend: GiteaBackend | None = None,
         **kwargs: Any,
     ) -> None:
         self.config = config
         self.project_cache_file = project_cache_file
+        self.backend = backend
         super().__init__(**kwargs)
 
     def run_deferred(self) -> None:
@@ -390,6 +398,11 @@ def run_deferred(self) -> None:
 
         atomic_write_file(self.project_cache_file, model_dump_project_cache(repos))
 
+        # Update the backend's in-memory gitea_projects set directly
+        if self.backend:
+            self.backend.update_projects({repo.full_name for repo in repos})
+            tlog.info(f"Updated backend gitea_projects with {len(repos)} projects")
+
     def run_post(self) -> Any:
         return util.SUCCESS
 

buildbot_nix/buildbot_nix/github_projects.py

@@ -192,10 +192,12 @@ def __init__(
         self,
         project_config: GitHubProjectConfig,
         app_config: GitHubAppInstallationConfig,
+        backend: GithubAppAuthBackend | None = None,
         **kwargs: Any,
     ) -> None:
         self.project_config = project_config
         self.app_config = app_config
+        self.backend = backend
         super().__init__(**kwargs)
 
     def run_deferred(self) -> None:
@@ -240,6 +242,13 @@ def run_deferred(self) -> None:
             self.app_config.project_id_map_name, json.dumps(project_id_map)
         )
 
+        # Update the backend's in-memory data directly
+        if self.backend:
+            self.backend.update_reload_data(installation_token_map, project_id_map)
+            tlog.info(
+                f"Updated backend with {len(installation_token_map)} installations and {len(project_id_map)} projects"
+            )
+
         tlog.info(
             f"Fetched {len(repos)} repositories from {len(installation_token_map.items())} installation tokens."
         )
@@ -373,6 +382,14 @@ def __init__(self, config: GitHubConfig) -> None:
             )
             self.project_id_map = {}
 
+    def update_reload_data(
+        self,
+        installation_tokens: dict[int, InstallationToken],
+        project_map: dict[str, int],
+    ) -> None:
+        self.installation_tokens = installation_tokens
+        self.project_id_map = project_map
+
     def get_general_token(self) -> RepoToken:
         return self.jwt_token
 
@@ -422,6 +439,7 @@ def create_reload_builder_steps(
             ReloadGithubInstallations(
                 project_config=config,
                 app_config=app_config,
+                backend=self,
             ),
             CreateGitHubInstallationHooks(
                 project_config=config,