diff --git a/src/main/java/org/jenkinsci/plugins/github_branch_source/GitHubAppCredentials.java b/src/main/java/org/jenkinsci/plugins/github_branch_source/GitHubAppCredentials.java
index 843178454..87e0a855a 100644
--- a/src/main/java/org/jenkinsci/plugins/github_branch_source/GitHubAppCredentials.java
+++ b/src/main/java/org/jenkinsci/plugins/github_branch_source/GitHubAppCredentials.java
@@ -115,6 +115,7 @@ public void setOwner(String owner) {
 
     @SuppressWarnings("deprecation") // preview features are required for GitHub app integration, GitHub api adds deprecated to all preview methods
     static AppInstallationToken generateAppInstallationToken(String appId, String appPrivateKey, String apiUrl, String owner) {
+        JenkinsJVM.checkJenkinsJVM();
         // We expect this to be fast but if anything hangs in here we do not want to block indefinitely
         try (Timeout timeout = Timeout.limit(30, TimeUnit.SECONDS)) {
             String jwtToken = createJWT(appId, appPrivateKey);
@@ -149,7 +150,8 @@ static AppInstallationToken generateAppInstallationToken(String appId, String ap
                 .create();
 
             long expiration = getExpirationSeconds(appInstallationToken);
-            AppInstallationToken token = new AppInstallationToken(appInstallationToken.getToken(),
+            AppInstallationToken token = new AppInstallationToken(
+                Secret.fromString(appInstallationToken.getToken()),
                 expiration);
             LOGGER.log(Level.FINER,
                 "Generated App Installation Token for app ID {0}",
@@ -185,7 +187,6 @@ private static long getExpirationSeconds(GHAppInstallationToken appInstallationT
     @NonNull
     @Override
     public Secret getPassword() {
-        String appInstallationToken;
         synchronized (this) {
             try {
                 if (cachedToken == null || cachedToken.isStale()) {
@@ -208,12 +209,11 @@ public Secret getPassword() {
                     throw e;
                 }
             }
-            appInstallationToken = cachedToken.getToken();
-        }
+            LOGGER.log(Level.FINEST, "Returned GitHub App Installation Token for app ID {0}", appID);
 
-        LOGGER.log(Level.FINEST, "Returned GitHub App Installation Token for app ID {0}", appID);
+            return cachedToken.getToken();
+        }
 
-        return Secret.fromString(appInstallationToken);
     }
 
     /**
@@ -265,7 +265,7 @@ static class AppInstallationToken implements Serializable {
          */
         static long NOT_STALE_MINIMUM_SECONDS = Duration.ofMinutes(1).getSeconds();
 
-        private final String token;
+        private final Secret token;
         private final long expirationEpochSeconds;
         private final long staleEpochSeconds;
 
@@ -281,7 +281,7 @@ static class AppInstallationToken implements Serializable {
          * @param token the token string
          * @param expirationEpochSeconds the time in epoch seconds that this token will expire
          */
-        public AppInstallationToken(String token, long expirationEpochSeconds) {
+        public AppInstallationToken(Secret token, long expirationEpochSeconds) {
             long now = Instant.now().getEpochSecond();
             long minimumAllowedAge = Math.max(1, NOT_STALE_MINIMUM_SECONDS);
             long maximumAllowedAge = Math.max(1, 1 + STALE_AFTER_SECONDS);
@@ -306,7 +306,7 @@ public AppInstallationToken(String token, long expirationEpochSeconds) {
             this.staleEpochSeconds = now + secondsUntilStale;
         }
 
-        public String getToken() {
+        public Secret getToken() {
             return token;
         }
 
@@ -405,7 +405,6 @@ public String getUsername() {
         public Secret getPassword() {
             JenkinsJVM.checkNotJenkinsJVM();
             try {
-                String appInstallationToken;
                 synchronized (this) {
                     try {
                         if (cachedToken == null || cachedToken.isStale()) {
@@ -427,12 +426,11 @@ public Secret getPassword() {
                             throw e;
                         }
                     }
-                    appInstallationToken = cachedToken.getToken();
-                }
+                    LOGGER.log(Level.FINEST, "Returned GitHub App Installation Token for app ID {0} on agent", appID);
 
-                LOGGER.log(Level.FINEST, "Returned GitHub App Installation Token for app ID {0} on agent", appID);
+                    return cachedToken.getToken();
+                }
 
-                return Secret.fromString(appInstallationToken);
             } catch (IOException | InterruptedException x) {
                 throw new RuntimeException(x);
             }
diff --git a/src/test/java/org/jenkinsci/plugins/github_branch_source/GithubAppCredentialsAppInstallationTokenTest.java b/src/test/java/org/jenkinsci/plugins/github_branch_source/GithubAppCredentialsAppInstallationTokenTest.java
index 2c36704e0..46504c548 100644
--- a/src/test/java/org/jenkinsci/plugins/github_branch_source/GithubAppCredentialsAppInstallationTokenTest.java
+++ b/src/test/java/org/jenkinsci/plugins/github_branch_source/GithubAppCredentialsAppInstallationTokenTest.java
@@ -1,5 +1,6 @@
 package org.jenkinsci.plugins.github_branch_source;
 
+import hudson.util.Secret;
 import org.junit.Test;
 
 import java.time.Duration;
@@ -17,24 +18,25 @@ public void testAppInstallationTokenStale() throws Exception {
         long now;
 
         now = Instant.now().getEpochSecond();
-        token = new GitHubAppCredentials.AppInstallationToken("", now);
+        Secret secret = Secret.fromString("secret-token");
+        token = new GitHubAppCredentials.AppInstallationToken(secret, now);
         assertThat(token.isStale(), is(false));
         assertThat(token.getTokenStaleEpochSeconds(), equalTo(now + GitHubAppCredentials.AppInstallationToken.NOT_STALE_MINIMUM_SECONDS));
 
         now = Instant.now().getEpochSecond();
-        token = new GitHubAppCredentials.AppInstallationToken("",
+        token = new GitHubAppCredentials.AppInstallationToken(secret,
             now + Duration.ofMinutes(15).getSeconds());
         assertThat(token.isStale(), is(false));
         assertThat(token.getTokenStaleEpochSeconds(), equalTo(now + GitHubAppCredentials.AppInstallationToken.NOT_STALE_MINIMUM_SECONDS));
 
         now = Instant.now().getEpochSecond();
-        token = new GitHubAppCredentials.AppInstallationToken("",
+        token = new GitHubAppCredentials.AppInstallationToken(secret,
             now + GitHubAppCredentials.AppInstallationToken.STALE_BEFORE_EXPIRATION_SECONDS + 2);
         assertThat(token.isStale(), is(false));
         assertThat(token.getTokenStaleEpochSeconds(), equalTo(now + GitHubAppCredentials.AppInstallationToken.NOT_STALE_MINIMUM_SECONDS));
 
         now = Instant.now().getEpochSecond();
-        token = new GitHubAppCredentials.AppInstallationToken("",
+        token = new GitHubAppCredentials.AppInstallationToken(secret,
             now + GitHubAppCredentials.AppInstallationToken.STALE_BEFORE_EXPIRATION_SECONDS + Duration
                 .ofMinutes(7)
                 .getSeconds());
@@ -43,7 +45,7 @@ public void testAppInstallationTokenStale() throws Exception {
             equalTo(now + Duration.ofMinutes(7).getSeconds()));
 
         now = Instant.now().getEpochSecond();
-        token = new GitHubAppCredentials.AppInstallationToken("",
+        token = new GitHubAppCredentials.AppInstallationToken(secret,
             now + Duration.ofMinutes(90).getSeconds());
         assertThat(token.isStale(), is(false));
         assertThat(token.getTokenStaleEpochSeconds(),
@@ -55,7 +57,7 @@ public void testAppInstallationTokenStale() throws Exception {
             GitHubAppCredentials.AppInstallationToken.NOT_STALE_MINIMUM_SECONDS = -10;
 
             now = Instant.now().getEpochSecond();
-            token = new GitHubAppCredentials.AppInstallationToken("", now);
+            token = new GitHubAppCredentials.AppInstallationToken(secret, now);
             assertThat(token.isStale(), is(false));
             assertThat(token.getTokenStaleEpochSeconds(), equalTo(now + 1));