From 08d62cab7439bb225ec5194bcd8b01bad3017885 Mon Sep 17 00:00:00 2001 From: spatten Date: Wed, 3 Sep 2025 18:00:30 -0700 Subject: [PATCH 1/8] add the setting to Organization --- src/App/Fossa/Analyze.hs | 5 ++++- src/Fossa/API/Types.hs | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/App/Fossa/Analyze.hs b/src/App/Fossa/Analyze.hs index 04138b5f09..fe42eae3b7 100644 --- a/src/App/Fossa/Analyze.hs +++ b/src/App/Fossa/Analyze.hs @@ -124,7 +124,7 @@ import Effect.Logger ( ) import Effect.ReadFS (ReadFS) import Errata (Errata (..)) -import Fossa.API.Types (Organization (Organization, orgSupportsReachability)) +import Fossa.API.Types (Organization (Organization, orgSnippetScanSourceCodeRetentionDays, orgSupportsReachability)) import Path (Abs, Dir, Path, toFilePath) import Path.IO (makeRelative) import Prettyprinter ( @@ -315,6 +315,9 @@ analyze cfg = Diag.context "fossa-analyze" $ do runFossaApiClient apiOpts . preflightChecks $ AnalyzeChecks revision metadata ) + -- TODO: just pass this into ficus. No need to log it here + let snippetScanSourceCodeRetentionDays = orgSnippetScanSourceCodeRetentionDays =<< orgInfo + logDebug $ "Snippet scan source code retention days: " <> pretty (show snippetScanSourceCodeRetentionDays) -- additional source units are built outside the standard strategy flow, because they either -- require additional information (eg API credentials), or they return additional information (eg user deps). vsiResults <- Diag.errorBoundaryIO . diagToDebug $ diff --git a/src/Fossa/API/Types.hs b/src/Fossa/API/Types.hs index d45089d39a..a00819a297 100644 --- a/src/Fossa/API/Types.hs +++ b/src/Fossa/API/Types.hs @@ -509,6 +509,7 @@ data Organization = Organization , orgSupportsReachability :: Bool , orgSupportsPreflightChecks :: Bool , orgSubscription :: Subscription + , orgSnippetScanSourceCodeRetentionDays :: Maybe Int } deriving (Eq, Ord, Show) @@ -535,6 +536,7 @@ blankOrganization = , orgSupportsReachability = False , orgSupportsPreflightChecks = False , orgSubscription = Free + , orgSnippetScanSourceCodeRetentionDays = Nothing } instance FromJSON Organization where @@ -556,6 +558,7 @@ instance FromJSON Organization where <*> obj .:? "supportsReachability" .!= False <*> obj .:? "supportsPreflightChecks" .!= False <*> obj .:? "subscription" .!= Free + <*> obj .:? "snippetScanSourceCodeRetentionDays" .!= Nothing data TokenType = Push From a85311bd49a1d4a70559fcf1adab4139cb67125b Mon Sep 17 00:00:00 2001 From: spatten Date: Thu, 4 Sep 2025 15:02:50 -0700 Subject: [PATCH 2/8] pass snippet-scan-retention-days to Ficus if it exists --- src/App/Fossa/Analyze.hs | 2 +- src/App/Fossa/Ficus/Analyze.hs | 12 ++++++++---- src/App/Fossa/Ficus/Types.hs | 1 + 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/App/Fossa/Analyze.hs b/src/App/Fossa/Analyze.hs index fe42eae3b7..bfe01cd292 100644 --- a/src/App/Fossa/Analyze.hs +++ b/src/App/Fossa/Analyze.hs @@ -351,7 +351,7 @@ analyze cfg = Diag.context "fossa-analyze" $ do then do logInfo "Running in VSI only mode, skipping snippet-scan" pure Nothing - else Diag.context "snippet-scanning" . runStickyLogger SevInfo $ analyzeWithFicus basedir maybeApiOpts revision $ Config.licenseScanPathFilters vendoredDepsOptions + else Diag.context "snippet-scanning" . runStickyLogger SevInfo $ analyzeWithFicus basedir maybeApiOpts revision (Config.licenseScanPathFilters vendoredDepsOptions) snippetScanSourceCodeRetentionDays let ficusResults = join . resultToMaybe $ maybeFicusResults maybeLernieResults <- Diag.errorBoundaryIO . diagToDebug $ diff --git a/src/App/Fossa/Ficus/Analyze.hs b/src/App/Fossa/Ficus/Analyze.hs index 634e060584..40074542b5 100644 --- a/src/App/Fossa/Ficus/Analyze.hs +++ b/src/App/Fossa/Ficus/Analyze.hs @@ -84,9 +84,10 @@ analyzeWithFicus :: Maybe ApiOpts -> ProjectRevision -> Maybe LicenseScanPathFilters -> + Maybe Int -> m (Maybe FicusSnippetScanResults) -analyzeWithFicus rootDir apiOpts revision filters = do - analyzeWithFicusMain rootDir apiOpts revision filters +analyzeWithFicus rootDir apiOpts revision filters snippetScanRetentionDays = do + analyzeWithFicusMain rootDir apiOpts revision filters snippetScanRetentionDays analyzeWithFicusMain :: ( Has Diagnostics sig m @@ -97,8 +98,9 @@ analyzeWithFicusMain :: Maybe ApiOpts -> ProjectRevision -> Maybe LicenseScanPathFilters -> + Maybe Int -> m (Maybe FicusSnippetScanResults) -analyzeWithFicusMain rootDir apiOpts revision filters = do +analyzeWithFicusMain rootDir apiOpts revision filters snippetScanRetentionDays = do logDebugWithTime "Preparing Ficus analysis configuration..." messages <- runFicus ficusConfig logDebugWithTime "runFicus completed, processing results..." @@ -118,6 +120,7 @@ analyzeWithFicusMain rootDir apiOpts revision filters = do , ficusConfigSecret = apiOptsApiKey <$> apiOpts , ficusConfigRevision = revision , ficusConfigFlags = [All $ FicusAllFlag SkipHiddenFiles, All $ FicusAllFlag Gitignore] + , ficusConfigSnippetScanRetentionDays = snippetScanRetentionDays } ficusMessagesToFicusSnippetScanResults :: FicusMessages -> Maybe FicusSnippetScanResults @@ -250,7 +253,8 @@ ficusCommand ficusConfig bin = do logDebug $ "Ficus command: " <> pretty (maskApiKeyInCommand $ renderCommand cmd) pure cmd where - configArgs endpoint = ["analyze", "--secret", secret, "--endpoint", endpoint, "--locator", locator, "--set", "all:skip-hidden-files", "--set", "all:gitignore", "--exclude", ".git", "--exclude", ".git/**"] ++ configExcludes ++ [targetDir] + snippetScanRetentionDays = ficusConfigSnippetScanRetentionDays ficusConfig + configArgs endpoint = ["analyze", "--secret", secret, "--endpoint", endpoint, "--locator", locator, "--set", "all:skip-hidden-files", "--set", "all:gitignore", "--exclude", ".git", "--exclude", ".git/**"] ++ configExcludes ++ [targetDir] ++ maybe [] (\days -> ["--snippet-scan-retention-days", toText days]) snippetScanRetentionDays targetDir = toText $ toFilePath $ ficusConfigRootDir ficusConfig secret = maybe "" (toText . unApiKey) $ ficusConfigSecret ficusConfig locator = renderLocator $ Locator "custom" (projectName $ ficusConfigRevision ficusConfig) (Just $ projectRevision $ ficusConfigRevision ficusConfig) diff --git a/src/App/Fossa/Ficus/Types.hs b/src/App/Fossa/Ficus/Types.hs index 4379d3f82f..2250b665db 100644 --- a/src/App/Fossa/Ficus/Types.hs +++ b/src/App/Fossa/Ficus/Types.hs @@ -140,6 +140,7 @@ data FicusConfig = FicusConfig , ficusConfigSecret :: Maybe ApiKey , ficusConfigRevision :: ProjectRevision -- TODO: get this from `projectRevision AnalyzeConfig` , ficusConfigFlags :: [FicusPerStrategyFlag] + , ficusConfigSnippetScanRetentionDays :: Maybe Int } deriving (Show, Eq, Generic) From 0962898d98d7c33141ddb044b56b77f98005bba1 Mon Sep 17 00:00:00 2001 From: spatten Date: Thu, 4 Sep 2025 15:36:42 -0700 Subject: [PATCH 3/8] fix test fixtures --- integration-test/Analysis/FicusSpec.hs | 2 +- test/App/Fossa/API/BuildLinkSpec.hs | 8 ++++---- test/Test/Fixtures.hs | 3 +++ 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/integration-test/Analysis/FicusSpec.hs b/integration-test/Analysis/FicusSpec.hs index a30a387a2c..e124bdad93 100644 --- a/integration-test/Analysis/FicusSpec.hs +++ b/integration-test/Analysis/FicusSpec.hs @@ -51,7 +51,7 @@ spec = do testDataExists <- PIO.doesDirExist testDataDir testDataExists `shouldBe` True - result <- runStack . runDiagnostics . ignoreStickyLogger . ignoreLogger . runExecIO . runReadFSIO $ analyzeWithFicus testDataDir apiOpts revision Nothing + result <- runStack . runDiagnostics . ignoreStickyLogger . ignoreLogger . runExecIO . runReadFSIO $ analyzeWithFicus testDataDir apiOpts revision Nothing Nothing case result of Success _warnings analysisResult -> do diff --git a/test/App/Fossa/API/BuildLinkSpec.hs b/test/App/Fossa/API/BuildLinkSpec.hs index d2b6116927..00abcf5a63 100644 --- a/test/App/Fossa/API/BuildLinkSpec.hs +++ b/test/App/Fossa/API/BuildLinkSpec.hs @@ -42,7 +42,7 @@ spec = do describe "SAML URL builder" $ do it' "should render simple locators" $ do let locator = Locator "fetcher123" "project123" $ Just "revision123" - org = Just $ Organization (OrgId 1) True False True CLILicenseScan True True True False False False False [] False False API.Free + org = Just $ Organization (OrgId 1) True False True CLILicenseScan True True True False False False False [] False False API.Free Nothing revision = ProjectRevision "" "not this revision" $ Just "master123" actual <- getBuildURLWithOrg org revision Fixtures.apiOpts locator @@ -50,7 +50,7 @@ spec = do it' "should render git@ locators" $ do let locator = Locator "fetcher@123/abc" "git@github.com/user/repo" $ Just "revision@123/abc" - org = Just $ Organization (OrgId 103) True False True CLILicenseScan True True True False False False False [] False False API.Free + org = Just $ Organization (OrgId 103) True False True CLILicenseScan True True True False False False False [] False False API.Free Nothing revision = ProjectRevision "not this project name" "not this revision" $ Just "weird--branch" actual <- getBuildURLWithOrg org revision Fixtures.apiOpts locator @@ -58,7 +58,7 @@ spec = do it' "should render full url correctly" $ do let locator = Locator "a" "b" $ Just "c" - org = Just $ Organization (OrgId 33) True False True CLILicenseScan True True True False False False False [] False False API.Free + org = Just $ Organization (OrgId 33) True False True CLILicenseScan True True True False False False False [] False False API.Free Nothing revision = ProjectRevision "" "not this revision" $ Just "master" actual <- getBuildURLWithOrg org revision Fixtures.apiOpts locator @@ -75,7 +75,7 @@ spec = do describe "Fossa URL Builder" $ it' "should render from API info" $ do GetApiOpts `returnsOnce` Fixtures.apiOpts - GetOrganization `returnsOnce` Organization (OrgId 1) True False True CLILicenseScan True True True False False False False [] False False API.Free + GetOrganization `returnsOnce` Organization (OrgId 1) True False True CLILicenseScan True True True False False False False [] False False API.Free Nothing let locator = Locator "fetcher123" "project123" $ Just "revision123" revision = ProjectRevision "" "not this revision" $ Just "master123" actual <- getFossaBuildUrl revision locator diff --git a/test/Test/Fixtures.hs b/test/Test/Fixtures.hs index 9995212811..cc6e37c93c 100644 --- a/test/Test/Fixtures.hs +++ b/test/Test/Fixtures.hs @@ -143,6 +143,7 @@ organization = , orgSupportsReachability = False , orgSupportsPreflightChecks = False , orgSubscription = Free + , orgSnippetScanSourceCodeRetentionDays = Nothing } organizationWithPreflightChecks :: API.Organization @@ -164,6 +165,7 @@ organizationWithPreflightChecks = , orgSupportsReachability = False , orgSupportsPreflightChecks = True , orgSubscription = Free + , orgSnippetScanSourceCodeRetentionDays = Nothing } organizationWithPremiumSubscription :: API.Organization @@ -185,6 +187,7 @@ organizationWithPremiumSubscription = , orgSupportsReachability = False , orgSupportsPreflightChecks = True , orgSubscription = Premium + , orgSnippetScanSourceCodeRetentionDays = Nothing } pushToken :: API.TokenTypeResponse From 038f3d4be4633cb15ecb6306b3492e729c8ecd25 Mon Sep 17 00:00:00 2001 From: spatten Date: Fri, 5 Sep 2025 12:20:06 -0700 Subject: [PATCH 4/8] fix order of args in ficus invocation --- src/App/Fossa/Ficus/Analyze.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/App/Fossa/Ficus/Analyze.hs b/src/App/Fossa/Ficus/Analyze.hs index 40074542b5..cc9d8e1e64 100644 --- a/src/App/Fossa/Ficus/Analyze.hs +++ b/src/App/Fossa/Ficus/Analyze.hs @@ -254,7 +254,7 @@ ficusCommand ficusConfig bin = do pure cmd where snippetScanRetentionDays = ficusConfigSnippetScanRetentionDays ficusConfig - configArgs endpoint = ["analyze", "--secret", secret, "--endpoint", endpoint, "--locator", locator, "--set", "all:skip-hidden-files", "--set", "all:gitignore", "--exclude", ".git", "--exclude", ".git/**"] ++ configExcludes ++ [targetDir] ++ maybe [] (\days -> ["--snippet-scan-retention-days", toText days]) snippetScanRetentionDays + configArgs endpoint = ["analyze", "--secret", secret, "--endpoint", endpoint, "--locator", locator, "--set", "all:skip-hidden-files", "--set", "all:gitignore", "--exclude", ".git", "--exclude", ".git/**"] ++ configExcludes ++ maybe [] (\days -> ["--snippet-scan-retention-days", toText days]) snippetScanRetentionDays ++ [targetDir] targetDir = toText $ toFilePath $ ficusConfigRootDir ficusConfig secret = maybe "" (toText . unApiKey) $ ficusConfigSecret ficusConfig locator = renderLocator $ Locator "custom" (projectName $ ficusConfigRevision ficusConfig) (Just $ projectRevision $ ficusConfigRevision ficusConfig) From c27d4f87c34a3753dffcacde3c569e48ac97e978 Mon Sep 17 00:00:00 2001 From: spatten Date: Tue, 9 Sep 2025 16:07:12 -0700 Subject: [PATCH 5/8] remove unnecessary logging --- src/App/Fossa/Analyze.hs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/App/Fossa/Analyze.hs b/src/App/Fossa/Analyze.hs index bfe01cd292..87f72fe28b 100644 --- a/src/App/Fossa/Analyze.hs +++ b/src/App/Fossa/Analyze.hs @@ -315,9 +315,6 @@ analyze cfg = Diag.context "fossa-analyze" $ do runFossaApiClient apiOpts . preflightChecks $ AnalyzeChecks revision metadata ) - -- TODO: just pass this into ficus. No need to log it here - let snippetScanSourceCodeRetentionDays = orgSnippetScanSourceCodeRetentionDays =<< orgInfo - logDebug $ "Snippet scan source code retention days: " <> pretty (show snippetScanSourceCodeRetentionDays) -- additional source units are built outside the standard strategy flow, because they either -- require additional information (eg API credentials), or they return additional information (eg user deps). vsiResults <- Diag.errorBoundaryIO . diagToDebug $ From f912f39c2bc0559de19c1288730b5885996f8582 Mon Sep 17 00:00:00 2001 From: spatten Date: Wed, 10 Sep 2025 11:31:52 -0700 Subject: [PATCH 6/8] fix bug --- src/App/Fossa/Analyze.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/App/Fossa/Analyze.hs b/src/App/Fossa/Analyze.hs index 87f72fe28b..4e55d68e16 100644 --- a/src/App/Fossa/Analyze.hs +++ b/src/App/Fossa/Analyze.hs @@ -348,7 +348,7 @@ analyze cfg = Diag.context "fossa-analyze" $ do then do logInfo "Running in VSI only mode, skipping snippet-scan" pure Nothing - else Diag.context "snippet-scanning" . runStickyLogger SevInfo $ analyzeWithFicus basedir maybeApiOpts revision (Config.licenseScanPathFilters vendoredDepsOptions) snippetScanSourceCodeRetentionDays + else Diag.context "snippet-scanning" . runStickyLogger SevInfo $ analyzeWithFicus basedir maybeApiOpts revision (Config.licenseScanPathFilters vendoredDepsOptions) (orgSnippetScanSourceCodeRetentionDays =<< orgInfo) let ficusResults = join . resultToMaybe $ maybeFicusResults maybeLernieResults <- Diag.errorBoundaryIO . diagToDebug $ From d50771cf6d02a2f91695f577848a94a25043155b Mon Sep 17 00:00:00 2001 From: spatten Date: Wed, 10 Sep 2025 12:44:56 -0700 Subject: [PATCH 7/8] update a test --- integration-test/Analysis/FicusSpec.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-test/Analysis/FicusSpec.hs b/integration-test/Analysis/FicusSpec.hs index e124bdad93..55d177a3fc 100644 --- a/integration-test/Analysis/FicusSpec.hs +++ b/integration-test/Analysis/FicusSpec.hs @@ -51,7 +51,7 @@ spec = do testDataExists <- PIO.doesDirExist testDataDir testDataExists `shouldBe` True - result <- runStack . runDiagnostics . ignoreStickyLogger . ignoreLogger . runExecIO . runReadFSIO $ analyzeWithFicus testDataDir apiOpts revision Nothing Nothing + result <- runStack . runDiagnostics . ignoreStickyLogger . ignoreLogger . runExecIO . runReadFSIO $ analyzeWithFicus testDataDir apiOpts revision Nothing (Just 10) case result of Success _warnings analysisResult -> do From 02d1ffbabfa3f01ff6dbc42fe013e7d0c059d623 Mon Sep 17 00:00:00 2001 From: spatten Date: Thu, 11 Sep 2025 12:02:20 -0700 Subject: [PATCH 8/8] update the changelog --- Changelog.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Changelog.md b/Changelog.md index 7dc7fb0e1e..c0503b5a16 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,4 +1,8 @@ # FOSSA CLI Changelog +## 3.11.9 + +- Get the snippet-scan-retention-days setting from an organization, and pass it into Ficus ([#1586](https://github.com/fossas/fossa-cli/pull/1586)) + ## 3.11.8 - Update latest version of a dependency for `--x-snippet-scan`. This change outputs errors from the snippet scanning CLI so that we can debug more easily ([#1587](https://github.com/fossas/fossa-cli/pull/1587))