diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs index ded4320cba8c..6509e1b1cd1c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs @@ -59,7 +59,7 @@ public override void Populate(TextWriter trapFile) { // In this case, we don't extract the attribute again, as it was extracted using * ID // originally and we re-use that. - if (Context.OnlyScaffold && (ReportingLocation is null || !ReportingLocation.IsInSource)) + if (OnlyScaffold && (ReportingLocation is null || !ReportingLocation.IsInSource)) { return; } @@ -67,7 +67,7 @@ public override void Populate(TextWriter trapFile) var type = Type.Create(Context, Symbol.AttributeClass); trapFile.attributes(this, kind, type.TypeRef, entity); - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntity.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntity.cs index 2002fe0f1d7a..0196de3d2c71 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntity.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntity.cs @@ -25,6 +25,8 @@ protected CachedEntity(Context context) : base(context) public abstract void Populate(TextWriter trapFile); public abstract bool NeedsPopulation { get; } + + public abstract bool OnlyScaffold { get; } } /// @@ -70,8 +72,6 @@ protected static void WriteLocationsToTrap(Action writeAction, } } - public override bool NeedsPopulation { get; } - public override int GetHashCode() => Symbol is null ? 0 : Symbol.GetHashCode(); public override bool Equals(object? obj) @@ -81,6 +81,8 @@ public override bool Equals(object? obj) } public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel; + + public override bool OnlyScaffold => Context.OnlyScaffold; } /// diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedSymbol.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedSymbol.cs index 92861e97fdd8..a9dcc21d9119 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedSymbol.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedSymbol.cs @@ -140,7 +140,9 @@ public ExpressionSyntax? ExpressionBody public virtual bool IsSourceDeclaration => Symbol.IsSourceDeclaration(); - public override bool NeedsPopulation => Context.Defines(Symbol); + // When extracting in overlay mode we always need to populate to ensure that + // all transitive dependencies are extracted. + //public override bool NeedsPopulation => Context.Defines(Symbol) || Context.IsOverlayMode; public Location Location => Context.CreateLocation(ReportingLocation); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs index af579a47dc59..5494c86d3365 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs @@ -10,7 +10,7 @@ private CommentBlock(Context cx, Comments.CommentBlock init) public override void Populate(TextWriter trapFile) { - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } @@ -31,7 +31,7 @@ public override void WriteId(EscapingTextWriter trapFile) public void BindTo(Label entity, CommentBinding binding) { - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs index f7db5dbe2942..436b919c405b 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs @@ -21,7 +21,7 @@ private CommentLine(Context cx, Microsoft.CodeAnalysis.Location loc, CommentLine public override void Populate(TextWriter trapFile) { - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs index 2c3b25b2e1c4..475b903f2fdf 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs @@ -37,7 +37,7 @@ public override void Populate(TextWriter trapFile) } ExtractCompilerGenerated(trapFile); - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } @@ -60,7 +60,7 @@ protected override void ExtractInitializers(TextWriter trapFile) // Do not extract initializers for constructed types. // Extract initializers for constructors with a body, primary constructors // and default constructors for classes and structs declared in source code. - if (Block is null && ExpressionBody is null && !MakeSynthetic || Context.OnlyScaffold) + if (Block is null && ExpressionBody is null && !MakeSynthetic || OnlyScaffold) { return; } @@ -187,7 +187,7 @@ Symbol.ContainingType.TypeKind is TypeKind.Class or TypeKind.Struct && /// private bool IsBestSourceLocation => ReportingLocation is not null && Context.IsLocationInContext(ReportingLocation); - private bool MakeSynthetic => (IsPrimary || (IsDefault && IsBestSourceLocation)) && !Context.OnlyScaffold; + private bool MakeSynthetic => (IsPrimary || (IsDefault && IsBestSourceLocation)) && !OnlyScaffold; [return: NotNullIfNotNull(nameof(constructor))] public static new Constructor? Create(Context cx, IMethodSymbol? constructor) diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs index bbd90989617c..fb6e2f8087bc 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs @@ -20,6 +20,10 @@ public override void WriteId(EscapingTextWriter trapFile) trapFile.Write(";event"); } + public override bool NeedsPopulation => Context.Defines(Symbol) || Context.IsOverlayMode; + + public override bool OnlyScaffold => base.OnlyScaffold || !Context.Defines(Symbol); + public override void Populate(TextWriter trapFile) { PopulateNullability(trapFile, Symbol.GetAnnotatedType()); @@ -50,7 +54,7 @@ public override void Populate(TextWriter trapFile) TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier!.Name, this, explicitInterface); } - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs index 9a010aad3760..d9c130172003 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs @@ -21,7 +21,7 @@ private Field(Context cx, IFieldSymbol init) // Do not populate backing fields. // Populate Tuple fields. public override bool NeedsPopulation => - (base.NeedsPopulation && !Symbol.IsImplicitlyDeclared) || Symbol.ContainingType.IsTupleType; + ((Context.Defines(Symbol) || Context.IsOverlayMode) && !Symbol.IsImplicitlyDeclared) || Symbol.ContainingType.IsTupleType; public override void Populate(TextWriter trapFile) { @@ -49,7 +49,7 @@ public override void Populate(TextWriter trapFile) } } - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs index 870c2eb76500..d1385b53d7ad 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs @@ -54,7 +54,7 @@ public override void Populate(TextWriter trapFile) TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier!.Name, this, explicitInterface); } - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs index 22174f7e9456..20d37d242612 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs @@ -20,6 +20,9 @@ public sealed override void WriteQuotedId(EscapingTextWriter trapFile) public override void Populate(TextWriter trapFile) { } + // Populated by calling "PopulateManual". + public override bool NeedsPopulation => false; + public void PopulateManual(Expression parent, bool isVar) { var trapFile = Context.TrapWriter.Writer; @@ -41,7 +44,7 @@ public void PopulateManual(Expression parent, bool isVar) trapFile.localvars(this, Kinds.VariableKind.None, Symbol.Name, @var, Type.Create(Context, parent.Type).TypeRef, parent); } - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs index c1b0f1a65bcb..7149c0044261 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs @@ -37,6 +37,10 @@ protected void PopulateParameters() } } + public override bool NeedsPopulation => Context.Defines(Symbol) || Context.IsOverlayMode; + + public override bool OnlyScaffold => base.OnlyScaffold || !Context.Defines(Symbol); + /// /// Extracts constructor initializers. /// diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs index 3eaafdca23bf..8b9a991ee02a 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs @@ -48,7 +48,7 @@ public override void Populate(TextWriter trapFile) trapFile.parent_namespace_declaration(this, parent); } - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs index 22bcd1dce2c8..ee4162bbcce8 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs @@ -39,7 +39,7 @@ public override void Populate(TextWriter trapFile) ExtractRefReturn(trapFile, Symbol, this); ExtractCompilerGenerated(trapFile); - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs index 49ef9a4a6e9a..7b31f8b9859d 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs @@ -115,7 +115,7 @@ public override void Populate(TextWriter trapFile) var type = Type.Create(Context, Symbol.Type); trapFile.@params(this, Name, type.TypeRef, Ordinal, ParamKind, Parent!, Original); - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PreprocessorDirective.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PreprocessorDirective.cs index 4b0da499b804..379f6a5ef057 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PreprocessorDirective.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PreprocessorDirective.cs @@ -17,7 +17,7 @@ public sealed override void Populate(TextWriter trapFile) var compilation = Compilation.Create(Context); trapFile.preprocessor_directive_compilation(this, compilation); - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs index d48d778cb75a..2f42df052d5c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs @@ -36,6 +36,10 @@ public override void WriteId(EscapingTextWriter trapFile) trapFile.Write(";property"); } + public override bool NeedsPopulation => Context.Defines(Symbol) || Context.IsOverlayMode; + + public override bool OnlyScaffold => base.OnlyScaffold || !Context.Defines(Symbol); + public override void Populate(TextWriter trapFile) { PopulateAttributes(); @@ -68,7 +72,7 @@ public override void Populate(TextWriter trapFile) TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier!.Name, this, explicitInterface); } - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs index b1413b206ae8..09487dfef500 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs @@ -19,7 +19,7 @@ public override void Populate(TextWriter trapFile) trapFile.has_modifiers(this, Modifier.Create(Context, "public")); trapFile.parent_namespace(this, Namespace.Create(Context, Context.Compilation.GlobalNamespace)); - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs index dcf2bffe095f..7afac6f37c26 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs @@ -81,7 +81,7 @@ public override void Populate(TextWriter trapFile) } // Class location - if ((!Symbol.IsGenericType || Symbol.IsReallyUnbound()) && !Context.OnlyScaffold) + if ((!Symbol.IsGenericType || Symbol.IsReallyUnbound()) && !OnlyScaffold) { WriteLocationsToTrap(trapFile.type_location, this, Locations); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs index f20be262e30d..e7609a26ace2 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs @@ -51,7 +51,7 @@ public override void Populate(TextWriter trapFile) trapFile.tuple_element(this, index++, element); } - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs index 3e79a8f81018..4cdd85682aab 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs @@ -17,7 +17,7 @@ protected Type(Context cx, ITypeSymbol? init) #nullable restore warnings public override bool NeedsPopulation => - base.NeedsPopulation || Symbol.TypeKind == TypeKind.Dynamic || Symbol.TypeKind == TypeKind.TypeParameter; + Context.Defines(Symbol) || Context.IsOverlayMode || Symbol.TypeKind == TypeKind.Dynamic || Symbol.TypeKind == TypeKind.TypeParameter; public static bool ConstructedOrParentIsConstructed(INamedTypeSymbol symbol) { @@ -222,7 +222,7 @@ protected void PopulateType(TextWriter trapFile, bool constructUnderlyingTupleTy private IEnumerable GetBaseTypeDeclarations() { - if (!IsSourceDeclaration || !Symbol.FromSource() || Context.OnlyScaffold) + if (!IsSourceDeclaration || !Symbol.FromSource() || OnlyScaffold) { return Enumerable.Empty(); } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs index 7d117dcd427a..2f174a66d46c 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs @@ -26,7 +26,7 @@ public override void Populate(TextWriter trapFile) var parentNs = Namespace.Create(Context, Symbol.TypeParameterKind == TypeParameterKind.Method ? Context.Compilation.GlobalNamespace : Symbol.ContainingNamespace); trapFile.parent_namespace(this, parentNs); - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs index fc9358ffc2dc..0621925c79ae 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs @@ -29,7 +29,7 @@ public override void Populate(TextWriter trapFile) ContainingType.PopulateGenerics(); Overrides(trapFile); - if (Context.OnlyScaffold) + if (OnlyScaffold) { return; } @@ -49,7 +49,7 @@ public override void Populate(TextWriter trapFile) } } - public override bool NeedsPopulation => Context.Defines(Symbol) || IsImplicitOperator(out _); + public override bool NeedsPopulation => base.NeedsPopulation || IsImplicitOperator(out _); public override Type ContainingType { diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs index 1ba8827c9d29..5a42b30a63a5 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs @@ -78,6 +78,14 @@ public void AnalyseTree(SyntaxTree tree) /// public void AnalyseReferences() { + // Only source files can be considered changed in overlay mode. + // The transitive dependencies to references are extracted, when the + // changed files are extracted. + if (OverlayInfo.IsOverlayMode) + { + return; + } + foreach (var assembly in compilation.References.OfType()) { extractionTasks.Add(() => DoAnalyseReferenceAssembly(assembly)); @@ -197,9 +205,18 @@ private void DoExtractTree(SyntaxTree tree) using var trapWriter = transformedSourcePath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false); var currentTaskId = IncrementTaskCount(); - ReportProgressTaskStarted(currentTaskId, sourcePath); var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), OverlayInfo, addAssemblyTrapPrefix); + + // If the file is not changed, the transitive dependency extraction is handled + // when the changed file is extracted. + if (cx.OnlyScaffold) + { + return; + } + + ReportProgressTaskStarted(currentTaskId, sourcePath); + // Ensure that the file itself is populated in case the source file is totally empty var root = tree.GetRoot(); Entities.File.Create(cx, root.SyntaxTree.FilePath); diff --git a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs index c37521652046..5edfdb4333c1 100644 --- a/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs +++ b/csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs @@ -30,11 +30,18 @@ public class Context public bool ShouldAddAssemblyTrapPrefix { get; } /// - /// Holds if trap only should be created for types and member signatures (and not for expressions and statements). + /// Holds if TRAP only should be created for types and member signatures (and not for expressions and statements). /// This is the case for all unchanged files, when running in overlay mode. /// public bool OnlyScaffold { get; } + /// + /// Holds if the extractor is running in overlay mode. + /// + public bool IsOverlayMode { get; } + + public bool OnlyScaffoldSymbol(ISymbol symbol) => OnlyScaffold || !Defines(symbol); + public IList TrapStackSuffix { get; } = new List(); private int GetNewId() => TrapWriter.IdCounter++; @@ -536,6 +543,7 @@ public Context(ExtractionContext extractionContext, Compilation c, TrapWriter tr ShouldAddAssemblyTrapPrefix = shouldAddAssemblyTrapPrefix; Compilation = c; this.scope = scope; + IsOverlayMode = overlayInfo.IsOverlayMode; OnlyScaffold = overlayInfo.IsOverlayMode && ( IsAssemblyScope || (scope is SourceScope ss && overlayInfo.OnlyMakeScaffold(ss.SourceTree.FilePath)));