C#: Narrow scaffolding to (transitive) dependencies.

csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs

@@ -59,15 +59,15 @@ public override void Populate(TextWriter trapFile)
         {
             // In this case, we don't extract the attribute again, as it was extracted using * ID
             // originally and we re-use that.
-            if (Context.OnlyScaffold && (ReportingLocation is null || !ReportingLocation.IsInSource))
+            if (OnlyScaffold && (ReportingLocation is null || !ReportingLocation.IsInSource))
             {
                 return;
             }
 
             var type = Type.Create(Context, Symbol.AttributeClass);
             trapFile.attributes(this, kind, type.TypeRef, entity);
 
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedEntity.cs

@@ -25,6 +25,8 @@ protected CachedEntity(Context context) : base(context)
         public abstract void Populate(TextWriter trapFile);
 
         public abstract bool NeedsPopulation { get; }
+
+        public abstract bool OnlyScaffold { get; }
     }
 
     /// <summary>
@@ -70,8 +72,6 @@ protected static void WriteLocationsToTrap<T1>(Action<T1, Location> writeAction,
             }
         }
 
-        public override bool NeedsPopulation { get; }
-
         public override int GetHashCode() => Symbol is null ? 0 : Symbol.GetHashCode();
 
         public override bool Equals(object? obj)
@@ -81,6 +81,8 @@ public override bool Equals(object? obj)
         }
 
         public override TrapStackBehaviour TrapStackBehaviour => TrapStackBehaviour.NoLabel;
+
+        public override bool OnlyScaffold => Context.OnlyScaffold;
     }
 
     /// <summary>

csharp/extractor/Semmle.Extraction.CSharp/Entities/Base/CachedSymbol.cs

@@ -140,7 +140,9 @@ public ExpressionSyntax? ExpressionBody
 
         public virtual bool IsSourceDeclaration => Symbol.IsSourceDeclaration();
 
-        public override bool NeedsPopulation => Context.Defines(Symbol);
+        // When extracting in overlay mode we always need to populate to ensure that
+        // all transitive dependencies are extracted.
+        //public override bool NeedsPopulation => Context.Defines(Symbol) || Context.IsOverlayMode;
 
         public Location Location => Context.CreateLocation(ReportingLocation);
     }

csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentBlock.cs

@@ -10,7 +10,7 @@ private CommentBlock(Context cx, Comments.CommentBlock init)
 
         public override void Populate(TextWriter trapFile)
         {
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }
@@ -31,7 +31,7 @@ public override void WriteId(EscapingTextWriter trapFile)
 
         public void BindTo(Label entity, CommentBinding binding)
         {
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/CommentLine.cs

@@ -21,7 +21,7 @@ private CommentLine(Context cx, Microsoft.CodeAnalysis.Location loc, CommentLine
 
         public override void Populate(TextWriter trapFile)
         {
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Constructor.cs

@@ -37,7 +37,7 @@ public override void Populate(TextWriter trapFile)
             }
             ExtractCompilerGenerated(trapFile);
 
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }
@@ -60,7 +60,7 @@ protected override void ExtractInitializers(TextWriter trapFile)
             // Do not extract initializers for constructed types.
             // Extract initializers for constructors with a body, primary constructors
             // and default constructors for classes and structs declared in source code.
-            if (Block is null && ExpressionBody is null && !MakeSynthetic || Context.OnlyScaffold)
+            if (Block is null && ExpressionBody is null && !MakeSynthetic || OnlyScaffold)
             {
                 return;
             }
@@ -187,7 +187,7 @@ Symbol.ContainingType.TypeKind is TypeKind.Class or TypeKind.Struct &&
         /// </summary>
         private bool IsBestSourceLocation => ReportingLocation is not null && Context.IsLocationInContext(ReportingLocation);
 
-        private bool MakeSynthetic => (IsPrimary || (IsDefault && IsBestSourceLocation)) && !Context.OnlyScaffold;
+        private bool MakeSynthetic => (IsPrimary || (IsDefault && IsBestSourceLocation)) && !OnlyScaffold;
 
         [return: NotNullIfNotNull(nameof(constructor))]
         public static new Constructor? Create(Context cx, IMethodSymbol? constructor)

csharp/extractor/Semmle.Extraction.CSharp/Entities/Event.cs

@@ -20,6 +20,10 @@ public override void WriteId(EscapingTextWriter trapFile)
             trapFile.Write(";event");
         }
 
+        public override bool NeedsPopulation => Context.Defines(Symbol) || Context.IsOverlayMode;
+
+        public override bool OnlyScaffold => base.OnlyScaffold || !Context.Defines(Symbol);
+
         public override void Populate(TextWriter trapFile)
         {
             PopulateNullability(trapFile, Symbol.GetAnnotatedType());
@@ -50,7 +54,7 @@ public override void Populate(TextWriter trapFile)
                     TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier!.Name, this, explicitInterface);
             }
 
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Field.cs

@@ -21,7 +21,7 @@ private Field(Context cx, IFieldSymbol init)
         // Do not populate backing fields.
         // Populate Tuple fields.
         public override bool NeedsPopulation =>
-            (base.NeedsPopulation && !Symbol.IsImplicitlyDeclared) || Symbol.ContainingType.IsTupleType;
+            ((Context.Defines(Symbol) || Context.IsOverlayMode) && !Symbol.IsImplicitlyDeclared) || Symbol.ContainingType.IsTupleType;
 
         public override void Populate(TextWriter trapFile)
         {
@@ -49,7 +49,7 @@ public override void Populate(TextWriter trapFile)
                 }
             }
 
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Indexer.cs

@@ -54,7 +54,7 @@ public override void Populate(TextWriter trapFile)
                     TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier!.Name, this, explicitInterface);
             }
 
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/LocalVariable.cs

@@ -20,6 +20,9 @@ public sealed override void WriteQuotedId(EscapingTextWriter trapFile)
 
         public override void Populate(TextWriter trapFile) { }
 
+        // Populated by calling "PopulateManual".
+        public override bool NeedsPopulation => false;
+
         public void PopulateManual(Expression parent, bool isVar)
         {
             var trapFile = Context.TrapWriter.Writer;
@@ -41,7 +44,7 @@ public void PopulateManual(Expression parent, bool isVar)
                 trapFile.localvars(this, Kinds.VariableKind.None, Symbol.Name, @var, Type.Create(Context, parent.Type).TypeRef, parent);
             }
 
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Method.cs

@@ -37,6 +37,10 @@ protected void PopulateParameters()
             }
         }
 
+        public override bool NeedsPopulation => Context.Defines(Symbol) || Context.IsOverlayMode;
+
+        public override bool OnlyScaffold => base.OnlyScaffold || !Context.Defines(Symbol);
+
         /// <summary>
         /// Extracts constructor initializers.
         /// </summary>

csharp/extractor/Semmle.Extraction.CSharp/Entities/NamespaceDeclaration.cs

@@ -48,7 +48,7 @@ public override void Populate(TextWriter trapFile)
                 trapFile.parent_namespace_declaration(this, parent);
             }
 
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/OrdinaryMethod.cs

@@ -39,7 +39,7 @@ public override void Populate(TextWriter trapFile)
             ExtractRefReturn(trapFile, Symbol, this);
             ExtractCompilerGenerated(trapFile);
 
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Parameter.cs

@@ -115,7 +115,7 @@ public override void Populate(TextWriter trapFile)
             var type = Type.Create(Context, Symbol.Type);
             trapFile.@params(this, Name, type.TypeRef, Ordinal, ParamKind, Parent!, Original);
 
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PreprocessorDirective.cs

@@ -17,7 +17,7 @@ public sealed override void Populate(TextWriter trapFile)
             var compilation = Compilation.Create(Context);
             trapFile.preprocessor_directive_compilation(this, compilation);
 
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Property.cs

@@ -36,6 +36,10 @@ public override void WriteId(EscapingTextWriter trapFile)
             trapFile.Write(";property");
         }
 
+        public override bool NeedsPopulation => Context.Defines(Symbol) || Context.IsOverlayMode;
+
+        public override bool OnlyScaffold => base.OnlyScaffold || !Context.Defines(Symbol);
+
         public override void Populate(TextWriter trapFile)
         {
             PopulateAttributes();
@@ -68,7 +72,7 @@ public override void Populate(TextWriter trapFile)
                     TypeMention.Create(Context, syntax.ExplicitInterfaceSpecifier!.Name, this, explicitInterface);
             }
 
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/DynamicType.cs

@@ -19,7 +19,7 @@ public override void Populate(TextWriter trapFile)
 
             trapFile.has_modifiers(this, Modifier.Create(Context, "public"));
             trapFile.parent_namespace(this, Namespace.Create(Context, Context.Compilation.GlobalNamespace));
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs

@@ -81,7 +81,7 @@ public override void Populate(TextWriter trapFile)
             }
 
             // Class location
-            if ((!Symbol.IsGenericType || Symbol.IsReallyUnbound()) && !Context.OnlyScaffold)
+            if ((!Symbol.IsGenericType || Symbol.IsReallyUnbound()) && !OnlyScaffold)
             {
                 WriteLocationsToTrap(trapFile.type_location, this, Locations);
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TupleType.cs

@@ -51,7 +51,7 @@ public override void Populate(TextWriter trapFile)
                     trapFile.tuple_element(this, index++, element);
             }
 
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/Type.cs

@@ -17,7 +17,7 @@ protected Type(Context cx, ITypeSymbol? init)
 #nullable restore warnings
 
         public override bool NeedsPopulation =>
-            base.NeedsPopulation || Symbol.TypeKind == TypeKind.Dynamic || Symbol.TypeKind == TypeKind.TypeParameter;
+            Context.Defines(Symbol) || Context.IsOverlayMode || Symbol.TypeKind == TypeKind.Dynamic || Symbol.TypeKind == TypeKind.TypeParameter;
 
         public static bool ConstructedOrParentIsConstructed(INamedTypeSymbol symbol)
         {
@@ -222,7 +222,7 @@ protected void PopulateType(TextWriter trapFile, bool constructUnderlyingTupleTy
 
         private IEnumerable<BaseTypeSyntax> GetBaseTypeDeclarations()
         {
-            if (!IsSourceDeclaration || !Symbol.FromSource() || Context.OnlyScaffold)
+            if (!IsSourceDeclaration || !Symbol.FromSource() || OnlyScaffold)
             {
                 return Enumerable.Empty<BaseTypeSyntax>();
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/TypeParameter.cs

@@ -26,7 +26,7 @@ public override void Populate(TextWriter trapFile)
             var parentNs = Namespace.Create(Context, Symbol.TypeParameterKind == TypeParameterKind.Method ? Context.Compilation.GlobalNamespace : Symbol.ContainingNamespace);
             trapFile.parent_namespace(this, parentNs);
 
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }

csharp/extractor/Semmle.Extraction.CSharp/Entities/UserOperator.cs

@@ -29,7 +29,7 @@ public override void Populate(TextWriter trapFile)
             ContainingType.PopulateGenerics();
             Overrides(trapFile);
 
-            if (Context.OnlyScaffold)
+            if (OnlyScaffold)
             {
                 return;
             }
@@ -49,7 +49,7 @@ public override void Populate(TextWriter trapFile)
             }
         }
 
-        public override bool NeedsPopulation => Context.Defines(Symbol) || IsImplicitOperator(out _);
+        public override bool NeedsPopulation => base.NeedsPopulation || IsImplicitOperator(out _);
 
         public override Type ContainingType
         {

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs

@@ -78,6 +78,14 @@ public void AnalyseTree(SyntaxTree tree)
         /// </summary>
         public void AnalyseReferences()
         {
+            // Only source files can be considered changed in overlay mode.
+            // The transitive dependencies to references are extracted, when the
+            // changed files are extracted.
+            if (OverlayInfo.IsOverlayMode)
+            {
+                return;
+            }
+
             foreach (var assembly in compilation.References.OfType<PortableExecutableReference>())
             {
                 extractionTasks.Add(() => DoAnalyseReferenceAssembly(assembly));
@@ -197,9 +205,18 @@ private void DoExtractTree(SyntaxTree tree)
                 using var trapWriter = transformedSourcePath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
 
                 var currentTaskId = IncrementTaskCount();
-                ReportProgressTaskStarted(currentTaskId, sourcePath);
 
                 var cx = new Context(ExtractionContext, compilation, trapWriter, new SourceScope(tree), OverlayInfo, addAssemblyTrapPrefix);
+
+                // If the file is not changed, the transitive dependency extraction is handled
+                // when the changed file is extracted.
+                if (cx.OnlyScaffold)
+                {
+                    return;
+                }
+
+                ReportProgressTaskStarted(currentTaskId, sourcePath);
+
                 // Ensure that the file itself is populated in case the source file is totally empty
                 var root = tree.GetRoot();
                 Entities.File.Create(cx, root.SyntaxTree.FilePath);

csharp/extractor/Semmle.Extraction.CSharp/Extractor/Context.cs

@@ -30,11 +30,18 @@ public class Context
         public bool ShouldAddAssemblyTrapPrefix { get; }
 
         /// <summary>
-        /// Holds if trap only should be created for types and member signatures (and not for expressions and statements).
+        /// Holds if TRAP only should be created for types and member signatures (and not for expressions and statements).
         /// This is the case for all unchanged files, when running in overlay mode.
         /// </summary>
         public bool OnlyScaffold { get; }
 
+        /// <summary>
+        /// Holds if the extractor is running in overlay mode.
+        /// </summary>
+        public bool IsOverlayMode { get; }
+
+        public bool OnlyScaffoldSymbol(ISymbol symbol) => OnlyScaffold || !Defines(symbol);
+
         public IList<object> TrapStackSuffix { get; } = new List<object>();
 
         private int GetNewId() => TrapWriter.IdCounter++;
@@ -536,6 +543,7 @@ public Context(ExtractionContext extractionContext, Compilation c, TrapWriter tr
             ShouldAddAssemblyTrapPrefix = shouldAddAssemblyTrapPrefix;
             Compilation = c;
             this.scope = scope;
+            IsOverlayMode = overlayInfo.IsOverlayMode;
             OnlyScaffold = overlayInfo.IsOverlayMode && (
                 IsAssemblyScope
                 || (scope is SourceScope ss && overlayInfo.OnlyMakeScaffold(ss.SourceTree.FilePath)));