fix: scoping

CHANGELOG.md

@@ -1,5 +1,9 @@
 # News
 
+## v1.0.0 - dev
+
+- **(breaking)** Support for proper Julia scoping of variables inside of resumable functions.
+
 ## v0.6.10 - 2024-09-08
 
 - Add `length` support by allowing `@resumable length=ex function [...]`
@@ -39,7 +43,7 @@
   * introduction of `@yieldfrom` to delegate to another resumable function or iterator (similar to [Python's `yield from`](https://peps.python.org/pep-0380/))
   * resumable functions are now allowed to return values, so that `r = @yieldfrom f` also stores the return value of `f` in `r`
 
-* 2023: v0.6.2 
+* 2023: v0.6.2
   * Julia v1.10 compatibility fix
   * resumable functions can now dispatch on types
 

docs/src/manual.md

@@ -49,7 +49,7 @@ DocTestSetup = quote
 
   @resumable function basic_example()
     @yield "Initial call"
-    @yield 
+    @yield
     "Final call"
   end
 end
@@ -58,7 +58,7 @@ end
 ```julia
 @resumable function basic_example()
   @yield "Initial call"
-  @yield 
+  @yield
   "Final call"
 end
 ```
@@ -373,6 +373,7 @@ DocTestSetup = quote
   @resumable function arrays_of_tuples()
     for u in [[(1,2),(3,4)], [(5,6),(7,8)]]
       for i in 1:2
+        local val
         let i=i
           val = [a[i] for a in u]
         end
@@ -413,4 +414,4 @@ DocTestSetup = nothing
 - In a `try` block only top level `@yield` statements are allowed.
 - In a `catch` or a `finally` block a `@yield` statement is not allowed.
 - An anonymous function can not contain a `@yield` statement.
-- If a `FiniteStateMachineIterator` object is used in more than one `for` loop, only the `state` variable is reinitialised. A `@resumable function` that alters its arguments will use the modified values as initial parameters.
+- If a `FiniteStateMachineIterator` object is used in more than one `for` loop, only the `state` variable is reinitialised. A `@resumable function` that alters its arguments will use the modified values as initial parameters.

src/macro.jl

@@ -73,6 +73,7 @@ macro resumable(ex::Expr...)
 
   # The function that executes a step of the finite state machine
   func_def = splitdef(expr)
+  @debug func_def[:body]|>MacroTools.striplines
   rtype = :rtype in keys(func_def) ? func_def[:rtype] : Any
   args, kwargs, arg_dict = get_args(func_def)
   params = ((get_param_name(param) for param in func_def[:whereparams])...,)
@@ -82,7 +83,26 @@ macro resumable(ex::Expr...)
   func_def[:body] = postwalk(transform_arg_yieldfrom, func_def[:body])
   func_def[:body] = postwalk(transform_yieldfrom, func_def[:body])
   func_def[:body] = postwalk(x->transform_for(x, ui8), func_def[:body])
+  @debug func_def[:body]|>MacroTools.striplines
+
+  # Scoping fixes
+
+  # :name is :(fA::A) if it is an overloading call function (fA::A)(...)
+  # ...
+  if func_def[:name] isa Expr
+    func_def[:name].head != :(::) && error("Unrecognized function name: $(func_def[:name].head)")
+    _name = func_def[:name].args[1]
+  else
+    _name = func_def[:name]
+  end
+
+  scope = ScopeTracker(0, __module__, [Dict(i =>i for i in vcat(args, kwargs, [_name], params...))])
+  func_def[:body] = scoping(copy(func_def[:body]), scope)
+  func_def[:body] = postwalk(x->transform_remove_local(x), func_def[:body])
+  @debug func_def[:body]|>MacroTools.striplines
+
   inferfn, slots = get_slots(copy(func_def), arg_dict, __module__)
+  @debug slots
 
   # check if the resumable function is a callable struct instance (a functional) that is referencing itself
   isfunctional = @capture(func_def[:name], functional_::T_) && inexpr(func_def[:body], functional)
@@ -113,7 +133,6 @@ macro resumable(ex::Expr...)
     fsmi._state = 0x00
     fsmi
   end
-
   # the bare/fallback version of the constructor supplies default slot type parameters
   # we only need to define this if there there are actually slot defaults to be filled
   if !isempty(slot_T)
@@ -139,7 +158,6 @@ macro resumable(ex::Expr...)
     end
   )
   @debug type_expr|>MacroTools.striplines
-
   # The "original" function that now is simply a wrapper around the construction of the finite state machine
   call_def = copy(func_def)
   call_def[:rtype] = nothing
@@ -209,7 +227,6 @@ macro resumable(ex::Expr...)
     call_expr = postwalk(x->x==:(ResumableFunctions.typed_fsmi) ? :(ResumableFunctions.typed_fsmi_fallback) : x, call_expr)
   end
   @debug func_expr|>MacroTools.striplines
-
   # The final expression:
   # - the finite state machine struct
   # - the function stepping through the states

src/transforms.jl

@@ -1,3 +1,11 @@
+"""
+Function that removes `local x` expression.
+"""
+function transform_remove_local(ex)
+  ex isa Expr && ex.head === :local && return Expr(:block)
+  return ex
+end
+
 """
 Function that replaces a variable
 """
@@ -67,25 +75,47 @@ end
 
 """
 Function that replaces a `for` loop by a corresponding `while` loop saving explicitly the *iterator* and its *state*.
+
+For loops of the form `for a, b, c; body; end` are denested.
 """
 function transform_for(expr, ui8::BoxedUInt8)
+  (expr isa Expr && expr.head === :for) || return expr
+  # test for simple for a in b expression
+  expr.args[1].head === :(=) && return transform_for_inner(expr, ui8)
+  # must be a complicated iteration
+  expr.args[1].head !== :block && error("Unrecognized for expression: $(expr.args[1])")
+  body = expr.args[2]
+  # denest, starting at the back
+  for a in reverse(expr.args[1].args)
+    body = Expr(:for, a, body)
+    # turn for into while loop
+    body = transform_for_inner(body, ui8)
+  end
+  return body
+end
+
+function transform_for_inner(expr, ui8::BoxedUInt8)
+  # turning for into while loops
   @capture(expr, for element_ in iterator_ body_ end) || return expr
+  localelement = Expr(:local, element)
   ui8.n += one(UInt8)
   next = Symbol("_iteratornext_", ui8.n)
   state = Symbol("_iterstate_", ui8.n)
   iterator_value = Symbol("_iterator_", ui8.n)
   label = Symbol("_iteratorlabel_", ui8.n)
   body = postwalk(x->transform_continue(x, label), :(begin $(body) end))
-  quote
+  res = quote
     $iterator_value = $iterator
     @nosave $next = iterate($iterator_value)
     while $next !== nothing
+      $localelement
       ($element, $state) = $next
       $body
       @label $label
       $next = iterate($iterator_value, $state)
     end
   end
+  res
 end
 
 
@@ -102,7 +132,9 @@ Function that replaces a variable `x` in an expression by `_fsmi.x` where `x` is
 """
 function transform_slots(expr, symbols)
   expr isa Expr || return expr
-  expr.head === :let && return transform_slots_let(expr, symbols)
+  #expr.head === :let && return transform_slots_let(expr, symbols)
+
+  # "Problematic" expressions all have been transformed into Expr(:kw,...)
   for i in 1:length(expr.args)
     expr.head === :kw && i === 1 && continue
     expr.head === Symbol("quote") && continue
@@ -111,29 +143,6 @@ function transform_slots(expr, symbols)
   expr
 end
 
-"""
-Function that handles `let` block
-"""
-function transform_slots_let(expr::Expr, symbols)
-  @capture(expr, let vars_; body_ end)
-  locals = Set{Symbol}()
-  (isa(vars, Expr) && vars.head==:(=))  || error("@resumable currently supports only single variable declarations in let blocks, i.e. only let blocks exactly of the form `let i=j; ...; end`. If you need multiple variables, please submit an issue on the issue tracker and consider contributing a patch.")
-  sym = vars.args[1].args[2].value
-  push!(locals, sym)
-  vars.args[1] = sym
-  body = postwalk(x->transform_let(x, locals), :(begin $(body) end))
-  :(let $vars; $body end)
-end
-
-"""
-Function that replaces a variable `_fsmi.x` in an expression by `x` where `x` is a variable declared in a `let` block.
-"""
-function transform_let(expr, symbols::Set{Symbol})
-  expr isa Expr || return expr
-  expr.head === :. || return expr
-  expr = expr.args[2].value in symbols ? :($(expr.args[2].value)) : expr
-end
-
 """
 Function that replaces a `arg = @yield ret` statement by
 ```julia