proc-macro-srv: Reimplement token trees via ropes

Author: Veykril

Cargo.lock

@@ -18,8 +18,6 @@ libloading.workspace = true
 memmap2.workspace = true
 temp-dir.workspace = true
 
-tt.workspace = true
-syntax-bridge.workspace = true
 paths.workspace = true
 # span = {workspace = true, default-features = false} does not work
 span = { path = "../span", version = "0.0.0", default-features = false}
@@ -38,8 +36,9 @@ expect-test.workspace = true
 proc-macro-test.path = "./proc-macro-test"
 
 [features]
+default = ["sysroot-abi"]
 sysroot-abi = []
-in-rust-tree = ["syntax-bridge/in-rust-tree", "tt/in-rust-tree", "sysroot-abi"]
+in-rust-tree = ["sysroot-abi"]
 
 [lints]
 workspace = true

crates/proc-macro-srv/Cargo.toml

@@ -12,8 +12,7 @@ use object::Object;
 use paths::{Utf8Path, Utf8PathBuf};
 
 use crate::{
-    PanicMessage, ProcMacroKind, ProcMacroSrvSpan, dylib::proc_macros::ProcMacros,
-    server_impl::TopSubtree,
+    PanicMessage, ProcMacroKind, ProcMacroSrvSpan, dylib::proc_macros::ProcMacros, tt::TokenStream,
 };
 
 pub(crate) struct Expander {
@@ -40,18 +39,18 @@ impl Expander {
     pub(crate) fn expand<S: ProcMacroSrvSpan>(
         &self,
         macro_name: &str,
-        macro_body: TopSubtree<S>,
-        attributes: Option<TopSubtree<S>>,
+        macro_body: TokenStream<S>,
+        attribute: Option<TokenStream<S>>,
         def_site: S,
         call_site: S,
         mixed_site: S,
-    ) -> Result<TopSubtree<S>, PanicMessage>
+    ) -> Result<TokenStream<S>, PanicMessage>
     where
         <S::Server as bridge::server::Types>::TokenStream: Default,
     {
         self.inner
             .proc_macros
-            .expand(macro_name, macro_body, attributes, def_site, call_site, mixed_site)
+            .expand(macro_name, macro_body, attribute, def_site, call_site, mixed_site)
     }
 
     pub(crate) fn list_macros(&self) -> impl Iterator<Item = (&str, ProcMacroKind)> {

crates/proc-macro-srv/src/dylib.rs

@@ -2,7 +2,7 @@
 
 use proc_macro::bridge;
 
-use crate::{ProcMacroKind, ProcMacroSrvSpan, server_impl::TopSubtree};
+use crate::{ProcMacroKind, ProcMacroSrvSpan, tt::TokenStream};
 
 #[repr(transparent)]
 pub(crate) struct ProcMacros([bridge::client::ProcMacro]);
@@ -17,18 +17,13 @@ impl ProcMacros {
     pub(crate) fn expand<S: ProcMacroSrvSpan>(
         &self,
         macro_name: &str,
-        macro_body: TopSubtree<S>,
-        attributes: Option<TopSubtree<S>>,
+        macro_body: TokenStream<S>,
+        attribute: Option<TokenStream<S>>,
         def_site: S,
         call_site: S,
         mixed_site: S,
-    ) -> Result<TopSubtree<S>, crate::PanicMessage> {
-        let parsed_body = crate::server_impl::TokenStream::with_subtree(macro_body);
-
-        let parsed_attributes = attributes
-            .map_or_else(crate::server_impl::TokenStream::default, |attr| {
-                crate::server_impl::TokenStream::with_subtree(attr)
-            });
+    ) -> Result<TokenStream<S>, crate::PanicMessage> {
+        let parsed_attributes = attribute.unwrap_or_default();
 
         for proc_macro in &self.0 {
             match proc_macro {
@@ -38,35 +33,29 @@ impl ProcMacros {
                     let res = client.run(
                         &bridge::server::SameThread,
                         S::make_server(call_site, def_site, mixed_site),
-                        parsed_body,
+                        macro_body,
                         cfg!(debug_assertions),
                     );
-                    return res
-                        .map(|it| it.into_subtree(call_site))
-                        .map_err(crate::PanicMessage::from);
+                    return res.map_err(crate::PanicMessage::from);
                 }
                 bridge::client::ProcMacro::Bang { name, client } if *name == macro_name => {
                     let res = client.run(
                         &bridge::server::SameThread,
                         S::make_server(call_site, def_site, mixed_site),
-                        parsed_body,
+                        macro_body,
                         cfg!(debug_assertions),
                     );
-                    return res
-                        .map(|it| it.into_subtree(call_site))
-                        .map_err(crate::PanicMessage::from);
+                    return res.map_err(crate::PanicMessage::from);
                 }
                 bridge::client::ProcMacro::Attr { name, client } if *name == macro_name => {
                     let res = client.run(
                         &bridge::server::SameThread,
                         S::make_server(call_site, def_site, mixed_site),
                         parsed_attributes,
-                        parsed_body,
+                        macro_body,
                         cfg!(debug_assertions),
                     );
-                    return res
-                        .map(|it| it.into_subtree(call_site))
-                        .map_err(crate::PanicMessage::from);
+                    return res.map_err(crate::PanicMessage::from);
                 }
                 _ => continue,
             }

crates/proc-macro-srv/src/dylib/proc_macros.rs

@@ -28,6 +28,7 @@ extern crate rustc_lexer;
 
 mod dylib;
 mod server_impl;
+mod tt;
 
 use std::{
     collections::{HashMap, hash_map::Entry},
@@ -43,8 +44,6 @@ use paths::{Utf8Path, Utf8PathBuf};
 use span::Span;
 use temp_dir::TempDir;
 
-use crate::server_impl::TokenStream;
-
 pub use crate::server_impl::token_id::SpanId;
 
 #[derive(Copy, Clone, Eq, PartialEq, Debug)]
@@ -81,12 +80,12 @@ impl ProcMacroSrv<'_> {
         env: &[(String, String)],
         current_dir: Option<impl AsRef<Path>>,
         macro_name: &str,
-        macro_body: tt::TopSubtree<S>,
-        attribute: Option<tt::TopSubtree<S>>,
+        macro_body: tt::TokenStream<S>,
+        attribute: Option<tt::TokenStream<S>>,
         def_site: S,
         call_site: S,
         mixed_site: S,
-    ) -> Result<Vec<tt::TokenTree<S>>, PanicMessage> {
+    ) -> Result<tt::TokenStream<S>, PanicMessage> {
         let snapped_env = self.env;
         let expander = self.expander(lib.as_ref()).map_err(|err| PanicMessage {
             message: Some(format!("failed to load macro: {err}")),
@@ -102,15 +101,7 @@ impl ProcMacroSrv<'_> {
                 .name(macro_name.to_owned())
                 .spawn_scoped(s, move || {
                     expander
-                        .expand(
-                            macro_name,
-                            server_impl::TopSubtree(macro_body.0.into_vec()),
-                            attribute.map(|it| server_impl::TopSubtree(it.0.into_vec())),
-                            def_site,
-                            call_site,
-                            mixed_site,
-                        )
-                        .map(|tt| tt.0)
+                        .expand(macro_name, macro_body, attribute, def_site, call_site, mixed_site)
                 });
             match thread.unwrap().join() {
                 Ok(res) => res,
@@ -157,8 +148,8 @@ impl ProcMacroSrv<'_> {
     }
 }
 
-pub trait ProcMacroSrvSpan: Copy + Send {
-    type Server: proc_macro::bridge::server::Server<TokenStream = TokenStream<Self>>;
+pub trait ProcMacroSrvSpan: Copy + Send + Sync {
+    type Server: proc_macro::bridge::server::Server<TokenStream = crate::tt::TokenStream<Self>>;
     fn make_server(call_site: Self, def_site: Self, mixed_site: Self) -> Self::Server;
 }