Make edition per-token, not per-file

More correctly, *also* per-token. Because as it turns out, while the top-level edition affects parsing (I think), the per-token edition affects escaping of identifiers/keywords.

crates/edition/src/lib.rs

@@ -2,6 +2,7 @@
 // This should live in a separate crate because we use it in both actual code and codegen.
 use std::fmt;
 
+// The syntax context stuff needs the discriminants to start from 0 and be consecutive.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
 #[repr(u8)]
 pub enum Edition {

crates/hir-def/src/body.rs

@@ -40,7 +40,8 @@ use crate::{
 pub struct HygieneId(pub(crate) span::SyntaxContextId);
 
 impl HygieneId {
-    pub const ROOT: Self = Self(span::SyntaxContextId::ROOT);
+    // The edition doesn't matter here, we only use this for comparisons and to lookup the macro.
+    pub const ROOT: Self = Self(span::SyntaxContextId::root(Edition::Edition2015));
 
     pub fn new(ctx: span::SyntaxContextId) -> Self {
         Self(ctx)

crates/hir-def/src/expander.rs

@@ -10,7 +10,7 @@ use hir_expand::{
     ExpandResult, HirFileId, InFile, Lookup, MacroCallId,
 };
 use limit::Limit;
-use span::SyntaxContextId;
+use span::{Edition, SyntaxContextId};
 use syntax::{ast, Parse};
 use triomphe::Arc;
 
@@ -60,7 +60,7 @@ impl Expander {
 
     pub fn syntax_context(&self) -> SyntaxContextId {
         // FIXME:
-        SyntaxContextId::ROOT
+        SyntaxContextId::root(Edition::CURRENT)
     }
 
     pub fn enter_expand<T: ast::AstNode>(

crates/hir-def/src/item_tree/tests.rs

@@ -270,7 +270,7 @@ m!();
             // AstId: 2
             pub macro m2 { ... }
 
-            // AstId: 3, SyntaxContext: 0, ExpandTo: Items
+            // AstId: 3, SyntaxContext: 2, ExpandTo: Items
             m!(...);
         "#]],
     );

crates/hir-def/src/macro_expansion_tests/mbe.rs

@@ -35,9 +35,9 @@ macro_rules! f {
     };
 }
 
-struct#0:[email protected]#1# MyTraitMap2#0:[email protected]#0# {#0:[email protected]#1#
-    map#0:[email protected]#1#:#0:[email protected]#1# #0:[email protected]#1#::#0:[email protected]#1#std#0:[email protected]#1#::#0:[email protected]#1#collections#0:[email protected]#1#::#0:[email protected]#1#HashSet#0:[email protected]#1#<#0:[email protected]#1#(#0:[email protected]#1#)#0:[email protected]#1#>#0:[email protected]#1#,#0:[email protected]#1#
-}#0:[email protected]#1#
+struct#0:[email protected]#4# MyTraitMap2#0:[email protected]#2# {#0:[email protected]#4#
+    map#0:[email protected]#4#:#0:[email protected]#4# #0:[email protected]#4#::#0:[email protected]#4#std#0:[email protected]#4#::#0:[email protected]#4#collections#0:[email protected]#4#::#0:[email protected]#4#HashSet#0:[email protected]#4#<#0:[email protected]#4#(#0:[email protected]#4#)#0:[email protected]#4#>#0:[email protected]#4#,#0:[email protected]#4#
+}#0:[email protected]#4#
 "#]],
     );
 }
@@ -75,12 +75,12 @@ macro_rules! f {
     };
 }
 
-fn#0:[email protected]#0# main#0:[email protected]#0#(#0:[email protected]#0#)#0:[email protected]#0# {#0:[email protected]#0#
-    1#0:[email protected]#0#;#0:[email protected]#0#
-    1.0#0:[email protected]#0#;#0:[email protected]#0#
-    (#0:[email protected]#0#(#0:[email protected]#0#1#0:[email protected]#0#,#0:[email protected]#0# )#0:[email protected]#0#,#0:[email protected]#0# )#0:[email protected]#0#.#0:[email protected]#0#0#0:[email protected]#0#.#0:[email protected]#0#0#0:[email protected]#0#;#0:[email protected]#0#
-    let#0:[email protected]#0# x#0:[email protected]#0# =#0:[email protected]#0# 1#0:[email protected]#0#;#0:[email protected]#0#
-}#0:[email protected]#0#
+fn#0:[email protected]#2# main#0:[email protected]#2#(#0:[email protected]#2#)#0:[email protected]#2# {#0:[email protected]#2#
+    1#0:[email protected]#2#;#0:[email protected]#2#
+    1.0#0:[email protected]#2#;#0:[email protected]#2#
+    (#0:[email protected]#2#(#0:[email protected]#2#1#0:[email protected]#2#,#0:[email protected]#2# )#0:[email protected]#2#,#0:[email protected]#2# )#0:[email protected]#2#.#0:[email protected]#2#0#0:[email protected]#2#.#0:[email protected]#2#0#0:[email protected]#2#;#0:[email protected]#2#
+    let#0:[email protected]#2# x#0:[email protected]#2# =#0:[email protected]#2# 1#0:[email protected]#2#;#0:[email protected]#2#
+}#0:[email protected]#2#
 
 
 "#]],
@@ -171,7 +171,7 @@ fn main(foo: ()) {
     }
 
     fn main(foo: ()) {
-        /* error: unresolved macro unresolved */"helloworld!"#0:[email protected]#0#;
+        /* error: unresolved macro unresolved */"helloworld!"#0:[email protected]#2#;
     }
 }
 
@@ -197,7 +197,7 @@ macro_rules! mk_struct {
 #[macro_use]
 mod foo;
 
-struct#1:[email protected]#1# Foo#0:[email protected]#0#(#1:[email protected]#1#u32#0:[email protected]#0#)#1:[email protected]#1#;#1:[email protected]#1#
+struct#1:[email protected]#4# Foo#0:[email protected]#2#(#1:[email protected]#4#u32#0:[email protected]#2#)#1:[email protected]#4#;#1:[email protected]#4#
 "#]],
     );
 }
@@ -423,10 +423,10 @@ m! { foo, bar }
 macro_rules! m {
     ($($i:ident),*) => ( impl Bar { $(fn $i() {})* } );
 }
-impl#\1# Bar#\1# {#\1#
-    fn#\1# foo#\0#(#\1#)#\1# {#\1#}#\1#
-    fn#\1# bar#\0#(#\1#)#\1# {#\1#}#\1#
-}#\1#
+impl#\4# Bar#\4# {#\4#
+    fn#\4# foo#\2#(#\4#)#\4# {#\4#}#\4#
+    fn#\4# bar#\2#(#\4#)#\4# {#\4#}#\4#
+}#\4#
 "#]],
     );
 }

crates/hir-def/src/macro_expansion_tests/mod.rs

@@ -358,6 +358,7 @@ impl ProcMacroExpander for IdentityWhenValidProcMacroExpander {
         let (parse, _) = syntax_bridge::token_tree_to_syntax_node(
             subtree,
             syntax_bridge::TopEntryPoint::MacroItems,
+            &mut |_| span::Edition::CURRENT,
             span::Edition::CURRENT,
         );
         if parse.errors().is_empty() {

crates/hir-def/src/macro_expansion_tests/proc_macros.rs

@@ -181,9 +181,9 @@ fn foo(&self) {
     self.0. 1;
 }
 
-fn#0:[email protected]#0# foo#0:[email protected]#0#(#0:[email protected]#0#&#0:[email protected]#0#self#0:[email protected]#0# )#0:[email protected]#0# {#0:[email protected]#0#
-    self#0:[email protected]#0# .#0:[email protected]#0#0#0:[email protected]#0#.#0:[email protected]#0#1#0:[email protected]#0#;#0:[email protected]#0#
-}#0:[email protected]#0#"#]],
+fn#0:[email protected]#2# foo#0:[email protected]#2#(#0:[email protected]#2#&#0:[email protected]#2#self#0:[email protected]#2# )#0:[email protected]#2# {#0:[email protected]#2#
+    self#0:[email protected]#2# .#0:[email protected]#2#0#0:[email protected]#2#.#0:[email protected]#2#1#0:[email protected]#2#;#0:[email protected]#2#
+}#0:[email protected]#2#"#]],
     );
 }
 

crates/hir-def/src/nameres/collector.rs

@@ -74,7 +74,7 @@ pub(super) fn collect_defs(db: &dyn DefDatabase, def_map: DefMap, tree_id: TreeI
 
     let proc_macros = if krate.is_proc_macro {
         db.proc_macros()
-            .for_crate(def_map.krate, db.syntax_context(tree_id.file_id()))
+            .for_crate(def_map.krate, db.syntax_context(tree_id.file_id(), krate.edition))
             .unwrap_or_default()
     } else {
         Default::default()

crates/hir-expand/src/builtin/derive_macro.rs

@@ -4,7 +4,7 @@ use intern::sym;
 use itertools::{izip, Itertools};
 use parser::SyntaxKind;
 use rustc_hash::FxHashSet;
-use span::{MacroCallId, Span, SyntaxContextId};
+use span::{Edition, MacroCallId, Span, SyntaxContextId};
 use stdx::never;
 use syntax_bridge::DocCommentDesugarMode;
 use tracing::debug;
@@ -33,7 +33,7 @@ macro_rules! register_builtin {
         }
 
         impl BuiltinDeriveExpander {
-            pub fn expander(&self) -> fn(Span, &tt::TopSubtree) -> ExpandResult<tt::TopSubtree>  {
+            pub fn expander(&self) -> fn(&dyn ExpandDatabase, Span, &tt::TopSubtree) -> ExpandResult<tt::TopSubtree>  {
                 match *self {
                     $( BuiltinDeriveExpander::$trait => $expand, )*
                 }
@@ -58,8 +58,8 @@ impl BuiltinDeriveExpander {
         tt: &tt::TopSubtree,
         span: Span,
     ) -> ExpandResult<tt::TopSubtree> {
-        let span = span_with_def_site_ctxt(db, span, id);
-        self.expander()(span, tt)
+        let span = span_with_def_site_ctxt(db, span, id, Edition::CURRENT);
+        self.expander()(db, span, tt)
     }
 }
 
@@ -226,8 +226,12 @@ struct AdtParam {
 }
 
 // FIXME: This whole thing needs a refactor. Each derive requires its special values, and the result is a mess.
-fn parse_adt(tt: &tt::TopSubtree, call_site: Span) -> Result<BasicAdtInfo, ExpandError> {
-    let (adt, tm) = to_adt_syntax(tt, call_site)?;
+fn parse_adt(
+    db: &dyn ExpandDatabase,
+    tt: &tt::TopSubtree,
+    call_site: Span,
+) -> Result<BasicAdtInfo, ExpandError> {
+    let (adt, tm) = to_adt_syntax(db, tt, call_site)?;
     parse_adt_from_syntax(&adt, &tm, call_site)
 }
 
@@ -382,12 +386,14 @@ fn parse_adt_from_syntax(
 }
 
 fn to_adt_syntax(
+    db: &dyn ExpandDatabase,
     tt: &tt::TopSubtree,
     call_site: Span,
 ) -> Result<(ast::Adt, span::SpanMap<SyntaxContextId>), ExpandError> {
-    let (parsed, tm) = syntax_bridge::token_tree_to_syntax_node(
+    let (parsed, tm) = crate::db::token_tree_to_syntax_node(
+        db,
         tt,
-        syntax_bridge::TopEntryPoint::MacroItems,
+        crate::ExpandTo::Items,
         parser::Edition::CURRENT_FIXME,
     );
     let macro_items = ast::MacroItems::cast(parsed.syntax_node())
@@ -446,12 +452,13 @@ fn name_to_token(
 /// where B1, ..., BN are the bounds given by `bounds_paths`. Z is a phantom type, and
 /// therefore does not get bound by the derived trait.
 fn expand_simple_derive(
+    db: &dyn ExpandDatabase,
     invoc_span: Span,
     tt: &tt::TopSubtree,
     trait_path: tt::TopSubtree,
     make_trait_body: impl FnOnce(&BasicAdtInfo) -> tt::TopSubtree,
 ) -> ExpandResult<tt::TopSubtree> {
-    let info = match parse_adt(tt, invoc_span) {
+    let info = match parse_adt(db, tt, invoc_span) {
         Ok(info) => info,
         Err(e) => {
             return ExpandResult::new(
@@ -520,14 +527,22 @@ fn expand_simple_derive_with_parsed(
     }
 }
 
-fn copy_expand(span: Span, tt: &tt::TopSubtree) -> ExpandResult<tt::TopSubtree> {
+fn copy_expand(
+    db: &dyn ExpandDatabase,
+    span: Span,
+    tt: &tt::TopSubtree,
+) -> ExpandResult<tt::TopSubtree> {
     let krate = dollar_crate(span);
-    expand_simple_derive(span, tt, quote! {span => #krate::marker::Copy }, |_| quote! {span =>})
+    expand_simple_derive(db, span, tt, quote! {span => #krate::marker::Copy }, |_| quote! {span =>})
 }
 
-fn clone_expand(span: Span, tt: &tt::TopSubtree) -> ExpandResult<tt::TopSubtree> {
+fn clone_expand(
+    db: &dyn ExpandDatabase,
+    span: Span,
+    tt: &tt::TopSubtree,
+) -> ExpandResult<tt::TopSubtree> {
     let krate = dollar_crate(span);
-    expand_simple_derive(span, tt, quote! {span => #krate::clone::Clone }, |adt| {
+    expand_simple_derive(db, span, tt, quote! {span => #krate::clone::Clone }, |adt| {
         if matches!(adt.shape, AdtShape::Union) {
             let star = tt::Punct { char: '*', spacing: ::tt::Spacing::Alone, span };
             return quote! {span =>
@@ -576,9 +591,13 @@ fn and_and(span: Span) -> tt::TopSubtree {
     quote! {span => #and& }
 }
 
-fn default_expand(span: Span, tt: &tt::TopSubtree) -> ExpandResult<tt::TopSubtree> {
+fn default_expand(
+    db: &dyn ExpandDatabase,
+    span: Span,
+    tt: &tt::TopSubtree,
+) -> ExpandResult<tt::TopSubtree> {
     let krate = &dollar_crate(span);
-    expand_simple_derive(span, tt, quote! {span => #krate::default::Default }, |adt| {
+    expand_simple_derive(db, span, tt, quote! {span => #krate::default::Default }, |adt| {
         let body = match &adt.shape {
             AdtShape::Struct(fields) => {
                 let name = &adt.name;
@@ -615,9 +634,13 @@ fn default_expand(span: Span, tt: &tt::TopSubtree) -> ExpandResult<tt::TopSubtre
     })
 }
 
-fn debug_expand(span: Span, tt: &tt::TopSubtree) -> ExpandResult<tt::TopSubtree> {
+fn debug_expand(
+    db: &dyn ExpandDatabase,
+    span: Span,
+    tt: &tt::TopSubtree,
+) -> ExpandResult<tt::TopSubtree> {
     let krate = &dollar_crate(span);
-    expand_simple_derive(span, tt, quote! {span => #krate::fmt::Debug }, |adt| {
+    expand_simple_derive(db, span, tt, quote! {span => #krate::fmt::Debug }, |adt| {
         let for_variant = |name: String, v: &VariantShape| match v {
             VariantShape::Struct(fields) => {
                 let for_fields = fields.iter().map(|it| {
@@ -687,9 +710,13 @@ fn debug_expand(span: Span, tt: &tt::TopSubtree) -> ExpandResult<tt::TopSubtree>
     })
 }
 
-fn hash_expand(span: Span, tt: &tt::TopSubtree) -> ExpandResult<tt::TopSubtree> {
+fn hash_expand(
+    db: &dyn ExpandDatabase,
+    span: Span,
+    tt: &tt::TopSubtree,
+) -> ExpandResult<tt::TopSubtree> {
     let krate = &dollar_crate(span);
-    expand_simple_derive(span, tt, quote! {span => #krate::hash::Hash }, |adt| {
+    expand_simple_derive(db, span, tt, quote! {span => #krate::hash::Hash }, |adt| {
         if matches!(adt.shape, AdtShape::Union) {
             // FIXME: Return expand error here
             return quote! {span =>};
@@ -734,14 +761,22 @@ fn hash_expand(span: Span, tt: &tt::TopSubtree) -> ExpandResult<tt::TopSubtree>
     })
 }
 
-fn eq_expand(span: Span, tt: &tt::TopSubtree) -> ExpandResult<tt::TopSubtree> {
+fn eq_expand(
+    db: &dyn ExpandDatabase,
+    span: Span,
+    tt: &tt::TopSubtree,
+) -> ExpandResult<tt::TopSubtree> {
     let krate = dollar_crate(span);
-    expand_simple_derive(span, tt, quote! {span => #krate::cmp::Eq }, |_| quote! {span =>})
+    expand_simple_derive(db, span, tt, quote! {span => #krate::cmp::Eq }, |_| quote! {span =>})
 }
 
-fn partial_eq_expand(span: Span, tt: &tt::TopSubtree) -> ExpandResult<tt::TopSubtree> {
+fn partial_eq_expand(
+    db: &dyn ExpandDatabase,
+    span: Span,
+    tt: &tt::TopSubtree,
+) -> ExpandResult<tt::TopSubtree> {
     let krate = dollar_crate(span);
-    expand_simple_derive(span, tt, quote! {span => #krate::cmp::PartialEq }, |adt| {
+    expand_simple_derive(db, span, tt, quote! {span => #krate::cmp::PartialEq }, |adt| {
         if matches!(adt.shape, AdtShape::Union) {
             // FIXME: Return expand error here
             return quote! {span =>};
@@ -811,9 +846,13 @@ fn self_and_other_patterns(
     (self_patterns, other_patterns)
 }
 
-fn ord_expand(span: Span, tt: &tt::TopSubtree) -> ExpandResult<tt::TopSubtree> {
+fn ord_expand(
+    db: &dyn ExpandDatabase,
+    span: Span,
+    tt: &tt::TopSubtree,
+) -> ExpandResult<tt::TopSubtree> {
     let krate = &dollar_crate(span);
-    expand_simple_derive(span, tt, quote! {span => #krate::cmp::Ord }, |adt| {
+    expand_simple_derive(db, span, tt, quote! {span => #krate::cmp::Ord }, |adt| {
         fn compare(
             krate: &tt::Ident,
             left: tt::TopSubtree,
@@ -869,9 +908,13 @@ fn ord_expand(span: Span, tt: &tt::TopSubtree) -> ExpandResult<tt::TopSubtree> {
     })
 }
 
-fn partial_ord_expand(span: Span, tt: &tt::TopSubtree) -> ExpandResult<tt::TopSubtree> {
+fn partial_ord_expand(
+    db: &dyn ExpandDatabase,
+    span: Span,
+    tt: &tt::TopSubtree,
+) -> ExpandResult<tt::TopSubtree> {
     let krate = &dollar_crate(span);
-    expand_simple_derive(span, tt, quote! {span => #krate::cmp::PartialOrd }, |adt| {
+    expand_simple_derive(db, span, tt, quote! {span => #krate::cmp::PartialOrd }, |adt| {
         fn compare(
             krate: &tt::Ident,
             left: tt::TopSubtree,
@@ -932,8 +975,12 @@ fn partial_ord_expand(span: Span, tt: &tt::TopSubtree) -> ExpandResult<tt::TopSu
     })
 }
 
-fn coerce_pointee_expand(span: Span, tt: &tt::TopSubtree) -> ExpandResult<tt::TopSubtree> {
-    let (adt, _span_map) = match to_adt_syntax(tt, span) {
+fn coerce_pointee_expand(
+    db: &dyn ExpandDatabase,
+    span: Span,
+    tt: &tt::TopSubtree,
+) -> ExpandResult<tt::TopSubtree> {
+    let (adt, _span_map) = match to_adt_syntax(db, tt, span) {
         Ok(it) => it,
         Err(err) => {
             return ExpandResult::new(tt::TopSubtree::empty(tt::DelimSpan::from_single(span)), err);