feat: alternate PRQL-native implementation of append by:name

prqlc/bindings/prqlc-python/src/lib.rs

@@ -232,15 +232,15 @@ mod test {
     fn debug_prql_lineage() {
         assert_snapshot!(
             debug::prql_lineage(r#"from a | select { beta, gamma }"#).unwrap(),
-            @r#"{"frames":[["1:9-31",{"columns":[{"Single":{"name":["a","beta"],"target_id":120,"target_name":null}},{"Single":{"name":["a","gamma"],"target_id":121,"target_name":null}}],"inputs":[{"id":118,"name":"a","table":["default_db","a"]}]}]],"nodes":[{"id":118,"kind":"Ident","span":"1:0-6","ident":{"Ident":["default_db","a"]},"parent":123},{"id":120,"kind":"Ident","span":"1:18-22","ident":{"Ident":["this","a","beta"]},"targets":[118],"parent":122},{"id":121,"kind":"Ident","span":"1:24-29","ident":{"Ident":["this","a","gamma"]},"targets":[118],"parent":122},{"id":122,"kind":"Tuple","span":"1:16-31","children":[120,121],"parent":123},{"id":123,"kind":"TransformCall: Select","span":"1:9-31","children":[118,122]}],"ast":{"name":"Project","stmts":[{"VarDef":{"kind":"Main","name":"main","value":{"Pipeline":{"exprs":[{"FuncCall":{"name":{"Ident":["from"],"span":"1:0-4"},"args":[{"Ident":["a"],"span":"1:5-6"}]},"span":"1:0-6"},{"FuncCall":{"name":{"Ident":["select"],"span":"1:9-15"},"args":[{"Tuple":[{"Ident":["beta"],"span":"1:18-22"},{"Ident":["gamma"],"span":"1:24-29"}],"span":"1:16-31"}]},"span":"1:9-31"}]},"span":"1:0-31"}},"span":"1:0-31"}]}}"#
+            @r###"{"frames":[["1:9-31",{"columns":[{"Single":{"name":["a","beta"],"target_id":123,"target_name":null}},{"Single":{"name":["a","gamma"],"target_id":124,"target_name":null}}],"inputs":[{"id":121,"name":"a","table":["default_db","a"]}]}]],"nodes":[{"id":121,"kind":"Ident","span":"1:0-6","ident":{"Ident":["default_db","a"]},"parent":126},{"id":123,"kind":"Ident","span":"1:18-22","ident":{"Ident":["this","a","beta"]},"targets":[121],"parent":125},{"id":124,"kind":"Ident","span":"1:24-29","ident":{"Ident":["this","a","gamma"]},"targets":[121],"parent":125},{"id":125,"kind":"Tuple","span":"1:16-31","children":[123,124],"parent":126},{"id":126,"kind":"TransformCall: Select","span":"1:9-31","children":[121,125]}],"ast":{"name":"Project","stmts":[{"VarDef":{"kind":"Main","name":"main","value":{"Pipeline":{"exprs":[{"FuncCall":{"name":{"Ident":["from"],"span":"1:0-4"},"args":[{"Ident":["a"],"span":"1:5-6"}]},"span":"1:0-6"},{"FuncCall":{"name":{"Ident":["select"],"span":"1:9-15"},"args":[{"Tuple":[{"Ident":["beta"],"span":"1:18-22"},{"Ident":["gamma"],"span":"1:24-29"}],"span":"1:16-31"}]},"span":"1:9-31"}]},"span":"1:0-31"}},"span":"1:0-31"}]}}"###
         );
     }
 
     #[test]
     fn debug_pl_to_lineage() {
         assert_snapshot!(
             prql_to_pl(r#"from a | select { beta, gamma }"#).and_then(|x| debug::pl_to_lineage(&x)).unwrap(),
-            @r#"{"frames":[["1:9-31",{"columns":[{"Single":{"name":["a","beta"],"target_id":120,"target_name":null}},{"Single":{"name":["a","gamma"],"target_id":121,"target_name":null}}],"inputs":[{"id":118,"name":"a","table":["default_db","a"]}]}]],"nodes":[{"id":118,"kind":"Ident","span":"1:0-6","ident":{"Ident":["default_db","a"]},"parent":123},{"id":120,"kind":"Ident","span":"1:18-22","ident":{"Ident":["this","a","beta"]},"targets":[118],"parent":122},{"id":121,"kind":"Ident","span":"1:24-29","ident":{"Ident":["this","a","gamma"]},"targets":[118],"parent":122},{"id":122,"kind":"Tuple","span":"1:16-31","children":[120,121],"parent":123},{"id":123,"kind":"TransformCall: Select","span":"1:9-31","children":[118,122]}],"ast":{"name":"Project","stmts":[{"VarDef":{"kind":"Main","name":"main","value":{"Pipeline":{"exprs":[{"FuncCall":{"name":{"Ident":["from"],"span":"1:0-4"},"args":[{"Ident":["a"],"span":"1:5-6"}]},"span":"1:0-6"},{"FuncCall":{"name":{"Ident":["select"],"span":"1:9-15"},"args":[{"Tuple":[{"Ident":["beta"],"span":"1:18-22"},{"Ident":["gamma"],"span":"1:24-29"}],"span":"1:16-31"}]},"span":"1:9-31"}]},"span":"1:0-31"}},"span":"1:0-31"}]}}"#
+            @r###"{"frames":[["1:9-31",{"columns":[{"Single":{"name":["a","beta"],"target_id":123,"target_name":null}},{"Single":{"name":["a","gamma"],"target_id":124,"target_name":null}}],"inputs":[{"id":121,"name":"a","table":["default_db","a"]}]}]],"nodes":[{"id":121,"kind":"Ident","span":"1:0-6","ident":{"Ident":["default_db","a"]},"parent":126},{"id":123,"kind":"Ident","span":"1:18-22","ident":{"Ident":["this","a","beta"]},"targets":[121],"parent":125},{"id":124,"kind":"Ident","span":"1:24-29","ident":{"Ident":["this","a","gamma"]},"targets":[121],"parent":125},{"id":125,"kind":"Tuple","span":"1:16-31","children":[123,124],"parent":126},{"id":126,"kind":"TransformCall: Select","span":"1:9-31","children":[121,125]}],"ast":{"name":"Project","stmts":[{"VarDef":{"kind":"Main","name":"main","value":{"Pipeline":{"exprs":[{"FuncCall":{"name":{"Ident":["from"],"span":"1:0-4"},"args":[{"Ident":["a"],"span":"1:5-6"}]},"span":"1:0-6"},{"FuncCall":{"name":{"Ident":["select"],"span":"1:9-15"},"args":[{"Tuple":[{"Ident":["beta"],"span":"1:18-22"},{"Ident":["gamma"],"span":"1:24-29"}],"span":"1:16-31"}]},"span":"1:9-31"}]},"span":"1:0-31"}},"span":"1:0-31"}]}}"###
         );
     }
 }

prqlc/prqlc/src/cli/test.rs

@@ -424,7 +424,7 @@ fn compare_directories(dir1: &Path, dir2: &Path) {
 fn debug() {
     assert_cmd_snapshot!(prqlc_command()
         .args(["debug", "lineage"])
-        .pass_stdin("from tracks | select {artist, album}"), @"
+        .pass_stdin("from tracks | select {artist, album}"), @r###"
     success: true
     exit_code: 0
     ----- stdout -----
@@ -435,61 +435,61 @@ fn debug() {
           name:
           - tracks
           - artist
-          target_id: 120
+          target_id: 123
           target_name: null
         - !Single
           name:
           - tracks
           - album
-          target_id: 121
+          target_id: 124
           target_name: null
         inputs:
-        - id: 118
+        - id: 121
           name: tracks
           table:
           - default_db
           - tracks
     nodes:
-    - id: 118
+    - id: 121
       kind: Ident
       span: 1:0-11
       ident: !Ident
       - default_db
       - tracks
-      parent: 123
-    - id: 120
+      parent: 126
+    - id: 123
       kind: Ident
       span: 1:22-28
       ident: !Ident
       - this
       - tracks
       - artist
       targets:
-      - 118
-      parent: 122
-    - id: 121
+      - 121
+      parent: 125
+    - id: 124
       kind: Ident
       span: 1:30-35
       ident: !Ident
       - this
       - tracks
       - album
       targets:
-      - 118
-      parent: 122
-    - id: 122
+      - 121
+      parent: 125
+    - id: 125
       kind: Tuple
       span: 1:21-36
       children:
-      - 120
-      - 121
-      parent: 123
-    - id: 123
+      - 123
+      - 124
+      parent: 126
+    - id: 126
       kind: 'TransformCall: Select'
       span: 1:14-36
       children:
-      - 118
-      - 122
+      - 121
+      - 125
     ast:
       name: Project
       stmts:
@@ -528,7 +528,7 @@ fn debug() {
         span: 1:0-36
 
     ----- stderr -----
-    ");
+    "###);
 
     // Don't test the output of this, since on one min-versions check it had
     // different results, and didn't repro on Mac. It having different results

prqlc/prqlc/src/semantic/module.rs

@@ -134,6 +134,33 @@ impl Module {
         ns.names.get(&fq_ident.name)
     }
 
+    /// Recursively list all idents within this module. Useful for debugging.
+    pub fn all_names(&self, prefix: Option<&Ident>) -> Vec<Ident> {
+        let mut rv = Vec::new();
+
+        for (name, decl) in &self.names {
+            let name = match prefix {
+                Some(p) => p.clone() + Ident::from_name(name),
+                None => Ident::from_name(name),
+            };
+            rv.push(name.clone());
+
+            match &decl.kind {
+                DeclKind::Module(inner) => {
+                    rv.extend(inner.all_names(Some(&name)));
+                }
+                DeclKind::LayeredModules(stack) => {
+                    for inner in stack {
+                        rv.extend(inner.all_names(Some(&name)));
+                    }
+                }
+                _ => {}
+            }
+        }
+
+        rv
+    }
+
     pub fn lookup(&self, ident: &Ident) -> HashSet<Ident> {
         fn lookup_in(module: &Module, ident: Ident) -> HashSet<Ident> {
             let (prefix, ident) = ident.pop_front();
@@ -217,6 +244,7 @@ impl Module {
                     None => {
                         namespace.redirects.push(Ident::from_name(input_name));
 
+                        log::trace!("find_input_by_name {input_name} {:#?}", lineage.inputs);
                         let input = lineage.find_input_by_name(input_name).unwrap();
                         let order = lineage.inputs.iter().position(|i| i.id == input.id);
                         let order = order.unwrap();

prqlc/prqlc/src/semantic/resolver/expr.rs

@@ -75,6 +75,15 @@
         let r = match node.kind {
             pl::ExprKind::Ident(ident) => {
                 log::debug!("resolving ident {ident}...");
+                log::trace!(
+                    "... candidates: {:#?}",
+                    self.root_mod
+                        .module
+                        .all_names(None)
+                        .into_iter()
+                        .map(|i| i.to_string())
+                        .collect::<Vec<_>>()
+                );
                 let fq_ident = self
                     .resolve_ident(&ident)
                     .map_err(|e| e.with_span(node.span))?;
@@ -311,6 +320,55 @@
         }
         res
     }
+
+    pub fn construct_wildcard_from_lineage(
+        &mut self,
+        prefix: &[&String],
+        expr: &Box<pl::Expr>,
+    ) -> Vec<pl::Expr> {
+        let Some(lineage) = &expr.lineage else {
+            return vec![];
+        };
+
+        log::trace!("construct_wildcard_from_lineage: {lineage:#?}");
+
+        lineage
+            .columns
+            .iter()
+            .filter_map(|col| match col {
+                pl::LineageColumn::All { input_id, .. } => Some(pl::Expr {
+                    id: Some(self.id.gen()),
+                    target_id: Some(*input_id),
+                    flatten: true,
+                    ty: Some(Ty::new(TyKind::Tuple(vec![TyTupleField::Wildcard(None)]))),
+                    ..pl::Expr::new(pl::Ident::from_name(NS_SELF))
+                }),
+                pl::LineageColumn::Single {
+                    target_id,
+                    target_name: Some(target_name),
+                    ..
+                } => Some(pl::Expr {
+                    id: Some(self.id.gen()),
+                    target_id: Some(*target_id),
+                    alias: Some(target_name.to_string()),
+                    ..pl::Expr::new(pl::Ident::from_path(
+                        [prefix.to_vec(), vec![target_name]].concat(),
+                    ))
+                }),
+                pl::LineageColumn::Single {
+                    target_id,
+                    name: Some(ident),
+                    ..
+                } => Some(pl::Expr {
+                    id: Some(self.id.gen()),
+                    target_id: Some(*target_id),
+                    alias: Some(ident.name.to_string()),
+                    ..pl::Expr::new(ident.clone())
+                }),
+                _ => None,
+            })
+            .collect()
+    }
 }
 
 fn ty_of_lineage(lineage: &pl::Lineage) -> Ty {

prqlc/prqlc/src/semantic/resolver/names.rs

@@ -6,6 +6,7 @@
 use crate::ir::decl::{Decl, DeclKind, Module};
 use crate::ir::pl::{Expr, ExprKind};
 use crate::pr::Ident;
+use crate::pr::TyKind;
 use crate::semantic::{NS_INFER, NS_INFER_MODULE, NS_SELF, NS_THAT, NS_THIS};
 use crate::Error;
 use crate::Result;
@@ -238,18 +239,76 @@
                     ..Expr::new(ExprKind::Tuple(fields))
                 };
                 let cols_expr = DeclKind::Expr(Box::new(cols_expr));
-                let save_as = "_wildcard_match";
+                let save_as = format!("_wildcard_match_{module_fq_self}");
                 self.root_mod
                     .module
                     .names
                     .insert(save_as.to_string(), cols_expr.into());
 
                 // Then we can return ident to that decl.
-                Ok(Ident::from_name(save_as))
+                return Ok(Ident::from_name(save_as));
             }
-            0 => Err(Error::new_simple(format!("Unknown relation {ident}"))),
-            _ => Err(ambiguous_error(decls, Some(&ident.name))),
+            0 => {} // fallthrough
+            _ => return Err(ambiguous_error(decls, Some(&ident.name))),
+        };
+
+        // Second pass - look for an Expr referenced by the ident; if
+        // it has an inferred ty that is relation-shaped, return that.
+        let ident = ident.clone().pop().unwrap();
+        let decls = self.root_mod.module.lookup(&ident);
+        log::trace!("resolve_ident_wildcard pass 2 decls: {decls:?}");
+
+        match decls.len() {
+            1 => {
+                let relation_fq = decls.into_iter().next().unwrap();
+
+                // Retrieve the expr type
+                let decl = self.root_mod.module.get(&relation_fq).unwrap();
+                let decl_kind = decl.kind.clone();
+
+                let ty_tuple = (|| {
+                    let DeclKind::Expr(expr) = &decl_kind else {
+                        return None;
+                    };
+                    let Some(ty) = &expr.ty else { return None };
+                    let TyKind::Array(Some(ty)) = &ty.kind else {
+                        return None;
+                    };
+                    let TyKind::Tuple(tup) = &ty.kind else {
+                        return None;
+                    };
+                    Some(tup)
+                })();
+
+                if let (DeclKind::Expr(expr), Some(ty_tuple)) = (&decl_kind, ty_tuple) {
+                    log::trace!("ty_tuple is {ty_tuple:#?}");
+
+                    let prefix = relation_fq.iter().collect_vec();
+                    let fields = self.construct_wildcard_from_lineage(&prefix, expr);
+                    log::trace!("resolve_ident_wildcard pass 2 fields: {fields:?}");
+
+                    // This is just a workaround to return an Expr from this function.
+                    // We wrap the expr into DeclKind::Expr and save it into the root module.
+                    let cols_expr = Expr {
+                        flatten: true,
+                        ..Expr::new(ExprKind::Tuple(fields))
+                    };
+                    let cols_expr = DeclKind::Expr(Box::new(cols_expr));
+                    let save_as = format!("_wildcard_match_{relation_fq}");
+                    self.root_mod
+                        .module
+                        .names
+                        .insert(save_as.to_string(), cols_expr.into());
+
+                    // Then we can return ident to that decl.
+                    return Ok(Ident::from_name(save_as));
+                }
+            }
+            0 => {} // fallthrough
+            _ => return Err(ambiguous_error(decls, Some(&ident.name))),
         }
+
+        return Err(Error::new_simple(format!("Unknown relation {ident}")));
     }
 }
 

prqlc/prqlc/src/semantic/resolver/snapshots/prqlc__semantic__resolver__test__append_union_different_tables.snap

@@ -7,21 +7,21 @@ columns:
       name:
         - employees
         - name
-      target_id: 134
+      target_id: 137
       target_name: ~
   - Single:
       name:
         - employees
         - salary
-      target_id: 135
+      target_id: 138
       target_name: ~
 inputs:
-  - id: 132
+  - id: 135
     name: employees
     table:
       - default_db
       - employees
-  - id: 121
+  - id: 124
     name: managers
     table:
       - default_db

prqlc/prqlc/src/semantic/resolver/snapshots/prqlc__semantic__resolver__test__frames_and_names-2.snap

@@ -4,18 +4,18 @@ expression: "resolve_lineage(r#\"\n            from table_1\n            join cu
 ---
 columns:
   - All:
-      input_id: 119
+      input_id: 122
       except: []
   - All:
-      input_id: 116
+      input_id: 119
       except: []
 inputs:
-  - id: 119
+  - id: 122
     name: table_1
     table:
       - default_db
       - table_1
-  - id: 116
+  - id: 119
     name: customers
     table:
       - default_db

prqlc/prqlc/src/semantic/resolver/snapshots/prqlc__semantic__resolver__test__frames_and_names-3.snap

@@ -7,26 +7,26 @@ columns:
       name:
         - e
         - emp_no
-      target_id: 129
+      target_id: 132
       target_name: ~
   - Single:
       name:
         - e
         - gender
-      target_id: 130
+      target_id: 133
       target_name: ~
   - Single:
       name:
         - emp_salary
-      target_id: 148
+      target_id: 151
       target_name: ~
 inputs:
-  - id: 122
+  - id: 125
     name: e
     table:
       - default_db
       - employees
-  - id: 119
+  - id: 122
     name: salaries
     table:
       - default_db