From 5dcdd2f4daf88a76cf571176f3d961b5997c08de Mon Sep 17 00:00:00 2001 From: Jille Timmermans Date: Tue, 27 Feb 2024 11:26:41 +0100 Subject: [PATCH 1/5] feat(compiler): Support subqueries in the FROM clause issue #2989, #2400 and probably others --- internal/compiler/query_catalog.go | 49 +++++++++++++++-- internal/compiler/resolve.go | 22 ++++++++ .../testdata/join_alias/mysql/go/query.sql.go | 54 +++++++++++++++++++ .../testdata/join_alias/mysql/query.sql | 6 +++ 4 files changed, 127 insertions(+), 4 deletions(-) diff --git a/internal/compiler/query_catalog.go b/internal/compiler/query_catalog.go index 80b59d876c..96b42be7f3 100644 --- a/internal/compiler/query_catalog.go +++ b/internal/compiler/query_catalog.go @@ -9,13 +9,15 @@ import ( ) type QueryCatalog struct { - catalog *catalog.Catalog - ctes map[string]*Table - embeds rewrite.EmbedSet + catalog *catalog.Catalog + ctes map[string]*Table + fromClauses map[string]*Table + embeds rewrite.EmbedSet } func (comp *Compiler) buildQueryCatalog(c *catalog.Catalog, node ast.Node, embeds rewrite.EmbedSet) (*QueryCatalog, error) { var with *ast.WithClause + var from *ast.List switch n := node.(type) { case *ast.DeleteStmt: with = n.WithClause @@ -23,12 +25,20 @@ func (comp *Compiler) buildQueryCatalog(c *catalog.Catalog, node ast.Node, embed with = n.WithClause case *ast.UpdateStmt: with = n.WithClause + from = n.FromClause case *ast.SelectStmt: with = n.WithClause + from = n.FromClause default: with = nil + from = nil + } + qc := &QueryCatalog{ + catalog: c, + ctes: map[string]*Table{}, + fromClauses: map[string]*Table{}, + embeds: embeds, } - qc := &QueryCatalog{catalog: c, ctes: map[string]*Table{}, embeds: embeds} if with != nil { for _, item := range with.Ctes.Items { if cte, ok := item.(*ast.CommonTableExpr); ok { @@ -60,6 +70,37 @@ func (comp *Compiler) buildQueryCatalog(c *catalog.Catalog, node ast.Node, embed } } } + if from != nil { + for _, item := range from.Items { + if rs, ok := item.(*ast.RangeSubselect); ok { + cols, err := comp.outputColumns(qc, rs.Subquery) + if err != nil { + return nil, err + } + var names []string + if rs.Alias.Colnames != nil { + for _, item := range rs.Alias.Colnames.Items { + if val, ok := item.(*ast.String); ok { + names = append(names, val.Str) + } else { + names = append(names, "") + } + } + } + rel := &ast.TableName{Name: *rs.Alias.Aliasname} + for i := range cols { + cols[i].Table = rel + if len(names) > i { + cols[i].Name = names[i] + } + } + qc.fromClauses[*rs.Alias.Aliasname] = &Table{ + Rel: rel, + Columns: cols, + } + } + } + } return qc, nil } diff --git a/internal/compiler/resolve.go b/internal/compiler/resolve.go index b1fbb1990e..9a82aa325c 100644 --- a/internal/compiler/resolve.go +++ b/internal/compiler/resolve.go @@ -80,6 +80,28 @@ func (comp *Compiler) resolveCatalogRefs(qc *QueryCatalog, rvs []*ast.RangeVar, aliasMap[*rv.Alias.Aliasname] = fqn } } + for _, f := range qc.fromClauses { + catCols := make([]*catalog.Column, 0, len(f.Columns)) + for _, col := range f.Columns { + catCols = append(catCols, &catalog.Column{ + Name: col.Name, + Type: ast.TypeName{Name: col.DataType}, + IsNotNull: col.NotNull, + IsUnsigned: col.Unsigned, + IsArray: col.IsArray, + ArrayDims: col.ArrayDims, + Comment: col.Comment, + Length: col.Length, + }) + } + + if err := indexTable(catalog.Table{ + Rel: f.Rel, + Columns: catCols, + }); err != nil { + return nil, err + } + } // resolve a table for an embed for _, embed := range embeds { diff --git a/internal/endtoend/testdata/join_alias/mysql/go/query.sql.go b/internal/endtoend/testdata/join_alias/mysql/go/query.sql.go index 836f62c6d4..e6d565560b 100644 --- a/internal/endtoend/testdata/join_alias/mysql/go/query.sql.go +++ b/internal/endtoend/testdata/join_alias/mysql/go/query.sql.go @@ -80,3 +80,57 @@ func (q *Queries) AliasJoin(ctx context.Context, id uint64) ([]AliasJoinRow, err } return items, nil } + +const columnAlias = `-- name: ColumnAlias :many +SELECT n FROM (SELECT 1 AS n) AS x WHERE n <= ? +` + +func (q *Queries) ColumnAlias(ctx context.Context, n int32) ([]int32, error) { + rows, err := q.db.QueryContext(ctx, columnAlias, n) + if err != nil { + return nil, err + } + defer rows.Close() + var items []int32 + for rows.Next() { + var n int32 + if err := rows.Scan(&n); err != nil { + return nil, err + } + items = append(items, n) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + +const subqueryAlias = `-- name: SubqueryAlias :many +SELECT n FROM (SELECT 1 AS n) AS x WHERE x.n <= ? +` + +func (q *Queries) SubqueryAlias(ctx context.Context, n int32) ([]int32, error) { + rows, err := q.db.QueryContext(ctx, subqueryAlias, n) + if err != nil { + return nil, err + } + defer rows.Close() + var items []int32 + for rows.Next() { + var n int32 + if err := rows.Scan(&n); err != nil { + return nil, err + } + items = append(items, n) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} diff --git a/internal/endtoend/testdata/join_alias/mysql/query.sql b/internal/endtoend/testdata/join_alias/mysql/query.sql index 9b087bcae7..82bb2e5cd5 100644 --- a/internal/endtoend/testdata/join_alias/mysql/query.sql +++ b/internal/endtoend/testdata/join_alias/mysql/query.sql @@ -9,3 +9,9 @@ SELECT * FROM foo f JOIN bar b ON b.id = f.id WHERE f.id = ?; + +-- name: SubqueryAlias :many +SELECT * FROM (SELECT 1 AS n) AS x WHERE x.n <= ?; + +-- name: ColumnAlias :many +SELECT * FROM (SELECT 1 AS n) AS x WHERE n <= ?; From 462e9b08a46a408e62d7524109da15a62bebd816 Mon Sep 17 00:00:00 2001 From: Jille Timmermans Date: Fri, 5 Apr 2024 00:10:50 +0200 Subject: [PATCH 2/5] fix a crash --- internal/compiler/resolve.go | 40 +++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/internal/compiler/resolve.go b/internal/compiler/resolve.go index 9a82aa325c..49fe52e898 100644 --- a/internal/compiler/resolve.go +++ b/internal/compiler/resolve.go @@ -80,26 +80,28 @@ func (comp *Compiler) resolveCatalogRefs(qc *QueryCatalog, rvs []*ast.RangeVar, aliasMap[*rv.Alias.Aliasname] = fqn } } - for _, f := range qc.fromClauses { - catCols := make([]*catalog.Column, 0, len(f.Columns)) - for _, col := range f.Columns { - catCols = append(catCols, &catalog.Column{ - Name: col.Name, - Type: ast.TypeName{Name: col.DataType}, - IsNotNull: col.NotNull, - IsUnsigned: col.Unsigned, - IsArray: col.IsArray, - ArrayDims: col.ArrayDims, - Comment: col.Comment, - Length: col.Length, - }) - } + if qc != nil { + for _, f := range qc.fromClauses { + catCols := make([]*catalog.Column, 0, len(f.Columns)) + for _, col := range f.Columns { + catCols = append(catCols, &catalog.Column{ + Name: col.Name, + Type: ast.TypeName{Name: col.DataType}, + IsNotNull: col.NotNull, + IsUnsigned: col.Unsigned, + IsArray: col.IsArray, + ArrayDims: col.ArrayDims, + Comment: col.Comment, + Length: col.Length, + }) + } - if err := indexTable(catalog.Table{ - Rel: f.Rel, - Columns: catCols, - }); err != nil { - return nil, err + if err := indexTable(catalog.Table{ + Rel: f.Rel, + Columns: catCols, + }); err != nil { + return nil, err + } } } From b1772b5cd2bee779b3ae22943e9b8c4b6aec6030 Mon Sep 17 00:00:00 2001 From: Jille Timmermans Date: Fri, 5 Apr 2024 00:14:21 +0200 Subject: [PATCH 3/5] regen a test --- .../select_subquery_alias/postgresql/pgx/go/query.sql.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/internal/endtoend/testdata/select_subquery_alias/postgresql/pgx/go/query.sql.go b/internal/endtoend/testdata/select_subquery_alias/postgresql/pgx/go/query.sql.go index 8744642eb0..09ceb6e0b3 100644 --- a/internal/endtoend/testdata/select_subquery_alias/postgresql/pgx/go/query.sql.go +++ b/internal/endtoend/testdata/select_subquery_alias/postgresql/pgx/go/query.sql.go @@ -24,19 +24,19 @@ where amounts.last_balance < $2 ` type FindWalletsParams struct { - Column1 pgtype.Text - Column2 pgtype.Numeric + Type string + LastBalance int32 } type FindWalletsRow struct { ID int64 Address string Balance pgtype.Numeric - TotalBalance pgtype.Numeric + TotalBalance int64 } func (q *Queries) FindWallets(ctx context.Context, arg FindWalletsParams) ([]FindWalletsRow, error) { - rows, err := q.db.Query(ctx, findWallets, arg.Column1, arg.Column2) + rows, err := q.db.Query(ctx, findWallets, arg.Type, arg.LastBalance) if err != nil { return nil, err } From 7d1b0c648ee30d22d698109bb2e229303043494e Mon Sep 17 00:00:00 2001 From: Jille Timmermans Date: Fri, 5 Apr 2024 19:15:58 +0000 Subject: [PATCH 4/5] Manually update test to mismatch `sqlc generate` but match the CI's desire :( --- .../select_subquery_alias/postgresql/pgx/go/query.sql.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/endtoend/testdata/select_subquery_alias/postgresql/pgx/go/query.sql.go b/internal/endtoend/testdata/select_subquery_alias/postgresql/pgx/go/query.sql.go index 09ceb6e0b3..0cb154dba8 100644 --- a/internal/endtoend/testdata/select_subquery_alias/postgresql/pgx/go/query.sql.go +++ b/internal/endtoend/testdata/select_subquery_alias/postgresql/pgx/go/query.sql.go @@ -25,14 +25,14 @@ where amounts.last_balance < $2 type FindWalletsParams struct { Type string - LastBalance int32 + LastBalance pgtype.Numeric } type FindWalletsRow struct { ID int64 Address string Balance pgtype.Numeric - TotalBalance int64 + TotalBalance pgtype.Numeric } func (q *Queries) FindWallets(ctx context.Context, arg FindWalletsParams) ([]FindWalletsRow, error) { From 178e44932239c3049c1224d5a5dff07561e86f02 Mon Sep 17 00:00:00 2001 From: Jille Timmermans Date: Tue, 16 Apr 2024 12:13:39 +0200 Subject: [PATCH 5/5] Bugfixes in handling unnamed subqueries --- internal/compiler/output_columns.go | 11 +++++-- internal/compiler/query_catalog.go | 12 ++++++-- .../testdata/join_alias/mysql/go/query.sql.go | 29 ++++++++++++++++++- .../testdata/join_alias/mysql/query.sql | 3 ++ 4 files changed, 48 insertions(+), 7 deletions(-) diff --git a/internal/compiler/output_columns.go b/internal/compiler/output_columns.go index dbdbe252b3..4e9c8b02d2 100644 --- a/internal/compiler/output_columns.go +++ b/internal/compiler/output_columns.go @@ -3,6 +3,7 @@ package compiler import ( "errors" "fmt" + "math/rand" "github.com/sqlc-dev/sqlc/internal/sql/ast" "github.com/sqlc-dev/sqlc/internal/sql/astutils" @@ -596,10 +597,14 @@ func (c *Compiler) sourceTables(qc *QueryCatalog, node ast.Node) ([]*Table, erro if err != nil { return nil, err } + rel := &ast.TableName{} + if n.Alias != nil && n.Alias.Aliasname != nil { + rel.Name = *n.Alias.Aliasname + } else { + rel.Name = fmt.Sprintf("unnamed_subquery_%d", rand.Int63()) + } tables = append(tables, &Table{ - Rel: &ast.TableName{ - Name: *n.Alias.Aliasname, - }, + Rel: rel, Columns: cols, }) diff --git a/internal/compiler/query_catalog.go b/internal/compiler/query_catalog.go index 96b42be7f3..29abace6e2 100644 --- a/internal/compiler/query_catalog.go +++ b/internal/compiler/query_catalog.go @@ -2,6 +2,7 @@ package compiler import ( "fmt" + "math/rand" "github.com/sqlc-dev/sqlc/internal/sql/ast" "github.com/sqlc-dev/sqlc/internal/sql/catalog" @@ -78,7 +79,7 @@ func (comp *Compiler) buildQueryCatalog(c *catalog.Catalog, node ast.Node, embed return nil, err } var names []string - if rs.Alias.Colnames != nil { + if rs.Alias != nil && rs.Alias.Colnames != nil { for _, item := range rs.Alias.Colnames.Items { if val, ok := item.(*ast.String); ok { names = append(names, val.Str) @@ -87,14 +88,19 @@ func (comp *Compiler) buildQueryCatalog(c *catalog.Catalog, node ast.Node, embed } } } - rel := &ast.TableName{Name: *rs.Alias.Aliasname} + rel := &ast.TableName{} + if rs.Alias != nil && rs.Alias.Aliasname != nil { + rel.Name = *rs.Alias.Aliasname + } else { + rel.Name = fmt.Sprintf("unaliased_table_%d", rand.Int63()) + } for i := range cols { cols[i].Table = rel if len(names) > i { cols[i].Name = names[i] } } - qc.fromClauses[*rs.Alias.Aliasname] = &Table{ + qc.fromClauses[rel.Name] = &Table{ Rel: rel, Columns: cols, } diff --git a/internal/endtoend/testdata/join_alias/mysql/go/query.sql.go b/internal/endtoend/testdata/join_alias/mysql/go/query.sql.go index e6d565560b..1ac7681d5e 100644 --- a/internal/endtoend/testdata/join_alias/mysql/go/query.sql.go +++ b/internal/endtoend/testdata/join_alias/mysql/go/query.sql.go @@ -82,7 +82,7 @@ func (q *Queries) AliasJoin(ctx context.Context, id uint64) ([]AliasJoinRow, err } const columnAlias = `-- name: ColumnAlias :many -SELECT n FROM (SELECT 1 AS n) AS x WHERE n <= ? +SELECT n FROM (SELECT 1 AS n) WHERE n <= ? ` func (q *Queries) ColumnAlias(ctx context.Context, n int32) ([]int32, error) { @@ -108,6 +108,33 @@ func (q *Queries) ColumnAlias(ctx context.Context, n int32) ([]int32, error) { return items, nil } +const columnAndQueryAlias = `-- name: ColumnAndQueryAlias :many +SELECT n FROM (SELECT 1 AS n) AS x WHERE n <= ? +` + +func (q *Queries) ColumnAndQueryAlias(ctx context.Context, n int32) ([]int32, error) { + rows, err := q.db.QueryContext(ctx, columnAndQueryAlias, n) + if err != nil { + return nil, err + } + defer rows.Close() + var items []int32 + for rows.Next() { + var n int32 + if err := rows.Scan(&n); err != nil { + return nil, err + } + items = append(items, n) + } + if err := rows.Close(); err != nil { + return nil, err + } + if err := rows.Err(); err != nil { + return nil, err + } + return items, nil +} + const subqueryAlias = `-- name: SubqueryAlias :many SELECT n FROM (SELECT 1 AS n) AS x WHERE x.n <= ? ` diff --git a/internal/endtoend/testdata/join_alias/mysql/query.sql b/internal/endtoend/testdata/join_alias/mysql/query.sql index 82bb2e5cd5..77ed3cd3ad 100644 --- a/internal/endtoend/testdata/join_alias/mysql/query.sql +++ b/internal/endtoend/testdata/join_alias/mysql/query.sql @@ -14,4 +14,7 @@ WHERE f.id = ?; SELECT * FROM (SELECT 1 AS n) AS x WHERE x.n <= ?; -- name: ColumnAlias :many +SELECT * FROM (SELECT 1 AS n) WHERE n <= ?; + +-- name: ColumnAndQueryAlias :many SELECT * FROM (SELECT 1 AS n) AS x WHERE n <= ?;