feat: push --exclude down to driver query level

cmd/cmd.go

@@ -19,7 +19,7 @@ func getSchemaFromJSONorDSN(c *config.Config) (*schema.Schema, error) {
 		}
 		return s, nil
 	}
-	s, err := datasource.Analyze(c.DSN)
+	s, err := datasource.Analyze(c.DSN, datasource.WithTableExcludes(c.PushDownableTableExcludes()))
 	if err != nil {
 		return nil, err
 	}

cmd/doc.go

@@ -69,7 +69,7 @@ var docCmd = &cobra.Command{
 			return err
 		}
 
-		s, err := datasource.Analyze(c.DSN)
+		s, err := datasource.Analyze(c.DSN, datasource.WithTableExcludes(c.PushDownableTableExcludes()))
 		if err != nil {
 			return err
 		}

config/config.go

@@ -549,6 +549,17 @@ func (c *Config) MergeAdditionalData(s *schema.Schema) error {
 	return nil
 }
 
+// PushDownableTableExcludes returns the exclude patterns that are safe to
+// push down to the driver as a query-level filter. When include or label
+// filters are in play, distance traversal can re-pull excluded tables, so
+// in that case we let the post-fetch schema.Filter handle everything.
+func (c *Config) PushDownableTableExcludes() []string {
+	if len(c.Include) > 0 || len(c.includeLabels) > 0 {
+		return nil
+	}
+	return c.Exclude
+}
+
 // FilterTables filter tables from schema.Schema using include: and exclude: and includeLabels.
 func (c *Config) FilterTables(s *schema.Schema) error {
 	return s.Filter(&schema.FilterOption{

datasource/datasource.go

@@ -41,8 +41,28 @@ var supportDriversWithDburl = []string{
 	"clickhouse",
 }
 
+// AnalyzeOption configures Analyze.
+type AnalyzeOption func(*analyzeOptions)
+
+type analyzeOptions struct {
+	tableExcludes []string
+}
+
+// WithTableExcludes hints to filterer-capable drivers to push down the
+// given exclude patterns at the query stage. Patterns use the same glob
+// syntax as the --exclude flag.
+func WithTableExcludes(patterns []string) AnalyzeOption {
+	return func(o *analyzeOptions) {
+		o.tableExcludes = patterns
+	}
+}
+
 // Analyze database.
-func Analyze(dsn config.DSN) (_ *schema.Schema, err error) {
+func Analyze(dsn config.DSN, analyzeOpts ...AnalyzeOption) (_ *schema.Schema, err error) {
+	ao := &analyzeOptions{}
+	for _, opt := range analyzeOpts {
+		opt(ao)
+	}
 	defer func() {
 		err = errors.WithStack(err)
 	}()
@@ -155,6 +175,9 @@ func Analyze(dsn config.DSN) (_ *schema.Schema, err error) {
 	default:
 		return s, fmt.Errorf("unsupported driver '%s'", u.Driver)
 	}
+	if f, ok := driver.(drivers.TableFilterer); ok && len(ao.tableExcludes) > 0 {
+		f.SetTableExcludes(ao.tableExcludes)
+	}
 	err = driver.Analyze(s)
 	if err != nil {
 		return nil, err

drivers/drivers.go

@@ -12,3 +12,12 @@ type Driver interface {
 
 // Option is the type for change Config.
 type Option func(Driver) error
+
+// TableFilterer is implemented by drivers that can apply table-name
+// exclusion at the query stage as a perf optimization. The post-fetch
+// schema.Filter remains authoritative; drivers that under-fetch will
+// silently lose tables, so implementations must only push down patterns
+// they can faithfully translate.
+type TableFilterer interface {
+	SetTableExcludes(patterns []string)
+}

drivers/postgres/postgres.go

@@ -19,8 +19,9 @@ var reVersion = regexp.MustCompile(`([0-9]+(\.[0-9]+)*)`)
 
 // Postgres struct.
 type Postgres struct {
-	db     *sql.DB
-	rsMode bool
+	db            *sql.DB
+	rsMode        bool
+	tableExcludes []string
 }
 
 // New return new Postgres.
@@ -31,6 +32,14 @@ func New(db *sql.DB) *Postgres {
 	}
 }
 
+// SetTableExcludes pushes table-name exclude patterns into the table
+// listing query. Patterns use the same glob syntax as the --exclude flag;
+// only patterns that translate cleanly to SQL LIKE are pushed down, the
+// rest fall through to schema.Filter post-fetch.
+func (p *Postgres) SetTableExcludes(patterns []string) {
+	p.tableExcludes = patterns
+}
+
 // Analyze PostgreSQL database schema.
 func (p *Postgres) Analyze(s *schema.Schema) (err error) {
 	defer func() {
@@ -98,24 +107,7 @@ func (p *Postgres) Analyze(s *schema.Schema) (err error) {
 	fullTableNames := []string{}
 
 	// tables
-	tableRows, err := p.db.Query(`
-SELECT
-    cls.oid AS oid,
-    cls.relname AS table_name,
-    CASE
-        WHEN cls.relkind IN ('r', 'p') THEN 'BASE TABLE'
-        WHEN cls.relkind = 'v' THEN 'VIEW'
-        WHEN cls.relkind = 'm' THEN 'MATERIALIZED VIEW'
-        WHEN cls.relkind = 'f' THEN 'FOREIGN TABLE'
-    END AS table_type,
-    ns.nspname AS table_schema,
-    descr.description AS table_comment
-FROM pg_class AS cls
-INNER JOIN pg_namespace AS ns ON cls.relnamespace = ns.oid
-LEFT JOIN pg_description AS descr ON cls.oid = descr.objoid AND descr.objsubid = 0
-WHERE ns.nspname NOT IN ('pg_catalog', 'information_schema')
-AND cls.relkind IN ('r', 'p', 'v', 'f', 'm')
-ORDER BY oid`)
+	tableRows, err := p.db.Query(p.queryForTables())
 	if err != nil {
 		return errors.WithStack(err)
 	}
@@ -339,29 +331,39 @@ ORDER BY tgrelid
 	s.Tables = tables
 
 	// Relations
+	validRelations := []*schema.Relation{}
 	for _, r := range relations {
 		strColumns, strParentTable, strParentColumns, err := parseFK(r.Def)
 		if err != nil {
 			return err
 		}
-		for _, c := range strColumns {
-			column, err := r.Table.FindColumnByName(c)
-			if err != nil {
-				return err
-			}
-			r.Columns = append(r.Columns, column)
-			column.ParentRelations = append(column.ParentRelations, r)
-		}
 
 		dn, err := detectFullTableName(strParentTable, s.Driver.Meta.SearchPaths, fullTableNames)
 		if err != nil {
+			if len(p.tableExcludes) > 0 {
+				// Parent table was filtered out at the query stage; drop the relation.
+				continue
+			}
 			return err
 		}
 		strParentTable = dn
 		parentTable, err := s.FindTableByName(strParentTable)
 		if err != nil {
+			if len(p.tableExcludes) > 0 {
+				continue
+			}
 			return err
 		}
+
+		for _, c := range strColumns {
+			column, err := r.Table.FindColumnByName(c)
+			if err != nil {
+				return err
+			}
+			r.Columns = append(r.Columns, column)
+			column.ParentRelations = append(column.ParentRelations, r)
+		}
+
 		r.ParentTable = parentTable
 		for _, c := range strParentColumns {
 			column, err := parentTable.FindColumnByName(c)
@@ -371,9 +373,10 @@ ORDER BY tgrelid
 			r.ParentColumns = append(r.ParentColumns, column)
 			column.ChildRelations = append(column.ChildRelations, r)
 		}
+		validRelations = append(validRelations, r)
 	}
 
-	s.Relations = relations
+	s.Relations = validRelations
 
 	// referenced tables of view
 	for _, t := range s.Tables {
@@ -712,6 +715,68 @@ GROUP BY cls.relname, idx.indexrelid, descr.description
 ORDER BY idx.indexrelid`
 }
 
+func (p *Postgres) queryForTables() string {
+	baseQuery := `
+SELECT
+    cls.oid AS oid,
+    cls.relname AS table_name,
+    CASE
+        WHEN cls.relkind IN ('r', 'p') THEN 'BASE TABLE'
+        WHEN cls.relkind = 'v' THEN 'VIEW'
+        WHEN cls.relkind = 'm' THEN 'MATERIALIZED VIEW'
+        WHEN cls.relkind = 'f' THEN 'FOREIGN TABLE'
+    END AS table_type,
+    ns.nspname AS table_schema,
+    descr.description AS table_comment
+FROM pg_class AS cls
+INNER JOIN pg_namespace AS ns ON cls.relnamespace = ns.oid
+LEFT JOIN pg_description AS descr ON cls.oid = descr.objoid AND descr.objsubid = 0
+WHERE ns.nspname NOT IN ('pg_catalog', 'information_schema')
+AND cls.relkind IN ('r', 'p', 'v', 'f', 'm')`
+
+	for _, like := range tableExcludeLikes(p.tableExcludes) {
+		baseQuery += fmt.Sprintf("\nAND cls.relname NOT LIKE %s ESCAPE '\\' AND (ns.nspname || '.' || cls.relname) NOT LIKE %s ESCAPE '\\'", like, like)
+	}
+
+	baseQuery += `
+ORDER BY oid`
+	return baseQuery
+}
+
+// tableExcludeLikes converts each glob pattern that contains only `*` and
+// literal characters into a SQL-quoted LIKE pattern (with `_` and `%`
+// escaped). Patterns containing other wildcard characters are dropped;
+// schema.Filter will still exclude those tables post-fetch.
+func tableExcludeLikes(patterns []string) []string {
+	var out []string
+	for _, p := range patterns {
+		like, ok := globToLike(p)
+		if !ok {
+			continue
+		}
+		out = append(out, "'"+strings.ReplaceAll(like, "'", "''")+"'")
+	}
+	return out
+}
+
+func globToLike(pattern string) (string, bool) {
+	var b strings.Builder
+	for _, r := range pattern {
+		switch r {
+		case '*':
+			b.WriteByte('%')
+		case '?':
+			return "", false
+		case '%', '_', '\\':
+			b.WriteByte('\\')
+			b.WriteRune(r)
+		default:
+			b.WriteRune(r)
+		}
+	}
+	return b.String(), true
+}
+
 func detectFullTableName(name string, searchPaths, fullTableNames []string) (_ string, err error) {
 	defer func() {
 		err = errors.WithStack(err)

drivers/postgres/postgres_unit_test.go

@@ -0,0 +1,66 @@
+package postgres
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestGlobToLike(t *testing.T) {
+	tests := []struct {
+		name     string
+		in       string
+		wantLike string
+		wantOK   bool
+	}{
+		{"plain name", "users", "users", true},
+		{"trailing star", "events_*", `events\_%`, true},
+		{"leading star", "*_archive", `%\_archive`, true},
+		{"escapes percent", "10%off", `10\%off`, true},
+		{"escapes backslash", `a\b`, `a\\b`, true},
+		{"schema qualified", "public.events_*", `public.events\_%`, true},
+		{"rejects question mark", "foo?", "", false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, ok := globToLike(tt.in)
+			if ok != tt.wantOK {
+				t.Fatalf("globToLike(%q) ok = %v, want %v", tt.in, ok, tt.wantOK)
+			}
+			if got != tt.wantLike {
+				t.Errorf("globToLike(%q) = %q, want %q", tt.in, got, tt.wantLike)
+			}
+		})
+	}
+}
+
+func TestQueryForTablesNoExcludes(t *testing.T) {
+	p := &Postgres{}
+	q := p.queryForTables()
+	if strings.Contains(q, "NOT LIKE") {
+		t.Errorf("query unexpectedly contains NOT LIKE clause when no excludes set:\n%s", q)
+	}
+	if !strings.Contains(q, "ORDER BY oid") {
+		t.Errorf("query missing ORDER BY clause:\n%s", q)
+	}
+}
+
+func TestQueryForTablesWithExcludes(t *testing.T) {
+	p := &Postgres{}
+	p.SetTableExcludes([]string{"events_*", "public.logs_*", "ignored?glob"})
+	q := p.queryForTables()
+
+	wants := []string{
+		`cls.relname NOT LIKE 'events\_%' ESCAPE '\'`,
+		`(ns.nspname || '.' || cls.relname) NOT LIKE 'events\_%' ESCAPE '\'`,
+		`cls.relname NOT LIKE 'public.logs\_%' ESCAPE '\'`,
+	}
+	for _, w := range wants {
+		if !strings.Contains(q, w) {
+			t.Errorf("query missing expected clause %q\nfull query:\n%s", w, q)
+		}
+	}
+	// The ?-glob pattern should be dropped entirely.
+	if strings.Contains(q, "ignored") {
+		t.Errorf("query unexpectedly contains pushed-down ?-glob pattern:\n%s", q)
+	}
+}