diff --git a/ibis/backends/__init__.py b/ibis/backends/__init__.py index f1c06f1a4adb..830ebfde1959 100644 --- a/ibis/backends/__init__.py +++ b/ibis/backends/__init__.py @@ -48,6 +48,43 @@ class TablesAccessor(collections.abc.Mapping): def __init__(self, backend: BaseBackend) -> None: self._backend = backend + def _execute_if_exists( + self, method_name: str, database=None, like=None + ) -> list[str]: + """Executes method if it exists and it doesn't raise a NotImplementedError, else returns an empty list.""" + try: + method = getattr(self._backend.ddl, method_name) + return method(database=database, like=like) + except NotImplementedError: + pass + return [] + + def _gather_tables(self, database=None, like=None) -> list[str]: + """Gathers table names using the list_* methods available on the backend.""" + # TODO: break this down into views/tables to be more explicit in repr (see #9859) + # list_* methods that might exist on a given backends. + list_methods = [ + "list_tables", + "list_temp_tables", + "list_views", + "list_temp_views", + "list_foreign_tables", + "list_materialized_views", + ] + tables = [] + for method_name in list_methods: + tables.extend( + self._execute_if_exists(method_name, database=database, like=like) + ) + return tables + + def __call__(self, database=None, like=None): + return self._gather_tables(database, like) + + @property + def _tables(self) -> list[str]: + return self._gather_tables() + def __getitem__(self, name) -> ir.Table: try: return self._backend.table(name) @@ -63,29 +100,179 @@ def __getattr__(self, name) -> ir.Table: raise AttributeError(name) from exc def __iter__(self) -> Iterator[str]: - return iter(sorted(self._backend.list_tables())) + return iter(sorted(self._tables)) def __len__(self) -> int: - return len(self._backend.list_tables()) + return len(self._tables) def __dir__(self) -> list[str]: o = set() o.update(dir(type(self))) o.update( name - for name in self._backend.list_tables() + for name in self._tables if name.isidentifier() and not keyword.iskeyword(name) ) return list(o) def __repr__(self) -> str: - tables = self._backend.list_tables() rows = ["Tables", "------"] - rows.extend(f"- {name}" for name in sorted(tables)) + rows.extend(f"- {name}" for name in sorted(self._tables)) return "\n".join(rows) def _ipython_key_completions_(self) -> list[str]: - return self._backend.list_tables() + return self._tables + + +class DDLAccessor: + """ddl accessor list views.""" + + def __init__(self, backend: BaseBackend) -> None: + self._backend = backend + + def _raise_if_not_implemented(self, method_name: str): + try: + getattr(self._backend, method_name) + except AttributeError as e: + if f"has no attribute '{method_name}'" in str(e): + raise NotImplementedError( + f"The method {method_name} is not implemented for the {self._backend.name} backend" + ) + + def list_tables( + self, like: str | None = None, database: tuple[str, str] | str | None = None + ) -> list[str]: + """Return the list of table names in a database via the backend's implementation. + + ::: {.callout-note} + ## Ibis does not use the word `schema` to refer to database hierarchy. + + A collection of tables is referred to as a `database`. + A collection of `database` is referred to as a `catalog`. + + These terms are mapped onto the corresponding features in each + backend (where available), regardless of whether the backend itself + uses the same terminology. + ::: + + Parameters + ---------- + like + A pattern to use for listing tables. + database + Database to list tables from. Default behavior is to show tables in + the current database. + """ + + self._raise_if_not_implemented("_list_tables") + return self._backend._list_tables(like=like, database=database) + + def list_temp_tables( + self, like: str | None = None, database: tuple[str, str] | str | None = None + ) -> list[str]: + """Return the list of temporary table names in a database via the backend's implementation. + + ::: {.callout-note} + ## Ibis does not use the word `schema` to refer to database hierarchy. + + A collection of tables is referred to as a `database`. + A collection of `database` is referred to as a `catalog`. + + These terms are mapped onto the corresponding features in each + backend (where available), regardless of whether the backend itself + uses the same terminology. + ::: + + Parameters + ---------- + like + A pattern to use for listing tables. + database + Database to list tables from. Default behavior is to show tables in + the current database. + """ + + self._raise_if_not_implemented("_list_temp_tables") + return self._backend._list_temp_tables(like=like, database=database) + + def list_views( + self, like: str | None = None, database: tuple[str, str] | str | None = None + ) -> list[str]: + """Return the list of view names in a database via the backend's implementation. + + ::: {.callout-note} + ## Ibis does not use the word `schema` to refer to database hierarchy. + + A collection of tables is referred to as a `database`. + A collection of `database` is referred to as a `catalog`. + + These terms are mapped onto the corresponding features in each + backend (where available), regardless of whether the backend itself + uses the same terminology. + ::: + + Parameters + ---------- + like + A pattern to use for listing tables. + database + Database to list tables from. Default behavior is to show tables in + the current database. + """ + + self._raise_if_not_implemented("_list_views") + return self._backend._list_views(like=like, database=database) + + def list_temp_views( + self, like: str | None = None, database: tuple[str, str] | str | None = None + ) -> list[str]: + """Return the list of temporary view names in a database via the backend's implementation. + + ::: {.callout-note} + ## Ibis does not use the word `schema` to refer to database hierarchy. + + A collection of tables is referred to as a `database`. + A collection of `database` is referred to as a `catalog`. + + These terms are mapped onto the corresponding features in each + backend (where available), regardless of whether the backend itself + uses the same terminology. + ::: + + Parameters + ---------- + like + A pattern to use for listing tables. + database + Database to list tables from. Default behavior is to show tables in + the current database. + """ + + self._raise_if_not_implemented("_list_temp_views") + return self._backend._list_temp_views(like=like, database=database) + + def list_materialized_views( + self, like: str | None = None, database: tuple[str, str] | str | None = None + ) -> list[str]: + self._raise_if_not_implemented("_list_materialized_views") + return self._backend._list_materialized_views(like=like, database=database) + + def list_foreign_tables( + self, like: str | None = None, database: tuple[str, str] | str | None = None + ) -> list[str]: + """Return the list of foreign table names via the backend's implementation. + + Parameters + ---------- + like + A pattern to use for listing tables. + database + Database to list foreign tables from. Default behavior is to show tables in + the current database. + """ + + self._raise_if_not_implemented("_list_foreign_tables") + return self._backend._list_foreign_tables(like=like, database=database) class _FileIOHandler: @@ -947,44 +1134,6 @@ def _filter_with_like(values: Iterable[str], like: str | None = None) -> list[st pattern = re.compile(like) return sorted(filter(pattern.findall, values)) - @abc.abstractmethod - def list_tables( - self, like: str | None = None, database: tuple[str, str] | str | None = None - ) -> list[str]: - """Return the list of table names in the current database. - - For some backends, the tables may be files in a directory, - or other equivalent entities in a SQL database. - - ::: {.callout-note} - ## Ibis does not use the word `schema` to refer to database hierarchy. - - A collection of tables is referred to as a `database`. - A collection of `database` is referred to as a `catalog`. - - These terms are mapped onto the corresponding features in each - backend (where available), regardless of whether the backend itself - uses the same terminology. - ::: - - Parameters - ---------- - like - A pattern in Python's regex format. - database - The database from which to list tables. - If not provided, the current database is used. - For backends that support multi-level table hierarchies, you can - pass in a dotted string path like `"catalog.database"` or a tuple of - strings like `("catalog", "database")`. - - Returns - ------- - list[str] - The list of the table names that match the pattern `like`. - - """ - @abc.abstractmethod def table( self, name: str, database: tuple[str, str] | str | None = None @@ -1035,6 +1184,11 @@ def tables(self): """ return TablesAccessor(self) + @property + def ddl(self): + """A ddl accessor.""" + return DDLAccessor(self) + @property @abc.abstractmethod def version(self) -> str: diff --git a/ibis/backends/clickhouse/__init__.py b/ibis/backends/clickhouse/__init__.py index be3334373594..74b5e66b5639 100644 --- a/ibis/backends/clickhouse/__init__.py +++ b/ibis/backends/clickhouse/__init__.py @@ -30,6 +30,7 @@ from ibis.backends.clickhouse.converter import ClickHousePandasData from ibis.backends.sql import SQLBackend from ibis.backends.sql.compilers.base import C +from ibis.util import deprecated if TYPE_CHECKING: from collections.abc import Iterable, Iterator, Mapping @@ -206,38 +207,111 @@ def list_databases(self, like: str | None = None) -> list[str]: databases = [] return self._filter_with_like(databases, like) - def list_tables( - self, like: str | None = None, database: str | None = None + def _list_query_constructor(self, col: str, where_predicates: list) -> str: + """Helper function to construct sqlglot queries for _list_* methods.""" + + sg_query = ( + sg.select(col) + .from_(sg.table("tables", db="system")) + .where(*where_predicates) + ).sql(self.name) + + return sg_query + + def _list_objects( + self, + like: str | None, + database: tuple[str, str] | str | None, + object_type: str, + is_temp: bool = False, ) -> list[str]: - """List the tables in the database. + """Generic method to list objects like tables or views.""" - Parameters - ---------- - like - A pattern to use for listing tables. - database - Database to list tables from. Default behavior is to show tables in - the current database. - """ + table_loc = self._to_sqlglot_table(database) - query = sg.select(C.name).from_(sg.table("tables", db="system")) + # when is_temp=True the table is in memory and no database is assigned + database = table_loc.db or ("" if is_temp else self.current_database) + col = "name" - if database is None: - database = self.compiler.f.currentDatabase() + if object_type == "Table": # we have to check is not a view or mat view + where_predicates = [ + C.database.eq(sge.convert(database)), + C.engine.isin("View", "MaterializedView").not_(), + C.is_temporary.eq(int(is_temp)), + ] else: - database = sge.convert(database) - - query = query.where(C.database.eq(database).or_(C.is_temporary)) + where_predicates = [ + C.database.eq(sge.convert(database)), + C.engine.eq(object_type), + C.is_temporary.eq(int(is_temp)), + ] - with self._safe_raw_sql(query) as result: + sql = self._list_query_constructor(col, where_predicates) + with self._safe_raw_sql(sql) as result: results = result.result_columns if results: (tables,) = results else: tables = [] + return self._filter_with_like(tables, like) + def _list_tables( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List physical tables.""" + + return self._list_objects(like, database, "Table") + + def _list_temp_tables( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List temporary tables.""" + + return self._list_objects(like, database, "Table", is_temp=True) + + def _list_views( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List views.""" + + return self._list_objects(like, database, "View") + + def _list_materialized_views( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List views.""" + + return self._list_objects(like, database, "MaterializedView") + + @deprecated(as_of="10.0", instead="use the con.tables") + def list_tables( + self, like: str | None = None, database: str | None = None + ) -> list[str]: + table_loc = self._to_sqlglot_table(database) + + database = self.current_database + if table_loc is not None: + database = table_loc.db or database + + tables_and_views = list( + set(self._list_tables(like=like, database=database)) + | set(self._list_temp_tables(like=like, database=database)) + | set(self._list_views(like=like, database=database)) + | set(self._list_materialized_views(like=like, database=database)) + ) + + return tables_and_views + def _normalize_external_tables(self, external_tables=None) -> ExternalData | None: """Merge registered external tables with any new external tables.""" external_data = ExternalData() diff --git a/ibis/backends/clickhouse/tests/test_client.py b/ibis/backends/clickhouse/tests/test_client.py index 270b305f3aeb..fcde3af85f7e 100644 --- a/ibis/backends/clickhouse/tests/test_client.py +++ b/ibis/backends/clickhouse/tests/test_client.py @@ -193,8 +193,8 @@ def test_get_schema_using_query(con, query, expected_schema): def test_list_tables_database(con): - tables = con.list_tables() - tables2 = con.list_tables(database=con.current_database) + tables = con.ddl.list_tables() + tables2 = con.ddl.list_tables(database=con.current_database) # some overlap, but not necessarily identical because # a database may have temporary tables added/removed between list_tables # calls @@ -216,7 +216,7 @@ def tmpcon(worker_id): def test_list_tables_empty_database(tmpcon): - assert not tmpcon.list_tables() + assert not tmpcon.ddl.list_tables() @pytest.mark.parametrize("temp", [True, False], ids=["temp", "no_temp"]) @@ -453,3 +453,25 @@ def test_query_cache(con, method_name): method(settings={"ooze_query_cash": True}) assert result == expected + + +def test_list_materialized_views(con): + con.create_table( + "my_table", + {"id": [1, 2, 3], "val": ["a", "b", "c"]}, + schema=ibis.schema({"id": "!int", "val": "!str"}), + overwrite=True, + ) + + # drop mat view if exists, then create + # handles materialized views as tables for dropping purposes + con.raw_sql("""DROP TABLE IF EXISTS mat_view_example""") + # it doesn't like both statements in one + con.raw_sql( + """CREATE MATERIALIZED VIEW mat_view_example ENGINE=MergeTree() ORDER BY id AS SELECT * FROM my_table""" + ) + + assert "mat_view_example" in con.ddl.list_materialized_views() + assert "mat_view_example" not in con.ddl.list_views() + assert "mat_view_example" not in con.ddl.list_tables() + assert "mat_view_example" not in con.ddl.list_temp_tables() diff --git a/ibis/backends/datafusion/__init__.py b/ibis/backends/datafusion/__init__.py index 8ea440b3cdfa..57fbe02989fa 100644 --- a/ibis/backends/datafusion/__init__.py +++ b/ibis/backends/datafusion/__init__.py @@ -99,7 +99,7 @@ def do_connect( ... "diamonds": "ci/ibis-testing-data/csv/diamonds.csv", ... } >>> con = ibis.datafusion.connect(config) - >>> con.list_tables() + >>> con.tables() ['astronauts', 'diamonds'] >>> con.table("diamonds") DatabaseTable: diamonds @@ -293,6 +293,58 @@ def drop_database( with self._safe_raw_sql(sge.Drop(kind="SCHEMA", this=db_name, exists=force)): pass + def _list_query_constructor(self, col: str, where_predicates: list) -> str: + """Helper function to construct sqlglot queries for _list_* methods.""" + + sg_query = ( + sg.select(col) + .from_(sg.table("tables", db="information_schema")) + .where(*where_predicates) + .order_by("table_name") # in datafusion we ordered the tables + ).sql(self.name) + + return sg_query + + def _list_objects( + self, + like: str | None, + database: tuple[str, str] | str | None, + object_type: str, + ) -> list[str]: + """Generic method to list objects like tables or views.""" + + database = database or "public" + + col = "table_name" + where_predicates = [ + C.table_schema.eq(sge.convert(database)), + C.table_type.eq(object_type), + ] + + sql = self._list_query_constructor(col, where_predicates) + out = self.raw_sql(sql).to_pydict() + + return self._filter_with_like(out[col], like) + + def _list_tables( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List physical tables.""" + + return self._list_objects(like, database, "BASE TABLE") + + def _list_views( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List views.""" + + return self._list_objects(like, database, "VIEW") + + @deprecated(as_of="10.0", instead="use the con.tables") def list_tables( self, like: str | None = None, @@ -313,16 +365,14 @@ def list_tables( The list of the table names that match the pattern `like`. """ database = database or "public" - query = ( - sg.select("table_name") - .from_("information_schema.tables") - .where(sg.column("table_schema").eq(sge.convert(database))) - .order_by("table_name") - ) - return self._filter_with_like( - self.raw_sql(query).to_pydict()["table_name"], like + + tables_and_views = list( + set(self._list_tables(like=like, database=database)) + | set(self._list_views(like=like, database=database)) ) + return tables_and_views + def get_schema( self, table_name: str, diff --git a/ibis/backends/datafusion/tests/test_connect.py b/ibis/backends/datafusion/tests/test_connect.py index 6b3773f8370f..21ae02262812 100644 --- a/ibis/backends/datafusion/tests/test_connect.py +++ b/ibis/backends/datafusion/tests/test_connect.py @@ -20,19 +20,20 @@ def name_to_path(data_dir): def test_none_config(): config = None conn = ibis.datafusion.connect(config) - assert conn.list_tables() == [] + assert conn.ddl.list_tables() == [] + assert conn.ddl.list_views() == [] def test_str_config(name_to_path): config = {name: str(path) for name, path in name_to_path.items()} conn = ibis.datafusion.connect(config) - assert sorted(conn.list_tables()) == sorted(name_to_path) + assert sorted(conn.tables) == sorted(name_to_path) def test_path_config(name_to_path): config = name_to_path conn = ibis.datafusion.connect(config) - assert sorted(conn.list_tables()) == sorted(name_to_path) + assert sorted(conn.tables) == sorted(name_to_path) def test_context_config(name_to_path): @@ -40,4 +41,4 @@ def test_context_config(name_to_path): for name, path in name_to_path.items(): ctx.register_parquet(name, str(path)) conn = ibis.datafusion.connect(ctx) - assert sorted(conn.list_tables()) == sorted(name_to_path) + assert sorted(conn.tables) == sorted(name_to_path) diff --git a/ibis/backends/duckdb/__init__.py b/ibis/backends/duckdb/__init__.py index 0bdaa57a0f66..f1e0d409e9f9 100644 --- a/ibis/backends/duckdb/__init__.py +++ b/ibis/backends/duckdb/__init__.py @@ -979,81 +979,101 @@ def read_delta( self.con.register(table_name, delta_table.to_pyarrow_dataset()) return self.table(table_name) - def list_tables( + def _list_query_constructor(self, col: str, where_predicates: list) -> str: + """Helper function to construct sqlglot queries for _list_* methods.""" + + sg_query = ( + sg.select(col) + .from_(sg.table("tables", db="information_schema")) + .where(*where_predicates) + ).sql(self.name) + + return sg_query + + def _list_objects( + self, + like: str | None, + database: tuple[str, str] | str | None, + object_type: str, + is_temp: bool = False, + ) -> list[str]: + """Generic method to list objects like tables or views.""" + + table_loc = self._to_sqlglot_table(database) + + catalog = table_loc.catalog or ("temp" if is_temp else self.current_catalog) + database = table_loc.db or self.current_database + + col = "table_name" + where_predicates = [ + C.table_catalog.eq(sge.convert(catalog)), + C.table_schema.eq(sge.convert(database)), + C.table_type.eq(object_type), + ] + + sql = self._list_query_constructor(col, where_predicates) + out = self.con.execute(sql).fetch_arrow_table() + + return self._filter_with_like(out[col].to_pylist(), like) + + def _list_tables( self, like: str | None = None, database: tuple[str, str] | str | None = None, ) -> list[str]: - """List tables and views. + """List physical tables.""" - ::: {.callout-note} - ## Ibis does not use the word `schema` to refer to database hierarchy. + return self._list_objects(like, database, "BASE TABLE") - A collection of tables is referred to as a `database`. - A collection of `database` is referred to as a `catalog`. + def _list_temp_tables( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List temporary tables.""" - These terms are mapped onto the corresponding features in each - backend (where available), regardless of whether the backend itself - uses the same terminology. - ::: + return self._list_objects(like, database, "LOCAL TEMPORARY", is_temp=True) - Parameters - ---------- - like - Regex to filter by table/view name. - database - Database location. If not passed, uses the current database. + def _list_views( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List views.""" - By default uses the current `database` (`self.current_database`) and - `catalog` (`self.current_catalog`). + return self._list_objects(like, database, "VIEW") - To specify a table in a separate catalog, you can pass in the - catalog and database as a string `"catalog.database"`, or as a tuple of - strings `("catalog", "database")`. + def _list_temp_views( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List temporary views.""" - Returns - ------- - list[str] - List of table and view names. + return self._list_objects(like, database, "VIEW", is_temp=True) - Examples - -------- - >>> import ibis - >>> con = ibis.duckdb.connect() - >>> foo = con.create_table("foo", schema=ibis.schema(dict(a="int"))) - >>> con.list_tables() - ['foo'] - >>> bar = con.create_view("bar", foo) - >>> con.list_tables() - ['bar', 'foo'] - >>> con.create_database("my_database") - >>> con.list_tables(database="my_database") - [] - >>> con.raw_sql("CREATE TABLE my_database.baz (a INTEGER)") # doctest: +ELLIPSIS - - >>> con.list_tables(database="my_database") - ['baz'] + @deprecated(as_of="10.0", instead="use the con.tables") + def list_tables( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List tables and views.""" - """ table_loc = self._to_sqlglot_table(database) - catalog = table_loc.catalog or self.current_catalog - database = table_loc.db or self.current_database + database = self.current_database + if table_loc is not None: + database = table_loc.db or database - col = "table_name" - sql = ( - sg.select(col) - .from_(sg.table("tables", db="information_schema")) - .distinct() - .where( - C.table_catalog.isin(sge.convert(catalog), sge.convert("temp")), - C.table_schema.eq(sge.convert(database)), - ) - .sql(self.dialect) + tables_and_views = list( + set(self._list_tables(like=like, database=database)) + | set(self._list_temp_tables(like=like, database=database)) + | set(self._list_views(like=like, database=database)) + | set(self._list_temp_views(like=like, database=database)) ) - out = self.con.execute(sql).fetch_arrow_table() - return self._filter_with_like(out[col].to_pylist(), like) + return tables_and_views def read_postgres( self, uri: str, *, table_name: str | None = None, database: str = "public" @@ -1266,10 +1286,10 @@ def attach_sqlite( ... ) # doctest: +ELLIPSIS <...> >>> con = ibis.connect("duckdb://") - >>> con.list_tables() + >>> con.tables() [] >>> con.attach_sqlite("/tmp/attach_sqlite.db") - >>> con.list_tables() + >>> con.tables() ['t'] """ diff --git a/ibis/backends/duckdb/tests/test_catalog.py b/ibis/backends/duckdb/tests/test_catalog.py index a59c04ee3743..a5a96d99b81a 100644 --- a/ibis/backends/duckdb/tests/test_catalog.py +++ b/ibis/backends/duckdb/tests/test_catalog.py @@ -36,8 +36,8 @@ def test_read_write_external_catalog(con, external_duckdb_file, monkeypatch): assert "ext" in con.list_catalogs() assert "main" in con.list_databases(catalog="ext") - assert "starwars" in con.list_tables(database="ext.main") - assert "starwars" not in con.list_tables() + assert "starwars" in con.ddl.list_tables(database="ext.main") + assert "starwars" not in con.ddl.list_tables() starwars = con.table("starwars", database="ext.main") tm.assert_frame_equal(starwars.to_pandas(), starwars_df) @@ -47,8 +47,8 @@ def test_read_write_external_catalog(con, external_duckdb_file, monkeypatch): _ = con.create_table("t2", obj=t, database="ext.main") - assert "t2" in con.list_tables(database="ext.main") - assert "t2" not in con.list_tables() + assert "t2" in con.ddl.list_tables(database="ext.main") + assert "t2" not in con.ddl.list_tables() table = con.table("t2", database="ext.main") @@ -60,8 +60,8 @@ def test_read_write_external_catalog(con, external_duckdb_file, monkeypatch): _ = con.create_table("t2", obj=t_overwrite, database="ext.main", overwrite=True) - assert "t2" in con.list_tables(database="ext.main") - assert "t2" not in con.list_tables() + assert "t2" in con.ddl.list_tables(database="ext.main") + assert "t2" not in con.ddl.list_tables() table = con.table("t2", database="ext.main") diff --git a/ibis/backends/duckdb/tests/test_client.py b/ibis/backends/duckdb/tests/test_client.py index f02af4f040b6..a0483e537460 100644 --- a/ibis/backends/duckdb/tests/test_client.py +++ b/ibis/backends/duckdb/tests/test_client.py @@ -289,12 +289,12 @@ def test_list_tables(con): "diamonds", "functional_alltypes", "win", - }.issubset(con.list_tables()) + }.issubset(con.tables) icecream_table = ["ice_cream"] - assert con.list_tables(database="shops") == icecream_table - assert con.list_tables(database=("shops",)) == icecream_table + assert con.ddl.list_tables(database="shops") == icecream_table + assert con.ddl.list_tables(database=("shops",)) == icecream_table def test_settings_repr(): @@ -308,16 +308,16 @@ def test_connect_named_in_memory_db(): con_named_db = ibis.duckdb.connect(":memory:mydb") con_named_db.create_table("ork", schema=ibis.schema(dict(bork="int32"))) - assert "ork" in con_named_db.list_tables() + assert "ork" in con_named_db.tables con_named_db_2 = ibis.duckdb.connect(":memory:mydb") - assert "ork" in con_named_db_2.list_tables() + assert "ork" in con_named_db_2.tables unnamed_memory_db = ibis.duckdb.connect(":memory:") - assert "ork" not in unnamed_memory_db.list_tables() + assert "ork" not in unnamed_memory_db.tables default_memory_db = ibis.duckdb.connect() - assert "ork" not in default_memory_db.list_tables() + assert "ork" not in default_memory_db.tables @pytest.mark.parametrize( @@ -416,7 +416,7 @@ def test_read_csv_with_types(tmp_path, input, all_varchar): def test_memtable_doesnt_leak(con, monkeypatch): monkeypatch.setattr(ibis.options, "default_backend", con) name = "memtable_doesnt_leak" - assert name not in con.list_tables() + assert name not in con.ddl.list_tables() df = ibis.memtable({"a": [1, 2, 3]}, name=name).execute() - assert name not in con.list_tables() + assert name not in con.ddl.list_tables() assert len(df) == 3 diff --git a/ibis/backends/duckdb/tests/test_geospatial.py b/ibis/backends/duckdb/tests/test_geospatial.py index 2c748c1d9f48..d62004751c61 100644 --- a/ibis/backends/duckdb/tests/test_geospatial.py +++ b/ibis/backends/duckdb/tests/test_geospatial.py @@ -263,12 +263,9 @@ def test_geospatial_flip_coordinates(geotable): def test_create_table_geospatial_types(geotable, con): name = ibis.util.gen_name("geotable") - - # con = ibis.get_backend(geotable) - t = con.create_table(name, geotable, temp=True) - assert t.op().name in con.list_tables() + assert t.op().name in con.tables assert any(map(methodcaller("is_geospatial"), t.schema().values())) diff --git a/ibis/backends/duckdb/tests/test_register.py b/ibis/backends/duckdb/tests/test_register.py index 68ac9cbae9d5..0ea223743e76 100644 --- a/ibis/backends/duckdb/tests/test_register.py +++ b/ibis/backends/duckdb/tests/test_register.py @@ -292,7 +292,7 @@ def test_attach_sqlite(data_dir, tmp_path): con = ibis.duckdb.connect() con.attach_sqlite(test_db_path) - assert set(con.list_tables()) >= { + assert set(con.tables) >= { "functional_alltypes", "awards_players", "batting", @@ -304,7 +304,7 @@ def test_attach_sqlite(data_dir, tmp_path): # overwrite existing sqlite_db and force schema to all strings con.attach_sqlite(test_db_path, overwrite=True, all_varchar=True) - assert set(con.list_tables()) >= { + assert set(con.tables) >= { "functional_alltypes", "awards_players", "batting", diff --git a/ibis/backends/mysql/__init__.py b/ibis/backends/mysql/__init__.py index 2490bc275ef3..b806c69f0672 100644 --- a/ibis/backends/mysql/__init__.py +++ b/ibis/backends/mysql/__init__.py @@ -24,7 +24,8 @@ from ibis import util from ibis.backends import CanCreateDatabase from ibis.backends.sql import SQLBackend -from ibis.backends.sql.compilers.base import STAR, TRUE, C +from ibis.backends.sql.compilers.base import STAR, C +from ibis.util import deprecated if TYPE_CHECKING: from collections.abc import Mapping @@ -124,7 +125,7 @@ def do_connect( >>> password = os.environ.get("IBIS_TEST_MYSQL_PASSWORD", "ibis") >>> database = os.environ.get("IBIS_TEST_MYSQL_DATABASE", "ibis_testing") >>> con = ibis.mysql.connect(database=database, host=host, user=user, password=password) - >>> con.list_tables() # doctest: +ELLIPSIS + >>> con.tables() # doctest: +ELLIPSIS [...] >>> t = con.table("functional_alltypes") >>> t @@ -309,62 +310,95 @@ def raw_sql(self, query: str | sg.Expression, **kwargs: Any) -> Any: con.commit() return cursor - # TODO: disable positional arguments - def list_tables( + def _list_query_constructor(self, col: str, where_predicates: list) -> str: + """Helper function to construct sqlglot queries for _list_* methods.""" + + sg_query = ( + sg.select(col) + .from_(sg.table("tables", db="information_schema")) + .where(*where_predicates) + ).sql(self.name) + + return sg_query + + def _list_objects( + self, + like: str | None, + database: tuple[str, str] | str | None, + object_type: str, + ) -> list[str]: + """Generic method to list objects like tables or views.""" + + table_loc = self._to_sqlglot_table(database) + + ## having an issue as it seem mysql doesn't have a self.current_catalog + ## not clear to me why, my guess is it doesn't support catalogs but unclear + # catalog = table_loc.catalog or self.current_catalog + database = table_loc.db or self.current_database + + col = "table_name" + where_predicates = [ + # C.table_catalog.eq(sge.convert(catalog)), + C.table_schema.eq(sge.convert(database)), + C.table_type.eq(object_type), + ] + + sql = self._list_query_constructor(col, where_predicates) + + with self._safe_raw_sql(sql) as cur: + out = cur.fetchall() + + return self._filter_with_like(map(itemgetter(0), out), like) + + def _list_tables( self, like: str | None = None, database: tuple[str, str] | str | None = None, ) -> list[str]: - """List the tables in the database. + """List physical tables.""" - ::: {.callout-note} - ## Ibis does not use the word `schema` to refer to database hierarchy. + return self._list_objects(like, database, "BASE TABLE") - A collection of tables is referred to as a `database`. - A collection of `database` is referred to as a `catalog`. + def _list_temp_tables( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List temporary tables.""" - These terms are mapped onto the corresponding features in each - backend (where available), regardless of whether the backend itself - uses the same terminology. - ::: + return self._list_objects(like, database, "TEMPORARY") - Parameters - ---------- - like - A pattern to use for listing tables. - database - Database to list tables from. Default behavior is to show tables in - the current database (`self.current_database`). - """ - if database is not None: - table_loc = database - else: - table_loc = self.current_database + def _list_views( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List views.""" - table_loc = self._to_sqlglot_table(table_loc) + return self._list_objects(like, database, "VIEW") - conditions = [TRUE] + # TODO: disable positional arguments + @deprecated(as_of="10.0", instead="use the con.tables") + def list_tables( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List the tables and views in the database.""" - if (sg_cat := table_loc.args["catalog"]) is not None: - sg_cat.args["quoted"] = False - if (sg_db := table_loc.args["db"]) is not None: - sg_db.args["quoted"] = False - if table_loc.catalog or table_loc.db: - conditions = [C.table_schema.eq(sge.convert(table_loc.sql(self.name)))] + table_loc = self._to_sqlglot_table(database) - col = "table_name" - sql = ( - sg.select(col) - .from_(sg.table("tables", db="information_schema")) - .distinct() - .where(*conditions) - .sql(self.name) - ) + database = self.current_database + if table_loc is not None: + database = table_loc.db or database - with self._safe_raw_sql(sql) as cur: - out = cur.fetchall() + tables_and_views = list( + set(self._list_tables(like=like, database=database)) + | set(self._list_temp_tables(like=like, database=database)) + | set(self._list_views(like=like, database=database)) + ) - return self._filter_with_like(map(itemgetter(0), out), like) + return tables_and_views def execute( self, expr: ir.Expr, limit: str | None = "default", **kwargs: Any diff --git a/ibis/backends/mysql/tests/test_client.py b/ibis/backends/mysql/tests/test_client.py index 831363d71362..bcac77d36b03 100644 --- a/ibis/backends/mysql/tests/test_client.py +++ b/ibis/backends/mysql/tests/test_client.py @@ -234,10 +234,10 @@ def test_list_tables(con): "event", "func", } - assert con.list_tables() + assert con.tables - assert mysql_tables.issubset(con.list_tables(database="mysql")) - assert mysql_tables.issubset(con.list_tables(database=("mysql",))) + assert mysql_tables.issubset(con.ddl.list_tables(database="mysql")) + assert mysql_tables.issubset(con.ddl.list_tables(database=("mysql",))) def test_invalid_port(): diff --git a/ibis/backends/postgres/__init__.py b/ibis/backends/postgres/__init__.py index ddc3d4e169c4..57f9b8b4a83b 100644 --- a/ibis/backends/postgres/__init__.py +++ b/ibis/backends/postgres/__init__.py @@ -23,7 +23,8 @@ from ibis import util from ibis.backends import CanCreateDatabase, CanListCatalog from ibis.backends.sql import SQLBackend -from ibis.backends.sql.compilers.base import TRUE, C, ColGen, F +from ibis.backends.sql.compilers.base import C, ColGen, F +from ibis.util import deprecated if TYPE_CHECKING: from collections.abc import Callable @@ -229,7 +230,7 @@ def do_connect( >>> password = os.environ.get("IBIS_TEST_POSTGRES_PASSWORD", "postgres") >>> database = os.environ.get("IBIS_TEST_POSTGRES_DATABASE", "ibis_testing") >>> con = ibis.postgres.connect(database=database, host=host, user=user, password=password) - >>> con.list_tables() # doctest: +ELLIPSIS + >>> con.tables() # doctest: +ELLIPSIS [...] >>> t = con.table("functional_alltypes") >>> t @@ -299,86 +300,182 @@ def _session_temp_db(self) -> str | None: return res[0] return res - def list_tables( + def _list_query_constructor( + self, col: str, where_predicates: list, tov: str + ) -> str: + """Helper function to construct sqlglot queries for _list_* methods.""" + + sg_query = ( + sg.select(col) + .from_(sg.table(tov, db="information_schema")) + .where(*where_predicates) + ).sql(self.name) + + return sg_query + + def _list_objects( + self, + like: str | None, + database: tuple[str, str] | str | None, + object_type: str, + is_temp: bool = False, + ) -> list[str]: + """Generic method to list objects like tables, temporary tables or views. + + (not used for temporary views) + """ + + table_loc = self._to_sqlglot_table(database) + + catalog = table_loc.catalog or self.current_catalog + # temporary tables and views are in a separate postgres schema + # that are of the form pg_temp_* where * is a number + database = table_loc.db or ("pg_temp_%" if is_temp else self.current_database) + col = "table_name" + + where_predicates = [ + C.table_catalog.eq(sge.convert(catalog)), + C.table_schema.like(sge.convert(database)) + if is_temp + else C.table_schema.eq(sge.convert(database)), + C.table_type.eq(object_type), + ] + + sql = self._list_query_constructor(col, where_predicates, tov="tables") + + with self._safe_raw_sql(sql) as cur: + out = cur.fetchall() + return self._filter_with_like(map(itemgetter(0), out), like) + + def _list_tables( self, like: str | None = None, database: tuple[str, str] | str | None = None, ) -> list[str]: - """List the tables in the database. + """List physical tables.""" - ::: {.callout-note} - ## Ibis does not use the word `schema` to refer to database hierarchy. + return self._list_objects(like, database, "BASE TABLE") - A collection of tables is referred to as a `database`. - A collection of `database` is referred to as a `catalog`. + def _list_temp_tables( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List temporary tables.""" - These terms are mapped onto the corresponding features in each - backend (where available), regardless of whether the backend itself - uses the same terminology. - ::: + # Avoid the problem of having temp views showing on the + # when listing table_type "LOCAL TEMPORARY" in information schema table - Parameters - ---------- - like - A pattern to use for listing tables. - database - Database to list tables from. Default behavior is to show tables in - the current database. - """ - - if database is not None: - table_loc = database - else: - table_loc = (self.current_catalog, self.current_database) - - table_loc = self._to_sqlglot_table(table_loc) - - conditions = [TRUE] - - if (db := table_loc.args["db"]) is not None: - db.args["quoted"] = False - db = db.sql(dialect=self.name) - conditions.append(C.table_schema.eq(sge.convert(db))) - if (catalog := table_loc.args["catalog"]) is not None: - catalog.args["quoted"] = False - catalog = catalog.sql(dialect=self.name) - conditions.append(C.table_catalog.eq(sge.convert(catalog))) - - sql = ( - sg.select("table_name") - .from_(sg.table("tables", db="information_schema")) - .distinct() - .where(*conditions) - .sql(self.dialect) + temp_tables_and_views = set( + self._list_objects(like, database, "LOCAL TEMPORARY", is_temp=True) ) + temp_views_only = set(self._list_temp_views(like, database)) + + # I don't like this ^ but I the only other way I found of doing it is + # looking into pg_class where relkind is 'r' + # and relnamespace is in the pg_namespace table where we can search for + # nspname like 'pg_temp_%' (this involves two different tables, not sure is a good idea) + # the sql is + # SELECT + # relname AS table_name + # FROM + # pg_class + # WHERE + # relkind = 'r' -- 'r' stands for ordinary tables + # AND relnamespace IN ( + # SELECT oid + # FROM pg_namespace + # WHERE nspname LIKE 'pg_temp_%' + # ); + + return list(temp_tables_and_views - temp_views_only) + + def _list_views( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List views.""" + + return self._list_objects(like, database, "VIEW") + + def _list_temp_views( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List temporary views.""" + + table_loc = self._to_sqlglot_table(database) + + catalog = table_loc.catalog or self.current_catalog + # temporary tables and views are in a separate postgres table_schema that + # are of the form pg_temp_* where * is a number + database = table_loc.db or "pg_temp_%" + col = "table_name" + + # information_schema.views doesn't have a table_type to check + # we guarantee it's a temp view because the table_schema is pg_temp_* and + # we are checking only for views + where_predicates = [ + C.table_catalog.eq(sge.convert(catalog)), + C.table_schema.like(sge.convert(database)), + ] + + sql = self._list_query_constructor(col, where_predicates, tov="views") with self._safe_raw_sql(sql) as cur: out = cur.fetchall() - # Include temporary tables only if no database has been explicitly specified - # to avoid temp tables showing up in all calls to `list_tables` - if db == "public": - out += self._fetch_temp_tables() - return self._filter_with_like(map(itemgetter(0), out), like) - def _fetch_temp_tables(self): - # postgres temporary tables are stored in a separate schema - # so we need to independently grab them and return them along with - # the existing results - - sql = ( - sg.select("table_name") - .from_(sg.table("tables", db="information_schema")) - .distinct() - .where(C.table_type.eq(sge.convert("LOCAL TEMPORARY"))) - .sql(self.dialect) + def _list_foreign_tables( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + table_loc = self._to_sqlglot_table(database) + + catalog = table_loc.catalog or self.current_catalog + database = table_loc.db or self.current_database + + col = "foreign_table_name" + where_predicates = [ + C.foreign_table_catalog.eq(sge.convert(catalog)), + C.foreign_table_schema.like(sge.convert(database)), + ] + sql = self._list_query_constructor( + col, where_predicates=where_predicates, tov="foreign_tables" ) with self._safe_raw_sql(sql) as cur: out = cur.fetchall() - return out + return self._filter_with_like(map(itemgetter(0), out), like) + + @deprecated(as_of="10.0", instead="use the con.tables") + def list_tables( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List the tables in the database.""" + + table_loc = self._to_sqlglot_table(database) + + database = self.current_database + if table_loc is not None: + database = table_loc.db or database + + tables_and_views = list( + set(self._list_tables(like=like, database=database)) + | set(self._list_temp_tables(like=like, database=database)) + | set(self._list_views(like=like, database=database)) + | set(self._list_temp_views(like=like, database=database)) + | set(self._list_foreign_tables(like=like, database=database)) + ) + + return tables_and_views def list_catalogs(self, like=None) -> list[str]: # http://dba.stackexchange.com/a/1304/58517 diff --git a/ibis/backends/postgres/tests/test_client.py b/ibis/backends/postgres/tests/test_client.py index c996c33f33ac..212245b47201 100644 --- a/ibis/backends/postgres/tests/test_client.py +++ b/ibis/backends/postgres/tests/test_client.py @@ -66,14 +66,14 @@ def test_simple_aggregate_execute(alltypes): def test_list_tables(con): - assert len(con.list_tables(like="functional")) == 1 - assert {"astronauts", "batting", "diamonds"} <= set(con.list_tables()) + assert len(con.tables(like="functional")) == 1 + assert {"astronauts", "batting", "diamonds"} <= set(con.tables) _ = con.create_table("tempy", schema=ibis.schema(dict(id="int")), temp=True) - assert "tempy" in con.list_tables() + assert "tempy" in con.tables # temp tables only show up when database='public' (or default) - assert "tempy" not in con.list_tables(database="tiger") + assert "tempy" not in con.tables(database="tiger") def test_compile_toplevel(assert_sql): @@ -196,7 +196,7 @@ def test_unknown_column_type(con, col): def test_insert_with_cte(con): X = con.create_table("X", schema=ibis.schema(dict(id="int")), temp=True) - assert "X" in con.list_tables() + assert "X" in con.tables expr = X.join(X.mutate(a=X["id"] + 1), ["id"]) Y = con.create_table("Y", expr, temp=True) assert Y.execute().empty @@ -432,3 +432,27 @@ def enum_table(con): def test_enum_table(con, enum_table): t = con.table(enum_table) assert t.mood.type() == dt.unknown + + +def test_list_foreign_table(con): + sql = """CREATE EXTENSION IF NOT EXISTS postgres_fdw; + + CREATE SERVER foreign_server + FOREIGN DATA WRAPPER postgres_fdw; + + CREATE USER MAPPING FOR CURRENT_USER + SERVER foreign_server; + + CREATE FOREIGN TABLE my_foreign_table ( + id INTEGER, + name VARCHAR + ) + SERVER foreign_server; + """ + + con.raw_sql(sql) + foreign_tables = con.ddl.list_foreign_tables() + + assert isinstance(foreign_tables, list) + assert "my_foreign_table" in foreign_tables + assert "my_foreign_table" not in con.ddl.list_tables() diff --git a/ibis/backends/sqlite/__init__.py b/ibis/backends/sqlite/__init__.py index a5c074aefbd8..e6c2ef5a6f32 100644 --- a/ibis/backends/sqlite/__init__.py +++ b/ibis/backends/sqlite/__init__.py @@ -21,6 +21,7 @@ from ibis.backends.sql.compilers.base import C, F from ibis.backends.sqlite.converter import SQLitePandasData from ibis.backends.sqlite.udf import ignore_nulls, register_all +from ibis.util import deprecated if TYPE_CHECKING: from collections.abc import Iterator, Mapping @@ -159,48 +160,108 @@ def list_databases(self, like: str | None = None) -> list[str]: return sorted(self._filter_with_like(results, like)) - def list_tables( - self, like: str | None = None, database: str | None = None + def _list_query_constructor(self, col: str, where_predicates: list) -> str: + """Helper function to construct sqlglot queries for _list_* methods.""" + + sg_query = ( + sg.select(col).from_(F.pragma_table_list()).where(*where_predicates) + ).sql(self.name) + + return sg_query + + def _list_objects( + self, + like: str | None, + database: tuple[str, str] | str | None, + object_type: str, + is_temp: bool = False, ) -> list[str]: - """List the tables in the database. + """Generic method to list objects like tables or views.""" - If `database` is None, the current database is used, and temporary - tables are included in the result. + table_loc = self._to_sqlglot_table(database) - Parameters - ---------- - like - A pattern to use for listing tables. - database - Database to list tables from. Default behavior is to show tables in - the current database. - """ - if database is None: - database = "main" - schemas = [database, "temp"] - else: - schemas = [database] + # sqlite doesn't support catalogs as far as I can tell + # all temp tables are in the "temp" sqlite schema + database = table_loc.db or ("temp" if is_temp else self.current_database) + # needs to check what db I get if main or nothing when db none - sql = ( - sg.select(C.name) - .from_(F.pragma_table_list()) - .where( - C.schema.isin(*map(sge.convert, schemas)), - C.type.isin(sge.convert("table"), sge.convert("view")), - ~C.name.isin( + col = "name" + where_predicates = [ + C.schema.eq(sge.convert(database)), + C.type.eq(object_type), + ~( + C.name.isin( sge.convert("sqlite_schema"), sge.convert("sqlite_master"), sge.convert("sqlite_temp_schema"), sge.convert("sqlite_temp_master"), - ), - ) - .sql(self.dialect) - ) + ) + ), + ] + + sql = self._list_query_constructor(col, where_predicates) with self._safe_raw_sql(sql) as cur: results = [r[0] for r in cur.fetchall()] return sorted(self._filter_with_like(results, like)) + def _list_tables( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List physical tables.""" + + return self._list_objects(like, database, "table") + + def _list_temp_tables( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List temporary tables.""" + + return self._list_objects(like, database, "table", is_temp=True) + + def _list_views( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List views.""" + + return self._list_objects(like, database, "view") + + def _list_temp_views( + self, + like: str | None = None, + database: tuple[str, str] | str | None = None, + ) -> list[str]: + """List temporary views.""" + + return self._list_objects(like, database, "view", is_temp=True) + + @deprecated(as_of="10.0", instead="use the con.tables") + def list_tables( + self, like: str | None = None, database: str | None = None + ) -> list[str]: + """List tables and views.""" + + table_loc = self._to_sqlglot_table(database) + + database = self.current_database + if table_loc is not None: + database = table_loc.db or database + + tables_and_views = list( + set(self._list_tables(like=like, database=database)) + | set(self._list_temp_tables(like=like, database=database)) + | set(self._list_views(like=like, database=database)) + | set(self._list_temp_views(like=like, database=database)) + ) + + return tables_and_views + def _parse_type(self, typ: str, nullable: bool) -> dt.DataType: typ = typ.lower() try: @@ -426,7 +487,7 @@ def attach(self, name: str, path: str | Path) -> None: >>> con1 = ibis.sqlite.connect("/tmp/original.db") >>> con2 = ibis.sqlite.connect("/tmp/new.db") >>> con1.attach("new", "/tmp/new.db") - >>> con1.list_tables(database="new") + >>> con1.tables(database="new") [] """ with self.begin() as cur: diff --git a/ibis/backends/sqlite/tests/test_client.py b/ibis/backends/sqlite/tests/test_client.py index f039cd5fabba..a83b46e9a2d3 100644 --- a/ibis/backends/sqlite/tests/test_client.py +++ b/ibis/backends/sqlite/tests/test_client.py @@ -19,13 +19,13 @@ def test_attach_file(tmp_path): client = ibis.sqlite.connect() - assert not client.list_tables() + assert not client.tables() client.attach("baz", Path(dbpath)) client.attach("bar", dbpath) - foo_tables = client.list_tables(database="baz") - bar_tables = client.list_tables(database="bar") + foo_tables = client.ddl.list_tables(database="baz") + bar_tables = client.ddl.list_tables(database="bar") assert foo_tables == ["test"] assert foo_tables == bar_tables @@ -93,5 +93,5 @@ def test_has_operation(con): def test_list_temp_tables_by_default(con): name = ibis.util.gen_name("sqlite_temp_table") con.create_table(name, schema={"a": "int"}, temp=True) - assert name in con.list_tables(database="temp") - assert name in con.list_tables() + assert name in con.ddl.list_temp_tables(database="temp") + assert name in con.tables() diff --git a/ibis/backends/tests/test_api.py b/ibis/backends/tests/test_api.py index 108a194ecb0c..29ad03995074 100644 --- a/ibis/backends/tests/test_api.py +++ b/ibis/backends/tests/test_api.py @@ -1,10 +1,15 @@ from __future__ import annotations +import contextlib + import pytest +import sqlglot as sg +import sqlglot.expressions as sge from pytest import param import ibis.expr.types as ir from ibis.backends.conftest import TEST_TABLES +from ibis.backends.sql.compilers.base import STAR from ibis.backends.tests.errors import PyDruidProgrammingError @@ -53,9 +58,9 @@ def test_catalog_consistency(backend, con): assert current_catalog in catalogs -def test_list_tables(con): - tables = con.list_tables() - assert isinstance(tables, list) +def test_con_tables(con): + tables = con.tables + # only table that is guaranteed to be in all backends key = "functional_alltypes" assert key in tables or key.upper() in tables @@ -76,7 +81,7 @@ def test_tables_accessor_mapping(con): # temporary might pop into existence in parallel test runs, in between the # first `list_tables` call and the second, so we check that there's a # non-empty intersection - assert TEST_TABLES.keys() & set(map(str.lower, con.list_tables())) + assert TEST_TABLES.keys() & set(map(str.lower, con.ddl.list_tables())) assert TEST_TABLES.keys() & set(map(str.lower, con.tables)) @@ -136,3 +141,82 @@ def test_unbind(alltypes, expr_fn): assert "Unbound" not in repr(expr) assert "Unbound" in repr(expr.unbind()) + + +def test_list_tables(ddl_con): + # should check only physical tables + table_name = "functional_alltypes" + tables = ddl_con.ddl.list_tables() + assert isinstance(tables, list) + assert table_name in tables + + assert table_name not in ddl_con.ddl.list_views() + + # not all backends have list_temp_tables + with contextlib.suppress(NotImplementedError): + assert table_name not in ddl_con.ddl.list_temp_tables() + + # not all backends have list_temp_views + with contextlib.suppress(NotImplementedError): + assert table_name not in ddl_con.ddl.list_temp_views() + + +def test_list_views(ddl_con, temp_view): + # temp_view: view name + expr = ddl_con.table("functional_alltypes") + ddl_con.create_view(temp_view, expr) + + views = ddl_con.ddl.list_views() + + assert isinstance(views, list) + assert temp_view in views + assert temp_view not in ddl_con.ddl.list_tables() + + # not all backends have list_temp_tables + with contextlib.suppress(NotImplementedError): + assert temp_view not in ddl_con.ddl.list_temp_tables() + + # not all backends have list_temp_views + with contextlib.suppress(NotImplementedError): + assert temp_view not in ddl_con.ddl.list_temp_views() + + +@pytest.mark.never( + "datafusion", reason="datafusion does not support temporary views on sql" +) +def test_list_temp_tables(ddl_con): + expr = ddl_con.table("functional_alltypes") + temp_table_name = "all_types_temp" + ddl_con.create_table(temp_table_name, expr, temp=True) + temp_tables = ddl_con.ddl.list_temp_tables() + + assert isinstance(temp_tables, list) + assert temp_table_name in temp_tables + assert temp_table_name not in ddl_con.ddl.list_views() + assert temp_table_name not in ddl_con.ddl.list_tables() + + # not all backends have list_temp_views + with contextlib.suppress(NotImplementedError): + assert temp_table_name not in ddl_con.ddl.list_temp_views() + + +@pytest.mark.never( + ["datafusion", "mysql", "clickhouse"], reason="does not support temporary views" +) +def test_list_temp_views(ddl_con): + # TODO: replace raw_sql with create_temp + + temp_view = sge.Create( + this=sg.table("temp_view_example"), + kind="VIEW", + expression=sg.select(STAR).from_(sg.table("functional_alltypes")), + properties=sge.Properties(expressions=[sge.TemporaryProperty()]), + ) + ddl_con.raw_sql(temp_view.sql(ddl_con.dialect)) + temporary_views = ddl_con.ddl.list_temp_views() + + assert isinstance(temporary_views, list) + assert "temp_view_example" in temporary_views + assert "temp_view_example" not in ddl_con.ddl.list_tables() + assert "temp_view_example" not in ddl_con.ddl.list_views() + assert "temp_view_example" not in ddl_con.ddl.list_temp_tables() diff --git a/ibis/backends/tests/test_client.py b/ibis/backends/tests/test_client.py index 069d6a525b2a..c73578097779 100644 --- a/ibis/backends/tests/test_client.py +++ b/ibis/backends/tests/test_client.py @@ -324,7 +324,7 @@ def test_create_temporary_table_from_schema(con_no_data, new_schema): con_no_data.reconnect() # verify table no longer exist after reconnect - assert temp_table not in con_no_data.list_tables() + assert temp_table not in con_no_data.tables @mark.notimpl( @@ -387,7 +387,7 @@ def test_nullable_input_output(con, temp_table): def test_create_drop_view(ddl_con, temp_view): # setup table_name = "functional_alltypes" - tables = ddl_con.list_tables() + tables = ddl_con.tables if table_name in tables or (table_name := table_name.upper()) in tables: expr = ddl_con.table(table_name) @@ -399,7 +399,7 @@ def test_create_drop_view(ddl_con, temp_view): # create a new view ddl_con.create_view(temp_view, expr) # check if the view was created - assert temp_view in ddl_con.list_tables() + assert temp_view in ddl_con.ddl.list_views() t_expr = ddl_con.table(table_name) v_expr = ddl_con.table(temp_view) @@ -442,7 +442,7 @@ def employee_data_1_temp_table(backend, con, test_employee_schema): _create_temp_table_with_schema( backend, con, temp_table_name, test_employee_schema, data=test_employee_data_1 ) - assert temp_table_name in con.list_tables() + assert temp_table_name in con.tables yield temp_table_name con.drop_table(temp_table_name, force=True) @@ -697,7 +697,7 @@ def test_unsigned_integer_type(con, temp_table): schema=ibis.schema(dict(a="uint8", b="uint16", c="uint32", d="uint64")), overwrite=True, ) - assert temp_table in con.list_tables() + assert temp_table in con.tables @pytest.mark.backend @@ -1003,7 +1003,7 @@ def test_create_table_in_memory(con, obj, table_name, monkeypatch): t = con.create_table(table_name, obj()) try: - assert table_name in con.list_tables() + assert table_name in con.ddl.list_tables() assert pa.table({"a": ["a"], "b": [1]}).equals(t.to_pyarrow()) finally: con.drop_table(table_name, force=True) diff --git a/ibis/backends/tests/test_expr_caching.py b/ibis/backends/tests/test_expr_caching.py index 75aba279553e..5fb5a47ef584 100644 --- a/ibis/backends/tests/test_expr_caching.py +++ b/ibis/backends/tests/test_expr_caching.py @@ -93,7 +93,7 @@ def test_persist_expression_multiple_refs(backend, con, alltypes): assert op not in con._cache_op_to_entry # assert that table has been dropped - assert name not in con.list_tables() + assert name not in con.tables @mark.notimpl(["datafusion", "flink", "impala", "trino", "druid"]) @@ -115,7 +115,7 @@ def test_persist_expression_repeated_cache(alltypes, con): del nested_cached_table, cached_table - assert name not in con.list_tables() + assert name not in con.tables @mark.notimpl(["datafusion", "flink", "impala", "trino", "druid"]) diff --git a/ibis/backends/tests/test_register.py b/ibis/backends/tests/test_register.py index 02df85ef1cb0..82e746d38491 100644 --- a/ibis/backends/tests/test_register.py +++ b/ibis/backends/tests/test_register.py @@ -101,7 +101,7 @@ def test_register_csv(con, data_dir, fname, in_table_name, out_table_name): with pytest.warns(FutureWarning, match="v9.1"): table = con.register(fname, table_name=in_table_name) - assert any(out_table_name in t for t in con.list_tables()) + assert any(out_table_name in t for t in con.tables) if con.name != "datafusion": table.count().execute() @@ -218,7 +218,7 @@ def test_register_parquet( with pytest.warns(FutureWarning, match="v9.1"): table = con.register(f"parquet://{fname.name}", table_name=in_table_name) - assert any(out_table_name in t for t in con.list_tables()) + assert any(out_table_name in t for t in con.tables) if con.name != "datafusion": table.count().execute() @@ -263,7 +263,7 @@ def test_register_iterator_parquet( table_name=None, ) - assert any("ibis_read_parquet" in t for t in con.list_tables()) + assert any("ibis_read_parquet" in t for t in con.tables) assert table.count().execute() diff --git a/ibis/backends/tests/test_signatures.py b/ibis/backends/tests/test_signatures.py index d1065867c0bf..d9be7cf74900 100644 --- a/ibis/backends/tests/test_signatures.py +++ b/ibis/backends/tests/test_signatures.py @@ -10,6 +10,7 @@ CanCreateDatabase, CanListCatalog, CanListDatabase, + DDLAccessor, ) from ibis.backends.sql import SQLBackend from ibis.backends.tests.signature.typecheck import compatible @@ -100,7 +101,7 @@ def _scrape_methods(modules, params): ), ), "list_tables": pytest.param( - BaseBackend, + DDLAccessor, # Replacing BaseBackend with DDLAccessor "list_tables", marks=pytest.mark.notyet(["flink"]), ), diff --git a/ibis/examples/tests/test_examples.py b/ibis/examples/tests/test_examples.py index 40d1e2bf4afe..a2eeee4affd0 100644 --- a/ibis/examples/tests/test_examples.py +++ b/ibis/examples/tests/test_examples.py @@ -84,7 +84,7 @@ def test_non_example(): def test_backend_arg(): con = ibis.duckdb.connect() t = ibis.examples.penguins.fetch(backend=con) - assert t.get_name() in con.list_tables() + assert t.get_name() in con.tables @pytest.mark.duckdb