diff --git a/api/src/main/java/org/apache/iceberg/catalog/SupportsNamespaces.java b/api/src/main/java/org/apache/iceberg/catalog/SupportsNamespaces.java index 7c3af5fe5745..409de896dcb6 100644 --- a/api/src/main/java/org/apache/iceberg/catalog/SupportsNamespaces.java +++ b/api/src/main/java/org/apache/iceberg/catalog/SupportsNamespaces.java @@ -22,8 +22,10 @@ import java.util.Map; import java.util.Set; import org.apache.iceberg.exceptions.AlreadyExistsException; +import org.apache.iceberg.exceptions.ForbiddenException; import org.apache.iceberg.exceptions.NamespaceNotEmptyException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; +import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList; import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap; /** @@ -74,6 +76,28 @@ default List listNamespaces() { return listNamespaces(Namespace.empty()); } + /** + * List namespaces from the catalog recursively. + * + *

If an object such as a table, view, or function exists, its parent namespaces must also + * exist and must be returned by this discovery method. For example, if table a.b.t exists, this + * method must return ["a"] in the result array. + * + * @return a List of namespace {@link Namespace} names + */ + default List listNamespacesRecursively() { + ImmutableList.Builder namespaces = ImmutableList.builder(); + try { + for (Namespace ns : listNamespaces(Namespace.empty())) { + namespaces.add(ns); + namespaces.addAll(listNamespacesRecursively(ns)); + } + } catch (NoSuchNamespaceException | ForbiddenException e) { + // Skip the namespace if it has been deleted concurrently or access is forbidden + } + return namespaces.build(); + } + /** * List child namespaces from the namespace. * @@ -104,6 +128,48 @@ default List listNamespaces() { */ List listNamespaces(Namespace namespace) throws NoSuchNamespaceException; + /** + * List child namespaces from the namespace recursively. + * + *

For two existing tables named 'a.b.c.table' and 'a.b.d.table', this method returns: + * + *

+ * + * + * + * + * + * + * + * @return a List of child {@link Namespace} names from the given namespace + * @throws NoSuchNamespaceException If the namespace does not exist (optional) + */ + default List listNamespacesRecursively(Namespace namespace) + throws NoSuchNamespaceException { + ImmutableList.Builder namespaces = ImmutableList.builder(); + try { + for (Namespace ns : listNamespaces(namespace)) { + namespaces.add(ns); + namespaces.addAll(listNamespacesRecursively(ns)); + } + } catch (NoSuchNamespaceException | ForbiddenException e) { + // Skip the namespace if it has been deleted concurrently or access is forbidden + } + return namespaces.build(); + } + /** * Load metadata properties for a namespace. * diff --git a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java index 3a1e62260aae..c1692808513f 100644 --- a/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java +++ b/core/src/main/java/org/apache/iceberg/rest/CatalogHandlers.java @@ -70,6 +70,7 @@ import org.apache.iceberg.catalog.ViewCatalog; import org.apache.iceberg.exceptions.AlreadyExistsException; import org.apache.iceberg.exceptions.CommitFailedException; +import org.apache.iceberg.exceptions.ForbiddenException; import org.apache.iceberg.exceptions.NoSuchNamespaceException; import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.exceptions.NoSuchViewException; @@ -77,6 +78,7 @@ import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.relocated.com.google.common.base.Preconditions; import org.apache.iceberg.relocated.com.google.common.collect.Iterables; +import org.apache.iceberg.relocated.com.google.common.collect.Lists; import org.apache.iceberg.relocated.com.google.common.collect.Maps; import org.apache.iceberg.relocated.com.google.common.collect.Sets; import org.apache.iceberg.rest.RESTCatalogProperties.SnapshotMode; @@ -302,6 +304,16 @@ private static Pair, String> paginate(List list, String pageToken public static ListNamespacesResponse listNamespaces( SupportsNamespaces catalog, Namespace parent) { + return listNamespaces(catalog, parent, false); + } + + public static ListNamespacesResponse listNamespacesRecursively( + SupportsNamespaces catalog, Namespace parent) { + return listNamespaces(catalog, parent, true); + } + + private static ListNamespacesResponse listNamespaces( + SupportsNamespaces catalog, Namespace parent, boolean recursive) { List results; if (parent.isEmpty()) { results = catalog.listNamespaces(); @@ -309,9 +321,31 @@ public static ListNamespacesResponse listNamespaces( results = catalog.listNamespaces(parent); } + if (recursive) { + results = listNamespacesRecursively(catalog, results); + } + return ListNamespacesResponse.builder().addAll(results).build(); } + private static List listNamespacesRecursively( + SupportsNamespaces catalog, List namespaces) { + List allNamespaces = Lists.newArrayList(namespaces); + + for (Namespace namespace : namespaces) { + try { + List children = catalog.listNamespaces(namespace); + if (!children.isEmpty()) { + allNamespaces.addAll(listNamespacesRecursively(catalog, children)); + } + } catch (NoSuchNamespaceException | ForbiddenException e) { + // Skip the namespace if it has been deleted concurrently or access is forbidden + } + } + + return allNamespaces; + } + public static ListNamespacesResponse listNamespaces( SupportsNamespaces catalog, Namespace parent, String pageToken, String pageSize) { List results; diff --git a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java index c7b5b5d41c74..7c12ee01f9f8 100644 --- a/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java +++ b/core/src/main/java/org/apache/iceberg/rest/RESTSessionCatalog.java @@ -751,6 +751,15 @@ public void createNamespace( @Override public List listNamespaces(SessionContext context, Namespace namespace) { + return listNamespaces(context, namespace, false); + } + + public List listNamespacesRecursively(SessionContext context, Namespace namespace) { + return listNamespaces(context, namespace, true); + } + + private List listNamespaces( + SessionContext context, Namespace namespace, boolean recursive) { if (!endpoints.contains(Endpoint.V1_LIST_NAMESPACES)) { return ImmutableList.of(); } @@ -760,6 +769,10 @@ public List listNamespaces(SessionContext context, Namespace namespac queryParams.put("parent", RESTUtil.namespaceToQueryParam(namespace, namespaceSeparator)); } + if (recursive) { + queryParams.put("recursive", "true"); + } + ImmutableList.Builder namespaces = ImmutableList.builder(); String pageToken = ""; if (pageSize != null) { diff --git a/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java b/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java index 8c6dc52b1575..770e5c65f0ba 100644 --- a/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java +++ b/core/src/test/java/org/apache/iceberg/rest/RESTCatalogAdapter.java @@ -190,6 +190,7 @@ public T handleRequest( } else { ns = Namespace.empty(); } + boolean recursive = PropertyUtil.propertyAsBoolean(vars, "recursive", false); String pageToken = PropertyUtil.propertyAsString(vars, "pageToken", null); String pageSize = PropertyUtil.propertyAsString(vars, "pageSize", null); @@ -199,6 +200,10 @@ public T handleRequest( responseType, CatalogHandlers.listNamespaces(asNamespaceCatalog, ns, pageToken, pageSize)); } else { + if (recursive) { + return castResponse( + responseType, CatalogHandlers.listNamespacesRecursively(asNamespaceCatalog, ns)); + } return castResponse( responseType, CatalogHandlers.listNamespaces(asNamespaceCatalog, ns)); } diff --git a/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java b/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java index e4fa156059d8..cccb9382741e 100644 --- a/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java +++ b/core/src/test/java/org/apache/iceberg/rest/TestRESTCatalog.java @@ -3549,6 +3549,60 @@ private void runConfigurableNamespaceSeparatorTest( any()); } + @Test + public void testListNamespacesRecursively() throws IOException { + InMemoryCatalog backend = new InMemoryCatalog(); + backend.initialize( + "backend", ImmutableMap.of(CatalogProperties.WAREHOUSE_LOCATION, temp.toString())); + RESTCatalogAdapter adapter = Mockito.spy(new RESTCatalogAdapter(backend)); + RESTCatalog catalog = catalog(adapter); + + try { + // Create a namespace hierarchy: + // - ns1 + // - ns1.child1 + // - ns1.child2 + // - ns1.child2.grandchild + // - ns2 + Namespace ns1 = Namespace.of("ns1"); + Namespace ns1Child1 = Namespace.of("ns1", "child1"); + Namespace ns1Child2 = Namespace.of("ns1", "child2"); + Namespace ns1Child2Grandchild = Namespace.of("ns1", "child2", "grandchild"); + Namespace ns2 = Namespace.of("ns2"); + + catalog.createNamespace(ns1); + catalog.createNamespace(ns1Child1); + catalog.createNamespace(ns1Child2); + catalog.createNamespace(ns1Child2Grandchild); + catalog.createNamespace(ns2); + + // recursive listing - should get all namespaces in the hierarchy + assertThat(catalog.listNamespacesRecursively()) + .containsExactlyInAnyOrder(ns1, ns1Child1, ns1Child2, ns1Child2Grandchild, ns2); + + // non-recursive listing - should only get direct children + assertThat(catalog.listNamespaces()).containsExactlyInAnyOrder(ns1, ns2); + + // recursive listing of ns1 - should get all descendants of ns1 + assertThat(catalog.listNamespacesRecursively(ns1)) + .containsExactlyInAnyOrder(ns1Child1, ns1Child2, ns1Child2Grandchild); + + // non-recursive listing of ns1 - should only get direct children + assertThat(catalog.listNamespaces(ns1)).containsExactlyInAnyOrder(ns1Child1, ns1Child2); + + // recursive listing of ns1.child2 - should only get direct children because + // nested namespaces don't exist + assertThat(catalog.listNamespacesRecursively(ns1Child2)).containsExactly(ns1Child2Grandchild); + + // non-recursive listing of ns1.child2 - should only get direct children + assertThat(catalog.listNamespaces(ns1Child2)).containsExactly(ns1Child2Grandchild); + assertThat(catalog.listNamespacesRecursively(ns1Child2)).containsExactly(ns1Child2Grandchild); + } finally { + catalog.close(); + backend.close(); + } + } + private RESTCatalog createCatalogWithIdempAdapter(ConfigResponse cfg, boolean expectOnMutations) { RESTCatalogAdapter adapter = Mockito.spy( diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 2435cd43f0e5..183c1fc36063 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -280,6 +280,13 @@ paths: schema: type: string example: "accounting%1Ftax" + - name: recursive + in: query + description: An optional flag indicating whether to list namespaces recursively. + required: false + schema: + type: boolean + default: false responses: 200: $ref: '#/components/responses/ListNamespacesResponse'