Normalize legacy FieldConfig indexes into indexes

pinot-common/src/test/java/org/apache/pinot/common/utils/config/TableConfigSerDeUtilsTest.java

@@ -242,6 +242,10 @@ public void testSerDe()
           new FieldConfig("column3", FieldConfig.EncodingType.RAW, Collections.emptyList(),
               FieldConfig.CompressionCodec.SNAPPY, null));
       TableConfig tableConfig = tableConfigBuilder.setFieldConfigList(fieldConfigList).build();
+      String tableConfigJson = tableConfig.toJsonString();
+      assertTrue(tableConfigJson.contains("\"indexes\""));
+      assertFalse(tableConfigJson.contains("\"indexTypes\""));
+      assertFalse(tableConfigJson.contains("\"indexType\""));
 
       checkFieldConfig(tableConfig);
 

pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexType.java

@@ -31,10 +31,8 @@
 import org.apache.pinot.segment.spi.V1Constants;
 import org.apache.pinot.segment.spi.creator.IndexCreationContext;
 import org.apache.pinot.segment.spi.index.AbstractIndexType;
-import org.apache.pinot.segment.spi.index.ColumnConfigDeserializer;
 import org.apache.pinot.segment.spi.index.FieldIndexConfigs;
 import org.apache.pinot.segment.spi.index.FstIndexConfig;
-import org.apache.pinot.segment.spi.index.IndexConfigDeserializer;
 import org.apache.pinot.segment.spi.index.IndexHandler;
 import org.apache.pinot.segment.spi.index.IndexReaderConstraintException;
 import org.apache.pinot.segment.spi.index.IndexReaderFactory;
@@ -44,7 +42,6 @@
 import org.apache.pinot.segment.spi.index.reader.TextIndexReader;
 import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
 import org.apache.pinot.segment.spi.store.SegmentDirectory;
-import org.apache.pinot.spi.config.table.FieldConfig;
 import org.apache.pinot.spi.config.table.TableConfig;
 import org.apache.pinot.spi.data.FieldSpec;
 import org.apache.pinot.spi.data.Schema;
@@ -91,12 +88,6 @@ public String getPrettyName() {
     return INDEX_DISPLAY_NAME;
   }
 
-  @Override
-  protected ColumnConfigDeserializer<FstIndexConfig> createDeserializerForLegacyConfigs() {
-    return IndexConfigDeserializer.fromIndexTypes(FieldConfig.IndexType.FST,
-        (tableConfig, fieldConfig) -> new FstIndexConfig(false));
-  }
-
   @Override
   public FSTIndexCreator createIndexCreator(IndexCreationContext context, FstIndexConfig indexConfig)
       throws IOException {

pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/fst/IFSTIndexType.java

@@ -30,10 +30,8 @@
 import org.apache.pinot.segment.spi.V1Constants;
 import org.apache.pinot.segment.spi.creator.IndexCreationContext;
 import org.apache.pinot.segment.spi.index.AbstractIndexType;
-import org.apache.pinot.segment.spi.index.ColumnConfigDeserializer;
 import org.apache.pinot.segment.spi.index.FieldIndexConfigs;
 import org.apache.pinot.segment.spi.index.FstIndexConfig;
-import org.apache.pinot.segment.spi.index.IndexConfigDeserializer;
 import org.apache.pinot.segment.spi.index.IndexHandler;
 import org.apache.pinot.segment.spi.index.IndexReaderConstraintException;
 import org.apache.pinot.segment.spi.index.IndexReaderFactory;
@@ -43,7 +41,6 @@
 import org.apache.pinot.segment.spi.index.reader.TextIndexReader;
 import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
 import org.apache.pinot.segment.spi.store.SegmentDirectory;
-import org.apache.pinot.spi.config.table.FieldConfig;
 import org.apache.pinot.spi.config.table.TableConfig;
 import org.apache.pinot.spi.data.FieldSpec;
 import org.apache.pinot.spi.data.Schema;
@@ -87,12 +84,6 @@ public String getPrettyName() {
     return INDEX_DISPLAY_NAME;
   }
 
-  @Override
-  protected ColumnConfigDeserializer<FstIndexConfig> createDeserializerForLegacyConfigs() {
-    return IndexConfigDeserializer.fromIndexTypes(FieldConfig.IndexType.IFST,
-        (tableConfig, fieldConfig) -> new FstIndexConfig(false));
-  }
-
   @Override
   public FSTIndexCreator createIndexCreator(IndexCreationContext context, FstIndexConfig indexConfig)
       throws IOException {

pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/h3/H3IndexType.java

@@ -19,6 +19,7 @@
 
 package org.apache.pinot.segment.local.segment.index.h3;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.base.Preconditions;
 import java.io.IOException;
 import java.io.UncheckedIOException;
@@ -51,11 +52,11 @@
 import org.apache.pinot.segment.spi.index.reader.H3IndexResolution;
 import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
 import org.apache.pinot.segment.spi.store.SegmentDirectory;
-import org.apache.pinot.spi.config.table.FieldConfig;
 import org.apache.pinot.spi.config.table.TableConfig;
 import org.apache.pinot.spi.data.FieldSpec;
 import org.apache.pinot.spi.data.FieldSpec.DataType;
 import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.JsonUtils;
 
 
 public class H3IndexType extends AbstractIndexType<H3IndexConfig, H3IndexReader, GeoSpatialIndexCreator> {
@@ -94,9 +95,18 @@ public String getPrettyName() {
   }
 
   @Override
-  protected ColumnConfigDeserializer<H3IndexConfig> createDeserializerForLegacyConfigs() {
-    return IndexConfigDeserializer.fromIndexTypes(FieldConfig.IndexType.H3,
-        (tableConfig, fieldConfig) -> new H3IndexConfig(fieldConfig.getProperties()));
+  protected ColumnConfigDeserializer<H3IndexConfig> createDeserializer() {
+    return IndexConfigDeserializer.fromIndexes(getPrettyName(), (tableConfig, fieldConfig) -> {
+      JsonNode indexNode = fieldConfig.getIndexes().get(getPrettyName());
+      if (IndexConfigDeserializer.isEnabledOnlyConfig(indexNode)) {
+        return new H3IndexConfig(fieldConfig.getProperties());
+      }
+      try {
+        return JsonUtils.jsonNodeToObject(indexNode, H3IndexConfig.class);
+      } catch (IOException e) {
+        throw new UncheckedIOException(e);
+      }
+    });
   }
 
   @Override

pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/inverted/InvertedIndexType.java

@@ -51,7 +51,6 @@
 import org.apache.pinot.segment.spi.index.reader.SortedIndexReader;
 import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
 import org.apache.pinot.segment.spi.store.SegmentDirectory;
-import org.apache.pinot.spi.config.table.FieldConfig;
 import org.apache.pinot.spi.config.table.IndexConfig;
 import org.apache.pinot.spi.config.table.TableConfig;
 import org.apache.pinot.spi.data.FieldSpec;
@@ -96,13 +95,9 @@ public String getPrettyName() {
 
   @Override
   protected ColumnConfigDeserializer<IndexConfig> createDeserializerForLegacyConfigs() {
-    ColumnConfigDeserializer<IndexConfig> fromInvertedIndexColumns =
-        IndexConfigDeserializer.fromCollection(tableConfig -> tableConfig.getIndexingConfig().getInvertedIndexColumns(),
-            (acum, column) -> acum.put(column, IndexConfig.ENABLED));
-    ColumnConfigDeserializer<IndexConfig> fromFieldConfigs =
-        IndexConfigDeserializer.fromIndexTypes(FieldConfig.IndexType.INVERTED,
-            (tableConfig, fieldConfig) -> IndexConfig.ENABLED);
-    return fromInvertedIndexColumns.withFallbackAlternative(fromFieldConfigs);
+    return IndexConfigDeserializer.fromCollection(
+        tableConfig -> tableConfig.getIndexingConfig().getInvertedIndexColumns(),
+        (acum, column) -> acum.put(column, IndexConfig.ENABLED));
   }
 
   public DictionaryBasedInvertedIndexCreator createIndexCreator(IndexCreationContext context)

pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/json/JsonIndexType.java

@@ -48,7 +48,6 @@
 import org.apache.pinot.segment.spi.index.reader.JsonIndexReader;
 import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
 import org.apache.pinot.segment.spi.store.SegmentDirectory;
-import org.apache.pinot.spi.config.table.FieldConfig;
 import org.apache.pinot.spi.config.table.JsonIndexConfig;
 import org.apache.pinot.spi.config.table.TableConfig;
 import org.apache.pinot.spi.data.FieldSpec;
@@ -100,10 +99,7 @@ protected ColumnConfigDeserializer<JsonIndexConfig> createDeserializerForLegacyC
     ColumnConfigDeserializer<JsonIndexConfig> fromJsonIndexColumns =
         IndexConfigDeserializer.fromCollection(tableConfig -> tableConfig.getIndexingConfig().getJsonIndexColumns(),
             (accum, column) -> accum.put(column, JsonIndexConfig.DEFAULT));
-    ColumnConfigDeserializer<JsonIndexConfig> fromFieldConfigs =
-        IndexConfigDeserializer.fromIndexTypes(FieldConfig.IndexType.JSON,
-            (tableConfig, fieldConfig) -> JsonIndexConfig.DEFAULT);
-    return fromJsonIndexConfigs.withFallbackAlternative(fromJsonIndexColumns).withFallbackAlternative(fromFieldConfigs);
+    return fromJsonIndexConfigs.withFallbackAlternative(fromJsonIndexColumns);
   }
 
   @Override

pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/range/RangeIndexType.java

@@ -19,6 +19,7 @@
 
 package org.apache.pinot.segment.local.segment.index.range;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.base.Preconditions;
 import java.io.IOException;
 import java.util.Collections;
@@ -48,7 +49,6 @@
 import org.apache.pinot.segment.spi.index.reader.RangeIndexReader;
 import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
 import org.apache.pinot.segment.spi.store.SegmentDirectory;
-import org.apache.pinot.spi.config.table.FieldConfig;
 import org.apache.pinot.spi.config.table.TableConfig;
 import org.apache.pinot.spi.data.FieldSpec;
 import org.apache.pinot.spi.data.FieldSpec.DataType;
@@ -93,6 +93,16 @@ public String getPrettyName() {
     return INDEX_DISPLAY_NAME;
   }
 
+  @Override
+  protected ColumnConfigDeserializer<RangeIndexConfig> createDeserializer() {
+    return IndexConfigDeserializer.fromIndexes(getPrettyName(), (tableConfig, fieldConfig) -> {
+      JsonNode indexNode = fieldConfig.getIndexes().get(getPrettyName());
+      Boolean disabled = indexNode.has("disabled") ? indexNode.get("disabled").asBoolean() : null;
+      Integer version = indexNode.has("version") ? indexNode.get("version").asInt() : getRangeIndexVersion(tableConfig);
+      return new RangeIndexConfig(disabled, version);
+    }).withExclusiveAlternative(createDeserializerForLegacyConfigs());
+  }
+
   @Override
   protected ColumnConfigDeserializer<RangeIndexConfig> createDeserializerForLegacyConfigs() {
     ColumnConfigDeserializer<RangeIndexConfig> fromRangeIndexColumns = (tableConfig, schema) -> {
@@ -110,10 +120,7 @@ protected ColumnConfigDeserializer<RangeIndexConfig> createDeserializerForLegacy
       }
       return result;
     };
-    ColumnConfigDeserializer<RangeIndexConfig> fromIndexTypes =
-        IndexConfigDeserializer.fromIndexTypes(FieldConfig.IndexType.RANGE,
-            (tableConfig, fieldConfig) -> new RangeIndexConfig(getRangeIndexVersion(tableConfig)));
-    return fromRangeIndexColumns.withFallbackAlternative(fromIndexTypes);
+    return fromRangeIndexColumns;
   }
 
   private static int getRangeIndexVersion(TableConfig tableConfig) {

pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/text/TextIndexType.java

@@ -19,10 +19,12 @@
 
 package org.apache.pinot.segment.local.segment.index.text;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.Lists;
 import java.io.File;
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.List;
 import java.util.Map;
 import javax.annotation.Nullable;
@@ -48,10 +50,10 @@
 import org.apache.pinot.segment.spi.index.reader.TextIndexReader;
 import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
 import org.apache.pinot.segment.spi.store.SegmentDirectory;
-import org.apache.pinot.spi.config.table.FieldConfig;
 import org.apache.pinot.spi.config.table.TableConfig;
 import org.apache.pinot.spi.data.FieldSpec;
 import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.JsonUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -95,9 +97,19 @@ public String getPrettyName() {
   }
 
   @Override
-  protected ColumnConfigDeserializer<TextIndexConfig> createDeserializerForLegacyConfigs() {
-    return IndexConfigDeserializer.fromIndexTypes(FieldConfig.IndexType.TEXT,
-        (tableConfig, fieldConfig) -> new TextIndexConfigBuilder().withProperties(fieldConfig.getProperties()).build());
+  protected ColumnConfigDeserializer<TextIndexConfig> createDeserializer() {
+    return IndexConfigDeserializer.fromIndexes(getPrettyName(), (tableConfig, fieldConfig) -> {
+      JsonNode indexNode = fieldConfig.getIndexes().get(getPrettyName());
+      TextIndexConfig indexConfig;
+      try {
+        indexConfig = JsonUtils.jsonNodeToObject(indexNode, TextIndexConfig.class);
+      } catch (IOException e) {
+        throw new UncheckedIOException(e);
+      }
+      return IndexConfigDeserializer.isEnabledOnlyConfig(indexNode)
+          ? new TextIndexConfigBuilder(indexConfig).withProperties(fieldConfig.getProperties()).build()
+          : indexConfig;
+    });
   }
 
   @Override

pinot-segment-local/src/main/java/org/apache/pinot/segment/local/segment/index/vector/VectorIndexType.java

@@ -18,9 +18,11 @@
  */
 package org.apache.pinot.segment.local.segment.index.vector;
 
+import com.fasterxml.jackson.databind.JsonNode;
 import com.google.common.base.Preconditions;
 import java.io.File;
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.List;
 import java.util.Map;
 import javax.annotation.Nullable;
@@ -51,10 +53,10 @@
 import org.apache.pinot.segment.spi.index.reader.VectorIndexReader;
 import org.apache.pinot.segment.spi.store.SegmentDirectory;
 import org.apache.pinot.segment.spi.store.SegmentDirectoryPaths;
-import org.apache.pinot.spi.config.table.FieldConfig;
 import org.apache.pinot.spi.config.table.TableConfig;
 import org.apache.pinot.spi.data.FieldSpec;
 import org.apache.pinot.spi.data.Schema;
+import org.apache.pinot.spi.utils.JsonUtils;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -120,9 +122,18 @@ public String getPrettyName() {
   }
 
   @Override
-  protected ColumnConfigDeserializer<VectorIndexConfig> createDeserializerForLegacyConfigs() {
-    return IndexConfigDeserializer.fromIndexTypes(FieldConfig.IndexType.VECTOR,
-        (tableConfig, fieldConfig) -> new VectorIndexConfig(fieldConfig.getProperties()));
+  protected ColumnConfigDeserializer<VectorIndexConfig> createDeserializer() {
+    return IndexConfigDeserializer.fromIndexes(getPrettyName(), (tableConfig, fieldConfig) -> {
+      JsonNode indexNode = fieldConfig.getIndexes().get(getPrettyName());
+      if (IndexConfigDeserializer.isEnabledOnlyConfig(indexNode)) {
+        return new VectorIndexConfig(fieldConfig.getProperties());
+      }
+      try {
+        return JsonUtils.jsonNodeToObject(indexNode, VectorIndexConfig.class);
+      } catch (IOException e) {
+        throw new UncheckedIOException(e);
+      }
+    });
   }
 
   @Override

pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/H3IndexTest.java

@@ -60,7 +60,6 @@
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertTrue;
 
 
 public class H3IndexTest implements PinotBuffersAfterMethodCheckRule {
@@ -431,7 +430,7 @@ public void oldToNewConfConversion()
           .filter(fc -> fc.getName().equals("dimStr"))
           .collect(Collectors.toList()).get(0);
       assertNotNull(fieldConfig.getIndexes().get(H3IndexType.INDEX_DISPLAY_NAME));
-      assertTrue(fieldConfig.getIndexTypes().isEmpty());
+      Assert.assertEquals(fieldConfig.getIndexType(), FieldConfig.IndexType.H3);
     }
 
     @Test
@@ -454,7 +453,7 @@ public void testConvertToUpdatedFormat()
           .filter(fc -> fc.getName().equals("location_st_point"))
           .collect(Collectors.toList()).get(0);
       Assert.assertEquals(fieldConfig.getEncodingType(), RAW);
-      assertTrue(fieldConfig.getIndexTypes().isEmpty());
+      Assert.assertEquals(fieldConfig.getIndexType(), FieldConfig.IndexType.H3);
       assertNull(fieldConfig.getProperties());
       JsonNode node = fieldConfig.getIndexes().get(H3IndexType.INDEX_DISPLAY_NAME);
       Assert.assertEquals(node.toString(), "{\"disabled\":false,\"resolution\":[5,6,13]}");

pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/JsonIndexTest.java

@@ -1054,7 +1054,7 @@ public void oldToNewConfConversion() {
       assertNull(_tableConfig.getIndexingConfig().getJsonIndexColumns());
       assertNull(_tableConfig.getIndexingConfig().getJsonIndexConfigs());
       assertNotNull(fieldConfig.getIndexes().get(JsonIndexType.INDEX_DISPLAY_NAME));
-      assertTrue(fieldConfig.getIndexTypes().isEmpty());
+      assertEquals(fieldConfig.getIndexType(), FieldConfig.IndexType.JSON);
       assertNull(fieldConfig.getProperties());
     }
   }

pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/RangeIndexTest.java

@@ -32,7 +32,6 @@
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertTrue;
 
 public class RangeIndexTest {
   public static class ConfTest extends AbstractSerdeIndexContract {
@@ -48,7 +47,7 @@ public void oldToNewConfConversion() {
       JsonNode indexConfig = fieldConfig.getIndexes().get(RangeIndexType.INDEX_DISPLAY_NAME);
       assertNotNull(indexConfig);
       assertEquals(RangeIndexConfig.DEFAULT.getVersion(), indexConfig.get("version").asInt());
-      assertTrue(fieldConfig.getIndexTypes().isEmpty());
+      assertEquals(fieldConfig.getIndexType(), FieldConfig.IndexType.RANGE);
       assertNull(fieldConfig.getProperties());
     }
   }

pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/TextIndexTest.java

@@ -31,7 +31,6 @@
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertTrue;
 
 public class TextIndexTest {
   public static class ConfTest extends AbstractSerdeIndexContract {
@@ -51,7 +50,7 @@ public void oldToNewConfConversion()
       JsonNode indexConfig = fieldConfig.getIndexes().get(TextIndexType.INDEX_DISPLAY_NAME);
       assertNotNull(indexConfig);
       assertFalse(indexConfig.get("disabled").asBoolean());
-      assertTrue(fieldConfig.getIndexTypes().isEmpty());
+      assertEquals(fieldConfig.getIndexType(), FieldConfig.IndexType.TEXT);
       assertNull(fieldConfig.getProperties());
     }
   }

pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/VectorIndexTest.java

@@ -29,7 +29,6 @@
 import static org.testng.Assert.assertFalse;
 import static org.testng.Assert.assertNotNull;
 import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertTrue;
 
 public class VectorIndexTest {
   public static class ConfTest extends AbstractSerdeIndexContract {
@@ -59,7 +58,7 @@ public void testConvertToUpdatedFormat()
       JsonNode indexConfig = fieldConfig.getIndexes().get(VectorIndexType.INDEX_DISPLAY_NAME);
       assertNotNull(indexConfig);
       assertFalse(indexConfig.get("disabled").asBoolean());
-      assertTrue(fieldConfig.getIndexTypes().isEmpty());
+      Assert.assertEquals(fieldConfig.getIndexType(), FieldConfig.IndexType.VECTOR);
       assertNull(fieldConfig.getProperties());
       Assert.assertEquals(indexConfig.toString(),
           "{\"disabled\":false,\"vectorIndexType\":\"HNSW\",\"vectorDimension\":1536,"

pinot-segment-local/src/test/java/org/apache/pinot/segment/local/segment/index/fst/FstIndexTypeTest.java

@@ -95,7 +95,7 @@ public void oldToNewConfConversion()
           .filter(fc -> fc.getName().equals("dimStr"))
           .collect(Collectors.toList()).get(0);
       assertNotNull(fieldConfig.getIndexes().get(FstIndexType.INDEX_DISPLAY_NAME));
-      assertTrue(fieldConfig.getIndexTypes().isEmpty());
+      Assert.assertEquals(fieldConfig.getIndexType(), FieldConfig.IndexType.FST);
     }
   }