chore: env management overhaul

web-common/src/features/add-data/form/ConnectorForm.svelte

@@ -1,7 +1,7 @@
 <script lang="ts">
   import { createConnectorForm } from "@rilldata/web-common/features/sources/modal/FormValidation.ts";
   import { getConnectorSchema } from "@rilldata/web-common/features/sources/modal/connector-schemas.ts";
-  import { getConnectorYamlPreview } from "./yaml-preview.ts";
+  import { getConnectorYAML } from "./connector-source-yaml-generator.ts";
   import AddDataFormStructure from "@rilldata/web-common/features/add-data/form/AddDataFormStructure.svelte";
   import { queryClient } from "@rilldata/web-common/lib/svelte-query/globalQueryClient.ts";
   import { useRuntimeClient } from "@rilldata/web-common/runtime-client/v2";
@@ -17,6 +17,7 @@
   import { navigateToFile } from "@rilldata/web-common/layout/navigation/editor-routing";
   import { getConnectorDriverForSchema } from "@rilldata/web-common/features/add-data/manager/steps/utils.ts";
   import type { AddDataStateManager } from "@rilldata/web-common/features/add-data/manager/AddDataStateManager.svelte.ts";
+  import { getEnvFileStore } from "@rilldata/web-common/features/env-management/env-file-store.ts";
 
   export let stateManager: AddDataStateManager;
   export let step: CreateConnectorStep;
@@ -27,12 +28,15 @@
   export let onBack: () => void;
   export let onClose: () => void;
 
-  export let cachedFormValues: Record<string, unknown>;
-  export let connectorName: string;
-  export let cachedEnvBlob: string;
-
+  const envStore = getEnvFileStore();
   const runtimeClient = useRuntimeClient();
 
+  const {
+    name: connectorName,
+    formValues: cachedFormValues,
+    envEditSession: cachedEnvEditSession,
+  } = connectorFormCache.getOrCreate(step.schema, step.connectorId, envStore);
+
   $: connectorDriver = getConnectorDriverForSchema(step.schema);
 
   const superFormsParams = createConnectorForm({
@@ -49,7 +53,7 @@
           schemaName: step.schema,
           formValues: form.data,
           validate: true,
-          existingEnvBlob: cachedEnvBlob,
+          envEditSession: cachedEnvEditSession,
         });
 
         connectorFormCache.updateFormValues(step.connectorId, form.data);
@@ -67,11 +71,11 @@
 
   $: schema = getConnectorSchema(step.schema);
   $: yamlPreview = connectorDriver
-    ? getConnectorYamlPreview({
+    ? getConnectorYAML({
         connector: connectorDriver,
         formValues: $form,
         schema,
-        existingEnvBlob: cachedEnvBlob,
+        envEditSession: cachedEnvEditSession,
       })
     : "";
 
@@ -87,14 +91,20 @@
       schemaName: step.schema,
       formValues: $form,
       validate: false,
-      existingEnvBlob: cachedEnvBlob,
+      envEditSession: cachedEnvEditSession,
     });
     onClose();
     return navigateToFile(addLeadingSlash(connectorPath));
   }
 
   async function cleanupAndBack() {
-    await maybeDeleteConnector(runtimeClient, queryClient, connectorName);
+    await maybeDeleteConnector(
+      runtimeClient,
+      queryClient,
+      connectorName,
+      cachedEnvEditSession,
+    );
+    connectorFormCache.delete(step.connectorId);
 
     onBack();
   }

web-common/src/features/add-data/form/ImportTableForm.svelte

@@ -30,6 +30,9 @@
   } from "@rilldata/web-common/features/add-data/manager/steps/utils.ts";
   import DatabaseExplorer from "@rilldata/web-common/features/connectors/explorer/DatabaseExplorer.svelte";
   import { ConnectorExplorerStore } from "@rilldata/web-common/features/connectors/explorer/connector-explorer-store.ts";
+  import { getEnvFileStore } from "@rilldata/web-common/features/env-management/env-file-store.ts";
+  import { EnvEditSession } from "@rilldata/web-common/features/env-management/env-edit-session.ts";
+  import { getConnectorSchema } from "@rilldata/web-common/features/sources/modal/connector-schemas.ts";
 
   export let config: AddDataConfig;
   export let step: ExploreConnectorStep;
@@ -40,6 +43,13 @@
 
   const runtimeClient = useRuntimeClient();
 
+  const envStore = getEnvFileStore();
+  const envEditSession = new EnvEditSession(
+    envStore,
+    step.connector,
+    getConnectorSchema(step.schema),
+  );
+
   $: connectorDriverQuery = getAnalyzedConnectorByName(
     runtimeClient,
     step.connector,
@@ -124,7 +134,7 @@
           connector: step.connector,
           importFrom,
           importTo: generateImportToConfig(importFrom),
-          envBlob: null,
+          envEditSession,
         } satisfies ImportStepConfig);
       },
       validationMethod: "onsubmit",

web-common/src/features/add-data/form/SourceForm.svelte

@@ -6,12 +6,10 @@
   } from "@rilldata/web-common/runtime-client";
   import { getConnectorSchema } from "@rilldata/web-common/features/sources/modal/connector-schemas.ts";
   import { onMount } from "svelte";
-  import { getSourceYamlPreview } from "./yaml-preview.ts";
+  import { getSourceYAML } from "./connector-source-yaml-generator.ts";
   import AddDataFormStructure from "@rilldata/web-common/features/add-data/form/AddDataFormStructure.svelte";
   import { queryClient } from "@rilldata/web-common/lib/svelte-query/globalQueryClient.ts";
   import { prepareSourceFormData } from "@rilldata/web-common/features/sources/sourceUtils.ts";
-  import { getSchemaSecretKeys } from "@rilldata/web-common/features/templates/schema-utils.ts";
-  import { updateDotEnvWithSecrets } from "@rilldata/web-common/features/connectors/code-utils.ts";
   import {
     type AddDataConfig,
     type CreateModelStep,
@@ -30,9 +28,10 @@
   import {
     getConnectorDriverForSchema,
     getImportStepsForSource,
-    maybeGetEnvContent,
   } from "@rilldata/web-common/features/add-data/manager/steps/utils.ts";
   import { maybeInitProject } from "@rilldata/web-common/features/add-data/manager/steps/connector.ts";
+  import { getEnvFileStore } from "@rilldata/web-common/features/env-management/env-file-store.ts";
+  import { EnvEditSession } from "@rilldata/web-common/features/env-management/env-edit-session.ts";
   import { setSubmitError } from "@rilldata/web-common/features/add-data/form/errors.ts";
   import type { AddDataStateManager } from "@rilldata/web-common/features/add-data/manager/AddDataStateManager.svelte.ts";
   import { setError, type SuperValidated } from "sveltekit-superforms";
@@ -52,15 +51,20 @@
     $connectorDriverQuery.data?.driver ??
     getConnectorDriverForSchema(step.schema);
 
+  const envStore = getEnvFileStore();
+  const envEditSession = new EnvEditSession(
+    envStore,
+    step.connector,
+    getConnectorSchema(step.schema),
+  );
+
   const importSteps = getImportStepsForSource(config);
 
   // Capture .env blob ONCE on mount for consistent conflict detection in YAML preview.
   // This prevents the preview from updating when Test and Connect writes to .env.
   // Use null to indicate "not yet loaded" vs "" for "loaded but empty"
-  let existingEnvBlob: string | null = null;
   let defaultOLAP = "duckdb";
   onMount(async () => {
-    existingEnvBlob = await maybeGetEnvContent();
     try {
       const runtimeInstance = await queryClient.fetchQuery({
         queryKey: getRuntimeServiceGetInstanceQueryKey(
@@ -95,12 +99,12 @@
 
   $: schema = getConnectorSchema(step.schema);
   $: yamlPreview = connectorDriver
-    ? getSourceYamlPreview({
+    ? getSourceYAML({
         connectorName: step.connector,
         connector: connectorDriver,
         formValues: $form,
         schema,
-        existingEnvBlob,
+        envEditSession,
         outputConnector: defaultOLAP,
       })
     : "";
@@ -117,27 +121,6 @@
 
     await maybeInitProject(runtimeClient);
 
-    const [rewrittenConnector, rewrittenFormValues] = prepareSourceFormData(
-      connectorDriver,
-      formValues,
-      { connectorInstanceName: step.connector },
-    );
-    const schema = getConnectorSchema(rewrittenConnector.name ?? "");
-    const schemaSecretKeys = schema
-      ? getSchemaSecretKeys(schema, { step: "source" })
-      : [];
-
-    // Create or update the `.env` file
-    const { newBlob } = await updateDotEnvWithSecrets(
-      runtimeClient,
-      queryClient,
-      rewrittenConnector,
-      rewrittenFormValues,
-      {
-        secretKeys: schemaSecretKeys,
-      },
-    );
-
     if (formValues.file) {
       // TODO: support multiple files upload
       const firstFile = formValues.file[0];
@@ -154,12 +137,20 @@
         throw e; // rethrow so that global error handler is triggered
       }
     }
-    const yaml = getSourceYamlPreview({
+
+    const [rewrittenConnector] = prepareSourceFormData(
+      connectorDriver,
+      formValues,
+      { connectorInstanceName: step.connector },
+    );
+    const schema = getConnectorSchema(rewrittenConnector.name ?? "");
+
+    const yaml = getSourceYAML({
       connectorName: step.connector,
       connector: connectorDriver,
       formValues,
       schema,
-      existingEnvBlob,
+      envEditSession,
       outputConnector: defaultOLAP,
     });
 
@@ -176,7 +167,7 @@
         importFrom,
         formValues.name as string | undefined,
       ),
-      envBlob: newBlob,
+      envEditSession,
     } satisfies ImportStepConfig;
 
     onSubmit(importConfig);