Add image description for grounding

src/code.gs

@@ -112,12 +112,27 @@ const GenAIApp = (function () {
           const response = UrlFetchApp.fetch(imageInput);
           const blob = response.getBlob();
           const base64Image = Utilities.base64Encode(blob.getBytes());
+          let mimeType = blob.getContentType();
+          if (!mimeType || !mimeType.startsWith("image/")) {
+            const lower = imageInput.toLowerCase();
+            if (lower.endsWith(".png")) {
+              mimeType = "image/png";
+            } else if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) {
+              mimeType = "image/jpeg";
+            } else if (lower.endsWith(".webp")) {
+              mimeType = "image/webp";
+            } else if (lower.endsWith(".gif")) {
+              mimeType = "image/gif";
+            } else {
+              throw new Error("Failed to identify a valid image MIME type. Please check the file format for Gemini.");
+            }
+          }
           contents.push({
             role: "user",
             parts: [
               {
-                inline_data: {
-                  mime_type: blob.getContentType(),
+                inlineData: {
+                  mime_type: mimeType,
                   data: base64Image
                 }
               }
@@ -422,6 +437,13 @@ const GenAIApp = (function () {
           knowledgeLink = [];
         }
 
+        // Gemini does not support using images together with vector stores (RAG) yet.
+        // Images must be analyzed first and replaced with text before RAG processing.
+        const ragCorpusIds = Object.keys(addedVectorStores);
+        if (ragCorpusIds.length > 0 && model.includes("gemini")) {
+          contents = this._convertImagesToText(contents);
+        }
+
         let payload;
         if (model.includes("gemini")) {
           payload = this._buildGeminiPayload(advancedParametersObject);
@@ -737,6 +759,100 @@ const GenAIApp = (function () {
         return payload;
       }
 
+      /**
+       * Replaces all image parts in a Gemini conversation with a text description
+       * generated by Gemini 3 Pro Preview (Vertex AI Vision).
+       *
+       * - Detects images (inlineData / fileData) across all messages
+       * - Sends them to Gemini Vision for analysis
+       * - Removes images from the conversation
+       * - Appends a new message containing the image analysis
+       *
+       * @param {Array<Object>} currentContents
+       *   Gemini conversation contents.
+       *
+       * @returns {Array<Object>}
+       *   Updated contents with images removed and a text analysis appended.
+       */
+      this._convertImagesToText = function (currentContents) {
+        if (!currentContents || currentContents.length === 0) return currentContents;
+
+        const hasImages = currentContents.some(c => {
+          const parts = Array.isArray(c.parts) ? c.parts : (c.parts ? [c.parts] : []);
+          return parts.some(p => p.inlineData || p.fileData);
+        });
+
+        if (!hasImages) return currentContents;
+
+        if (typeof verbose !== 'undefined' && verbose) {
+          console.log("[GenAIApp] - Images detected. Converting to text description...");
+        }
+
+        const imageParts = currentContents.flatMap(c => {
+          const parts = Array.isArray(c.parts) ? c.parts : (c.parts ? [c.parts] : []);
+          return parts.filter(p => p.inlineData || p.fileData);
+        });
+
+        if (imageParts.length === 0) return currentContents;
+
+        const descriptionPayload = {
+          contents: [{
+            role: "user",
+            parts: [
+              ...imageParts,
+              { text: "Analyze these images for a technical support request. Transcribe any error messages, logs, code snippets, or visible UI text exactly. Describe the visual context briefly." }
+            ]
+          }],
+          generationConfig: {
+            temperature: 0.2,
+            maxOutputTokens: 2000
+          }
+        };
+
+        const options = {
+          method: 'post',
+          contentType: 'application/json',
+          headers: {
+            'Authorization': 'Bearer ' + ScriptApp.getOAuthToken()
+          },
+          payload: JSON.stringify(descriptionPayload),
+          muteHttpExceptions: true
+      };
+
+        const modelForVision = "gemini-3-pro-preview";
+        const endpoint = `https://aiplatform.googleapis.com/v1/projects/${gcpProjectId}/locations/global/publishers/google/models/${modelForVision}:generateContent`;
+
+        const response = UrlFetchApp.fetch(endpoint, options);
+        const result = JSON.parse(response.getContentText());
+
+        let description = "";
+        if (result?.candidates?.[0]?.content?.parts?.[0]?.text) {
+          description = result.candidates[0].content.parts[0].text
+        } else if (result?.parts?.[0]?.text) {
+          description = result.parts[0].text;
+        } else {
+          description = "Image analysis returned no text.";
+        }
+
+        let newContents = JSON.parse(JSON.stringify(currentContents));
+        newContents.forEach(c => {
+          const parts = Array.isArray(c.parts) ? c.parts : [c.parts];
+          c.parts = parts.filter(p => !p.inlineData && !p.fileData);
+        });
+
+        newContents = newContents.filter(c => {
+          const parts = Array.isArray(c.parts) ? c.parts : (c.parts ? [c.parts] : []);
+          return parts.length > 0;
+        });
+
+        newContents.push({
+          role: "user",
+          parts: [{ text: `IMAGE ANALYSIS:\n${description}` }]
+        });  
+
+        return newContents;
+      }
+
       /**
        * Get a blob from a Google Drive file ID
        *