Add caption support with new models, vuex, and edit modal integration

contentcuration/automation/settings.py

@@ -0,0 +1,27 @@
+from enum import Enum
+from torch.cuda import is_available as is_gpu_available
+
+DEVICE = "cuda:0" if is_gpu_available() else "cpu"
+
+
+# [TRANSCRIPTION GENERATION]
+WHISPER_MODELS = dict(
+    TINY="openai/whisper-tiny",
+    BASE="openai/whisper-base",
+    SMALL="openai/whisper-small",
+    MEDIUM="openai/whisper-medium",
+    LARGE="openai/whisper-large",
+    LARGEV2="openai/whisper-large-v2",
+)
+
+
+DEV_TRANSCRIPTION_MODEL = WHISPER_MODELS['TINY']
+TRANSCRIPTION_MODEL = WHISPER_MODELS['TINY']
+
+class WhisperTask(Enum):
+    TRANSLATE = "translate"
+    TRANSCRIBE = "transcribe"
+
+# https://huggingface.co/docs/transformers/v4.29.1/en/generation_strategies#customize-text-generation
+MAX_TOKEN_LENGTH = 448
+CHUNK_LENGTH = 10

contentcuration/automation/urls.py

@@ -0,0 +1,10 @@
+from automation.views import TranscriptionsViewSet
+from django.urls import include, path
+from rest_framework import routers
+
+automation_router = routers.DefaultRouter()
+automation_router.register(r'transcribe', TranscriptionsViewSet, basename="transcribe")
+
+urlpatterns = [
+    path("api/automation/", include(automation_router.urls), name='automation'),
+]

contentcuration/automation/views.py

@@ -1,3 +1,10 @@
-# from django.shortcuts import render
+from rest_framework.viewsets import ViewSet
+from rest_framework.response import Response
+from rest_framework.permissions import AllowAny
 
-# Create your views here.
+class TranscriptionsViewSet(ViewSet):
+    def create(self, request):
+        permission_classes = [AllowAny]
+        return Response({
+            "OK":"OK"
+        })

contentcuration/contentcuration/constants/transcription_languages.py

@@ -0,0 +1,141 @@
+# The list of languages is obtained from the Whisper project on GitHub, specifically from the tokenizer module.
+# You can find the complete list of available languages in the tokenizer module:
+# https://github.com/openai/whisper/blob/main/whisper/tokenizer.py
+
+# The supported languages are stored in the 'LANGUAGES' dictionary in the format of language code and language name.
+# For example, the first element in the 'LANGUAGES' dictionary is ('en', 'english').
+
+# To add support for a new model, we also need to update the `supportedLanguageList` array in the frontend TranscriptionLanguages.js file.
+# https://github.com/learningequality/studio/blob/unstable/contentcuration/contentcuration/frontend/shared/leUtils/TranscriptionLanguages.js
+
+import json
+from typing import List, Dict
+
+import le_utils.resources as resources
+
+WHISPER_LANGUAGES = {
+    "en": "english",
+    "zh": "chinese",
+    "de": "german",
+    "es": "spanish",
+    "ru": "russian",
+    "ko": "korean",
+    "fr": "french",
+    "ja": "japanese",
+    "pt": "portuguese",
+    "tr": "turkish",
+    "pl": "polish",
+    "ca": "catalan",
+    "nl": "dutch",
+    "ar": "arabic",
+    "sv": "swedish",
+    "it": "italian",
+    "id": "indonesian",
+    "hi": "hindi",
+    "fi": "finnish",
+    "vi": "vietnamese",
+    "he": "hebrew",
+    "uk": "ukrainian",
+    "el": "greek",
+    "ms": "malay",
+    "cs": "czech",
+    "ro": "romanian",
+    "da": "danish",
+    "hu": "hungarian",
+    "ta": "tamil",
+    "no": "norwegian",
+    "th": "thai",
+    "ur": "urdu",
+    "hr": "croatian",
+    "bg": "bulgarian",
+    "lt": "lithuanian",
+    "la": "latin",
+    "mi": "maori",
+    "ml": "malayalam",
+    "cy": "welsh",
+    "sk": "slovak",
+    "te": "telugu",
+    "fa": "persian",
+    "lv": "latvian",
+    "bn": "bengali",
+    "sr": "serbian",
+    "az": "azerbaijani",
+    "sl": "slovenian",
+    "kn": "kannada",
+    "et": "estonian",
+    "mk": "macedonian",
+    "br": "breton",
+    "eu": "basque",
+    "is": "icelandic",
+    "hy": "armenian",
+    "ne": "nepali",
+    "mn": "mongolian",
+    "bs": "bosnian",
+    "kk": "kazakh",
+    "sq": "albanian",
+    "sw": "swahili",
+    "gl": "galician",
+    "mr": "marathi",
+    "pa": "punjabi",
+    "si": "sinhala",
+    "km": "khmer",
+    "sn": "shona",
+    "yo": "yoruba",
+    "so": "somali",
+    "af": "afrikaans",
+    "oc": "occitan",
+    "ka": "georgian",
+    "be": "belarusian",
+    "tg": "tajik",
+    "sd": "sindhi",
+    "gu": "gujarati",
+    "am": "amharic",
+    "yi": "yiddish",
+    "lo": "lao",
+    "uz": "uzbek",
+    "fo": "faroese",
+    "ht": "haitian creole",
+    "ps": "pashto",
+    "tk": "turkmen",
+    "nn": "nynorsk",
+    "mt": "maltese",
+    "sa": "sanskrit",
+    "lb": "luxembourgish",
+    "my": "myanmar",
+    "bo": "tibetan",
+    "tl": "tagalog",
+    "mg": "malagasy",
+    "as": "assamese",
+    "tt": "tatar",
+    "haw": "hawaiian",
+    "ln": "lingala",
+    "ha": "hausa",
+    "ba": "bashkir",
+    "jw": "javanese",
+    "su": "sundanese",
+}
+
+
+def _load_kolibri_languages() -> List[str]:
+    """Loads the language codes from languagelookup.json and returns them as a list."""
+    filepath = resources.__path__[0]
+    kolibri_languages = []
+    with open(f"{filepath}/languagelookup.json") as f:
+        kolibri_languages = list(json.load(f).keys())
+    return kolibri_languages
+
+
+def _load_model_languages(languages: Dict[str, str]) -> List[str]:
+    """Load languages supported by the speech-to-text model.
+    :param: languages: dict mapping language codes to language names"""
+    return list(languages.keys())
+
+
+def create_captions_languages() -> List[str]:
+    """Finds the intersection of Kolibri languages and model languages and returns it."""
+    kolibri_set = set(_load_kolibri_languages())
+    model_set = set(_load_model_languages(languages=WHISPER_LANGUAGES))
+    return list(kolibri_set.intersection(model_set))
+
+
+CAPTIONS_LANGUAGES = create_captions_languages()

contentcuration/contentcuration/dev_urls.py

@@ -1,5 +1,6 @@
 import urllib.parse
 
+from automation.urls import urlpatterns as automation_urlpatterns
 from django.conf import settings
 from django.contrib import admin
 from django.core.files.storage import default_storage
@@ -76,6 +77,8 @@ def file_server(request, storage_path=None):
     re_path(r"^content/(?P<storage_path>.+)$", file_server),
 ]
 
+urlpatterns += automation_urlpatterns
+
 if getattr(settings, "DEBUG_PANEL_ACTIVE", False):
 
     import debug_toolbar

contentcuration/contentcuration/frontend/channelEdit/components/CaptionsTab/CaptionsTab.vue

@@ -0,0 +1,148 @@
+<template>
+
+  <div class="captions-tab-section">
+    <div class="my-4 px-4 title">
+      <ContentNodeIcon
+        v-if="node && node.kind"
+        :kind="node.kind"
+        class="mr-1"
+      />
+      <h2
+        v-if="node && node.title"
+        class="headline mx-2 notranslate"
+        data-test="title"
+      >
+        {{ $tr('header', { fileName: node.title }) }}
+      </h2>
+    </div>
+
+    <div v-if="!isGeneratingCaptions" class="mb-2 mt-2 px-4">
+      <p class="my-2">
+        {{ $tr('captionGenerationHelpText') }}
+      </p>
+
+      <LanguageDropdown
+        ref="language"
+        v-model="selectedLanguage"
+        clearable
+        :required="true"
+        :excludeLanguages="excludedLanguages"
+      />
+
+      <VBtn
+        v-if="selectedLanguage && !isGeneratingCaptions"
+        flat
+        color="primary"
+        class="font-weight-bold ml-0"
+        @click="addCaption"
+      >
+        {{ $tr('generateBtn') }}
+      </VBtn>
+    </div>
+    <!-- TODO -->
+    <!-- est. time, time elapsed can be good -->
+    <p v-if="isGeneratingCaptions">
+      {{ $tr('generating', { fileName: node.title }) }}
+      <LoadingText />
+      <br>
+    </p>
+
+  </div>
+
+</template>
+
+<script>
+
+  import { mapActions, mapGetters } from 'vuex';
+  import { CaptionFile, uuid4 } from 'shared/data/resources';
+  import { notSupportedCaptionLanguages } from 'shared/leUtils/TranscriptionLanguages';
+  import ContentNodeIcon from 'shared/views/ContentNodeIcon';
+  import LoadingText from 'shared/views/LoadingText';
+  import LanguageDropdown from 'shared/views/LanguageDropdown';
+
+  export default {
+    name: 'CaptionsTab',
+    components: {
+      ContentNodeIcon,
+      LanguageDropdown,
+      LoadingText,
+    },
+    props: {
+      nodeId: {
+        type: String,
+        required: true,
+      },
+    },
+    data() {
+      return {
+        selectedLanguage: null,
+      };
+    },
+    computed: {
+      ...mapGetters('contentNode', ['getContentNode']),
+      ...mapGetters('caption', ['isGeneratingGetter']),
+      ...mapGetters('file', ['getContentNodeFiles']),
+      // ...mapState('caption', ['captionFilesMap', 'captionCuesMap']),
+      node() {
+        return this.getContentNode(this.nodeId);
+      },
+      excludedLanguages() {
+        // excludeLanguages requires array of ids
+        return notSupportedCaptionLanguages.map(l => l.id);
+      },
+      isGeneratingCaptions() {
+        return this.isGeneratingGetter(this.nodeId);
+      },
+    },
+    methods: {
+      ...mapActions('caption', ['addCaptionFile']),
+      addCaption() {
+        const id = uuid4();
+        const fileId = this.getLongestDurationFileId();
+        const language = this.selectedLanguage;
+        if (!language && !fileId) return;
+
+        this.addCaptionFile({
+          id: id,
+          file_id: fileId,
+          language: language,
+          nodeId: this.nodeId,
+        });
+
+        CaptionFile.waitForCaptionCueGeneration(id).then(() => {
+          this.selectedLanguage = null;
+        });
+      },
+      getLongestDurationFileId() {
+        const files = this.getContentNodeFiles(this.nodeId);
+        const { id } = files.reduce((max, file) => (file.duration > max.duration ? file : max), {
+          duration: 0,
+        });
+        return id;
+      },
+    },
+    $trs: {
+      header: 'Add Captions for {fileName}',
+      generateBtn: 'Generate Captions',
+      captionGenerationHelpText:
+        'Select a language to automatically generate captions for this video',
+      generating: 'Generating caption for file: {fileName}',
+    },
+  };
+
+</script>
+
+<style lang="less" scoped>
+
+  .captions-tab-section {
+    height: 80vh;
+    vertical-align: text-top;
+  }
+
+  .title {
+    display: inline-flex;
+    align-items: center;
+    justify-content: flex-start;
+  }
+
+</style>

contentcuration/contentcuration/frontend/channelEdit/components/edit/EditModal.vue

@@ -340,6 +340,7 @@
               // (especially marking nodes as (in)complete)
               vm.loadFiles({ contentnode__in: childrenNodesIds }),
               vm.loadAssessmentItems({ contentnode__in: childrenNodesIds }),
+              vm.loadCaptions({ contentnode__in: childrenNodesIds }),
             ];
           } else {
             // no need to load assessment items or files as topics have none
@@ -394,6 +395,7 @@
       ]),
       ...mapActions('file', ['loadFiles', 'updateFile']),
       ...mapActions('assessmentItem', ['loadAssessmentItems', 'updateAssessmentItems']),
+      ...mapActions('caption', ['loadCaptions']),
       ...mapMutations('contentNode', { enableValidation: 'ENABLE_VALIDATION_ON_NODES' }),
       closeModal() {
         this.promptUploading = false;