Release v3.11.0

* fix setting of continous covariates in case of multiple settings and add a test

* update test

* Optimizing tidyCovariates for large data. Fixes #308 (#309)

Co-authored-by: ginberg <ginberg@gmail.com>
Co-authored-by: Admin_mschuemi <Admin_mschuemi@its.jnj.com>

* cran submission file

* add params to getDBCovariateData (#312)

* #292 add function to replace cohort schema and table

* #292 add params

* add test

* doc

* v3.11.0

* update news

---------

Co-authored-by: Martijn Schuemie <schuemie@ohdsi.org>
Co-authored-by: Admin_mschuemi <Admin_mschuemi@its.jnj.com>

Author: 3 people

CRAN-SUBMISSION

@@ -1,3 +1,3 @@
-Version: 3.8.0
-Date: 2025-03-19 15:22:25 UTC
-SHA: c0961a155c6fba22f3b5e4825b599f2410ed529b
+Version: 3.11.0
+Date: 2025-09-01 15:11:02 UTC
+SHA: 784c4a8dcda6fc0301c8b0823efdfccb1291745e

DESCRIPTION

@@ -1,8 +1,8 @@
 Package: FeatureExtraction
 Type: Package
 Title: Generating Features for a Cohort
-Version: 3.10.0
-Date: 2025-05-08
+Version: 3.11.0
+Date: 2025-09-01
 Authors@R: c(
   person("Martijn", "Schuemie", , "schuemie@ohdsi.org", role = c("aut")),
   person("Marc", "Suchard", role = c("aut")),

NEWS.md

@@ -1,3 +1,10 @@
+FeatureExtraction 3.11.0
+=======================
+
+- Improve tidyCovariates performance when using Andromeda version >= 1.0.0 (#308)
+- Fix error in merging covariateContinuous to multiple features in getDbCovariateData (#306)
+- Add arguments to getDbCovariateData to support a custom covariate cohort schema/table (#292)
+
 FeatureExtraction 3.10.0
 =======================
 

R/GetCovariates.R

@@ -69,6 +69,8 @@
 #' @param tempEmulationSchema    Some database platforms like Oracle and Impala do not truly support
 #'                               temp tables. To emulate temp tables, provide a schema with write
 #'                               privileges where temp tables can be created.
+#' @param covariateCohortDatabaseSchema The database schema where the cohorts used to define the covariates can be found.
+#' @param covariateCohortTable          The table where the cohorts used to define the covariates can be found.
 #'
 #' @return
 #' Returns an object of type \code{covariateData}, containing information on the covariates.
@@ -113,7 +115,9 @@ getDbCovariateData <- function(connectionDetails = NULL,
                                covariateSettings,
                                aggregated = FALSE,
                                minCharacterizationMean = 0,
-                               tempEmulationSchema = getOption("sqlRenderTempEmulationSchema")) {
+                               tempEmulationSchema = getOption("sqlRenderTempEmulationSchema"),
+                               covariateCohortDatabaseSchema = NULL,
+                               covariateCohortTable = NULL) {
   if (is.null(connectionDetails) && is.null(connection)) {
     stop("Need to provide either connectionDetails or connection")
   }
@@ -181,6 +185,13 @@ getDbCovariateData <- function(connectionDetails = NULL,
       hasData <- function(data) {
         return(!is.null(data) && (data %>% count() %>% pull()) > 0)
       }
+      if (!is.null(covariateCohortDatabaseSchema) && !is.null(covariateCohortTable)) {
+        covariateSettings <- replaceCovariateSettingsCohortSchemaTable(
+          covariateSettings,
+          covariateCohortDatabaseSchema,
+          covariateCohortTable
+        )
+      }
       for (i in 1:length(covariateSettings)) {
         fun <- attr(covariateSettings[[i]], "fun")
         args <- list(
@@ -209,10 +220,11 @@ getDbCovariateData <- function(connectionDetails = NULL,
           if (hasData(covariateData$covariatesContinuous)) {
             if (hasData(tempCovariateData$covariatesContinuous)) {
               Andromeda::appendToTable(covariateData$covariatesContinuous, tempCovariateData$covariatesContinuous)
-            } else if (hasData(tempCovariateData$covariatesContinuous)) {
-              covariateData$covariatesContinuous <- tempCovariateData$covariatesContinuous
             }
+          } else if (hasData(tempCovariateData$covariatesContinuous)) {
+            covariateData$covariatesContinuous <- tempCovariateData$covariatesContinuous
           }
+
           Andromeda::appendToTable(covariateData$covariateRef, tempCovariateData$covariateRef)
           Andromeda::appendToTable(covariateData$analysisRef, tempCovariateData$analysisRef)
           for (name in names(attr(tempCovariateData, "metaData"))) {

R/GetCovariatesFromOtherCohorts.R

@@ -325,3 +325,42 @@ warnIfPredefined <- function(analysisId, temporal = FALSE) {
     warning(sprintf("Analysis ID %d also used for prespecified analysis '%s'.", analysisId, preSpecAnalysis$analysisName))
   }
 }
+
+#' Utility function to set the cohort table & schema on createCohortBasedCovariateSettings
+#' with information from the execution settings
+#'
+#' @param covariateSettings An object of type \code{covariateSettings}
+#' @param covariateCohortDatabaseSchema The database schema where the cohorts used to define the covariates can be found.
+#' @param covariateCohortTable The table where the cohorts used to define the covariates can be found.
+#'
+#' @return
+#' An object of type \code{covariateSettings}
+#'
+replaceCovariateSettingsCohortSchemaTable <- function(covariateSettings,
+                                                      covariateCohortDatabaseSchema,
+                                                      covariateCohortTable) {
+  errorMessages <- checkmate::makeAssertCollection()
+  checkmate::assertList(covariateSettings, min.len = 1, add = errorMessages)
+  checkmate::assertCharacter(covariateCohortDatabaseSchema, add = errorMessages)
+  checkmate::assertCharacter(covariateCohortTable, add = errorMessages)
+  checkmate::reportAssertions(collection = errorMessages)
+
+  replaceProperties <- function(s) {
+    if (inherits(s, "covariateSettings") && "fun" %in% names(attributes(s))) {
+      if (attr(s, "fun") == "getDbCohortBasedCovariatesData") {
+        # Set the covariateCohortDatabaseSchema & covariateCohortTable values
+        s$covariateCohortDatabaseSchema <- covariateCohortDatabaseSchema
+        s$covariateCohortTable <- covariateCohortTable
+      }
+    }
+    return(s)
+  }
+  if (is.null(names(covariateSettings))) {
+    # List of lists
+    modifiedCovariateSettings <- lapply(covariateSettings, replaceProperties)
+  } else {
+    # Plain list
+    modifiedCovariateSettings <- replaceProperties(covariateSettings)
+  }
+  return(modifiedCovariateSettings)
+}

R/Normalization.R

@@ -179,19 +179,26 @@ tidyCovariateData <- function(covariateData,
       deleteCovariateIds <- c(deleteCovariateIds, toDelete$covariateId)
       ParallelLogger::logInfo("Removing ", nrow(toDelete), " infrequent covariates")
     }
-    if (length(deleteCovariateIds) > 0) {
-      newCovariates <- newCovariates %>%
-        filter(!.data$covariateId %in% deleteCovariateIds)
-    }
 
+    # When performing both filtering by covariate IDs and normalization, it is *much* faster
+    # to apply the filtering to the maxValuePerCovariateId table, and let the inner join
+    # apply the filtering to the covariate table (instead of filtering the covariate table
+    # directly).
     if (normalize) {
       ParallelLogger::logInfo("Normalizing covariates")
+      if (length(deleteCovariateIds) > 0) {
+        covariateData$maxValuePerCovariateId <- covariateData$maxValuePerCovariateId %>%
+          filter(!.data$covariateId %in% deleteCovariateIds)
+      }
       newCovariates <- newCovariates %>%
         inner_join(covariateData$maxValuePerCovariateId, by = "covariateId") %>%
         mutate(covariateValue = .data$covariateValue / .data$maxValue) %>%
         select(-.data$maxValue)
       metaData$normFactors <- covariateData$maxValuePerCovariateId %>%
         collect()
+    } else if (length(deleteCovariateIds) > 0) {
+      newCovariates <- newCovariates %>%
+        filter(!.data$covariateId %in% deleteCovariateIds)
     }
     newCovariateData$covariates <- newCovariates
     if (!is.null(covariateData$timeRef)) {