refactor(integration_tests): unify bridge models and clear dbt deprecations

integration_tests/dbt_project.yml

@@ -74,7 +74,8 @@ seeds:
     raw_data__lab_result:
       +post-hook: "{{ the_tuva_project.load_versioned_synthetic_seed('lab_result') }}"
     raw_data__medical_claim:
-      +batch_size: 200
+      +meta:
+        batch_size: 200
       +post-hook: "{{ the_tuva_project.load_versioned_synthetic_seed('medical_claim') }}"
     raw_data__observation:
       +post-hook: "{{ the_tuva_project.load_versioned_synthetic_seed('observation') }}"

integration_tests/macros/tuva_source.sql

@@ -0,0 +1,7 @@
+{% macro tuva_source(table_name) %}
+  {%- if var('use_synthetic_data', false) | as_bool -%}
+    {%- do return(ref('raw_data__' ~ table_name)) -%}
+  {%- else -%}
+    {%- do return(source('source_input', table_name)) -%}
+  {%- endif -%}
+{% endmacro %}

integration_tests/models/_sources.yml

@@ -0,0 +1,28 @@
+version: 2
+
+sources:
+  - name: source_input
+    database: "{{ var('input_database', target.database) }}"
+    schema: |
+      {%- if var('use_synthetic_data') == true -%}
+        {%- if var('tuva_schema_prefix',None) != None -%}{{var('tuva_schema_prefix')}}_raw_data
+        {%- else -%}raw_data{%- endif -%}
+      {%- else -%}
+        {{ var('input_schema', target.schema) }}
+      {%- endif -%}
+    tables:
+      - name: appointment
+      - name: condition
+      - name: eligibility
+      - name: encounter
+      - name: immunization
+      - name: lab_result
+      - name: location
+      - name: medical_claim
+      - name: medication
+      - name: observation
+      - name: patient
+      - name: pharmacy_claim
+      - name: practitioner
+      - name: procedure
+      - name: provider_attribution

integration_tests/models/appointment.sql

@@ -4,7 +4,7 @@
    )
 }}
 
-select
+{%- set tuva_columns -%}
       appointment_id
     , person_id
     , patient_id
@@ -39,5 +39,23 @@ select
     , normalized_cancellation_reason_code_type
     , normalized_cancellation_reason_code
     , normalized_cancellation_reason_description
+{%- endset -%}
+
+{# Uncomment the columns below to test extension columns passthrough feature #}
+{%- set tuva_extensions -%}
+    {# , source_appointment_type_code as x_temp_source_appointment_type_code #}
+    {# , start_datetime as x_temp_start_datetime #}
+    {# , reason as zzz_temp_reason #}
+{%- endset -%}
+
+{%- set tuva_metadata -%}
     , data_source
-from {{ ref('raw_data__appointment') }}
+    , file_name
+    , ingest_datetime
+{%- endset -%}
+
+select
+    {{ tuva_columns }}
+    {{ tuva_extensions }}
+    {{ tuva_metadata }}
+from {{ tuva_source('appointment') }}

integration_tests/models/condition.sql

@@ -4,37 +4,45 @@
    )
 }}
 
-{# Uncomment the synthetic extension columns below to test extension columns passthrough feature #}
-{%- set tuva_synthetic_extensions -%}
-    {# , cast(null as {{ dbt.type_string() }}) as x_temp_status #}
-    {# , cast(null as {{ dbt.type_string() }}) as x_temp_condition_type #}
-    {# , cast(null as {{ dbt.type_string() }}) as x_temp_source_code #}
-    {# , {{ try_to_cast_date('null', 'YYYY-MM-DD') }} as zzz_temp_recorded_date #}
+{%- set tuva_columns -%}
+      condition_id
+    , payer
+    , person_id
+    , patient_id
+    , encounter_id
+    , claim_id
+    , recorded_date
+    , onset_date
+    , resolved_date
+    , status
+    , condition_type
+    , source_code_type
+    , source_code
+    , source_description
+    , normalized_code_type
+    , normalized_code
+    , normalized_description
+    , condition_rank
+    , present_on_admit_code
+    , present_on_admit_description
 {%- endset -%}
 
-select {% if target.type == 'fabric' %} top 0 {% else %}{% endif %}
-  cast(null as {{ dbt.type_string() }}) as condition_id
-, cast(null as {{ dbt.type_string() }}) as payer
-, cast(null as {{ dbt.type_string() }}) as person_id
-, cast(null as {{ dbt.type_string() }}) as patient_id
-, cast(null as {{ dbt.type_string() }}) as encounter_id
-, cast(null as {{ dbt.type_string() }}) as claim_id
-, {{ try_to_cast_date('null', 'YYYY-MM-DD') }} as recorded_date
-, {{ try_to_cast_date('null', 'YYYY-MM-DD') }} as onset_date
-, {{ try_to_cast_date('null', 'YYYY-MM-DD') }} as resolved_date
-, cast(null as {{ dbt.type_string() }}) as status
-, cast(null as {{ dbt.type_string() }}) as condition_type
-, cast(null as {{ dbt.type_string() }}) as source_code_type
-, cast(null as {{ dbt.type_string() }}) as source_code
-, cast(null as {{ dbt.type_string() }}) as source_description
-, cast(null as {{ dbt.type_string() }}) as normalized_code_type
-, cast(null as {{ dbt.type_string() }}) as normalized_code
-, cast(null as {{ dbt.type_string() }}) as normalized_description
-, cast(null as {{ dbt.type_int() }}) as condition_rank
-, cast(null as {{ dbt.type_string() }}) as present_on_admit_code
-, cast(null as {{ dbt.type_string() }}) as present_on_admit_description
-{{ tuva_synthetic_extensions }}
-, cast(null as {{ dbt.type_string() }}) as data_source
-, cast(null as {{ dbt.type_string() }}) as file_name
-, cast(null as {{ dbt.type_timestamp() }}) as ingest_datetime
-{{ limit_zero() }}
+{# Uncomment the columns below to test extension columns passthrough feature #}
+{%- set tuva_extensions -%}
+    {# , status as x_temp_status #}
+    {# , condition_type as x_temp_condition_type #}
+    {# , source_code as x_temp_source_code #}
+    {# , recorded_date as zzz_temp_recorded_date #}
+{%- endset -%}
+
+{%- set tuva_metadata -%}
+    , data_source
+    , file_name
+    , ingest_datetime
+{%- endset -%}
+
+select
+    {{ tuva_columns }}
+    {{ tuva_extensions }}
+    {{ tuva_metadata }}
+from {{ tuva_source('condition') }}

integration_tests/models/eligibility.sql

@@ -71,4 +71,4 @@ select
     {{ tuva_columns }}
     {{ tuva_extensions }}
     {{ tuva_metadata }}
-from {{ ref('raw_data__eligibility') }}
+from {{ tuva_source('eligibility') }}

integration_tests/models/encounter.sql

@@ -4,42 +4,50 @@
    )
 }}
 
-{# Uncomment the synthetic extension columns below to test extension columns passthrough feature #}
-{%- set tuva_synthetic_extensions -%}
-    {# , cast(null as {{ dbt.type_string() }}) as x_temp_encounter_type #}
-    {# , {{ try_to_cast_date('null', 'YYYY-MM-DD') }} as x_temp_encounter_start_date #}
-    {# , cast(null as {{ dbt.type_string() }}) as zzz_temp_facility_name #}
+{%- set tuva_columns -%}
+      encounter_id
+    , person_id
+    , patient_id
+    , encounter_type
+    , encounter_start_date
+    , encounter_end_date
+    , length_of_stay
+    , admit_source_code
+    , admit_source_description
+    , admit_type_code
+    , admit_type_description
+    , discharge_disposition_code
+    , discharge_disposition_description
+    , attending_provider_id
+    , attending_provider_name
+    , facility_npi
+    , facility_name
+    , primary_diagnosis_code_type
+    , primary_diagnosis_code
+    , primary_diagnosis_description
+    , drg_code_type
+    , drg_code
+    , drg_description
+    , paid_amount
+    , allowed_amount
+    , charge_amount
 {%- endset -%}
 
-select {% if target.type == 'fabric' %} top 0 {% else %}{% endif %}
-  cast(null as {{ dbt.type_string() }}) as encounter_id
-, cast(null as {{ dbt.type_string() }}) as person_id
-, cast(null as {{ dbt.type_string() }}) as patient_id
-, cast(null as {{ dbt.type_string() }}) as encounter_type
-, {{ try_to_cast_date('null', 'YYYY-MM-DD') }} as encounter_start_date
-, {{ try_to_cast_date('null', 'YYYY-MM-DD') }} as encounter_end_date
-, cast(null as {{ dbt.type_int() }}) as length_of_stay
-, cast(null as {{ dbt.type_string() }}) as admit_source_code
-, cast(null as {{ dbt.type_string() }}) as admit_source_description
-, cast(null as {{ dbt.type_string() }}) as admit_type_code
-, cast(null as {{ dbt.type_string() }}) as admit_type_description
-, cast(null as {{ dbt.type_string() }}) as discharge_disposition_code
-, cast(null as {{ dbt.type_string() }}) as discharge_disposition_description
-, cast(null as {{ dbt.type_string() }}) as attending_provider_id
-, cast(null as {{ dbt.type_string() }}) as attending_provider_name
-, cast(null as {{ dbt.type_string() }}) as facility_npi
-, cast(null as {{ dbt.type_string() }}) as facility_name
-, cast(null as {{ dbt.type_string() }}) as primary_diagnosis_code_type
-, cast(null as {{ dbt.type_string() }}) as primary_diagnosis_code
-, cast(null as {{ dbt.type_string() }}) as primary_diagnosis_description
-, cast(null as {{ dbt.type_string() }}) as drg_code_type
-, cast(null as {{ dbt.type_string() }}) as drg_code
-, cast(null as {{ dbt.type_string() }}) as drg_description
-, cast(null as {{ dbt.type_float() }}) as paid_amount
-, cast(null as {{ dbt.type_float() }}) as allowed_amount
-, cast(null as {{ dbt.type_float() }}) as charge_amount
-{{ tuva_synthetic_extensions }}
-, cast(null as {{ dbt.type_string() }}) as data_source
-, cast(null as {{ dbt.type_string() }}) as file_name
-, cast(null as {{ dbt.type_timestamp() }}) as ingest_datetime
-{{ limit_zero() }}
+{# Uncomment the columns below to test extension columns passthrough feature #}
+{%- set tuva_extensions -%}
+    {# , encounter_type as x_temp_encounter_type #}
+    {# , encounter_start_date as x_temp_encounter_start_date #}
+    {# , facility_name as zzz_temp_facility_name #}
+{%- endset -%}
+
+{%- set tuva_metadata -%}
+    , data_source
+    , file_name
+    , ingest_datetime
+{%- endset -%}
+
+select
+    {{ tuva_columns }}
+    {{ tuva_extensions }}
+    {{ tuva_metadata }}
+from {{ tuva_source('encounter') }}

integration_tests/models/immunization.sql

@@ -44,4 +44,4 @@ select
     {{ tuva_columns }}
     {{ tuva_extensions }}
     {{ tuva_metadata }}
-from {{ ref('raw_data__immunization') }}
+from {{ tuva_source('immunization') }}

integration_tests/models/lab_result.sql

@@ -56,4 +56,4 @@ select
     {{ tuva_columns }}
     {{ tuva_extensions }}
     {{ tuva_metadata }}
-from {{ ref('raw_data__lab_result') }}
+from {{ tuva_source('lab_result') }}

integration_tests/models/location.sql

@@ -4,27 +4,35 @@
    )
 }}
 
-{# Uncomment the synthetic extension columns below to test extension columns passthrough feature #}
-{%- set tuva_synthetic_extensions -%}
-    {# , cast(null as {{ dbt.type_string() }}) as x_temp_state #}
-    {# , cast(null as {{ dbt.type_string() }}) as x_temp_parent_organization #}
-    {# , cast(null as {{ dbt.type_string() }}) as zzz_temp_facility_type #}
+{%- set tuva_columns -%}
+      location_id
+    , npi
+    , name
+    , facility_type
+    , parent_organization
+    , address
+    , city
+    , state
+    , zip_code
+    , latitude
+    , longitude
 {%- endset -%}
 
-select {% if target.type == 'fabric' %} top 0 {% else %}{% endif %}
-  cast(null as {{ dbt.type_string() }}) as location_id
-, cast(null as {{ dbt.type_string() }}) as npi
-, cast(null as {{ dbt.type_string() }}) as name
-, cast(null as {{ dbt.type_string() }}) as facility_type
-, cast(null as {{ dbt.type_string() }}) as parent_organization
-, cast(null as {{ dbt.type_string() }}) as address
-, cast(null as {{ dbt.type_string() }}) as city
-, cast(null as {{ dbt.type_string() }}) as state
-, cast(null as {{ dbt.type_string() }}) as zip_code
-, cast(null as {{ dbt.type_float() }}) as latitude
-, cast(null as {{ dbt.type_float() }}) as longitude
-{{ tuva_synthetic_extensions }}
-, cast(null as {{ dbt.type_string() }}) as data_source
-, cast(null as {{ dbt.type_string() }}) as file_name
-, cast(null as {{ dbt.type_timestamp() }}) as ingest_datetime
-{{ limit_zero() }}
+{# Uncomment the columns below to test extension columns passthrough feature #}
+{%- set tuva_extensions -%}
+    {# , state as x_temp_state #}
+    {# , parent_organization as x_temp_parent_organization #}
+    {# , facility_type as zzz_temp_facility_type #}
+{%- endset -%}
+
+{%- set tuva_metadata -%}
+    , data_source
+    , file_name
+    , ingest_datetime
+{%- endset -%}
+
+select
+    {{ tuva_columns }}
+    {{ tuva_extensions }}
+    {{ tuva_metadata }}
+from {{ tuva_source('location') }}

integration_tests/models/medical_claim.sql

@@ -168,4 +168,4 @@ select
     {{ tuva_columns }}
     {{ tuva_extensions }}
     {{ tuva_metadata }}
-from {{ ref('raw_data__medical_claim') }}
+from {{ tuva_source('medical_claim') }}