Update variable declarations (#59)

* Update variable declarations * fix variable declaration * fix variable declaration * fix variable declaration * Update + regen docs * PR fixes * PR fixes * PR fixes * PR fixes * PR fixes
fivetran · Jan 15, 2025 · 40846a9 · 40846a9
1 parent bd22af7
commit 40846a9
Show file tree

Hide file tree

Showing 30 changed files with 70 additions and 31 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,3 +1,17 @@
+# dbt_netsuite_source v0.11.1
+[PR #59](https://github.com/fivetran/dbt_netsuite_source/pull/59) includes the following updates:
+
+## Macro Updates
+- Introduced a local version of the `fill_pass_through_columns` macro that directly calls the variables within our staging models. This removes the existing string-to-variable conversion and leads to cleaner parsing.
+  - This new macro has no functional changes from the previous macro and will not require customers to make any changes on their end.
+- This new macro is applied to all staging models with passthrough column functionality, and replaces the existing `fill_pass_through_columns` macro.
+  - `stg_netsuite` and `stg_netsuite2` models impacted include: `accounts`, `classes`, `consolidated_exchange_rates`, `customers`, `departments`, `items`, `locations`, `subsidiaries`, `transaction_lines`, `transactions`, and `vendors`.
+- Similar changes are being made on a simultaneous release to the `persist_pass_through_columns` macro in the `dbt_netsuite` package. [See the release notes](https://github.com/fivetran/dbt_netsuite/releases/tag/v0.17.1) for more details.
+- The process for adding passthrough columns remains unchanged. [Consult the README](https://github.com/fivetran/dbt_netsuite?tab=readme-ov-file#passing-through-additional-fields) for more details.
+
+## Documentation Update
+- Moved badges at top of the README below the H1 header to be consistent with popular README formats.
+
 # dbt_netsuite_source v0.11.0
 [PR #57](https://github.com/fivetran/dbt_netsuite_source/pull/57) includes the following update: 
 

diff --git a/README.md b/README.md
@@ -1,4 +1,6 @@
-<p align="center">
+# Netsuite Source dbt Package ([Docs](https://fivetran.github.io/dbt_netsuite_source/))
+
+<p align="left">
     <a alt="License"
         href="https://github.com/fivetran/dbt_netsuite_source/blob/main/LICENSE">
         <img src="https://img.shields.io/badge/License-Apache%202.0-blue.svg" /></a>
@@ -13,7 +15,6 @@
         <img src="https://img.shields.io/badge/Fivetran_Quickstart_Compatible%3F-yes-green.svg" /></a>
 </p>
 
-# Netsuite Source dbt Package ([Docs](https://fivetran.github.io/dbt_netsuite_source/))
 ## What does this dbt package do?
 <!--section="netsuite_source_model"-->
 - Materializes [Netsuite staging tables](https://fivetran.github.io/dbt_netsuite_source/#!/overview/netsuite_source/models/?g_v=1&g_e=seeds) which leverage data in the format described by [this ERD](https://fivetran.com/docs/applications/netsuite#schemainformation). These staging tables clean, test, and prepare your Netsuite data from [Fivetran's connector](https://fivetran.com/docs/applications/netsuite) for analysis by doing the following:

diff --git a/dbt_project.yml b/dbt_project.yml
@@ -1,7 +1,7 @@
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 name: 'netsuite_source'
-version: '0.11.0'
+version: '0.11.1'
 
 models:
   netsuite_source:

diff --git a/docs/catalog.json b/docs/catalog.json
diff --git a/docs/manifest.json b/docs/manifest.json
diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml
@@ -1,6 +1,6 @@
 name: 'netsuite_source_integration_tests'
 
-version: '0.11.0'
+version: '0.11.1'
 
 profile: 'integration_tests'
 config-version: 2

diff --git a/macros/fill_pass_through_columns.sql b/macros/fill_pass_through_columns.sql
@@ -0,0 +1,23 @@
+{% macro fill_pass_through_columns(pass_through_variable) %}
+
+{{ adapter.dispatch('fill_pass_through_columns', 'netsuite_source') (pass_through_variable) }}
+
+{%- endmacro %}
+
+{% macro default__fill_pass_through_columns(pass_through_variable) %}
+
+{% if pass_through_variable %}
+    {% for field in pass_through_variable %}
+        {% if field is mapping %}
+            {% if field.transform_sql %}
+                , {{ field.transform_sql }} as {{ field.alias if field.alias else field.name }}
+            {% else %}
+                , {{ field.alias if field.alias else field.name }}
+            {% endif %}
+        {% else %}
+        , {{ field }}
+        {% endif %}
+    {% endfor %}
+{% endif %}
+
+{% endmacro %}
diff --git a/models/netsuite/stg_netsuite__accounts.sql b/models/netsuite/stg_netsuite__accounts.sql
@@ -41,7 +41,7 @@ final as (
         _fivetran_deleted
 
         --The below macro adds the fields defined within your accounts_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('accounts_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('accounts_pass_through_columns', [])) }}
 
     from fields
 )

diff --git a/models/netsuite/stg_netsuite__classes.sql b/models/netsuite/stg_netsuite__classes.sql
@@ -35,7 +35,7 @@ final as (
         _fivetran_deleted
 
         --The below macro adds the fields defined within your classes_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('classes_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('classes_pass_through_columns', [])) }}
 
     from fields
 )

diff --git a/models/netsuite/stg_netsuite__consolidated_exchange_rates.sql b/models/netsuite/stg_netsuite__consolidated_exchange_rates.sql
@@ -41,7 +41,7 @@ final as (
         _fivetran_deleted
 
         --The below macro adds the fields defined within your consolidated_exchange_rates_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('consolidated_exchange_rates_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('consolidated_exchange_rates_pass_through_columns', [])) }}
 
     from fields
 )

diff --git a/models/netsuite/stg_netsuite__customers.sql b/models/netsuite/stg_netsuite__customers.sql
@@ -41,7 +41,7 @@ final as (
         _fivetran_deleted
 
         --The below macro adds the fields defined within your customers_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('customers_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('customers_pass_through_columns', [])) }}
 
 
     from fields

diff --git a/models/netsuite/stg_netsuite__departments.sql b/models/netsuite/stg_netsuite__departments.sql
@@ -36,7 +36,7 @@ final as (
         _fivetran_deleted
 
         --The below macro adds the fields defined within your departments_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('departments_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('departments_pass_through_columns', [])) }}
 
     from fields
 )

diff --git a/models/netsuite/stg_netsuite__items.sql b/models/netsuite/stg_netsuite__items.sql
@@ -37,7 +37,7 @@ final as (
         _fivetran_deleted
 
         --The below macro adds the fields defined within your items_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('items_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('items_pass_through_columns', [])) }}
 
     from fields
 )

diff --git a/models/netsuite/stg_netsuite__locations.sql b/models/netsuite/stg_netsuite__locations.sql
@@ -38,7 +38,7 @@ final as (
         _fivetran_deleted
 
         --The below macro adds the fields defined within your locations_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('locations_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('locations_pass_through_columns', [])) }}
 
     from fields
 )

diff --git a/models/netsuite/stg_netsuite__subsidiaries.sql b/models/netsuite/stg_netsuite__subsidiaries.sql
@@ -38,7 +38,7 @@ final as (
         _fivetran_deleted
 
         --The below macro adds the fields defined within your subsidiaries_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('subsidiaries_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('subsidiaries_pass_through_columns', [])) }}
 
     from fields
 )

diff --git a/models/netsuite/stg_netsuite__transaction_lines.sql b/models/netsuite/stg_netsuite__transaction_lines.sql
@@ -44,7 +44,7 @@ final as (
         memo
 
         --The below macro adds the fields defined within your transaction_lines_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('transaction_lines_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('transaction_lines_pass_through_columns', [])) }}
 
     from fields
 )

diff --git a/models/netsuite/stg_netsuite__transactions.sql b/models/netsuite/stg_netsuite__transactions.sql
@@ -42,7 +42,7 @@ final as (
         _fivetran_deleted
 
         --The below macro adds the fields defined within your transactions_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('transactions_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('transactions_pass_through_columns', [])) }}
 
     from fields
 )

diff --git a/models/netsuite/stg_netsuite__vendors.sql b/models/netsuite/stg_netsuite__vendors.sql
@@ -37,7 +37,7 @@ final as (
         _fivetran_deleted
 
         --The below macro adds the fields defined within your vendors_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('vendors_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('vendors_pass_through_columns', [])) }}
 
     from fields
 )

diff --git a/models/netsuite2/stg_netsuite2__accounts.sql b/models/netsuite2/stg_netsuite2__accounts.sql
@@ -45,7 +45,8 @@ final as (
         _fivetran_deleted
 
         --The below macro adds the fields defined within your accounts_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('accounts_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('accounts_pass_through_columns', [])) }}
+
 
     from fields
 )

diff --git a/models/netsuite2/stg_netsuite2__classes.sql b/models/netsuite2/stg_netsuite2__classes.sql
@@ -30,7 +30,7 @@ final as (
         _fivetran_deleted
 
         --The below macro adds the fields defined within your classes_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('classes_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('classes_pass_through_columns', [])) }}
 
     from fields
 )

diff --git a/models/netsuite2/stg_netsuite2__consolidated_exchange_rates.sql b/models/netsuite2/stg_netsuite2__consolidated_exchange_rates.sql
@@ -33,7 +33,7 @@ final as (
         historicalrate as historical_rate
 
         --The below macro adds the fields defined within your consolidated_exchange_rates_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('consolidated_exchange_rates_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('consolidated_exchange_rates_pass_through_columns', [])) }}
 
     from fields
     where not coalesce(_fivetran_deleted, false)

diff --git a/models/netsuite2/stg_netsuite2__customers.sql b/models/netsuite2/stg_netsuite2__customers.sql
@@ -39,7 +39,7 @@ final as (
         cast(firstorderdate as date) as date_first_order_at
 
         --The below macro adds the fields defined within your customers_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('customers_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('customers_pass_through_columns', [])) }}
 
     from fields
     where not coalesce(_fivetran_deleted, false)

diff --git a/models/netsuite2/stg_netsuite2__departments.sql b/models/netsuite2/stg_netsuite2__departments.sql
@@ -31,7 +31,7 @@ final as (
         _fivetran_deleted
 
         --The below macro adds the fields defined within your departments_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('departments_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('departments_pass_through_columns', [])) }}
 
     from fields
 )

diff --git a/models/netsuite2/stg_netsuite2__entities.sql b/models/netsuite2/stg_netsuite2__entities.sql
@@ -34,7 +34,7 @@ final as (
         vendor as vendor_id
 
         --The below macro adds the fields defined within your entities_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('entities_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('entities_pass_through_columns', [])) }}
 
     from fields
 )

diff --git a/models/netsuite2/stg_netsuite2__items.sql b/models/netsuite2/stg_netsuite2__items.sql
@@ -42,8 +42,8 @@ final as (
         parent as parent_item_id
 
         --The below macro adds the fields defined within your items_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('items_pass_through_columns') }}
-        
+        {{ netsuite_source.fill_pass_through_columns(var('items_pass_through_columns', [])) }}
+
     from fields
     where not coalesce(_fivetran_deleted, false)
 )

diff --git a/models/netsuite2/stg_netsuite2__locations.sql b/models/netsuite2/stg_netsuite2__locations.sql
@@ -30,7 +30,7 @@ final as (
         subsidiary as subsidiary_id
 
         --The below macro adds the fields defined within your locations_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('locations_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('locations_pass_through_columns', [])) }}
 
     from fields
     where not coalesce(_fivetran_deleted, false)

diff --git a/models/netsuite2/stg_netsuite2__subsidiaries.sql b/models/netsuite2/stg_netsuite2__subsidiaries.sql
@@ -35,7 +35,7 @@ final as (
         currency as currency_id
 
         --The below macro adds the fields defined within your subsidiaries_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('subsidiaries_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('subsidiaries_pass_through_columns', [])) }}
 
     from fields
     where not coalesce(_fivetran_deleted, false)

diff --git a/models/netsuite2/stg_netsuite2__transaction_lines.sql b/models/netsuite2/stg_netsuite2__transaction_lines.sql
@@ -43,7 +43,7 @@ final as (
         netamount
 
         --The below macro adds the fields defined within your transaction_lines_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('transaction_lines_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('transaction_lines_pass_through_columns', [])) }}
 
     from fields
 )

diff --git a/models/netsuite2/stg_netsuite2__transactions.sql b/models/netsuite2/stg_netsuite2__transactions.sql
@@ -43,7 +43,7 @@ final as (
         reversaldefer = 'T' as is_reversal_defer
 
         --The below macro adds the fields defined within your transactions_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('transactions_pass_through_columns') }}
+        {{ netsuite_source.fill_pass_through_columns(var('transactions_pass_through_columns', [])) }}
 
     from fields
     where not coalesce(_fivetran_deleted, false)

diff --git a/models/netsuite2/stg_netsuite2__vendors.sql b/models/netsuite2/stg_netsuite2__vendors.sql
@@ -30,8 +30,8 @@ final as (
         category as vendor_category_id
 
         --The below macro adds the fields defined within your vendors_pass_through_columns variable into the staging model
-        {{ fivetran_utils.fill_pass_through_columns('vendors_pass_through_columns') }}
-        
+        {{ netsuite_source.fill_pass_through_columns(var('vendors_pass_through_columns', [])) }}
+
     from fields
     where not coalesce(_fivetran_deleted, false)
 )