StarRocks

Here's a sample configuration for a model using Starrocks with incremental materialization:

{{ config(
    materialized='incremental',
    incremental_strategy='insert_overwrite',      -- specify the incremental strategy
    unique_key=['order_id', 'date'],             -- composite unique key
    partition_by=['date'],                       -- date-based partitioning
    properties={
        "dynamic_partition.time_unit": "DAY",     -- partition by day
        "dynamic_partition.end": "3",             -- maintain 3 future partitions
        "dynamic_partition.prefix": "p",          -- partition name prefix
        "dynamic_partition.buckets": "4",         -- number of buckets per partition
        "dynamic_partition.create_history_partition": "true",
        "dynamic_partition.history_partition_num": "3",   -- maintain 3 historical partitions
        "replication_num": "3",                   -- number of replicas
        "storage_medium": "SSD"                   -- storage type
    }
) }}

SELECT
    id AS order_id,
    date(ordered_at) as date,
    order_total,
    ordered_at,
    now() as updated_at
FROM
    {{ source('ecom', 'raw_orders') }}
WHERE
    {% if is_incremental() %}
    date(ordered_at) = '{{ var("dt") }}'    -- process only current date's data
    {% endif %}

Last updated