[WIP] Testing

dbt/models/ccao/docs.md

@@ -1,3 +1,17 @@
+# additional_mydec_sales
+
+{% docs table_additional_mydec_sales %}
+Document numbers for sales that were recorded in MyDec but never made it into
+`iasworld.sales`. Used to inject these sales into `default.vw_pin_sale`
+directly from `sale.mydec`.
+
+One-time list generated in June 2026 from the Summary tab of an internal
+"Missing Sales" spreadsheet. Loaded by the `ccao-additional_mydec_sales.R`
+warehouse script.
+
+**Primary Key**: `doc_no`
+{% enddocs %}
+
 # cc_dli_senfrr
 
 {% docs table_cc_dli_senfrr %}

dbt/models/ccao/schema.yml

@@ -7,6 +7,9 @@ sources:
     tags:
       - load_manual
     tables:
+      - name: additional_mydec_sales
+        description: '{{ doc("table_additional_mydec_sales") }}'
+
       - name: commercial_valuation
         description: '{{ doc("table_commercial_valuation") }}'
         tags:

dbt/models/default/default.vw_pin_sale.sql

@@ -19,6 +19,71 @@ WITH town_class AS (
         AND par.deactivat IS NULL
 ),
 
+-- doc no's for these sales come from the ccao.additional_mydec_sales table.
+-- Constructing the rows here, upstream of all other logic, means the
+-- additional sales flow through the same dedupe windows and filters as
+-- iasworld sales
+sales_unioned AS (
+    SELECT
+        sales.parid,
+        sales.saledt,
+        sales.price,
+        sales.salekey,
+        sales.instruno,
+        sales.instrtyp,
+        sales.nopar,
+        sales.oldown,
+        sales.own1,
+        sales.saletype,
+        sales.deactivat,
+        sales.cur
+    FROM {{ source('iasworld', 'sales') }} AS sales
+    UNION ALL
+    SELECT
+        REPLACE(md.line_1_primary_pin, '-', '') AS parid,
+        -- Match the exact iasworld saledt string format so that the
+        -- dedupe windows below, which partition on raw saledt, treat
+        -- same-day sales from both sources as equal
+        CONCAT(md.line_4_instrument_date, ' 00:00:00.000') AS saledt,
+        CAST(md.line_11_full_consideration AS DECIMAL(10, 0)) AS price,
+        CAST(NULL AS DECIMAL(8, 0)) AS salekey,
+        md.document_number AS instruno,
+        -- Map mydec instrument types to iasworld deed type codes (see
+        -- the sale.deed_type seed) so that these sales pass downstream
+        -- deed type filters the same way iasworld sales do
+        CASE
+            WHEN md.line_5_instrument_type = 'Warranty Deed' THEN '01'
+            WHEN md.line_5_instrument_type = 'Trustee Deed' THEN '02'
+            WHEN md.line_5_instrument_type = 'Quit Claim Deed' THEN '03'
+            WHEN md.line_5_instrument_type = 'Executor Deed' THEN '04'
+            WHEN md.line_5_instrument_type = 'Beneficial interest'
+                THEN '06'
+            ELSE '05'
+        END AS instrtyp,
+        -- Non-multisale mydec sales are single-parcel by definition
+        CAST(1 AS DECIMAL(4, 0)) AS nopar,
+        NULLIF(TRIM(md.seller_name), '') AS oldown,
+        NULLIF(TRIM(md.buyer_name), '') AS own1,
+        CAST(NULL AS VARCHAR) AS saletype,
+        CAST(NULL AS VARCHAR) AS deactivat,
+        'Y' AS cur
+    FROM {{ source('sale', 'mydec') }} AS md
+    INNER JOIN {{ source('ccao', 'additional_mydec_sales') }} AS ams
+        ON md.document_number = ams.doc_no
+    -- Exclude doc nos already live in iasworld so we keep parity with
+    -- prod for sales that were ingested normally
+    LEFT JOIN (
+        SELECT DISTINCT NULLIF(REPLACE(instruno, 'D', ''), '') AS doc_no
+        FROM {{ source('iasworld', 'sales') }}
+        WHERE deactivat IS NULL
+            AND cur = 'Y'
+    ) AS ias
+        ON md.document_number = ias.doc_no
+    WHERE NOT md.is_multisale
+        AND md.line_11_full_consideration IS NOT NULL
+        AND ias.doc_no IS NULL
+),
+
 -- "nopar" isn't entirely accurate for sales associated with only one parcel,
 -- so we create our own counter
 calculated AS (
@@ -29,7 +94,7 @@ calculated AS (
         SELECT DISTINCT
             parid,
             NULLIF(REPLACE(instruno, 'D', ''), '') AS instruno
-        FROM {{ source('iasworld', 'sales') }}
+        FROM sales_unioned
         WHERE deactivat IS NULL
             AND cur = 'Y'
     )
@@ -128,7 +193,7 @@ unique_sales AS (
                 sales.instrtyp IN ('03', '04', '06') OR sales.instrtyp IS NULL,
                 FALSE
             ) AS sale_filter_deed_type
-        FROM {{ source('iasworld', 'sales') }} AS sales
+        FROM sales_unioned AS sales
         LEFT JOIN calculated
             ON NULLIF(REPLACE(sales.instruno, 'D', ''), '')
             = calculated.instruno
@@ -145,14 +210,15 @@ unique_sales AS (
             )
             AND tc.township_code IS NOT NULL
             AND sales.price IS NOT NULL
-    )
+    ) AS sales_calculated
     -- Only use max price by pin/sale date
     WHERE max_price = 1
         AND (bad_doc_no = 1 OR is_multisale = TRUE)
 ),
 
 mydec_sales AS (
-    SELECT * FROM (
+    SELECT *
+    FROM (
         SELECT
             REPLACE(document_number, 'D', '') AS doc_no,
             REPLACE(line_1_primary_pin, '-', '') AS pin,

dbt/models/sale/sale.vw_flag.sql

@@ -11,12 +11,12 @@ SELECT
     sf.sv_outlier_reason3,
     sf.run_id,
     sf.version
-FROM {{ source('sale', 'flag') }} AS sf
+FROM z_dev_miwagne_sale.flag AS sf
 INNER JOIN (
     SELECT
         meta_sale_document_num,
         MAX(version) AS max_version
-    FROM {{ source('sale', 'flag') }}
+    FROM z_dev_miwagne_sale.flag
     GROUP BY meta_sale_document_num
 ) AS mv
     ON sf.meta_sale_document_num = mv.meta_sale_document_num

dbt/models/sale/schema.yml

@@ -274,7 +274,7 @@ models:
           equals: >
             (
               SELECT COUNT(DISTINCT(meta_sale_document_num))
-              FROM {{ source('sale', 'flag') }}
+              FROM z_dev_miwagne_sale.flag
             )
       - unique_combination_of_columns:
           name: sale_vw_flag_unique_by_doc_no
@@ -430,7 +430,7 @@ models:
               SELECT COUNT(*)
               FROM (
                 SELECT DISTINCT(meta_sale_document_num) AS doc_no
-                FROM {{ source('sale', 'flag') }}
+                FROM z_dev_miwagne_sale.flag
                 UNION
                 SELECT DISTINCT(doc_no) AS doc_no
                 FROM {{ source('sale', 'flag_review') }}

etl/scripts-ccao-data-warehouse-us-east-1/ccao/ccao-additional_mydec_sales.R

@@ -0,0 +1,20 @@
+library(arrow)
+library(dplyr)
+library(openxlsx)
+
+# Declare output paths
+AWS_S3_WAREHOUSE_BUCKET <- Sys.getenv("AWS_S3_WAREHOUSE_BUCKET")
+output_bucket <- file.path(
+  AWS_S3_WAREHOUSE_BUCKET,
+  "ccao", "other", "additional_mydec_sales"
+)
+
+input_file <- "O:/CCAODATA/data/additional_mydec_sales/Missing Sales.xlsx"
+
+openxlsx::read.xlsx(input_file, sheet = "Summary") %>%
+  select(doc_no = `203.Document.Number`) %>%
+  mutate(
+    doc_no = gsub("\\D", "", as.character(doc_no)),
+    loaded_at = as.character(Sys.time())
+  ) %>%
+  write_parquet(file.path(output_bucket, "additional_mydec_sales.parquet"))