Skip to content

Commit c7983cb

Browse files
jeff-dudejeff-dude
authored
Update nft trades to match dex trades materialization design & ensure incremental / partition design in place (#1396)
* add block_date, update config block to partition by block_date and add to unique key, update incremental logic * change materialization to view on top of base incremental tables * clean up incremental logic filters * comment out sudoswap until tokenid array is exploded into rows * uncomment sudoswap in nft schema
1 parent 8aad705 commit c7983cb

File tree

9 files changed

+311
-289
lines changed

9 files changed

+311
-289
lines changed

‎spellbook/models/looksrare/ethereum/looksrare_ethereum_events.sql

Lines changed: 14 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
{{ config(
2-
alias ='events',
3-
materialized ='incremental',
4-
file_format ='delta',
5-
incremental_strategy='merge',
6-
unique_key='unique_trade_id'
7-
)
2+
alias = 'events',
3+
partition_by = ['block_date'],
4+
materialized = 'incremental',
5+
file_format = 'delta',
6+
incremental_strategy = 'merge',
7+
unique_key = ['block_date', 'unique_trade_id']
8+
)
89
}}
910

1011
WITH looks_rare AS (
@@ -104,6 +105,7 @@ SELECT DISTINCT
104105
'ethereum' as blockchain,
105106
'looksrare' as project,
106107
'v1' as version,
108+
TRY_CAST(date_trunc('DAY', looks_rare.block_time) AS date) AS block_date,
107109
looks_rare.block_time,
108110
token_id,
109111
tokens.name AS collection,
@@ -157,7 +159,12 @@ SELECT DISTINCT
157159
'looksrare' || '-' || tx_hash || '-' || token_id::string || '-' || seller::string || '-' || COALESCE(erc.contract_address, nft_contract_address) || '-' || looks_rare.evt_index::string || '-' || COALESCE(evt_type::string, 'Other') || '-' || COALESCE(case when erc.value_unique::string is null then '0' ELSE '1' end, '1') as unique_trade_id
158160
FROM looks_rare
159161
INNER JOIN {{ source('ethereum','transactions') }} tx ON tx_hash = tx.hash
160-
AND tx.block_time > '2022-01-01'
162+
{% if not is_incremental() %}
163+
AND tx.block_time > '2022-01-01'
164+
{% endif %}
165+
{% if is_incremental() %}
166+
AND TRY_CAST(date_trunc('DAY', tx.block_time) AS date) = TRY_CAST(date_trunc('DAY', looks_rare.block_time) AS date)
167+
{% endif %}
161168
LEFT JOIN erc_transfers erc ON erc.evt_tx_hash = tx_hash AND erc.token_id_erc = token_id
162169
LEFT JOIN {{ ref('tokens_ethereum_nft') }} tokens ON tokens.contract_address = nft_contract_address
163170
LEFT JOIN {{ ref('nft_ethereum_aggregators') }} agg ON agg.contract_address = tx.to

‎spellbook/models/magiceden/magiceden_events.sql

Lines changed: 41 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,45 @@
11
{{ config(
2-
alias ='events',
3-
materialized ='incremental',
4-
file_format ='delta',
5-
incremental_strategy='merge',
6-
unique_key='unique_trade_id'
2+
alias ='events'
73
)
84
}}
95

10-
SELECT blockchain,
11-
project,
12-
version,
13-
block_time,
14-
token_id,
15-
NULL::string as collection,
16-
amount_usd,
17-
token_standard,
18-
trade_type,
19-
number_of_items,
20-
NULL::string as trade_category,
21-
evt_type,
22-
seller,
23-
buyer,
24-
amount_original,
25-
amount_raw,
26-
currency_symbol,
27-
currency_contract,
28-
NULL::string as nft_contract_address,
29-
project_contract_address,
30-
NULL::string as aggregator_name,
31-
NULL::string as aggregator_address,
32-
tx_hash,
33-
block_number,
34-
NULL::string as tx_from,
35-
NULL::string as tx_to,
36-
platform_fee_amount_raw,
37-
platform_fee_amount,
38-
platform_fee_amount_usd,
39-
platform_fee_percentage,
40-
royalty_fee_amount_raw,
41-
royalty_fee_amount,
42-
royalty_fee_amount_usd,
43-
royalty_fee_percentage,
44-
NULL::string as royalty_fee_receive_address,
45-
royalty_fee_currency_symbol,
46-
unique_trade_id
47-
FROM {{ ref('magiceden_solana_events') }}
48-
49-
{% if is_incremental() %}
50-
-- this filter will only be applied on an incremental run
51-
WHERE block_time >= (select max(block_time) from {{ this }})
52-
{% endif %}
6+
SELECT
7+
blockchain,
8+
project,
9+
version,
10+
block_date,
11+
block_time,
12+
token_id,
13+
NULL::string as collection,
14+
amount_usd,
15+
token_standard,
16+
trade_type,
17+
number_of_items,
18+
NULL::string as trade_category,
19+
evt_type,
20+
seller,
21+
buyer,
22+
amount_original,
23+
amount_raw,
24+
currency_symbol,
25+
currency_contract,
26+
NULL::string as nft_contract_address,
27+
project_contract_address,
28+
NULL::string as aggregator_name,
29+
NULL::string as aggregator_address,
30+
tx_hash,
31+
block_number,
32+
NULL::string as tx_from,
33+
NULL::string as tx_to,
34+
platform_fee_amount_raw,
35+
platform_fee_amount,
36+
platform_fee_amount_usd,
37+
platform_fee_percentage,
38+
royalty_fee_amount_raw,
39+
royalty_fee_amount,
40+
royalty_fee_amount_usd,
41+
royalty_fee_percentage,
42+
NULL::string as royalty_fee_receive_address,
43+
royalty_fee_currency_symbol,
44+
unique_trade_id
45+
FROM {{ ref('magiceden_solana_events') }}

‎spellbook/models/magiceden/solana/magiceden_solana_events.sql

Lines changed: 21 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
{{ config(
2-
alias ='events',
3-
materialized ='incremental',
4-
file_format ='delta',
5-
incremental_strategy='merge',
6-
unique_key='unique_trade_id'
7-
)
2+
alias = 'events',
3+
partition_by = ['block_date'],
4+
materialized = 'incremental',
5+
file_format = 'delta',
6+
incremental_strategy = 'merge',
7+
unique_key = ['block_date', 'unique_trade_id']
8+
)
89
}}
910

1011
SELECT
@@ -14,6 +15,7 @@ SELECT
1415
WHEN (array_contains(account_keys, 'CMZYPASGWeTz7RNGHaRJfCq2XQ5pYK6nDvVQxzkH51zb')) THEN 'launchpad_v3'
1516
END as version,
1617
signatures[0] as tx_hash,
18+
block_date,
1719
block_time,
1820
block_slot::string as block_number,
1921
abs(post_balances[0] / 1e9 - pre_balances[0] / 1e9) * p.price AS amount_usd,
@@ -94,13 +96,16 @@ FROM {{ source('solana','transactions') }}
9496
LEFT JOIN prices.usd p
9597
ON p.minute = date_trunc('minute', block_time)
9698
AND p.symbol = 'SOL'
97-
WHERE (array_contains(account_keys, 'M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K') -- magic eden v2
98-
OR array_contains(account_keys, 'CMZYPASGWeTz7RNGHaRJfCq2XQ5pYK6nDvVQxzkH51zb'))
99-
AND success = 'True'
100-
AND block_date > '2022-01-05'
101-
AND block_slot > 114980355
102-
103-
{% if is_incremental() %}
104-
-- this filter will only be applied on an incremental run
105-
AND block_date >= (select max(block_time) from {{ this }})
106-
{% endif %}
99+
WHERE (
100+
array_contains(account_keys, 'M2mx93ekt1fmXSVkTrUL9xVFHkmME8HTUi5Cyc5aF7K') -- magic eden v2
101+
OR array_contains(account_keys, 'CMZYPASGWeTz7RNGHaRJfCq2XQ5pYK6nDvVQxzkH51zb')
102+
)
103+
AND success = 'True'
104+
{% if not is_incremental() %}
105+
AND block_date > '2022-01-05'
106+
AND block_slot > 114980355
107+
{% endif %}
108+
{% if is_incremental() %}
109+
-- this filter will only be applied on an incremental run
110+
AND block_date >= (select max(block_date) from {{ this }})
111+
{% endif %}

‎spellbook/models/opensea/ethereum/opensea_ethereum_events.sql

Lines changed: 85 additions & 91 deletions
Original file line numberDiff line numberDiff line change
@@ -1,94 +1,88 @@
11
{{ config(
2-
alias ='events',
3-
materialized ='incremental',
4-
file_format ='delta',
5-
incremental_strategy='merge',
6-
unique_key='unique_trade_id'
7-
)
2+
alias ='events'
3+
)
84
}}
95

10-
SELECT blockchain,
11-
project,
12-
version,
13-
block_time,
14-
token_id,
15-
collection,
16-
amount_usd,
17-
token_standard,
18-
trade_type,
19-
number_of_items,
20-
trade_category,
21-
evt_type,
22-
seller,
23-
buyer,
24-
amount_original,
25-
amount_raw,
26-
currency_symbol,
27-
currency_contract,
28-
nft_contract_address,
29-
project_contract_address,
30-
aggregator_name,
31-
aggregator_address,
32-
block_number,
33-
tx_hash,
34-
tx_from,
35-
tx_to,
36-
platform_fee_amount_raw,
37-
platform_fee_amount,
38-
platform_fee_amount_usd,
39-
platform_fee_percentage,
40-
royalty_fee_amount_raw,
41-
royalty_fee_amount,
42-
royalty_fee_amount_usd,
43-
royalty_fee_percentage,
44-
royalty_fee_receive_address,
45-
royalty_fee_currency_symbol,
46-
unique_trade_id
47-
FROM ({{ ref('opensea_v1_ethereum_events') }})
48-
{% if is_incremental() %}
49-
-- this filter will only be applied on an incremental run
50-
WHERE block_time > now() - interval 2 days
51-
{% endif %}
52-
UNION
53-
SELECT blockchain,
54-
'opensea' as project,
55-
'v3' as version,
56-
block_time,
57-
token_id,
58-
collection,
59-
amount_usd,
60-
token_standard,
61-
trade_type,
62-
number_of_items,
63-
trade_category,
64-
evt_type,
65-
seller,
66-
buyer,
67-
amount_original,
68-
amount_raw,
69-
currency_symbol,
70-
currency_contract,
71-
nft_contract_address,
72-
project_contract_address,
73-
aggregator_name,
74-
aggregator_address,
75-
block_number,
76-
tx_hash,
77-
tx_from,
78-
tx_to,
79-
platform_fee_amount_raw,
80-
platform_fee_amount,
81-
platform_fee_amount_usd,
82-
platform_fee_percentage,
83-
royalty_fee_amount_raw,
84-
royalty_fee_amount,
85-
royalty_fee_amount_usd,
86-
royalty_fee_percentage,
87-
royalty_fee_receive_address,
88-
royalty_fee_currency_symbol,
89-
unique_trade_id
90-
FROM ({{ ref('opensea_v3_ethereum_events') }})
91-
{% if is_incremental() %}
92-
-- this filter will only be applied on an incremental run
93-
WHERE block_time > now() - interval 2 days
94-
{% endif %}
6+
SELECT *
7+
FROM
8+
(
9+
SELECT
10+
blockchain,
11+
project,
12+
version,
13+
block_time,
14+
token_id,
15+
collection,
16+
amount_usd,
17+
token_standard,
18+
trade_type,
19+
number_of_items,
20+
trade_category,
21+
evt_type,
22+
seller,
23+
buyer,
24+
amount_original,
25+
amount_raw,
26+
currency_symbol,
27+
currency_contract,
28+
nft_contract_address,
29+
project_contract_address,
30+
aggregator_name,
31+
aggregator_address,
32+
tx_hash,
33+
block_number,
34+
tx_from,
35+
tx_to,
36+
platform_fee_amount_raw,
37+
platform_fee_amount,
38+
platform_fee_amount_usd,
39+
platform_fee_percentage,
40+
royalty_fee_amount_raw,
41+
royalty_fee_amount,
42+
royalty_fee_amount_usd,
43+
royalty_fee_percentage,
44+
royalty_fee_receive_address,
45+
royalty_fee_currency_symbol,
46+
unique_trade_id
47+
FROM ({{ ref('opensea_v1_ethereum_events') }})
48+
UNION
49+
SELECT
50+
blockchain,
51+
'opensea' as project,
52+
'v3' as version,
53+
block_time,
54+
token_id,
55+
collection,
56+
amount_usd,
57+
token_standard,
58+
trade_type,
59+
number_of_items,
60+
trade_category,
61+
evt_type,
62+
seller,
63+
buyer,
64+
amount_original,
65+
amount_raw,
66+
currency_symbol,
67+
currency_contract,
68+
nft_contract_address,
69+
project_contract_address,
70+
aggregator_name,
71+
aggregator_address,
72+
tx_hash,
73+
block_number,
74+
tx_from,
75+
tx_to,
76+
platform_fee_amount_raw,
77+
platform_fee_amount,
78+
platform_fee_amount_usd,
79+
platform_fee_percentage,
80+
royalty_fee_amount_raw,
81+
royalty_fee_amount,
82+
royalty_fee_amount_usd,
83+
royalty_fee_percentage,
84+
royalty_fee_receive_address,
85+
royalty_fee_currency_symbol,
86+
unique_trade_id
87+
FROM ({{ ref('opensea_v3_ethereum_events') }})
88+
)

0 commit comments

Comments
 (0)