Skip to content

Commit 5024638

Browse files
tspenovtspenov
authored
Merge branch 'master' into generalize-access-attempts
2 parents 7b69579 + c2ca93b commit 5024638

File tree

4 files changed

+179
-45
lines changed

4 files changed

+179
-45
lines changed

‎lib/sanbase/clickhouse/metric/metric_files/available_v2_metrics.json

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6270,6 +6270,78 @@
62706270
"data_type": "timeseries",
62716271
"docs_links": ["https://academy.santiment.net/metrics/sky-savings"]
62726272
},
6273+
{
6274+
"human_readable_name": "Pendle Total Markets TVL",
6275+
"name": "pendle_total_markets_tvl",
6276+
"metric": "pendle_total_markets_tvl",
6277+
"version": "2019-01-01",
6278+
"access": "restricted",
6279+
"selectors": ["slug"],
6280+
"min_plan": {
6281+
"SANAPI": "pro",
6282+
"SANBASE": "free"
6283+
},
6284+
"aggregation": "sum",
6285+
"min_interval": "1d",
6286+
"table": "daily_metrics_v2",
6287+
"has_incomplete_data": true,
6288+
"data_type": "timeseries",
6289+
"docs_links": ["https://academy.santiment.net/metrics/pendle-markets"]
6290+
},
6291+
{
6292+
"human_readable_name": "Pendle Underlying APY",
6293+
"name": "pendle_underlying_apy",
6294+
"metric": "pendle_underlying_apy",
6295+
"version": "2019-01-01",
6296+
"access": "restricted",
6297+
"selectors": ["slug"],
6298+
"min_plan": {
6299+
"SANAPI": "pro",
6300+
"SANBASE": "free"
6301+
},
6302+
"aggregation": "avg",
6303+
"min_interval": "1d",
6304+
"table": "daily_metrics_v2",
6305+
"has_incomplete_data": true,
6306+
"data_type": "timeseries",
6307+
"docs_links": ["https://academy.santiment.net/metrics/pendle-markets"]
6308+
},
6309+
{
6310+
"human_readable_name": "Pendle Implied APY",
6311+
"name": "pendle_implied_apy",
6312+
"metric": "pendle_implied_apy",
6313+
"version": "2019-01-01",
6314+
"access": "restricted",
6315+
"selectors": ["slug"],
6316+
"min_plan": {
6317+
"SANAPI": "pro",
6318+
"SANBASE": "free"
6319+
},
6320+
"aggregation": "avg",
6321+
"min_interval": "1d",
6322+
"table": "daily_metrics_v2",
6323+
"has_incomplete_data": true,
6324+
"data_type": "timeseries",
6325+
"docs_links": ["https://academy.santiment.net/metrics/pendle-markets"]
6326+
},
6327+
{
6328+
"human_readable_name": "Pendle Yield Spread",
6329+
"name": "pendle_yield_spread",
6330+
"metric": "pendle_yield_spread",
6331+
"version": "2019-01-01",
6332+
"access": "restricted",
6333+
"selectors": ["slug"],
6334+
"min_plan": {
6335+
"SANAPI": "pro",
6336+
"SANBASE": "free"
6337+
},
6338+
"aggregation": "avg",
6339+
"min_interval": "1d",
6340+
"table": "daily_metrics_v2",
6341+
"has_incomplete_data": true,
6342+
"data_type": "timeseries",
6343+
"docs_links": ["https://academy.santiment.net/metrics/pendle-markets"]
6344+
},
62736345
{
62746346
"human_readable_name": "Price volatility for the last {{sliding_window:human_readable}}",
62756347
"name": "price_volatility_{{sliding_window}}",

‎lib/sanbase/clickhouse/top_holders/metric_adapter.ex

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,10 @@ defmodule Sanbase.Clickhouse.TopHolders.MetricAdapter do
55
@behaviour Sanbase.Metric.Behaviour
66

77
import Sanbase.Clickhouse.TopHolders.SqlQuery
8-
import Sanbase.Utils.Transform, only: [maybe_unwrap_ok_value: 1]
8+
9+
import Sanbase.Utils.Transform,
10+
only: [maybe_fill_gaps_last_seen: 2, maybe_apply_function: 2, maybe_unwrap_ok_value: 1]
11+
912
import Sanbase.Utils.ErrorHandling, only: [not_implemented_function_for_metric_error: 2]
1013

1114
alias Sanbase.Project
@@ -62,13 +65,27 @@ defmodule Sanbase.Clickhouse.TopHolders.MetricAdapter do
6265
true <- chain_supported?(infr, slug, metric),
6366
{:ok, params} <-
6467
timeseries_data_params(selector, contract, infr, from, to, interval, decimals, opts) do
65-
timeseries_data_query(metric, params)
66-
|> ClickhouseRepo.query_transform(fn [timestamp, value] ->
67-
%{datetime: DateTime.from_unix!(timestamp), value: value}
68-
end)
68+
result =
69+
timeseries_data_query(metric, params)
70+
|> ClickhouseRepo.query_transform(fn [timestamp, value, has_changed] ->
71+
%{datetime: DateTime.from_unix!(timestamp), value: value, has_changed: has_changed}
72+
end)
73+
74+
case result do
75+
{:ok, list} -> {:ok, gap_fill_last_known(list)}
76+
{:error, error} -> {:error, error}
77+
end
6978
end
7079
end
7180

81+
defp gap_fill_last_known(list) do
82+
# Do not gap fill the leading missing values
83+
list = Enum.drop_while(list, &(&1.has_changed == 0))
84+
85+
{:ok, list} = maybe_fill_gaps_last_seen({:ok, list}, :value)
86+
list
87+
end
88+
7289
@impl Sanbase.Metric.Behaviour
7390
def timeseries_data_per_slug(metric, _selector, _from, _to, _interval, _opts) do
7491
not_implemented_function_for_metric_error("timeseries_data_per_slug", metric)

‎lib/sanbase/clickhouse/top_holders/top_holders_sql_query.ex

Lines changed: 77 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,13 @@
11
defmodule Sanbase.Clickhouse.TopHolders.SqlQuery do
22
@moduledoc false
33

4-
import Sanbase.DateTimeUtils
5-
import Sanbase.Metric.SqlQuery.Helper, only: [to_unix_timestamp: 3, aggregation: 3]
4+
import Sanbase.Metric.SqlQuery.Helper,
5+
only: [
6+
timerange_parameters: 3,
7+
to_unix_timestamp_from_number: 2,
8+
to_unix_timestamp: 3,
9+
aggregation: 3
10+
]
611

712
defguard has_labels(map)
813
when (is_map_key(map, :include_labels) and
@@ -19,38 +24,54 @@ defmodule Sanbase.Clickhouse.TopHolders.SqlQuery do
1924
{exclude_labels_str, excluded_labels_params} =
2025
exclude_labels_str_args(params, trailing_and: true)
2126

27+
{from, to, interval_sec, span} = timerange_parameters(params.from, params.to, params.interval)
28+
2229
sql = """
23-
SELECT dt, SUM(value) AS value
30+
SELECT dt, SUM(value), toUInt32(SUM(has_changed))
2431
FROM (
25-
SELECT * FROM (
26-
SELECT
27-
#{aggregation(params.aggregation, "value", "dt")} / {{decimals}} AS value,
28-
#{to_unix_timestamp(params.interval, "dt", argument_name: "interval")} AS dt,
29-
address
30-
FROM #{params.table} FINAL
31-
WHERE
32-
#{exclude_labels_str}
33-
contract = {{contract}} AND
34-
dt >= toDateTime({{from}}) AND
35-
dt < toDateTime({{to}}) AND
36-
rank IS NOT NULL AND rank > 0
37-
GROUP BY dt, address
38-
ORDER BY dt, value DESC
32+
SELECT
33+
#{to_unix_timestamp_from_number(params.interval, from_argument_name: "from")} AS dt,
34+
toFloat64(0) AS value,
35+
toUInt32(0) AS has_changed
36+
FROM numbers({{span}})
37+
38+
UNION ALL
39+
40+
SELECT dt, SUM(value) AS value, 1 AS has_changed
41+
FROM (
42+
SELECT * FROM (
43+
SELECT
44+
#{aggregation(params.aggregation, "value", "dt")} / {{decimals}} AS value,
45+
#{to_unix_timestamp(params.interval, "dt", argument_name: "interval")} AS dt,
46+
address
47+
FROM #{params.table} FINAL
48+
WHERE
49+
#{exclude_labels_str}
50+
contract = {{contract}} AND
51+
dt >= toDateTime({{from}}) AND
52+
dt < toDateTime({{to}}) AND
53+
rank IS NOT NULL AND rank > 0
54+
GROUP BY dt, address
55+
ORDER BY dt, value DESC
56+
)
57+
#{include_labels_str}
58+
LIMIT {{limit}} BY dt
3959
)
40-
#{include_labels_str}
41-
LIMIT {{limit}} BY dt
60+
GROUP BY dt
61+
ORDER BY dt
4262
)
4363
GROUP BY dt
4464
ORDER BY dt
4565
"""
4666

4767
params =
4868
%{
49-
interval: params.interval |> str_to_sec(),
69+
interval: interval_sec,
5070
contract: params.contract,
5171
limit: params.count,
52-
from: params.from |> DateTime.to_unix(),
53-
to: params.to |> DateTime.to_unix(),
72+
from: from,
73+
to: to,
74+
span: span,
5475
blockchain: params.blockchain,
5576
decimals: Sanbase.Math.ipow(10, params.decimals)
5677
}
@@ -63,35 +84,51 @@ defmodule Sanbase.Clickhouse.TopHolders.SqlQuery do
6384
def timeseries_data_query("amount_in_top_holders", params) do
6485
decimals = Sanbase.Math.ipow(10, params.decimals)
6586

87+
{from, to, interval_sec, span} = timerange_parameters(params.from, params.to, params.interval)
88+
6689
sql = """
67-
SELECT dt, SUM(value)
90+
SELECT dt, SUM(value), toUInt32(SUM(has_changed))
6891
FROM (
69-
SELECT * FROM (
70-
SELECT
71-
#{to_unix_timestamp(params.interval, "dt", argument_name: "interval")} AS dt,
72-
#{aggregation(params.aggregation, "value", "dt")} / #{decimals} AS value
73-
FROM #{params.table} FINAL
74-
WHERE
75-
contract = {{contract}} AND
76-
rank <= {{limit}} AND
77-
dt >= toDateTime({{from}}) AND
78-
dt < toDateTime({{to}}) AND
79-
rank IS NOT NULL AND rank > 0
80-
GROUP BY dt, address
81-
ORDER BY dt, value desc
92+
SELECT
93+
#{to_unix_timestamp_from_number(params.interval, from_argument_name: "from")} AS dt,
94+
toFloat64(0) AS value,
95+
toUInt32(0) AS has_changed
96+
FROM numbers({{span}})
97+
98+
UNION ALL
99+
100+
SELECT dt, SUM(value) AS value, 1 AS has_changed
101+
FROM (
102+
SELECT * FROM (
103+
SELECT
104+
#{to_unix_timestamp(params.interval, "dt", argument_name: "interval")} AS dt,
105+
#{aggregation(params.aggregation, "value", "dt")} / #{decimals} AS value
106+
FROM #{params.table} FINAL
107+
WHERE
108+
contract = {{contract}} AND
109+
rank <= {{limit}} AND
110+
dt >= toDateTime({{from}}) AND
111+
dt < toDateTime({{to}}) AND
112+
rank IS NOT NULL AND rank > 0
113+
GROUP BY dt, address
114+
ORDER BY dt, value desc
115+
)
116+
LIMIT {{limit}} BY dt
82117
)
83-
LIMIT {{limit}} BY dt
118+
GROUP BY dt
119+
ORDER BY dt
84120
)
85121
GROUP BY dt
86122
ORDER BY dt
87123
"""
88124

89125
params = %{
90-
interval: params.interval |> str_to_sec(),
126+
interval: interval_sec,
91127
contract: params.contract,
92128
limit: params.count,
93-
from: params.from |> DateTime.to_unix(),
94-
to: params.to |> DateTime.to_unix()
129+
from: from,
130+
to: to,
131+
span: span
95132
}
96133

97134
Sanbase.Clickhouse.Query.new(sql, params)

‎test/sanbase/billing/metric_access_level_test.exs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,10 @@ defmodule Sanbase.Billing.MetricAccessLevelTest do
200200
"sky_savings_withdrawals",
201201
"sky_savings_total_supplied",
202202
"sky_savings_apy",
203+
"pendle_total_markets_tvl",
204+
"pendle_underlying_apy",
205+
"pendle_implied_apy",
206+
"pendle_yield_spread",
203207
"holders_distribution_total",
204208
"holders_distribution_over_100",
205209
"nft_collection_avg_price_usd",
@@ -1666,6 +1670,10 @@ defmodule Sanbase.Billing.MetricAccessLevelTest do
16661670
"sky_savings_withdrawals",
16671671
"sky_savings_total_supplied",
16681672
"sky_savings_apy",
1673+
"pendle_total_markets_tvl",
1674+
"pendle_underlying_apy",
1675+
"pendle_implied_apy",
1676+
"pendle_yield_spread",
16691677
# ETH 2.0 stakers MVRV metrics
16701678
"eth2_stakers_realized_value_usd_365d",
16711679
"eth2_stakers_realized_value_usd_2y",

0 commit comments

Comments
 (0)