diff --git a/backend/api/filter/appfilter.go b/backend/api/filter/appfilter.go index ab2c1e43f..c6af0d760 100644 --- a/backend/api/filter/appfilter.go +++ b/backend/api/filter/appfilter.go @@ -1038,7 +1038,14 @@ func (af *AppFilter) getDeviceNames(ctx context.Context) (deviceNames []string, // getUDAttrKeys finds distinct user defined attribute // key and its types. func (af *AppFilter) getUDAttrKeys(ctx context.Context) (keytypes []event.UDKeyType, err error) { - stmt := sqlf.From("user_def_attrs"). + var table_name string + if af.Span { + table_name = "span_user_def_attrs" + } else { + table_name = "user_def_attrs" + } + + stmt := sqlf.From(table_name). Select("distinct key"). Select("toString(type) type"). Clause("prewhere app_id = toUUID(?)", af.AppID). diff --git a/backend/api/measure/event.go b/backend/api/measure/event.go index 49ab9211c..12fb91a26 100644 --- a/backend/api/measure/event.go +++ b/backend/api/measure/event.go @@ -623,6 +623,18 @@ func (e eventreq) validate() error { if err := e.spans[i].Validate(); err != nil { return err } + + // only process user defined attributes + // if the payload contains any. + // + // this check is super important to have + // because SDKs without support for user + // defined attributes won't ever send these. + if !e.spans[i].UserDefinedAttribute.Empty() { + if err := e.spans[i].UserDefinedAttribute.Validate(); err != nil { + return err + } + } } if e.size >= int64(maxBatchSize) { @@ -1227,7 +1239,9 @@ func (e eventreq) ingestSpans(ctx context.Context) error { Set(`attribute.device_manufacturer`, e.spans[i].Attributes.DeviceManufacturer). Set(`attribute.device_locale`, e.spans[i].Attributes.DeviceLocale). Set(`attribute.device_low_power_mode`, e.spans[i].Attributes.LowPowerModeEnabled). - Set(`attribute.device_thermal_throttling_enabled`, e.spans[i].Attributes.ThermalThrottlingEnabled) + Set(`attribute.device_thermal_throttling_enabled`, e.spans[i].Attributes.ThermalThrottlingEnabled). + // user defined attribute + Set(`user_defined_attribute`, e.spans[i].UserDefinedAttribute.Parameterize()) } return server.Server.ChPool.AsyncInsert(ctx, stmt.String(), false, stmt.Args()...) diff --git a/backend/api/span/span.go b/backend/api/span/span.go index 3cbc07965..471a45411 100644 --- a/backend/api/span/span.go +++ b/backend/api/span/span.go @@ -1,6 +1,7 @@ package span import ( + "backend/api/event" "backend/api/filter" "backend/api/platform" "backend/api/server" @@ -97,17 +98,18 @@ type SpanAttributes struct { } type SpanField struct { - AppID uuid.UUID `json:"app_id" binding:"required"` - SpanName string `json:"name" binding:"required"` - SpanID string `json:"span_id" binding:"required"` - ParentID string `json:"parent_id"` - TraceID string `json:"trace_id" binding:"required"` - SessionID uuid.UUID `json:"session_id" binding:"required"` - Status uint8 `json:"status" binding:"required"` - StartTime time.Time `json:"start_time" binding:"required"` - EndTime time.Time `json:"end_time" binding:"required"` - CheckPoints []CheckPointField `json:"checkpoints"` - Attributes SpanAttributes `json:"attributes"` + AppID uuid.UUID `json:"app_id" binding:"required"` + SpanName string `json:"name" binding:"required"` + SpanID string `json:"span_id" binding:"required"` + ParentID string `json:"parent_id"` + TraceID string `json:"trace_id" binding:"required"` + SessionID uuid.UUID `json:"session_id" binding:"required"` + Status uint8 `json:"status" binding:"required"` + StartTime time.Time `json:"start_time" binding:"required"` + EndTime time.Time `json:"end_time" binding:"required"` + CheckPoints []CheckPointField `json:"checkpoints"` + Attributes SpanAttributes `json:"attributes"` + UserDefinedAttribute event.UDAttribute `json:"user_defined_attribute" binding:"required"` } type RootSpanDisplay struct { @@ -138,6 +140,7 @@ type SpanDisplay struct { ThreadName string `json:"thread_name"` LowPowerModeEnabled bool `json:"device_low_power_mode"` ThermalThrottlingEnabled bool `json:"device_thermal_throttling_enabled"` + UserDefinedAttribute event.UDAttribute `json:"user_defined_attributes"` CheckPoints []CheckPointField `json:"checkpoints"` } @@ -427,21 +430,12 @@ func GetSpanInstancesWithFilter(ctx context.Context, spanName string, af *filter Select("attribute.device_manufacturer"). Select("attribute.device_model"). From("spans"). - Clause("prewhere app_id = toUUID(?) and span_name = ? and start_time >= ? and end_time <= ?", af.AppID, spanName, af.From, af.To). - OrderBy("start_time desc") + Clause("prewhere app_id = toUUID(?) and span_name = ? and start_time >= ? and end_time <= ?", af.AppID, spanName, af.From, af.To) if len(af.SpanStatuses) > 0 { stmt.Where("status").In(af.SpanStatuses) } - if af.Limit > 0 { - stmt.Limit(uint64(af.Limit) + 1) - } - - if af.Offset >= 0 { - stmt.Offset(uint64(af.Offset)) - } - if af.HasVersions() { selectedVersions, err := af.VersionPairs() if err != nil { @@ -488,6 +482,24 @@ func GetSpanInstancesWithFilter(ctx context.Context, spanName string, af *filter stmt.Where("attribute.device_name in ?", af.DeviceNames) } + if af.HasUDExpression() && !af.UDExpression.Empty() { + subQuery := sqlf.From("span_user_def_attrs"). + Select("span_id id"). + Where("app_id = toUUID(?)", af.AppID) + af.UDExpression.Augment(subQuery) + stmt.Clause("AND span_id in").SubQuery("(", ")", subQuery) + } + + stmt.OrderBy("start_time desc") + + if af.Limit > 0 { + stmt.Limit(uint64(af.Limit) + 1) + } + + if af.Offset >= 0 { + stmt.Offset(uint64(af.Offset)) + } + defer stmt.Close() rows, err := server.Server.ChPool.Query(ctx, stmt.String(), stmt.Args()...) @@ -590,6 +602,14 @@ func GetSpanMetricsPlotWithFilter(ctx context.Context, spanName string, af *filt stmt.Where("device_name in ?", af.DeviceNames) } + if af.HasUDExpression() && !af.UDExpression.Empty() { + subQuery := sqlf.From("span_user_def_attrs"). + Select("span_id id"). + Where("app_id = toUUID(?)", af.AppID) + af.UDExpression.Augment(subQuery) + stmt.Clause("AND span_id in").SubQuery("(", ")", subQuery) + } + stmt.GroupBy("app_version, datetime") stmt.OrderBy("datetime, tupleElement(app_version, 2) desc") @@ -639,6 +659,7 @@ func GetTrace(ctx context.Context, traceId string) (trace TraceDisplay, err erro Select("toString(attribute.thread_name)"). Select("attribute.device_low_power_mode"). Select("attribute.device_thermal_throttling_enabled"). + Select("user_defined_attribute"). From("spans"). Where("trace_id = ?", traceId). OrderBy("start_time desc") @@ -654,9 +675,10 @@ func GetTrace(ctx context.Context, traceId string) (trace TraceDisplay, err erro for rows.Next() { var rawCheckpoints [][]interface{} + var rawUserDefAttr map[string][]any span := SpanField{} - if err = rows.Scan(&span.AppID, &span.TraceID, &span.SessionID, &span.Attributes.UserID, &span.SpanID, &span.SpanName, &span.ParentID, &span.StartTime, &span.EndTime, &span.Status, &rawCheckpoints, &span.Attributes.AppVersion, &span.Attributes.AppBuild, &span.Attributes.OSName, &span.Attributes.OSVersion, &span.Attributes.DeviceManufacturer, &span.Attributes.DeviceModel, &span.Attributes.NetworkType, &span.Attributes.ThreadName, &span.Attributes.LowPowerModeEnabled, &span.Attributes.ThermalThrottlingEnabled); err != nil { + if err = rows.Scan(&span.AppID, &span.TraceID, &span.SessionID, &span.Attributes.UserID, &span.SpanID, &span.SpanName, &span.ParentID, &span.StartTime, &span.EndTime, &span.Status, &rawCheckpoints, &span.Attributes.AppVersion, &span.Attributes.AppBuild, &span.Attributes.OSName, &span.Attributes.OSVersion, &span.Attributes.DeviceManufacturer, &span.Attributes.DeviceModel, &span.Attributes.NetworkType, &span.Attributes.ThreadName, &span.Attributes.LowPowerModeEnabled, &span.Attributes.ThermalThrottlingEnabled, &rawUserDefAttr); err != nil { fmt.Println(err) return } @@ -665,7 +687,12 @@ func GetTrace(ctx context.Context, traceId string) (trace TraceDisplay, err erro return } - // Map rawCheckpoints into CheckPointField + // Map rawUserDefAttr + if len(rawUserDefAttr) > 0 { + span.UserDefinedAttribute.Scan(rawUserDefAttr) + } + + // Map rawCheckpoints for _, cp := range rawCheckpoints { rawName, _ := cp[0].(string) name := strings.ReplaceAll(rawName, "\u0000", "") @@ -699,6 +726,7 @@ func GetTrace(ctx context.Context, traceId string) (trace TraceDisplay, err erro span.Attributes.ThreadName, span.Attributes.LowPowerModeEnabled, span.Attributes.ThermalThrottlingEnabled, + span.UserDefinedAttribute, span.CheckPoints, } diff --git a/backend/cleanup/cleanup/cleanup.go b/backend/cleanup/cleanup/cleanup.go index f8acc3c93..360c76a24 100644 --- a/backend/cleanup/cleanup/cleanup.go +++ b/backend/cleanup/cleanup/cleanup.go @@ -63,6 +63,9 @@ func DeleteStaleData(ctx context.Context) { // delete user defined attributes deleteUserDefAttrs(ctx, appRetentions) + // delete span user defined attributes + deleteSpanUserDefAttrs(ctx, appRetentions) + // delete sessions deleteSessions(ctx, appRetentions) @@ -260,6 +263,31 @@ func deleteUserDefAttrs(ctx context.Context, retentions []AppRetention) { } } +// deleteSpanUserDefAttrs deletes stale span user defined attributes +// for each app's retention threshold. +func deleteSpanUserDefAttrs(ctx context.Context, retentions []AppRetention) { + errCount := 0 + for _, retention := range retentions { + stmt := sqlf. + DeleteFrom("span_user_def_attrs"). + Where("app_id = toUUID(?)", retention.AppID). + Where("end_of_month < ?", retention.Threshold) + + if err := server.Server.ChPool.Exec(ctx, stmt.String(), stmt.Args()...); err != nil { + errCount += 1 + fmt.Printf("Failed to delete stale span user defined attributes for app id %q: %v\n", retention.AppID, err) + stmt.Close() + continue + } + + stmt.Close() + } + + if errCount < 1 { + fmt.Println("Successfully deleted stale span user defined attributes") + } +} + // deleteSessions deletes stale sessions for each // app's retention threshold. func deleteSessions(ctx context.Context, retentions []AppRetention) { diff --git a/frontend/dashboard/app/[teamId]/traces/page.tsx b/frontend/dashboard/app/[teamId]/traces/page.tsx index 55988aa4c..19ef03903 100644 --- a/frontend/dashboard/app/[teamId]/traces/page.tsx +++ b/frontend/dashboard/app/[teamId]/traces/page.tsx @@ -76,7 +76,7 @@ export default function TracesOverview({ params }: { params: { teamId: string } showLocales={true} showDeviceManufacturers={true} showDeviceNames={true} - showUdAttrs={false} + showUdAttrs={true} showFreeText={false} onFiltersChanged={(updatedFilters) => setFilters(updatedFilters)} />
diff --git a/frontend/dashboard/app/api/api_calls.ts b/frontend/dashboard/app/api/api_calls.ts index 653a4d39d..a4d1428ff 100644 --- a/frontend/dashboard/app/api/api_calls.ts +++ b/frontend/dashboard/app/api/api_calls.ts @@ -753,6 +753,7 @@ export const emptyTrace = { "end_time": "", "duration": 0, "thread_name": "", + "user_defined_attributes": null, "checkpoints": [ { "name": "", diff --git a/frontend/dashboard/app/components/trace_viz.tsx b/frontend/dashboard/app/components/trace_viz.tsx index b55f0cd2c..289a3aebc 100644 --- a/frontend/dashboard/app/components/trace_viz.tsx +++ b/frontend/dashboard/app/components/trace_viz.tsx @@ -30,6 +30,7 @@ interface Span { leftOffset?: number width?: number visibility?: SpanVisibility + user_defined_attributes?: Map | null checkpoints: Checkpoint[] | null } @@ -289,6 +290,12 @@ const TraceViz: React.FC = ({ inputTrace }) => {

Span Status

{selectedSpan.status === 0 ? "Unset" : selectedSpan.status === 1 ? "Okay" : "Error"}

+ {selectedSpan.user_defined_attributes !== null && selectedSpan.user_defined_attributes !== undefined && Object.entries(selectedSpan.user_defined_attributes!).map(([attrKey, attrValue]) => ( +
+

{attrKey}

+

{attrValue?.toString()}

+
+ ))}

Checkpoints

{selectedSpan.checkpoints !== null && selectedSpan.checkpoints.length > 0 ? ": " : ": []"}

diff --git a/frontend/dashboard/app/components/user_def_attr_selector.tsx b/frontend/dashboard/app/components/user_def_attr_selector.tsx index 80be3217d..4aee4421b 100644 --- a/frontend/dashboard/app/components/user_def_attr_selector.tsx +++ b/frontend/dashboard/app/components/user_def_attr_selector.tsx @@ -204,7 +204,7 @@ const UserDefAttrSelector: React.FC = ({ attrs, ops, o
{isOpen && ( -
+
window.innerWidth / 2 ? 'right-0' : 'left-0'}`}>
0 +group by app_id, end_of_month, app_version, os_version, + key, type, value, span_id, session_id +order by app_id; + + +-- migrate:down +drop view if exists span_user_def_attrs_mv; diff --git a/self-host/clickhouse/schema.sql b/self-host/clickhouse/schema.sql index ba501b9fb..eb873c813 100644 --- a/self-host/clickhouse/schema.sql +++ b/self-host/clickhouse/schema.sql @@ -680,6 +680,75 @@ ORDER BY device_low_power_mode ASC, device_thermal_throttling_enabled ASC; +CREATE TABLE default.span_user_def_attrs +( + `app_id` UUID COMMENT 'associated app id' CODEC(LZ4), + `span_id` FixedString(16) COMMENT 'id of the span' CODEC(ZSTD(3)), + `session_id` UUID COMMENT 'id of the session' CODEC(LZ4), + `end_of_month` DateTime COMMENT 'last day of the month' CODEC(DoubleDelta, ZSTD(3)), + `app_version` Tuple( + LowCardinality(String), + LowCardinality(String)) COMMENT 'composite app version' CODEC(ZSTD(3)), + `os_version` Tuple( + LowCardinality(String), + LowCardinality(String)) COMMENT 'composite os version' CODEC(ZSTD(3)), + `key` LowCardinality(String) COMMENT 'key of the user defined attribute' CODEC(ZSTD(3)), + `type` Enum8('string' = 1, 'int64' = 2, 'float64' = 3, 'bool' = 4) COMMENT 'type of the user defined attribute' CODEC(ZSTD(3)), + `value` String COMMENT 'value of the user defined attribute' CODEC(ZSTD(3)), + INDEX end_of_month_minmax_idx end_of_month TYPE minmax GRANULARITY 2, + INDEX key_bloom_idx key TYPE bloom_filter(0.05) GRANULARITY 1, + INDEX key_set_idx key TYPE set(1000) GRANULARITY 2, + INDEX session_bloom_idx session_id TYPE bloom_filter GRANULARITY 2 +) +ENGINE = ReplacingMergeTree +PARTITION BY toYYYYMM(end_of_month) +ORDER BY (app_id, end_of_month, app_version, os_version, key, type, value, span_id, session_id) +SETTINGS index_granularity = 8192 +COMMENT 'derived span user defined attributes'; + +CREATE MATERIALIZED VIEW default.span_user_def_attrs_mv TO default.span_user_def_attrs +( + `app_id` UUID, + `span_id` FixedString(16), + `session_id` UUID, + `end_of_month` Date, + `app_version` Tuple( + LowCardinality(String), + LowCardinality(String)), + `os_version` Tuple( + LowCardinality(String), + LowCardinality(String)), + `key` LowCardinality(String), + `type` Enum8('string' = 1, 'int64' = 2, 'float64' = 3, 'bool' = 4), + `value` String +) +AS SELECT DISTINCT + app_id, + span_id, + session_id, + toLastDayOfMonth(start_time) AS end_of_month, + attribute.app_version AS app_version, + attribute.os_version AS os_version, + arr_key AS key, + arr_val.1 AS type, + arr_val.2 AS value +FROM default.spans +ARRAY JOIN + mapKeys(user_defined_attribute) AS arr_key, + mapValues(user_defined_attribute) AS arr_val +WHERE length(user_defined_attribute) > 0 +GROUP BY + app_id, + end_of_month, + app_version, + os_version, + key, + type, + value, + span_id, + session_id +ORDER BY app_id ASC; + CREATE TABLE default.spans ( `app_id` UUID COMMENT 'unique id of the app' CODEC(ZSTD(3)), @@ -716,12 +785,17 @@ CREATE TABLE default.spans `attribute.device_locale` LowCardinality(String) COMMENT 'rfc 5646 locale string' CODEC(ZSTD(3)), `attribute.device_low_power_mode` Bool COMMENT 'true if device is in power saving mode' CODEC(ZSTD(3)), `attribute.device_thermal_throttling_enabled` Bool COMMENT 'true if device is has thermal throttling enabled' CODEC(ZSTD(3)), + `user_defined_attribute` Map(LowCardinality(String), Tuple( + Enum8('string' = 1, 'int64' = 2, 'float64' = 3, 'bool' = 4), + String)) CODEC(ZSTD(3)), INDEX span_name_bloom_idx span_name TYPE bloom_filter GRANULARITY 2, INDEX span_id_bloom_idx span_id TYPE bloom_filter GRANULARITY 2, INDEX trace_id_bloom_idx trace_id TYPE bloom_filter GRANULARITY 2, INDEX parent_id_bloom_idx parent_id TYPE bloom_filter GRANULARITY 2, INDEX start_time_minmax_idx start_time TYPE minmax GRANULARITY 2, - INDEX end_time_minmax_idx end_time TYPE minmax GRANULARITY 2 + INDEX end_time_minmax_idx end_time TYPE minmax GRANULARITY 2, + INDEX user_defined_attribute_key_bloom_idx mapKeys(user_defined_attribute) TYPE bloom_filter(0.01) GRANULARITY 16, + INDEX user_defined_attribute_key_minmax_idx mapKeys(user_defined_attribute) TYPE minmax GRANULARITY 16 ) ENGINE = MergeTree PARTITION BY toYYYYMMDD(start_time) @@ -847,4 +921,7 @@ INSERT INTO schema_migrations (version) VALUES ('20241128084916'), ('20241128085921'), ('20241204135555'), - ('20241210052709'); + ('20241210052709'), + ('20250204070350'), + ('20250204070357'), + ('20250204070548'); diff --git a/self-host/session-data/sh.measure.sample/0.9.0-SNAPSHOT.debug/394f892a-839d-43c2-a0eb-34e9d5cf834d.json b/self-host/session-data/sh.measure.sample/0.9.0-SNAPSHOT.debug/394f892a-839d-43c2-a0eb-34e9d5cf834d.json index 13f7bbe10..171811b6f 100644 --- a/self-host/session-data/sh.measure.sample/0.9.0-SNAPSHOT.debug/394f892a-839d-43c2-a0eb-34e9d5cf834d.json +++ b/self-host/session-data/sh.measure.sample/0.9.0-SNAPSHOT.debug/394f892a-839d-43c2-a0eb-34e9d5cf834d.json @@ -1199,6 +1199,12 @@ "device_low_power_mode": false, "device_thermal_throttling_enabled": false }, + "user_defined_attribute": { + "plan": "premium", + "client_id": 156039, + "revenue": 2000.00, + "onboarded": true + }, "checkpoints": [] }, { @@ -1232,6 +1238,12 @@ "device_low_power_mode": false, "device_thermal_throttling_enabled": false }, + "user_defined_attribute": { + "plan": "free", + "client_id": 143892, + "revenue": 0.00, + "onboarded": false + }, "checkpoints": [] }, {