From 1cf39076dc4a86ed54ff442dde27ca5c6a7b81ca Mon Sep 17 00:00:00 2001 From: composer <2789706336@qq.com> Date: Thu, 27 Feb 2025 01:02:20 +0800 Subject: [PATCH 1/5] feat: view, service_instance_id --- exporter/dorisexporter/exporter_logs.go | 16 +++++++++++ exporter/dorisexporter/exporter_metrics.go | 27 +++++++++++++++++++ exporter/dorisexporter/exporter_traces.go | 18 ++++++++++++- exporter/dorisexporter/metrics_model.go | 1 + exporter/dorisexporter/sql/logs_ddl.sql | 10 ++++--- exporter/dorisexporter/sql/logs_view.sql | 4 +++ .../sql/metrics_exponential_histogram_ddl.sql | 4 ++- .../dorisexporter/sql/metrics_gauge_ddl.sql | 4 ++- .../sql/metrics_histogram_ddl.sql | 4 ++- .../dorisexporter/sql/metrics_sum_ddl.sql | 4 ++- .../dorisexporter/sql/metrics_summary_ddl.sql | 4 ++- exporter/dorisexporter/sql/metrics_view.sql | 4 +++ exporter/dorisexporter/sql/traces_ddl.sql | 8 +++--- exporter/dorisexporter/sql/traces_view.sql | 4 +++ 14 files changed, 99 insertions(+), 13 deletions(-) create mode 100644 exporter/dorisexporter/sql/logs_view.sql create mode 100644 exporter/dorisexporter/sql/metrics_view.sql create mode 100644 exporter/dorisexporter/sql/traces_view.sql diff --git a/exporter/dorisexporter/exporter_logs.go b/exporter/dorisexporter/exporter_logs.go index 4404c01e99911..5b185236ef436 100644 --- a/exporter/dorisexporter/exporter_logs.go +++ b/exporter/dorisexporter/exporter_logs.go @@ -21,10 +21,14 @@ import ( //go:embed sql/logs_ddl.sql var logsDDL string +//go:embed sql/logs_view.sql +var logsView string + // dLog Log to Doris type dLog struct { ServiceName string `json:"service_name"` Timestamp string `json:"timestamp"` + ServiceInstanceID string `json:"service_instance_id"` TraceID string `json:"trace_id"` SpanID string `json:"span_id"` SeverityNumber int32 `json:"severity_number"` @@ -70,6 +74,12 @@ func (e *logsExporter) start(ctx context.Context, host component.Host) error { if err != nil { return err } + + view := fmt.Sprintf(logsView, e.cfg.Table.Logs, e.cfg.Table.Logs) + _, err = conn.ExecContext(ctx, view) + if err != nil { + e.logger.Warn("failed to create materialized view", zap.Error(err)) + } } go e.reporter.report() @@ -96,6 +106,11 @@ func (e *logsExporter) pushLogData(ctx context.Context, ld plog.Logs) error { if ok { serviceName = v.AsString() } + serviceInstance := "" + v, ok = resourceAttributes.Get(semconv.AttributeServiceInstanceID) + if ok { + serviceInstance = v.AsString() + } for j := 0; j < resourceLogs.ScopeLogs().Len(); j++ { scopeLogs := resourceLogs.ScopeLogs().At(j) @@ -106,6 +121,7 @@ func (e *logsExporter) pushLogData(ctx context.Context, ld plog.Logs) error { log := &dLog{ ServiceName: serviceName, Timestamp: e.formatTime(logRecord.Timestamp().AsTime()), + ServiceInstanceID: serviceInstance, TraceID: traceutil.TraceIDToHexOrEmptyString(logRecord.TraceID()), SpanID: traceutil.SpanIDToHexOrEmptyString(logRecord.SpanID()), SeverityNumber: int32(logRecord.SeverityNumber()), diff --git a/exporter/dorisexporter/exporter_metrics.go b/exporter/dorisexporter/exporter_metrics.go index d36122b589597..d1a9ab7251e32 100644 --- a/exporter/dorisexporter/exporter_metrics.go +++ b/exporter/dorisexporter/exporter_metrics.go @@ -5,6 +5,7 @@ package dorisexporter // import "github.com/open-telemetry/opentelemetry-collect import ( "context" + _ "embed" // for SQL file embedding "encoding/json" "errors" "fmt" @@ -25,6 +26,9 @@ var ddls = []string{ metricsSummaryDDL, } +//go:embed sql/metrics_view.sql +var metricsView string + type metricsExporter struct { *commonExporter } @@ -61,6 +65,23 @@ func (e *metricsExporter) start(ctx context.Context, host component.Host) error return err } } + + models := []metricModel{ + &metricModelGauge{}, + &metricModelSum{}, + &metricModelHistogram{}, + &metricModelExponentialHistogram{}, + &metricModelSummary{}, + } + + for _, model := range models { + table := e.cfg.Table.Metrics + model.tableSuffix() + view := fmt.Sprintf(metricsView, table, table) + _, err = conn.ExecContext(ctx, view) + if err != nil { + e.logger.Warn("failed to create materialized view", zap.Error(err)) + } + } } go e.reporter.report() @@ -158,6 +179,11 @@ func (e *metricsExporter) pushMetricData(ctx context.Context, md pmetric.Metrics if ok { serviceName = v.AsString() } + serviceInstance := "" + v, ok = resourceAttributes.Get(semconv.AttributeServiceInstanceID) + if ok { + serviceInstance = v.AsString() + } for j := 0; j < resourceMetric.ScopeMetrics().Len(); j++ { scopeMetric := resourceMetric.ScopeMetrics().At(j) @@ -167,6 +193,7 @@ func (e *metricsExporter) pushMetricData(ctx context.Context, md pmetric.Metrics dm := &dMetric{ ServiceName: serviceName, + ServiceInstanceID: serviceInstance, MetricName: metric.Name(), MetricDescription: metric.Description(), MetricUnit: metric.Unit(), diff --git a/exporter/dorisexporter/exporter_traces.go b/exporter/dorisexporter/exporter_traces.go index e6194f2d0a5cc..63a47e875d60a 100644 --- a/exporter/dorisexporter/exporter_traces.go +++ b/exporter/dorisexporter/exporter_traces.go @@ -21,10 +21,14 @@ import ( //go:embed sql/traces_ddl.sql var tracesDDL string +//go:embed sql/traces_view.sql +var tracesView string + // dTrace Trace to Doris type dTrace struct { ServiceName string `json:"service_name"` Timestamp string `json:"timestamp"` + ServiceInstanceID string `json:"service_instance_id"` TraceID string `json:"trace_id"` SpanID string `json:"span_id"` TraceState string `json:"trace_state"` @@ -75,7 +79,7 @@ func (e *tracesExporter) start(ctx context.Context, host component.Host) error { } e.client = client - if !e.cfg.CreateSchema { + if e.cfg.CreateSchema { conn, err := createDorisMySQLClient(e.cfg) if err != nil { return err @@ -92,6 +96,12 @@ func (e *tracesExporter) start(ctx context.Context, host component.Host) error { if err != nil { return err } + + view := fmt.Sprintf(tracesView, e.cfg.Table.Traces, e.cfg.Table.Traces) + _, err = conn.ExecContext(ctx, view) + if err != nil { + e.logger.Warn("failed to create materialized view", zap.Error(err)) + } } go e.reporter.report() @@ -118,6 +128,11 @@ func (e *tracesExporter) pushTraceData(ctx context.Context, td ptrace.Traces) er if ok { serviceName = v.AsString() } + serviceInstance := "" + v, ok = resourceAttributes.Get(semconv.AttributeServiceInstanceID) + if ok { + serviceInstance = v.AsString() + } for j := 0; j < resourceSpan.ScopeSpans().Len(); j++ { scopeSpan := resourceSpan.ScopeSpans().At(j) @@ -157,6 +172,7 @@ func (e *tracesExporter) pushTraceData(ctx context.Context, td ptrace.Traces) er trace := &dTrace{ ServiceName: serviceName, Timestamp: e.formatTime(span.StartTimestamp().AsTime()), + ServiceInstanceID: serviceInstance, TraceID: traceutil.TraceIDToHexOrEmptyString(span.TraceID()), SpanID: traceutil.SpanIDToHexOrEmptyString(span.SpanID()), TraceState: span.TraceState().AsRaw(), diff --git a/exporter/dorisexporter/metrics_model.go b/exporter/dorisexporter/metrics_model.go index a269508d172b0..18326fe0b05d1 100644 --- a/exporter/dorisexporter/metrics_model.go +++ b/exporter/dorisexporter/metrics_model.go @@ -34,6 +34,7 @@ func (m *metricModelCommon[T]) label() string { // dMetric Basic Metric type dMetric struct { ServiceName string `json:"service_name"` + ServiceInstanceID string `json:"service_instance_id"` MetricName string `json:"metric_name"` MetricDescription string `json:"metric_description"` MetricUnit string `json:"metric_unit"` diff --git a/exporter/dorisexporter/sql/logs_ddl.sql b/exporter/dorisexporter/sql/logs_ddl.sql index b69abf71a8b4e..291eefff049bb 100644 --- a/exporter/dorisexporter/sql/logs_ddl.sql +++ b/exporter/dorisexporter/sql/logs_ddl.sql @@ -1,7 +1,8 @@ CREATE TABLE IF NOT EXISTS %s ( - service_name VARCHAR(200), timestamp DATETIME(6), + service_name VARCHAR(200), + service_instance_id VARCHAR(200), trace_id VARCHAR(200), span_id STRING, severity_number INT, @@ -13,6 +14,7 @@ CREATE TABLE IF NOT EXISTS %s scope_version STRING, INDEX idx_service_name(service_name) USING INVERTED, INDEX idx_timestamp(timestamp) USING INVERTED, + INDEX idx_service_instance_id(service_instance_id) USING INVERTED, INDEX idx_trace_id(trace_id) USING INVERTED, INDEX idx_span_id(span_id) USING INVERTED, INDEX idx_severity_number(severity_number) USING INVERTED, @@ -24,7 +26,7 @@ CREATE TABLE IF NOT EXISTS %s INDEX idx_scope_version(scope_version) USING INVERTED ) ENGINE = OLAP -DUPLICATE KEY(service_name, timestamp) +DUPLICATE KEY(timestamp, service_name) PARTITION BY RANGE(timestamp) () -DISTRIBUTED BY HASH(trace_id) BUCKETS AUTO -%s; \ No newline at end of file +DISTRIBUTED BY RANDOM BUCKETS AUTO +%s; diff --git a/exporter/dorisexporter/sql/logs_view.sql b/exporter/dorisexporter/sql/logs_view.sql new file mode 100644 index 0000000000000..50636a0ca1054 --- /dev/null +++ b/exporter/dorisexporter/sql/logs_view.sql @@ -0,0 +1,4 @@ +CREATE MATERIALIZED VIEW %s_services AS +SELECT service_name, service_instance_id +FROM %s +GROUP BY service_name, service_instance_id; diff --git a/exporter/dorisexporter/sql/metrics_exponential_histogram_ddl.sql b/exporter/dorisexporter/sql/metrics_exponential_histogram_ddl.sql index ef81d9f599adb..d106b2ddb7a31 100644 --- a/exporter/dorisexporter/sql/metrics_exponential_histogram_ddl.sql +++ b/exporter/dorisexporter/sql/metrics_exponential_histogram_ddl.sql @@ -2,6 +2,7 @@ CREATE TABLE IF NOT EXISTS %s_exponential_histogram ( service_name VARCHAR(200), timestamp DATETIME(6), + service_instance_id VARCHAR(200), metric_name VARCHAR(200), metric_description STRING, metric_unit STRING, @@ -25,6 +26,7 @@ CREATE TABLE IF NOT EXISTS %s_exponential_histogram scope_version STRING, INDEX idx_service_name(service_name) USING INVERTED, INDEX idx_timestamp(timestamp) USING INVERTED, + INDEX idx_service_instance_id(service_instance_id) USING INVERTED, INDEX idx_metric_name(metric_name) USING INVERTED, INDEX idx_metric_description(metric_description) USING INVERTED, INDEX idx_metric_unit(metric_unit) USING INVERTED, @@ -43,5 +45,5 @@ CREATE TABLE IF NOT EXISTS %s_exponential_histogram ENGINE = OLAP DUPLICATE KEY(service_name, timestamp) PARTITION BY RANGE(timestamp) () -DISTRIBUTED BY HASH(metric_name) BUCKETS AUTO +DISTRIBUTED BY RANDOM BUCKETS AUTO %s; diff --git a/exporter/dorisexporter/sql/metrics_gauge_ddl.sql b/exporter/dorisexporter/sql/metrics_gauge_ddl.sql index 59fc9a142a7e4..7a92e0572dab8 100644 --- a/exporter/dorisexporter/sql/metrics_gauge_ddl.sql +++ b/exporter/dorisexporter/sql/metrics_gauge_ddl.sql @@ -2,6 +2,7 @@ CREATE TABLE IF NOT EXISTS %s_gauge ( service_name VARCHAR(200), timestamp DATETIME(6), + service_instance_id VARCHAR(200), metric_name VARCHAR(200), metric_description STRING, metric_unit STRING, @@ -14,6 +15,7 @@ CREATE TABLE IF NOT EXISTS %s_gauge scope_version STRING, INDEX idx_service_name(service_name) USING INVERTED, INDEX idx_timestamp(timestamp) USING INVERTED, + INDEX idx_service_instance_id(service_instance_id) USING INVERTED, INDEX idx_metric_name(metric_name) USING INVERTED, INDEX idx_metric_description(metric_description) USING INVERTED, INDEX idx_metric_unit(metric_unit) USING INVERTED, @@ -26,5 +28,5 @@ CREATE TABLE IF NOT EXISTS %s_gauge ENGINE = OLAP DUPLICATE KEY(service_name, timestamp) PARTITION BY RANGE(timestamp) () -DISTRIBUTED BY HASH(metric_name) BUCKETS AUTO +DISTRIBUTED BY RANDOM BUCKETS AUTO %s; \ No newline at end of file diff --git a/exporter/dorisexporter/sql/metrics_histogram_ddl.sql b/exporter/dorisexporter/sql/metrics_histogram_ddl.sql index 74f1b37ec60d7..642874136a55f 100644 --- a/exporter/dorisexporter/sql/metrics_histogram_ddl.sql +++ b/exporter/dorisexporter/sql/metrics_histogram_ddl.sql @@ -2,6 +2,7 @@ CREATE TABLE IF NOT EXISTS %s_histogram ( service_name VARCHAR(200), timestamp DATETIME(6), + service_instance_id VARCHAR(200), metric_name VARCHAR(200), metric_description STRING, metric_unit STRING, @@ -20,6 +21,7 @@ CREATE TABLE IF NOT EXISTS %s_histogram scope_version STRING, INDEX idx_service_name(service_name) USING INVERTED, INDEX idx_timestamp(timestamp) USING INVERTED, + INDEX idx_service_instance_id(service_instance_id) USING INVERTED, INDEX idx_metric_name(metric_name) USING INVERTED, INDEX idx_metric_description(metric_description) USING INVERTED, INDEX idx_metric_unit(metric_unit) USING INVERTED, @@ -34,5 +36,5 @@ CREATE TABLE IF NOT EXISTS %s_histogram ENGINE = OLAP DUPLICATE KEY(service_name, timestamp) PARTITION BY RANGE(timestamp) () -DISTRIBUTED BY HASH(metric_name) BUCKETS AUTO +DISTRIBUTED BY RANDOM BUCKETS AUTO %s; \ No newline at end of file diff --git a/exporter/dorisexporter/sql/metrics_sum_ddl.sql b/exporter/dorisexporter/sql/metrics_sum_ddl.sql index f8e082fda725d..128d10e417a4a 100644 --- a/exporter/dorisexporter/sql/metrics_sum_ddl.sql +++ b/exporter/dorisexporter/sql/metrics_sum_ddl.sql @@ -2,6 +2,7 @@ CREATE TABLE IF NOT EXISTS %s_sum ( service_name VARCHAR(200), timestamp DATETIME(6), + service_instance_id VARCHAR(200), metric_name VARCHAR(200), metric_description STRING, metric_unit STRING, @@ -16,6 +17,7 @@ CREATE TABLE IF NOT EXISTS %s_sum scope_version STRING, INDEX idx_service_name(service_name) USING INVERTED, INDEX idx_timestamp(timestamp) USING INVERTED, + INDEX idx_service_instance_id(service_instance_id) USING INVERTED, INDEX idx_metric_name(metric_name) USING INVERTED, INDEX idx_metric_description(metric_description) USING INVERTED, INDEX idx_metric_unit(metric_unit) USING INVERTED, @@ -29,5 +31,5 @@ CREATE TABLE IF NOT EXISTS %s_sum ENGINE = OLAP DUPLICATE KEY(service_name, timestamp) PARTITION BY RANGE(timestamp) () -DISTRIBUTED BY HASH(metric_name) BUCKETS AUTO +DISTRIBUTED BY RANDOM BUCKETS AUTO %s; \ No newline at end of file diff --git a/exporter/dorisexporter/sql/metrics_summary_ddl.sql b/exporter/dorisexporter/sql/metrics_summary_ddl.sql index 5138f02e4ba70..efa86d8a073ff 100644 --- a/exporter/dorisexporter/sql/metrics_summary_ddl.sql +++ b/exporter/dorisexporter/sql/metrics_summary_ddl.sql @@ -2,6 +2,7 @@ CREATE TABLE IF NOT EXISTS %s_summary ( service_name VARCHAR(200), timestamp DATETIME(6), + service_instance_id VARCHAR(200), metric_name VARCHAR(200), metric_description STRING, metric_unit STRING, @@ -15,6 +16,7 @@ CREATE TABLE IF NOT EXISTS %s_summary scope_version STRING, INDEX idx_service_name(service_name) USING INVERTED, INDEX idx_timestamp(timestamp) USING INVERTED, + INDEX idx_service_instance_id(service_instance_id) USING INVERTED, INDEX idx_metric_name(metric_name) USING INVERTED, INDEX idx_metric_description(metric_description) USING INVERTED, INDEX idx_metric_unit(metric_unit) USING INVERTED, @@ -28,5 +30,5 @@ CREATE TABLE IF NOT EXISTS %s_summary ENGINE = OLAP DUPLICATE KEY(service_name, timestamp) PARTITION BY RANGE(timestamp) () -DISTRIBUTED BY HASH(metric_name) BUCKETS AUTO +DISTRIBUTED BY RANDOM BUCKETS AUTO %s; \ No newline at end of file diff --git a/exporter/dorisexporter/sql/metrics_view.sql b/exporter/dorisexporter/sql/metrics_view.sql new file mode 100644 index 0000000000000..50636a0ca1054 --- /dev/null +++ b/exporter/dorisexporter/sql/metrics_view.sql @@ -0,0 +1,4 @@ +CREATE MATERIALIZED VIEW %s_services AS +SELECT service_name, service_instance_id +FROM %s +GROUP BY service_name, service_instance_id; diff --git a/exporter/dorisexporter/sql/traces_ddl.sql b/exporter/dorisexporter/sql/traces_ddl.sql index 42d48d82de659..fc02858aff41c 100644 --- a/exporter/dorisexporter/sql/traces_ddl.sql +++ b/exporter/dorisexporter/sql/traces_ddl.sql @@ -1,7 +1,8 @@ CREATE TABLE IF NOT EXISTS %s ( service_name VARCHAR(200), - timestamp DATETIME(6), + timestamp DATETIME(6), + service_instance_id VARCHAR(200), trace_id VARCHAR(200), span_id STRING, trace_state STRING, @@ -19,7 +20,8 @@ CREATE TABLE IF NOT EXISTS %s scope_name STRING, scope_version STRING, INDEX idx_service_name(service_name) USING INVERTED, - INDEX idx_timestamp(timestamp) USING INVERTED, + INDEX idx_timestamp(timestamp) USING INVERTED, + INDEX idx_service_instance_id(service_instance_id) USING INVERTED, INDEX idx_trace_id(trace_id) USING INVERTED, INDEX idx_span_id(span_id) USING INVERTED, INDEX idx_trace_state(trace_state) USING INVERTED, @@ -38,5 +40,5 @@ CREATE TABLE IF NOT EXISTS %s ENGINE = OLAP DUPLICATE KEY(service_name, timestamp) PARTITION BY RANGE(timestamp) () -DISTRIBUTED BY HASH(trace_id) BUCKETS AUTO +DISTRIBUTED BY RANDOM BUCKETS AUTO %s; diff --git a/exporter/dorisexporter/sql/traces_view.sql b/exporter/dorisexporter/sql/traces_view.sql new file mode 100644 index 0000000000000..8ce1a51ec939f --- /dev/null +++ b/exporter/dorisexporter/sql/traces_view.sql @@ -0,0 +1,4 @@ +CREATE MATERIALIZED VIEW %s_services AS +SELECT service_name, service_instance_id, span_name +FROM %s +GROUP BY service_name, service_instance_id, span_name; From 41e06503e3327c62d9c442912655688fc1675483 Mon Sep 17 00:00:00 2001 From: composer <2789706336@qq.com> Date: Thu, 27 Feb 2025 01:09:16 +0800 Subject: [PATCH 2/5] feat: trace graph --- exporter/dorisexporter/exporter_traces.go | 43 +++++++++++++++++++ .../dorisexporter/sql/traces_graph_ddl.sql | 21 +++++++++ .../dorisexporter/sql/traces_graph_job.sql | 20 +++++++++ 3 files changed, 84 insertions(+) create mode 100644 exporter/dorisexporter/sql/traces_graph_ddl.sql create mode 100644 exporter/dorisexporter/sql/traces_graph_job.sql diff --git a/exporter/dorisexporter/exporter_traces.go b/exporter/dorisexporter/exporter_traces.go index 63a47e875d60a..2fbb3e3364390 100644 --- a/exporter/dorisexporter/exporter_traces.go +++ b/exporter/dorisexporter/exporter_traces.go @@ -24,6 +24,12 @@ var tracesDDL string //go:embed sql/traces_view.sql var tracesView string +//go:embed sql/traces_graph_ddl.sql +var tracesGraphDDL string + +//go:embed sql/traces_graph_job.sql +var tracesGraphJob string + // dTrace Trace to Doris type dTrace struct { ServiceName string `json:"service_name"` @@ -102,6 +108,24 @@ func (e *tracesExporter) start(ctx context.Context, host component.Host) error { if err != nil { e.logger.Warn("failed to create materialized view", zap.Error(err)) } + + ddl = fmt.Sprintf(tracesGraphDDL, e.cfg.Table.Traces, e.cfg.propertiesStr()) + _, err = conn.ExecContext(ctx, ddl) + if err != nil { + return err + } + + dropJob := e.formatDropTraceGraphJob() + _, err = conn.ExecContext(ctx, dropJob) + if err != nil { + e.logger.Warn("failed to drop job", zap.Error(err)) + } + + job := e.formatTraceGraphJob() + _, err = conn.ExecContext(ctx, job) + if err != nil { + e.logger.Warn("failed to create job", zap.Error(err)) + } } go e.reporter.report() @@ -245,3 +269,22 @@ func (e *tracesExporter) pushTraceDataInternal(ctx context.Context, traces []*dT return fmt.Errorf("failed to push trace data, response:%s", string(body)) } + +func (e *tracesExporter) formatDropTraceGraphJob() string { + return fmt.Sprintf( + "DROP JOB where jobName = '%s:%s_graph_job';", + e.cfg.Database, + e.cfg.Table.Traces, + ) +} + +func (e *tracesExporter) formatTraceGraphJob() string { + return fmt.Sprintf( + tracesGraphJob, + e.cfg.Database, + e.cfg.Table.Traces, + e.cfg.Table.Traces, + e.cfg.Table.Traces, + e.cfg.Table.Traces, + ) +} diff --git a/exporter/dorisexporter/sql/traces_graph_ddl.sql b/exporter/dorisexporter/sql/traces_graph_ddl.sql new file mode 100644 index 0000000000000..8ea9202fd9950 --- /dev/null +++ b/exporter/dorisexporter/sql/traces_graph_ddl.sql @@ -0,0 +1,21 @@ +CREATE TABLE IF NOT EXISTS %s_graph +( + timestamp DATETIME(6), + caller_service_name VARCHAR(200), + caller_service_instance_id VARCHAR(200), + callee_service_name VARCHAR(200), + callee_service_instance_id VARCHAR(200), + count BIGINT, + error_count BIGINT, + INDEX idx_timestamp(timestamp) USING INVERTED, + INDEX idx_caller_service_name(caller_service_name) USING INVERTED, + INDEX idx_caller_service_instance_id(caller_service_instance_id) USING INVERTED, + INDEX idx_callee_service_name(callee_service_name) USING INVERTED, + INDEX idx_callee_service_instance_id(callee_service_instance_id) USING INVERTED, + INDEX count(count) USING INVERTED, + INDEX error_count(error_count) USING INVERTED +) +UNIQUE KEY(timestamp, caller_service_name, caller_service_instance_id, callee_service_name, callee_service_instance_id) +PARTITION BY RANGE(timestamp) () +DISTRIBUTED BY HASH(caller_service_name) BUCKETS AUTO +%s; diff --git a/exporter/dorisexporter/sql/traces_graph_job.sql b/exporter/dorisexporter/sql/traces_graph_job.sql new file mode 100644 index 0000000000000..6cd28dad746c6 --- /dev/null +++ b/exporter/dorisexporter/sql/traces_graph_job.sql @@ -0,0 +1,20 @@ +CREATE JOB `%s:%s_graph_job` +ON SCHEDULE EVERY 10 MINUTE +DO +INSERT INTO %s_graph +SELECT + date_trunc(t2.timestamp, 'MINUTE') as timestamp, + t1.service_name AS caller_service_name, + t1.service_instance_id AS caller_service_instance_id, + t2.service_name AS callee_service_name, + t2.service_instance_id AS callee_service_instance_id, + count(*) as count, + sum(if(t2.status_code = 'STATUS_CODE_ERROR', 1, 0)) as error_count +FROM %s t1 +JOIN %s t2 +ON t1.trace_id = t2.trace_id +AND t1.span_id != '' +AND t1.service_name != t2.service_name +AND t1.span_id = t2.parent_span_id +AND t2.timestamp >= minutes_sub(date_trunc(now(), 'MINUTE'), 10) +GROUP BY timestamp, caller_service_name, caller_service_instance_id, callee_service_name, callee_service_instance_id; From 7cb8f725071e06281be9376f969a3e8ce68cdf39 Mon Sep 17 00:00:00 2001 From: composer <2789706336@qq.com> Date: Thu, 27 Feb 2025 01:15:47 +0800 Subject: [PATCH 3/5] chore: chlog --- .chloggen/doris-optimize-schema.yaml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 .chloggen/doris-optimize-schema.yaml diff --git a/.chloggen/doris-optimize-schema.yaml b/.chloggen/doris-optimize-schema.yaml new file mode 100644 index 0000000000000..294b9887b3a25 --- /dev/null +++ b/.chloggen/doris-optimize-schema.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: dorisexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: "optimize schema: add 'service_instance_id', add materialized views, add trace_graph table and trace_graph job" + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] From 52384b86fb615d82fd49b11cebd00dcec523486a Mon Sep 17 00:00:00 2001 From: composer <2789706336@qq.com> Date: Thu, 27 Feb 2025 01:30:48 +0800 Subject: [PATCH 4/5] chore: PR number --- .chloggen/doris-optimize-schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/doris-optimize-schema.yaml b/.chloggen/doris-optimize-schema.yaml index 294b9887b3a25..526953b1ab0f3 100644 --- a/.chloggen/doris-optimize-schema.yaml +++ b/.chloggen/doris-optimize-schema.yaml @@ -10,7 +10,7 @@ component: dorisexporter note: "optimize schema: add 'service_instance_id', add materialized views, add trace_graph table and trace_graph job" # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. -issues: [] +issues: [38229] # (Optional) One or more lines of additional information to render under the primary note. # These lines will be padded with 2 spaces and then inserted directly into the document. From f783a1a3a9113ca21b001a2c46bb744c6eadd52c Mon Sep 17 00:00:00 2001 From: composer <2789706336@qq.com> Date: Thu, 27 Feb 2025 02:03:04 +0800 Subject: [PATCH 5/5] chore: rerun ci --- .chloggen/doris-optimize-schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.chloggen/doris-optimize-schema.yaml b/.chloggen/doris-optimize-schema.yaml index 526953b1ab0f3..ed29ae287ecbd 100644 --- a/.chloggen/doris-optimize-schema.yaml +++ b/.chloggen/doris-optimize-schema.yaml @@ -7,7 +7,7 @@ change_type: enhancement component: dorisexporter # A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). -note: "optimize schema: add 'service_instance_id', add materialized views, add trace_graph table and trace_graph job" +note: "optimize schema: add 'service_instance_id', add materialized views, add trace_graph table and trace_graph job." # Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. issues: [38229]