[stable-25-2] Solomon external data source: new listing algorithm using working api (#28043)

Author: evanevanevanevannnn

ydb/library/yql/providers/solomon/actors/dq_solomon_metrics_queue.cpp

@@ -30,6 +30,13 @@ struct TTimeseries {
     std::vector<double> Values;
 };
 
+struct TLabelValues {
+    TString Name;
+    bool Absent;
+    bool Truncated;
+    std::vector<TString> Values;
+};
+
 struct TMetricTimeRange {
     TSelectors Selectors;
     TString Program;

ydb/library/yql/providers/solomon/common/util.h

@@ -8,11 +8,10 @@ TSolomonConfiguration::TSolomonConfiguration()
 {
     REGISTER_SETTING(*this, _EnableReading);
     REGISTER_SETTING(*this, _EnableRuntimeListing);
+    REGISTER_SETTING(*this, _EnableSolomonClientPostApi);
     REGISTER_SETTING(*this, _TruePointsFindRange);
-    REGISTER_SETTING(*this, MetricsQueuePageSize);
-    REGISTER_SETTING(*this, MetricsQueuePrefetchSize);
+    REGISTER_SETTING(*this, _MaxListingPageSize);
     REGISTER_SETTING(*this, MetricsQueueBatchCountLimit);
-    REGISTER_SETTING(*this, SolomonClientDefaultReplica);
     REGISTER_SETTING(*this, ComputeActorBatchSize);
     REGISTER_SETTING(*this, MaxApiInflight);
 }

ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp

@@ -12,11 +12,10 @@ struct TSolomonSettings {
 
     NCommon::TConfSetting<bool, false> _EnableReading;
     NCommon::TConfSetting<bool, false> _EnableRuntimeListing;
+    NCommon::TConfSetting<bool, false> _EnableSolomonClientPostApi;
     NCommon::TConfSetting<ui64, false> _TruePointsFindRange;
-    NCommon::TConfSetting<ui64, false> MetricsQueuePageSize;
-    NCommon::TConfSetting<ui64, false> MetricsQueuePrefetchSize;
+    NCommon::TConfSetting<ui64, false> _MaxListingPageSize;
     NCommon::TConfSetting<ui64, false> MetricsQueueBatchCountLimit;
-    NCommon::TConfSetting<TString, false> SolomonClientDefaultReplica;
     NCommon::TConfSetting<ui64, false> ComputeActorBatchSize;
     NCommon::TConfSetting<ui64, false> MaxApiInflight;
 };

ydb/library/yql/providers/solomon/provider/yql_solomon_config.h

@@ -349,29 +349,24 @@ class TSolomonDqIntegration: public TDqIntegrationBase {
             source.AddRequiredLabelNames(labelAsString);
         }
 
-        auto defaultReplica = (source.GetClusterType() == NSo::NProto::CT_SOLOMON ? "sas" : "cloud-prod-a");
-
         auto& solomonConfig = State_->Configuration;
         auto& sourceSettings = *source.MutableSettings();
 
-        auto metricsQueuePageSize = solomonConfig->MetricsQueuePageSize.Get().OrElse(2000);
-        sourceSettings.insert({"metricsQueuePageSize", ToString(metricsQueuePageSize)});
-
-        auto metricsQueuePrefetchSize = solomonConfig->MetricsQueuePrefetchSize.Get().OrElse(4000);
-        sourceSettings.insert({"metricsQueuePrefetchSize", ToString(metricsQueuePrefetchSize)});
-
-        auto metricsQueueBatchCountLimit = solomonConfig->MetricsQueueBatchCountLimit.Get().OrElse(10);
+        auto metricsQueueBatchCountLimit = solomonConfig->MetricsQueueBatchCountLimit.Get().OrElse(500);
         sourceSettings.insert({"metricsQueueBatchCountLimit", ToString(metricsQueueBatchCountLimit)});
 
-        auto solomonClientDefaultReplica = solomonConfig->SolomonClientDefaultReplica.Get().OrElse(defaultReplica);
-        sourceSettings.insert({"solomonClientDefaultReplica", ToString(solomonClientDefaultReplica)});
+        auto enableSolomonClientPostApi = solomonConfig->_EnableSolomonClientPostApi.Get().OrElse(false);
+        sourceSettings.insert({"enableSolomonClientPostApi", ToString(enableSolomonClientPostApi)});
 
         auto computeActorBatchSize = solomonConfig->ComputeActorBatchSize.Get().OrElse(100);
         sourceSettings.insert({"computeActorBatchSize", ToString(computeActorBatchSize)});
 
         auto truePointsFindRange = solomonConfig->_TruePointsFindRange.Get().OrElse(301);
         sourceSettings.insert({"truePointsFindRange", ToString(truePointsFindRange)});
 
+        auto maxListingPageSize = solomonConfig->_MaxListingPageSize.Get().OrElse(20000);
+        sourceSettings.insert({"maxListingPageSize", ToString(maxListingPageSize)});
+
         auto maxApiInflight = solomonConfig->MaxApiInflight.Get().OrElse(40);
         sourceSettings.insert({"maxApiInflight", ToString(maxApiInflight)});
 

ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp

@@ -16,7 +16,7 @@ namespace {
 struct TLoadSolomonMetaRequest {
     NSo::ISolomonAccessorClient::TPtr SolomonClient;
     NThreading::TFuture<NSo::TGetLabelsResponse> LabelNamesRequest;
-    NThreading::TFuture<NSo::TListMetricsResponse> ListMetricsRequest;
+    NThreading::TFuture<NSo::TListMetricsLabelsResponse> LabelValuesRequest;
 };
 
 TMaybe<TString> ExtractSetting(const TExprNode& settings, const TString& settingName) {
@@ -93,25 +93,24 @@ class TSolomonLoadTableMetadataTransformer : public TGraphTransformerBase {
                     return TStatus::Error;
                 }
 
-                auto defaultReplica = (source.GetClusterType() == NSo::NProto::CT_SOLOMON ? "sas" : "cloud-prod-a");
-                auto solomonClientDefaultReplica = State_->Configuration->SolomonClientDefaultReplica.Get().OrElse(defaultReplica);
-                source.MutableSettings()->insert({ "solomonClientDefaultReplica", ToString(solomonClientDefaultReplica) });
+                auto enableSolomonClientPostApi = State_->Configuration->_EnableSolomonClientPostApi.Get().OrElse(false);
+                source.MutableSettings()->insert({ "enableSolomonClientPostApi", ToString(enableSolomonClientPostApi) });
 
                 auto providerFactory = CreateCredentialsProviderFactoryForStructuredToken(State_->CredentialsFactory, State_->Configuration->Tokens.at(clusterName));
                 auto credentialsProvider = providerFactory->CreateProvider();
 
                 auto solomonClient = NSo::ISolomonAccessorClient::Make(std::move(source), credentialsProvider);
                 auto labelNamesFuture = solomonClient->GetLabelNames(selectors, from, to);
-                auto listMetricsFuture = solomonClient->ListMetrics(selectors, from, to, 30, 0);
+                auto listMetricsLabelsFuture = solomonClient->ListMetricsLabels(selectors, from, to);
 
                 LabelNamesRequests_[soReadObject.Raw()] = {
                     .SolomonClient = solomonClient,
                     .LabelNamesRequest = labelNamesFuture,
-                    .ListMetricsRequest = listMetricsFuture
+                    .LabelValuesRequest = listMetricsLabelsFuture
                 };
 
                 futures.push_back(labelNamesFuture.IgnoreResult());
-                futures.push_back(listMetricsFuture.IgnoreResult());
+                futures.push_back(listMetricsLabelsFuture.IgnoreResult());
             }
         }
 
@@ -142,10 +141,10 @@ class TSolomonLoadTableMetadataTransformer : public TGraphTransformerBase {
                 return TStatus::Error;
             }
 
-            auto listMetricsValue = request.ListMetricsRequest.GetValue();
-            if (listMetricsValue.Status != NSo::EStatus::STATUS_OK) {
+            auto listMetricLabelsValue = request.LabelValuesRequest.GetValue();
+            if (listMetricLabelsValue.Status != NSo::EStatus::STATUS_OK) {
                 ctx.AddError(TIssue(ctx.GetPosition(node->Pos()),
-                        TStringBuilder() << "Failed to get total metrics count, details: " << listMetricsValue.Error));
+                        TStringBuilder() << "Failed to get total metrics count, details: " << listMetricLabelsValue.Error));
                 return TStatus::Error;
             }
 
@@ -161,7 +160,7 @@ class TSolomonLoadTableMetadataTransformer : public TGraphTransformerBase {
                     .RequiredLabelNames()
                         .Add(labelNames)
                         .Build()
-                    .TotalMetricsCount<TCoAtom>().Build(ToString(listMetricsValue.Result.TotalCount))
+                    .TotalMetricsCount<TCoAtom>().Build(ToString(listMetricLabelsValue.Result.TotalCount))
                 .Done().Ptr());
         }