Skip to content

Commit 048cd0a

Browse files
evanevanevanevannnnevanevanevanevannnn
committed
Solomon data source: change solomon api methods from get to post (#27884)
1 parent 9ca374c commit 048cd0a

File tree

7 files changed

+83
-43
lines changed

7 files changed

+83
-43
lines changed

‎ydb/library/yql/providers/solomon/provider/yql_solomon_config.cpp‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,10 @@ TSolomonConfiguration::TSolomonConfiguration()
88
{
99
REGISTER_SETTING(*this, _EnableReading);
1010
REGISTER_SETTING(*this, _EnableRuntimeListing);
11+
REGISTER_SETTING(*this, _EnableSolomonClientPostApi);
1112
REGISTER_SETTING(*this, _TruePointsFindRange);
1213
REGISTER_SETTING(*this, _MaxListingPageSize);
1314
REGISTER_SETTING(*this, MetricsQueueBatchCountLimit);
14-
REGISTER_SETTING(*this, SolomonClientDefaultReplica);
1515
REGISTER_SETTING(*this, ComputeActorBatchSize);
1616
REGISTER_SETTING(*this, MaxApiInflight);
1717
}

‎ydb/library/yql/providers/solomon/provider/yql_solomon_config.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,10 @@ struct TSolomonSettings {
1212

1313
NCommon::TConfSetting<bool, false> _EnableReading;
1414
NCommon::TConfSetting<bool, false> _EnableRuntimeListing;
15+
NCommon::TConfSetting<bool, false> _EnableSolomonClientPostApi;
1516
NCommon::TConfSetting<ui64, false> _TruePointsFindRange;
1617
NCommon::TConfSetting<ui64, false> _MaxListingPageSize;
1718
NCommon::TConfSetting<ui64, false> MetricsQueueBatchCountLimit;
18-
NCommon::TConfSetting<TString, false> SolomonClientDefaultReplica;
1919
NCommon::TConfSetting<ui64, false> ComputeActorBatchSize;
2020
NCommon::TConfSetting<ui64, false> MaxApiInflight;
2121
};

‎ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -349,16 +349,14 @@ class TSolomonDqIntegration: public TDqIntegrationBase {
349349
source.AddRequiredLabelNames(labelAsString);
350350
}
351351

352-
auto defaultReplica = (source.GetClusterType() == NSo::NProto::CT_SOLOMON ? "sas" : "cloud-prod-a");
353-
354352
auto& solomonConfig = State_->Configuration;
355353
auto& sourceSettings = *source.MutableSettings();
356354

357355
auto metricsQueueBatchCountLimit = solomonConfig->MetricsQueueBatchCountLimit.Get().OrElse(500);
358356
sourceSettings.insert({"metricsQueueBatchCountLimit", ToString(metricsQueueBatchCountLimit)});
359357

360-
auto solomonClientDefaultReplica = solomonConfig->SolomonClientDefaultReplica.Get().OrElse(defaultReplica);
361-
sourceSettings.insert({"solomonClientDefaultReplica", ToString(solomonClientDefaultReplica)});
358+
auto enableSolomonClientPostApi = solomonConfig->_EnableSolomonClientPostApi.Get().OrElse(false);
359+
sourceSettings.insert({"enableSolomonClientPostApi", ToString(enableSolomonClientPostApi)});
362360

363361
auto computeActorBatchSize = solomonConfig->ComputeActorBatchSize.Get().OrElse(100);
364362
sourceSettings.insert({"computeActorBatchSize", ToString(computeActorBatchSize)});

‎ydb/library/yql/providers/solomon/provider/yql_solomon_load_meta.cpp‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -93,9 +93,8 @@ class TSolomonLoadTableMetadataTransformer : public TGraphTransformerBase {
9393
return TStatus::Error;
9494
}
9595

96-
auto defaultReplica = (source.GetClusterType() == NSo::NProto::CT_SOLOMON ? "sas" : "cloud-prod-a");
97-
auto solomonClientDefaultReplica = State_->Configuration->SolomonClientDefaultReplica.Get().OrElse(defaultReplica);
98-
source.MutableSettings()->insert({ "solomonClientDefaultReplica", ToString(solomonClientDefaultReplica) });
96+
auto enableSolomonClientPostApi = State_->Configuration->_EnableSolomonClientPostApi.Get().OrElse(false);
97+
source.MutableSettings()->insert({ "enableSolomonClientPostApi", ToString(enableSolomonClientPostApi) });
9998

10099
auto providerFactory = CreateCredentialsProviderFactoryForStructuredToken(State_->CredentialsFactory, State_->Configuration->Tokens.at(clusterName));
101100
auto credentialsProvider = providerFactory->CreateProvider();

‎ydb/library/yql/providers/solomon/solomon_accessor/client/solomon_accessor_client.cpp‎

Lines changed: 63 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -302,12 +302,12 @@ TGetDataResponse ProcessGetDataResponse(NYdbGrpc::TGrpcStatus&& status, ReadResp
302302
class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable_shared_from_this<TSolomonAccessorClient> {
303303
public:
304304
TSolomonAccessorClient(
305-
const TString& defaultReplica,
305+
bool enableSolomonClientPostApi,
306306
ui64 maxListingPageSize,
307307
ui64 maxApiInflight,
308308
NYql::NSo::NProto::TDqSolomonSource&& settings,
309309
std::shared_ptr<NYdb::ICredentialsProvider> credentialsProvider)
310-
: DefaultReplica(defaultReplica)
310+
: EnableSolomonClientPostApi(enableSolomonClientPostApi)
311311
, MaxListingPageSize(maxListingPageSize)
312312
, Settings(std::move(settings))
313313
, CredentialsProvider(credentialsProvider) {
@@ -327,7 +327,7 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable
327327

328328
public:
329329
NThreading::TFuture<TGetLabelsResponse> GetLabelNames(const TSelectors& selectors, TInstant from, TInstant to) const override final {
330-
auto url = BuildGetLabelsUrl(selectors, from, to);
330+
auto [url, body] = BuildGetLabelsHttpParams(selectors, from, to);
331331

332332
auto resultPromise = NThreading::NewPromise<TGetLabelsResponse>();
333333

@@ -337,7 +337,8 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable
337337

338338
DoHttpRequest(
339339
std::move(cb),
340-
std::move(url)
340+
std::move(url),
341+
std::move(body)
341342
);
342343

343344
return resultPromise.GetFuture();
@@ -546,7 +547,7 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable
546547
}
547548
}
548549

549-
TString BuildGetLabelsUrl(const TSelectors& selectors, TInstant from, TInstant to) const {
550+
std::tuple<TString, TString> BuildGetLabelsHttpParams(const TSelectors& selectors, TInstant from, TInstant to) const {
550551
TUrlBuilder builder(GetHttpSolomonEndpoint());
551552

552553
builder.AddPathComponent("api");
@@ -556,12 +557,23 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable
556557
builder.AddPathComponent("sensors");
557558
builder.AddPathComponent("names");
558559

559-
builder.AddUrlParam("projectId", GetProjectId());
560-
builder.AddUrlParam("selectors", BuildSelectorsProgram(selectors));
561-
builder.AddUrlParam("from", from.ToString());
562-
builder.AddUrlParam("to", to.ToString());
560+
NJsonWriter::TBuf w;
561+
562+
if (EnableSolomonClientPostApi) {
563+
w.BeginObject()
564+
.UnsafeWriteKey("projectId").WriteString(GetProjectId())
565+
.UnsafeWriteKey("selectors").WriteString(BuildSelectorsProgram(selectors))
566+
.UnsafeWriteKey("from").WriteString(from.ToString())
567+
.UnsafeWriteKey("to").WriteString(to.ToString())
568+
.EndObject();
569+
} else {
570+
builder.AddUrlParam("projectId", GetProjectId());
571+
builder.AddUrlParam("selectors", BuildSelectorsProgram(selectors));
572+
builder.AddUrlParam("from", from.ToString());
573+
builder.AddUrlParam("to", to.ToString());
574+
}
563575

564-
return builder.Build();
576+
return { builder.Build(), w.Str() };
565577
}
566578

567579
std::tuple<TString, TString> BuildListMetricsHttpParams(const TSelectors& selectors, TInstant from, TInstant to) const {
@@ -573,13 +585,25 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable
573585
builder.AddPathComponent(Settings.GetProject());
574586
builder.AddPathComponent("sensors");
575587

576-
builder.AddUrlParam("projectId", GetProjectId());
577-
builder.AddUrlParam("selectors", BuildSelectorsProgram(selectors));
578-
builder.AddUrlParam("from", from.ToString());
579-
builder.AddUrlParam("to", to.ToString());
580-
builder.AddUrlParam("pageSize", ToString(MaxListingPageSize));
588+
NJsonWriter::TBuf w;
581589

582-
return { builder.Build(), "" };
590+
if (EnableSolomonClientPostApi) {
591+
w.BeginObject()
592+
.UnsafeWriteKey("projectId").WriteString(GetProjectId())
593+
.UnsafeWriteKey("selectors").WriteString(BuildSelectorsProgram(selectors))
594+
.UnsafeWriteKey("from").WriteString(from.ToString())
595+
.UnsafeWriteKey("to").WriteString(to.ToString())
596+
.UnsafeWriteKey("pageSize").WriteLongLong(MaxListingPageSize)
597+
.EndObject();
598+
} else {
599+
builder.AddUrlParam("projectId", GetProjectId());
600+
builder.AddUrlParam("selectors", BuildSelectorsProgram(selectors));
601+
builder.AddUrlParam("from", from.ToString());
602+
builder.AddUrlParam("to", to.ToString());
603+
builder.AddUrlParam("pageSize", ToString(MaxListingPageSize));
604+
}
605+
606+
return { builder.Build(), w.Str() };
583607
}
584608

585609
std::tuple<TString, TString> BuildListMetricsLabelsHttpParams(const TSelectors& selectors, TInstant from, TInstant to) const {
@@ -592,13 +616,25 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable
592616
builder.AddPathComponent("sensors");
593617
builder.AddPathComponent("labels");
594618

595-
builder.AddUrlParam("projectId", GetProjectId());
596-
builder.AddUrlParam("selectors", BuildSelectorsProgram(selectors));
597-
builder.AddUrlParam("from", from.ToString());
598-
builder.AddUrlParam("to", to.ToString());
599-
builder.AddUrlParam("limit", "100000");
619+
NJsonWriter::TBuf w;
600620

601-
return { builder.Build(), "" };
621+
if (EnableSolomonClientPostApi) {
622+
w.BeginObject()
623+
.UnsafeWriteKey("projectId").WriteString(GetProjectId())
624+
.UnsafeWriteKey("selectors").WriteString(BuildSelectorsProgram(selectors))
625+
.UnsafeWriteKey("from").WriteString(from.ToString())
626+
.UnsafeWriteKey("to").WriteString(to.ToString())
627+
.UnsafeWriteKey("limit").WriteLongLong(100000)
628+
.EndObject();
629+
} else {
630+
builder.AddUrlParam("projectId", GetProjectId());
631+
builder.AddUrlParam("selectors", BuildSelectorsProgram(selectors));
632+
builder.AddUrlParam("from", from.ToString());
633+
builder.AddUrlParam("to", to.ToString());
634+
builder.AddUrlParam("limit", "100000");
635+
}
636+
637+
return { builder.Build(), w.Str() };
602638
}
603639

604640
std::tuple<TString, TString> BuildGetPointsCountHttpParams(const TString& program, TInstant from, TInstant to) const {
@@ -692,7 +728,7 @@ class TSolomonAccessorClient : public ISolomonAccessorClient, public std::enable
692728
}
693729

694730
private:
695-
const TString DefaultReplica;
731+
const bool EnableSolomonClientPostApi;
696732
const ui64 MaxListingPageSize;
697733
const ui64 ListSizeLimit = 100 * 1024 * 1024 * 8;
698734
const NYql::NSo::NProto::TDqSolomonSource Settings;
@@ -713,9 +749,9 @@ ISolomonAccessorClient::Make(
713749
std::shared_ptr<NYdb::ICredentialsProvider> credentialsProvider) {
714750
const auto& settings = source.settings();
715751

716-
TString defaultReplica;
717-
if (auto it = settings.find("solomonClientDefaultReplica"); it != settings.end()) {
718-
defaultReplica = it->second;
752+
bool enableSolomonClientPostApi = false;
753+
if (auto it = settings.find("enableSolomonClientPostApi"); it != settings.end()) {
754+
enableSolomonClientPostApi = FromString<bool>(it->second);
719755
}
720756

721757
ui64 maxListingPageSize = 20000;
@@ -728,7 +764,7 @@ ISolomonAccessorClient::Make(
728764
maxApiInflight = FromString<ui64>(it->second);
729765
}
730766

731-
return std::make_shared<TSolomonAccessorClient>(defaultReplica, maxListingPageSize, maxApiInflight, std::move(source), credentialsProvider);
767+
return std::make_shared<TSolomonAccessorClient>(enableSolomonClientPostApi, maxListingPageSize, maxApiInflight, std::move(source), credentialsProvider);
732768
}
733769

734770
} // namespace NYql::NSo

‎ydb/library/yql/tools/solomon_emulator/lib/webapp.py‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,8 @@ async def data_write(self, request):
110110
return web.json_response({"writtenMetricsCount": shard.add_metrics(metrics_json)})
111111

112112
async def sensor_names(self, request):
113-
selectors, success = _parse_selectors(request.rel_url.query["selectors"])
113+
json = await request.json()
114+
selectors, success = _parse_selectors(json["selectors"])
114115

115116
if not success:
116117
return web.HTTPBadRequest(text="Invalid selectors")
@@ -128,7 +129,8 @@ async def sensor_names(self, request):
128129
return web.json_response({"names": result})
129130

130131
async def sensor_labels(self, request):
131-
selectors, success = _parse_selectors(request.rel_url.query["selectors"])
132+
json = await request.json()
133+
selectors, success = _parse_selectors(json["selectors"])
132134

133135
if not success:
134136
return web.HTTPBadRequest(text="Invalid selectors")
@@ -146,7 +148,8 @@ async def sensor_labels(self, request):
146148
return web.json_response({"labels": labels, "totalCount": totalCount})
147149

148150
async def sensors(self, request):
149-
selectors, success = _parse_selectors(request.rel_url.query["selectors"])
151+
json = await request.json()
152+
selectors, success = _parse_selectors(json["selectors"])
150153

151154
if not success:
152155
return web.HTTPBadRequest(text="Invalid selectors")
@@ -309,11 +312,11 @@ def _build_read_response(labels, type, timestamps, values):
309312

310313

311314
def create_web_app(emulator):
312-
webapp = web.Application(client_max_size=1024**3, handler_args={"max_line_size": 1024**3, "max_field_size": 1024**3})
315+
webapp = web.Application()
313316
webapp.add_routes([
314-
web.get("/api/v2/projects/{project}/sensors/names", emulator.sensor_names),
315-
web.get("/api/v2/projects/{project}/sensors/labels", emulator.sensor_labels),
316-
web.get("/api/v2/projects/{project}/sensors", emulator.sensors),
317+
web.post("/api/v2/projects/{project}/sensors/names", emulator.sensor_names),
318+
web.post("/api/v2/projects/{project}/sensors/labels", emulator.sensor_labels),
319+
web.post("/api/v2/projects/{project}/sensors", emulator.sensors),
317320
web.get("/metrics/get", emulator.metrics_get),
318321
web.get("/ping", emulator.get_ping),
319322
web.post("/api/v2/push", emulator.api_v2_push),

‎ydb/tests/solomon/reading/base.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,10 @@ def setup_class(cls):
7272
"name": "_EnableRuntimeListing",
7373
"value": "true"
7474
},
75+
{
76+
"name": "_EnableSolomonClientPostApi",
77+
"value": "true"
78+
},
7579
{
7680
"name": "_MaxListingPageSize",
7781
"value": 1000

0 commit comments

Comments
 (0)