[Do not merge] POC for Parallel Upload dashboard and MPU metrics

google-cloud-storage/src/main/java/com/google/cloud/storage/HttpStorageOptions.java

@@ -65,6 +65,7 @@ public class HttpStorageOptions extends StorageOptions {
   private transient RetryDependenciesAdapter retryDepsAdapter;
   private final BlobWriteSessionConfig blobWriteSessionConfig;
 
+  private final boolean enableHttpClientsMetrics;
   private transient OpenTelemetry openTelemetry;
 
   private HttpStorageOptions(Builder builder, StorageDefaults serviceDefaults) {
@@ -75,6 +76,7 @@ private HttpStorageOptions(Builder builder, StorageDefaults serviceDefaults) {
                 builder.storageRetryStrategy, defaults().getStorageRetryStrategy()));
     retryDepsAdapter = new RetryDependenciesAdapter();
     blobWriteSessionConfig = builder.blobWriteSessionConfig;
+    enableHttpClientsMetrics = builder.enableHttpClientsMetrics;
     openTelemetry = builder.openTelemetry;
   }
 
@@ -99,6 +101,15 @@ StorageRpc getStorageRpcV1() {
   @BetaApi
   @Override
   public OpenTelemetry getOpenTelemetry() {
+    if (openTelemetry == null || openTelemetry == OpenTelemetry.noop()) {
+      if (enableHttpClientsMetrics) {
+        openTelemetry =
+            OpenTelemetryBootstrappingUtils.getHttpOpenTelemetrySdk(
+                getProjectId(), getUniverseDomain(), getHost(), true);
+      } else {
+        return OpenTelemetry.noop();
+      }
+    }
     return openTelemetry;
   }
 
@@ -110,7 +121,11 @@ public HttpStorageOptions.Builder toBuilder() {
   @Override
   public int hashCode() {
     return Objects.hash(
-        retryAlgorithmManager, blobWriteSessionConfig, openTelemetry, baseHashCode());
+        retryAlgorithmManager,
+        blobWriteSessionConfig,
+        enableHttpClientsMetrics,
+        openTelemetry,
+        baseHashCode());
   }
 
   @Override
@@ -124,6 +139,7 @@ public boolean equals(Object o) {
     HttpStorageOptions that = (HttpStorageOptions) o;
     return Objects.equals(retryAlgorithmManager, that.retryAlgorithmManager)
         && Objects.equals(blobWriteSessionConfig, that.blobWriteSessionConfig)
+        && Objects.equals(enableHttpClientsMetrics, that.enableHttpClientsMetrics)
         && Objects.equals(openTelemetry, that.openTelemetry)
         && this.baseEquals(that);
   }
@@ -156,6 +172,7 @@ public static class Builder extends StorageOptions.Builder {
     private StorageRetryStrategy storageRetryStrategy;
     private BlobWriteSessionConfig blobWriteSessionConfig =
         HttpStorageDefaults.INSTANCE.getDefaultStorageWriterConfig();
+    private boolean enableHttpClientsMetrics = false;
     private OpenTelemetry openTelemetry = HttpStorageDefaults.INSTANCE.getDefaultOpenTelemetry();
 
     Builder() {}
@@ -165,6 +182,7 @@ public static class Builder extends StorageOptions.Builder {
       HttpStorageOptions hso = (HttpStorageOptions) options;
       this.storageRetryStrategy = hso.retryAlgorithmManager.retryStrategy;
       this.blobWriteSessionConfig = hso.blobWriteSessionConfig;
+      this.enableHttpClientsMetrics = hso.enableHttpClientsMetrics;
       this.openTelemetry = hso.getOpenTelemetry();
     }
 
@@ -317,6 +335,18 @@ public HttpStorageOptions.Builder setOpenTelemetry(OpenTelemetry openTelemetry)
       this.openTelemetry = openTelemetry;
       return this;
     }
+
+    /**
+     * Option for whether this client should emit internal Otel HTTP client metrics to 
+     * Cloud Monitoring. To enable metric reporting, set this to true. False by default.
+     *
+     * @since 2.62.0 This new api is in preview and is subject to breaking changes.
+     */
+    @BetaApi
+    public HttpStorageOptions.Builder setEnableHttpClientsMetrics(boolean enableHttpClientsMetrics) {
+      this.enableHttpClientsMetrics = enableHttpClientsMetrics;
+      return this;
+    }
   }
 
   public static final class HttpStorageDefaults extends StorageDefaults {
@@ -360,6 +390,11 @@ public BlobWriteSessionConfig getDefaultStorageWriterConfig() {
     public OpenTelemetry getDefaultOpenTelemetry() {
       return OpenTelemetry.noop();
     }
+
+    @BetaApi
+    public boolean getDefaultEnableHttpClientsMetrics() {
+      return false;
+    }
   }
 
   /**

google-cloud-storage/src/main/java/com/google/cloud/storage/OpenTelemetryBootstrappingUtils.java

@@ -28,6 +28,7 @@
 import com.google.common.collect.Iterables;
 import io.grpc.ManagedChannelBuilder;
 import io.grpc.opentelemetry.GrpcOpenTelemetry;
+import io.opentelemetry.api.OpenTelemetry;
 import io.opentelemetry.api.common.AttributeKey;
 import io.opentelemetry.api.common.Attributes;
 import io.opentelemetry.api.common.AttributesBuilder;
@@ -124,7 +125,7 @@ static ChannelConfigurator enableGrpcMetrics(
     String metricServiceEndpoint = getCloudMonitoringEndpoint(endpoint, universeDomain);
     SdkMeterProvider provider =
         createMeterProvider(
-            metricServiceEndpoint, projectIdToUse, detectedAttributes, shouldSuppressExceptions);
+            metricServiceEndpoint, projectIdToUse, detectedAttributes, shouldSuppressExceptions, true);
 
     OpenTelemetrySdk openTelemetrySdk =
         OpenTelemetrySdk.builder().setMeterProvider(provider).build();
@@ -142,6 +143,49 @@ static ChannelConfigurator enableGrpcMetrics(
     return otelConfigurator.andThen(channelConfigurator);
   }
 
+  @NonNull
+  static OpenTelemetry getHttpOpenTelemetrySdk(
+      String projectId,
+      String universeDomain,
+      String host,
+      boolean shouldSuppressExceptions) {
+    GCPResourceProvider resourceProvider = new GCPResourceProvider();
+    Attributes detectedAttributes = resourceProvider.getAttributes();
+
+    @Nullable
+    String detectedProjectId = detectedAttributes.get(AttributeKey.stringKey("cloud.account.id"));
+    if (projectId == null && detectedProjectId == null) {
+      log.warning(
+          "Unable to determine the Project ID in order to report metrics. No HTTP client metrics"
+              + " will be reported.");
+      return OpenTelemetry.noop();
+    }
+
+    String projectIdToUse = detectedProjectId == null ? projectId : detectedProjectId;
+    if (!projectIdToUse.equals(projectId)) {
+      log.warning(
+          "The Project ID configured for HTTP client metrics is "
+              + projectIdToUse
+              + ", but the Project ID of the storage client is "
+              + projectId
+              + ". Make sure that the service account in use has the required metric writing role "
+              + "(roles/monitoring.metricWriter) in the project "
+              + projectIdToUse
+              + ", or metrics will not be written.");
+    }
+
+    String metricServiceEndpoint = getCloudMonitoringEndpoint(host, universeDomain);
+    SdkMeterProvider provider =
+        createMeterProvider(
+            metricServiceEndpoint,
+            projectIdToUse,
+            detectedAttributes,
+            shouldSuppressExceptions,
+            false);
+
+    return OpenTelemetrySdk.builder().setMeterProvider(provider).build();
+  }
+
   @SuppressWarnings("rawtypes") // ManagedChannelBuilder
   @FunctionalInterface
   interface ChannelConfigurator extends ApiFunction<ManagedChannelBuilder, ManagedChannelBuilder> {
@@ -210,7 +254,8 @@ static SdkMeterProvider createMeterProvider(
       String metricServiceEndpoint,
       String projectIdToUse,
       Attributes detectedAttributes,
-      boolean shouldSuppressExceptions) {
+      boolean shouldSuppressExceptions,
+      boolean enableGrpcViews) {
 
     MonitoredResourceDescription monitoredResourceDescription =
         new MonitoredResourceDescription(
@@ -233,11 +278,13 @@ static SdkMeterProvider createMeterProvider(
 
     // This replaces the dots with slashes in each metric, which is the format needed for this
     // monitored resource
-    for (String metric :
-        ImmutableList.copyOf(Iterables.concat(METRICS_TO_ENABLE, METRICS_ENABLED_BY_DEFAULT))) {
-      providerBuilder.registerView(
-          InstrumentSelector.builder().setName(metric).build(),
-          View.builder().setName(metric.replace(".", "/")).build());
+    if (enableGrpcViews) {
+      for (String metric :
+          ImmutableList.copyOf(Iterables.concat(METRICS_TO_ENABLE, METRICS_ENABLED_BY_DEFAULT))) {
+        providerBuilder.registerView(
+            InstrumentSelector.builder().setName(metric).build(),
+            View.builder().setName(metric.replace(".", "/")).build());
+      }
     }
     MetricExporter exporter =
         shouldSuppressExceptions
@@ -274,18 +321,20 @@ static SdkMeterProvider createMeterProvider(
                 .build())
         .setResource(Resource.create(attributesBuilder.build()));
 
-    addHistogramView(
-        providerBuilder, latencyHistogramBoundaries(), "grpc/client/attempt/duration", "s");
-    addHistogramView(
-        providerBuilder,
-        sizeHistogramBoundaries(),
-        "grpc/client/attempt/rcvd_total_compressed_message_size",
-        "By");
-    addHistogramView(
-        providerBuilder,
-        sizeHistogramBoundaries(),
-        "grpc/client/attempt/sent_total_compressed_message_size",
-        "By");
+    if (enableGrpcViews) {
+      addHistogramView(
+          providerBuilder, latencyHistogramBoundaries(), "grpc/client/attempt/duration", "s");
+      addHistogramView(
+          providerBuilder,
+          sizeHistogramBoundaries(),
+          "grpc/client/attempt/rcvd_total_compressed_message_size",
+          "By");
+      addHistogramView(
+          providerBuilder,
+          sizeHistogramBoundaries(),
+          "grpc/client/attempt/sent_total_compressed_message_size",
+          "By");
+    }
 
     return providerBuilder.build();
   }