From 28cb4361bd77c83df64c8225811c473c17941f2c Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Tue, 18 Nov 2025 16:40:51 +0100 Subject: [PATCH 1/5] Crashtracker: dual ship to error tracking --- .../datadog/crashtracking/ConfigManager.java | 4 +- .../datadog/crashtracking/CrashUploader.java | 321 ++++++++++++++---- .../datadog/crashtracking/dto/CrashLog.java | 17 +- .../datadog/crashtracking/dto/ErrorData.java | 18 +- .../datadog/crashtracking/dto/OSInfo.java | 9 + .../datadog/crashtracking/dto/SigInfo.java | 29 ++ .../parsers/HotspotCrashLogParser.java | 49 ++- .../crashtracking/CrashUploaderTest.java | 208 ++++++++++-- .../sample-crash-for-telemetry-2.txt | 1 + .../sample-crash-for-telemetry-3.txt | 1 + .../sample-crash-for-telemetry.txt | 1 + .../golden/errortracking/sample-ping.txt | 1 + .../sample-crash-for-telemetry-2.txt | 0 .../sample-crash-for-telemetry-3.txt | 0 .../sample-crash-for-telemetry.txt | 0 .../sample-ping-for-telemetry.txt | 0 .../main/java/datadog/trace/api/Config.java | 15 +- 17 files changed, 560 insertions(+), 114 deletions(-) create mode 100644 dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/SigInfo.java create mode 100644 dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-2.txt create mode 100644 dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-3.txt create mode 100644 dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry.txt create mode 100644 dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-ping.txt rename dd-java-agent/agent-crashtracking/src/test/resources/golden/{ => telemetry}/sample-crash-for-telemetry-2.txt (100%) rename dd-java-agent/agent-crashtracking/src/test/resources/golden/{ => telemetry}/sample-crash-for-telemetry-3.txt (100%) rename dd-java-agent/agent-crashtracking/src/test/resources/golden/{ => telemetry}/sample-crash-for-telemetry.txt (100%) rename dd-java-agent/agent-crashtracking/src/test/resources/golden/{ => telemetry}/sample-ping-for-telemetry.txt (100%) diff --git a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/ConfigManager.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/ConfigManager.java index 8a4f681cc4f..2949315f13b 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/ConfigManager.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/ConfigManager.java @@ -122,8 +122,10 @@ private static String getBaseName(Path path) { return filename.substring(0, dotIndex); } - private static String getMergedTagsForSerialization(Config config) { + // @VisibleForTesting + static String getMergedTagsForSerialization(Config config) { return config.getMergedCrashTrackingTags().entrySet().stream() + .filter(e -> e.getValue() != null) .map(e -> e.getKey() + ":" + e.getValue()) .collect(Collectors.joining(",")); } diff --git a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java index f7dd4d2a17e..8b57575db59 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java @@ -9,12 +9,17 @@ import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY; import static datadog.trace.util.TraceUtils.normalizeServiceName; import static datadog.trace.util.TraceUtils.normalizeTagValue; +import static java.time.format.DateTimeFormatter.ISO_OFFSET_DATE_TIME; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import static java.util.concurrent.TimeUnit.SECONDS; import com.squareup.moshi.JsonWriter; import datadog.common.container.ContainerInfo; import datadog.common.version.VersionInfo; import datadog.communication.http.OkHttpUtils; import datadog.crashtracking.dto.CrashLog; +import datadog.crashtracking.dto.ErrorData; +import datadog.crashtracking.dto.OSInfo; import datadog.environment.SystemProperties; import datadog.trace.api.Config; import datadog.trace.api.DDTags; @@ -24,20 +29,21 @@ import edu.umd.cs.findbugs.annotations.NonNull; import java.io.*; import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.time.Instant; +import java.time.ZonedDateTime; import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.Scanner; -import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; import javax.annotation.Nonnull; import okhttp3.Call; +import okhttp3.Callback; +import okhttp3.Dispatcher; import okhttp3.HttpUrl; import okhttp3.MediaType; import okhttp3.OkHttpClient; @@ -56,6 +62,8 @@ public final class CrashUploader { static final String HEADER_DD_EVP_ORIGIN = "DD-EVP-ORIGIN"; static final String JAVA_TRACING_LIBRARY = "dd-trace-java"; static final String HEADER_DD_EVP_ORIGIN_VERSION = "DD-EVP-ORIGIN-VERSION"; + static final String HEADER_DD_EVP_SUBDOMAIN = "X-Datadog-EVP-Subdomain"; + static final String ERROR_TRACKING_INTAKE = "error-tracking-intake"; static final String HEADER_DD_TELEMETRY_API_VERSION = "DD-Telemetry-API-Version"; static final String TELEMETRY_API_VERSION = "v2"; static final String HEADER_DD_TELEMETRY_REQUEST_TYPE = "DD-Telemetry-Request-Type"; @@ -64,13 +72,49 @@ public final class CrashUploader { private static final MediaType APPLICATION_JSON = MediaType.get("application/json; charset=utf-8"); + private static final class CallResult implements Callback { + private final String kind; // for logging + + private CallResult(String kind) { + this.kind = kind; + } + + @Override + public void onFailure(Call call, IOException e) { + log.error("Failed to upload {} to {}, got exception", kind, call.request().url(), e); + } + + @Override + public void onResponse(Call call, Response response) throws IOException { + if (response.isSuccessful()) { + log.info( + "Successfully uploaded the {} to {}, code = {} \"{}\"", + kind, + call.request().url(), + response.code(), + response.message()); + } else { + log.error( + "Failed to upload {} to {}, code = {} \"{}\", body = \"{}\"", + kind, + call.request().url(), + response.code(), + response.message(), + response.body() != null ? response.body().string().trim() : ""); + } + } + } + private final Config config; private final ConfigManager.StoredConfig storedConfig; - private final OkHttpClient telemetryClient; private final HttpUrl telemetryUrl; + private final HttpUrl errorTrackingUrl; + private final OkHttpClient uploadClient; + private final Dispatcher dispatcher; private final boolean agentless; private final String tags; + private final long timeout; public CrashUploader(@Nonnull final ConfigManager.StoredConfig storedConfig) { this(Config.get(), storedConfig); @@ -80,9 +124,11 @@ public CrashUploader(@Nonnull final ConfigManager.StoredConfig storedConfig) { @NonNull final Config config, @Nonnull final ConfigManager.StoredConfig storedConfig) { this.config = config; this.storedConfig = storedConfig; - - telemetryUrl = HttpUrl.get(config.getFinalCrashTrackingTelemetryUrl()); - agentless = config.isCrashTrackingAgentless(); + this.telemetryUrl = HttpUrl.get(config.getFinalCrashTrackingTelemetryUrl()); + this.errorTrackingUrl = HttpUrl.get(config.getFinalCrashTrackingErrorTrackingUrl()); + this.agentless = config.isCrashTrackingAgentless(); + this.dispatcher = new Dispatcher(); + dispatcher.setMaxRequests(4); final StringBuilder tagsBuilder = new StringBuilder(storedConfig.tags != null ? storedConfig.tags : ""); @@ -98,23 +144,32 @@ public CrashUploader(@Nonnull final ConfigManager.StoredConfig storedConfig) { ConfigProvider configProvider = config.configProvider(); - telemetryClient = + this.timeout = + SECONDS.toMillis( + configProvider.getInteger( + CRASH_TRACKING_UPLOAD_TIMEOUT, CRASH_TRACKING_UPLOAD_TIMEOUT_DEFAULT)); + + uploadClient = OkHttpUtils.buildHttpClient( config, - null, /* dispatcher */ - telemetryUrl, + dispatcher, /* dispatcher */ + telemetryUrl, // will be overridden in each request true, /* retryOnConnectionFailure */ - null, /* maxRunningRequests */ + 4, /* maxRunningRequests */ // allows 1 blocked ping and two crash request configProvider.getString(CRASH_TRACKING_PROXY_HOST), configProvider.getInteger(CRASH_TRACKING_PROXY_PORT), configProvider.getString(CRASH_TRACKING_PROXY_USERNAME), configProvider.getString(CRASH_TRACKING_PROXY_PASSWORD), - TimeUnit.SECONDS.toMillis( - configProvider.getInteger( - CRASH_TRACKING_UPLOAD_TIMEOUT, CRASH_TRACKING_UPLOAD_TIMEOUT_DEFAULT))); + timeout); } public void notifyCrashStarted(String error) { + sendPingToTelemetry(error); + sendPingToErrorTracking(error); + } + + // @VisibleForTesting + void sendPingToTelemetry(String error) { // send a ping message to the telemetry to notify that the crash report started try (Buffer buf = new Buffer(); JsonWriter writer = JsonWriter.of(buf)) { @@ -128,27 +183,88 @@ public void notifyCrashStarted(String error) { "Crashtracker crash ping: " + (error != null ? error : "crash processing started")); writer.endObject(); handleCall(makeTelemetryRequest(makeTelemetryRequestBody(buf.readUtf8(), true)), "ping"); - } catch (Throwable t) { - log.error("Failed to send crash ping", t); + log.error("Failed to prepare the telemetry crash ping payload", t); } } - public void upload(@Nonnull Path file) throws IOException { - String uuid = storedConfig.reportUUID; - uploadToLogs(file); - uploadToTelemetry(file, uuid); + // @VisibleForTesting + void sendPingToErrorTracking(String error) { + try { + final CrashLog ping = + new CrashLog( + storedConfig.reportUUID, + false, + ZonedDateTime.now().format(ISO_OFFSET_DATE_TIME), + new ErrorData( + null, + "Crashtracker crash ping: " + + (error != null ? error : "crash processing started"), + null, + false), + null, + OSInfo.current(), + null, + null, + "1.0"); + handleCall(makeErrorTrackingRequest(makeErrorTrackingRequestBody(ping, true)), "ping"); + } catch (Throwable t) { + log.error("Failed to prepare the error tracking crash ping payload", t); + } } @SuppressForbidden - boolean uploadToLogs(@Nonnull Path file) { + public void upload(@Nonnull Path file) { + String fileContent; try { - uploadToLogs(new String(Files.readAllBytes(file), StandardCharsets.UTF_8), System.out); - } catch (IOException e) { - log.error("Failed to upload crash file: {}", file, e); - return false; + fileContent = new String(Files.readAllBytes(file), Charset.defaultCharset()); + } catch (Throwable t) { + log.error("Failed to collect information about the crash", t); + return; // cannot proceed further + } + + try { + uploadToLogs(fileContent, System.out); + } catch (Throwable t) { + log.error("Unable to print the error crash as a log message", t); + } + remoteUpload(fileContent, true, true); + } + + // @VisibleForTesting + void remoteUpload( + @Nonnull String fileContent, boolean sendToTelemetry, boolean sendToErrorTracking) { + final String uuid = storedConfig.reportUUID; + try { + CrashLog crashLog = CrashLogParser.fromHotspotCrashLog(uuid, fileContent); + if (sendToTelemetry) { + uploadToTelemetry(crashLog); + } + if (sendToErrorTracking) { + uploadToErrorTracking(crashLog); + } + } catch (Throwable t) { + log.error("Error while before sending remotely the crash report with uuid {}", uuid, t); + } + int remaining; + long deadline = MILLISECONDS.toNanos(timeout) + System.nanoTime(); + // container that crashed + while ((remaining = dispatcher.queuedCallsCount() + dispatcher.runningCallsCount()) > 0 + && deadline > System.nanoTime()) { + try { + Thread.sleep(100); // good enough for this purpose even if we overflow + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + } + } + if (remaining > 0) { + dispatcher.cancelAll(); + log.error( + SEND_TELEMETRY, + "Failed to fully send the crash report with UUID {}. Still {} calls remaining", + uuid, + remaining); } - return true; } void uploadToLogs(@Nonnull String message, @Nonnull PrintStream out) throws IOException { @@ -172,17 +288,15 @@ void uploadToLogs(@Nonnull String message, @Nonnull PrintStream out) throws IOEx writer.endObject(); writer.endObject(); } - out.println(buf.readByteString().utf8()); } } // @VisibleForTesting - @SuppressForbidden static String extractErrorKind(String fileContent) { Matcher matcher = ERROR_MESSAGE_PATTERN.matcher(fileContent); if (!matcher.find()) { - System.err.println("No match found for error.kind"); + log.error("No match found for error.kind"); return null; } @@ -210,11 +324,10 @@ static String extractErrorKind(String fileContent) { Pattern.DOTALL | Pattern.MULTILINE); // @VisibleForTesting - @SuppressForbidden static String extractErrorMessage(String fileContent) { Matcher matcher = ERROR_MESSAGE_PATTERN.matcher(fileContent); if (!matcher.find()) { - System.err.println("No match found for error.message"); + log.error("No match found for error.message"); return null; } return Arrays.stream(matcher.group().split(System.lineSeparator())) @@ -255,21 +368,11 @@ private String extractErrorStackTrace(String fileContent, boolean redact) { return ""; } - boolean uploadToTelemetry(@Nonnull Path file, String uuid) { + void uploadToTelemetry(@Nonnull CrashLog crashLog) { try { - String content = new String(Files.readAllBytes(file), Charset.defaultCharset()); - CrashLog crashLog = CrashLogParser.fromHotspotCrashLog(uuid, content); - if (crashLog == null || crashLog.incomplete) { - log.error(SEND_TELEMETRY, "Failed to parse crash log with uuid {} ", uuid); - } - if (crashLog == null) { - return false; - } handleCall(makeTelemetryRequest(makeTelemetryRequestBody(crashLog.toJson(), false)), "crash"); - return !crashLog.incomplete; } catch (Throwable t) { - log.error("Failed to upload crash file: {}", file, t); - return false; + log.error("Failed to make a telemetry request", t); } } @@ -287,7 +390,7 @@ private Call makeTelemetryRequest(@Nonnull RequestBody requestBody) throws IOExc headers.put(HEADER_DD_TELEMETRY_API_VERSION, TELEMETRY_API_VERSION); headers.put(HEADER_DD_TELEMETRY_REQUEST_TYPE, TELEMETRY_REQUEST_TYPE); - return telemetryClient.newCall( + return uploadClient.newCall( OkHttpUtils.prepareRequest(telemetryUrl, headers, config, agentless) .post(requestBody) .build()); @@ -351,6 +454,108 @@ private RequestBody makeTelemetryRequestBody(@Nonnull String payload, boolean is } } + void uploadToErrorTracking(@Nonnull CrashLog crashLog) { + try { + handleCall(makeErrorTrackingRequest(makeErrorTrackingRequestBody(crashLog, false)), "crash"); + } catch (Throwable t) { + log.error("Failed to make a error tracking request", t); + } + } + + private Call makeErrorTrackingRequest(@Nonnull RequestBody requestBody) throws IOException { + final Map headers = new HashMap<>(); + // Set chunked transfer + MediaType contentType = requestBody.contentType(); + if (contentType != null) { + headers.put("Content-Type", contentType.toString()); + } + headers.put("Content-Length", Long.toString(requestBody.contentLength())); + headers.put("Transfer-Encoding", "chunked"); + headers.put(HEADER_DD_EVP_ORIGIN, JAVA_TRACING_LIBRARY); + headers.put(HEADER_DD_EVP_ORIGIN_VERSION, VersionInfo.VERSION); + if (!agentless) { + headers.put(HEADER_DD_EVP_SUBDOMAIN, ERROR_TRACKING_INTAKE); + } + + return uploadClient.newCall( + OkHttpUtils.prepareRequest(errorTrackingUrl, headers, config, agentless) + .post(requestBody) + .build()); + } + + private RequestBody makeErrorTrackingRequestBody(@Nonnull CrashLog payload, boolean isPing) + throws IOException { + try (Buffer buf = new Buffer()) { + try (JsonWriter writer = JsonWriter.of(buf)) { + writer.beginObject(); + writer.name("timestamp").value(payload.timestamp); + writer.name("ddsource").value("crashtracker"); + // error payload + writer.name("error"); + payload.error.writeAsJson(writer); // flat write an already serialized json object + // signal info + if (payload.sigInfo != null) { + writer.name("sig_info"); + writer.beginObject(); + if (payload.sigInfo.address != null) { + writer.name("si_addr").value(payload.sigInfo.address); + } + if (payload.sigInfo.name != null) { + writer.name("si_signo_human_readable").value(payload.sigInfo.name); + writer.name("si_signo").value(payload.sigInfo.number); + } + writer.endObject(); + } + + // os info + if (payload.osInfo != null) { + writer.name("os_info"); + writer.beginObject(); + writer.name("architecture").value(payload.osInfo.architecture); + writer.name("bitness").value(payload.osInfo.bitness); + writer.name("os_type").value(payload.osInfo.osType); + writer + .name("version") + .value( + SystemProperties.get( + "os.version")); // this has been restructured under OsInfo so taking raw here + writer.endObject(); + } + // tags + writer.name("ddtags").value(tagsForErrorTracking(payload.uuid, isPing, payload.incomplete)); + writer.endObject(); + } + return RequestBody.create(APPLICATION_JSON, buf.readByteString()); + } + } + + private String tagsForErrorTracking(String uuid, boolean isPing, boolean incomplete) { + final StringBuilder tags = new StringBuilder(); + if (storedConfig.tags != null) { + // normally it does not happen + tags.append(storedConfig.tags); + } else { + tags.append("service:") + .append(normalizeServiceName(storedConfig.service)); // ensure the service name is there + } + if (isPing) { + tags.append(",").append("is_crash_ping:true"); + } else { + tags.append(",").append("is_crash:true"); + if (incomplete) { + tags.append(",").append("incomplete:true"); + } + } + tags.append(",").append("data_schema_version:1.0"); + tags.append(",").append("language_name:jvm"); + tags.append(",") + .append("language_version:") + .append(normalizeTagValue(SystemProperties.getOrDefault("java.version", "unknown"))); + tags.append(",").append("tracer_version:").append(normalizeTagValue(VersionInfo.VERSION)); + tags.append(",").append("uuid:").append(uuid); + return (tags.toString()); + } + private String tagsForPing(String uuid) { final StringBuilder tags = new StringBuilder("is_crash_ping:true"); tags.append(",").append("language_name:jvm"); @@ -364,34 +569,6 @@ private String tagsForPing(String uuid) { } private void handleCall(final Call call, String kind) { - try (Response response = call.execute()) { - handleSuccess(call, response, kind); - } catch (Throwable t) { - handleFailure(t, kind); - } - } - - private void handleSuccess(final Call call, final Response response, String kind) - throws IOException { - if (response.isSuccessful()) { - log.info( - "Successfully uploaded the crash {} to {}, code = {} \"{}\"", - kind, - call.request().url(), - response.code(), - response.message()); - } else { - log.error( - "Failed to upload crash {} to {}, code = {} \"{}\", body = \"{}\"", - kind, - call.request().url(), - response.code(), - response.message(), - response.body() != null ? response.body().string().trim() : ""); - } - } - - private void handleFailure(final Throwable exception, String kind) { - log.error("Failed to upload crash {}, got exception", kind, exception); + call.enqueue(new CallResult(kind)); } } diff --git a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/CrashLog.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/CrashLog.java index 0dbefd4ad8b..c1254052023 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/CrashLog.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/CrashLog.java @@ -36,6 +36,9 @@ public final class CrashLog { @Json(name = "version_id") public final int version = VERSION; + // not serialized + public final transient SigInfo sigInfo; + public CrashLog( String uuid, boolean incomplete, @@ -44,6 +47,7 @@ public CrashLog( Metadata metadata, OSInfo osInfo, ProcInfo procInfo, + SigInfo sigInfo, String dataSchemaVersion) { this.uuid = uuid != null ? uuid : RandomUtils.randomUUID().toString(); this.incomplete = incomplete; @@ -52,6 +56,7 @@ public CrashLog( this.metadata = metadata; this.osInfo = osInfo; this.procInfo = procInfo; + this.sigInfo = sigInfo; this.dataSchemaVersion = dataSchemaVersion; } @@ -79,13 +84,23 @@ public boolean equals(Object o) { && Objects.equals(metadata, crashLog.metadata) && Objects.equals(osInfo, crashLog.osInfo) && Objects.equals(procInfo, crashLog.procInfo) + && Objects.equals(sigInfo, crashLog.sigInfo) && Objects.equals(dataSchemaVersion, crashLog.dataSchemaVersion); } @Override public int hashCode() { return Objects.hash( - uuid, timestamp, incomplete, error, metadata, osInfo, procInfo, version, dataSchemaVersion); + uuid, + timestamp, + incomplete, + error, + metadata, + osInfo, + procInfo, + sigInfo, + version, + dataSchemaVersion); } public boolean equalsForTest(Object o) { diff --git a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/ErrorData.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/ErrorData.java index 9ccb9a96e0d..6d654571a29 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/ErrorData.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/ErrorData.java @@ -1,11 +1,18 @@ package datadog.crashtracking.dto; import com.squareup.moshi.Json; +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.JsonWriter; +import com.squareup.moshi.Moshi; +import java.io.IOException; import java.util.Objects; public final class ErrorData { + private static final JsonAdapter ADAPTER = + new Moshi.Builder().build().adapter(ErrorData.class); + @Json(name = "is_crash") - public final boolean isCrash = true; + public final boolean isCrash; public final String kind; public final String message; @@ -16,9 +23,14 @@ public final class ErrorData { public final StackTrace stack; public ErrorData(String kind, String message, StackTrace stack) { + this(kind, message, stack, true); + } + + public ErrorData(String kind, String message, StackTrace stack, boolean isCrash) { this.kind = kind; this.message = message; this.stack = stack; + this.isCrash = isCrash; } @Override @@ -41,4 +53,8 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(isCrash, kind, message, sourceType, stack); } + + public void writeAsJson(final JsonWriter writer) throws IOException { + ADAPTER.toJson(writer, this); + } } diff --git a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/OSInfo.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/OSInfo.java index 793ec27aa4f..237f6c38990 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/OSInfo.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/OSInfo.java @@ -1,6 +1,7 @@ package datadog.crashtracking.dto; import com.squareup.moshi.Json; +import datadog.environment.SystemProperties; import java.util.Objects; public final class OSInfo { @@ -38,4 +39,12 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(architecture, bitness, osType, version); } + + public static OSInfo current() { + return new OSInfo( + SystemProperties.get("os.arch"), + SystemProperties.get("sun.arch.data.model"), + SystemProperties.get("os.name"), + SemanticVersion.of(SystemProperties.get("os.version"))); + } } diff --git a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/SigInfo.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/SigInfo.java new file mode 100644 index 00000000000..c4cb00d6b65 --- /dev/null +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/SigInfo.java @@ -0,0 +1,29 @@ +package datadog.crashtracking.dto; + +import java.util.Objects; + +public class SigInfo { + public final int number; + public final String name; + public final String address; + + public SigInfo(int number, String name, String address) { + this.number = number; + this.name = name; + this.address = address; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof SigInfo)) return false; + SigInfo sigInfo = (SigInfo) o; + return number == sigInfo.number + && Objects.equals(name, sigInfo.name) + && Objects.equals(address, sigInfo.address); + } + + @Override + public int hashCode() { + return Objects.hash(number, name, address); + } +} diff --git a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java index 72fa2dd8535..15952d9951b 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java @@ -8,10 +8,9 @@ import datadog.crashtracking.dto.Metadata; import datadog.crashtracking.dto.OSInfo; import datadog.crashtracking.dto.ProcInfo; -import datadog.crashtracking.dto.SemanticVersion; +import datadog.crashtracking.dto.SigInfo; import datadog.crashtracking.dto.StackFrame; import datadog.crashtracking.dto.StackTrace; -import datadog.environment.SystemProperties; import java.time.OffsetDateTime; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; @@ -19,6 +18,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Locale; +import java.util.regex.Matcher; import java.util.regex.Pattern; public final class HotspotCrashLogParser { @@ -42,6 +42,7 @@ enum State { private static final Pattern PLUS_SPLITTER = Pattern.compile("\\+"); private static final Pattern SPACE_SPLITTER = Pattern.compile("\\s+"); private static final Pattern NEWLINE_SPLITTER = Pattern.compile("\n"); + private static final Pattern SIGNAL_PARSER = Pattern.compile("\\s*(\\w+) \\((\\w+)\\).*"); private StackFrame parseLine(String line) { String functionName = null; @@ -116,7 +117,7 @@ private StackFrame parseLine(String line) { } public CrashLog parse(String uuid, String crashLog) { - String signal = null; + SigInfo sigInfo = null; String pid = null; List frames = new ArrayList<>(); String datetime = null; @@ -137,13 +138,27 @@ public CrashLog parse(String uuid, String crashLog) { // break out of the message block state = State.HEADER; } else if (!"#".equals(line)) { - if (signal == null) { + if (sigInfo == null) { + String name = null, address = null; + int number = 0; // first non-empty line after the message is the signal - signal = - line.substring( - 3, - line.indexOf( - ' ', 3)); // # SIGSEGV (0xb) at pc=0x00007f8b1c0b3d7d, pid=1, tid=1 + final Matcher signalMatcher = SIGNAL_PARSER.matcher(line.substring(3)); + // # SIGSEGV (0xb) at pc=0x00007f8b1c0b3d7d, pid=1, tid=1 + if (signalMatcher.matches()) { + name = signalMatcher.group(1); + try { + number = Integer.decode(signalMatcher.group(2)); + } catch (Throwable ignored) { + } + } + + int pcIdx = line.indexOf("pc="); + if (pcIdx > -1) { + int endIdx = line.indexOf(',', pcIdx); + address = line.substring(pcIdx + 3, endIdx); + } + sigInfo = new SigInfo(number, name, address); + int pidIdx = line.indexOf("pid="); if (pidIdx > -1) { int endIdx = line.indexOf(',', pidIdx); @@ -194,21 +209,19 @@ public CrashLog parse(String uuid, String crashLog) { // incomplete crash log incomplete = true; } - String message = "Process terminated by signal " + (signal != null ? signal : "UNKNOWN"); + String message = "Process terminated by signal " + (sigInfo != null ? sigInfo.name : "UNKNOWN"); ErrorData error = - new ErrorData(signal, message, new StackTrace(frames.toArray(new StackFrame[0]))); + new ErrorData( + sigInfo != null ? sigInfo.name : null, + message, + new StackTrace(frames.toArray(new StackFrame[0]))); // We can not really extract the full metadata and os info from the crash log // This code assumes the parser is run on the same machine as the crash happened Metadata metadata = new Metadata("dd-trace-java", VersionInfo.VERSION, "java", null); - OSInfo osInfo = - new OSInfo( - SystemProperties.get("os.arch"), - SystemProperties.get("sun.arch.data.model"), - SystemProperties.get("os.name"), - SemanticVersion.of(SystemProperties.get("os.version"))); ProcInfo procInfo = pid != null ? new ProcInfo(pid) : null; - return new CrashLog(uuid, incomplete, datetime, error, metadata, osInfo, procInfo, "1.0"); + return new CrashLog( + uuid, incomplete, datetime, error, metadata, OSInfo.current(), procInfo, sigInfo, "1.0"); } static String dateTimeToISO(String datetime) { diff --git a/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/CrashUploaderTest.java b/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/CrashUploaderTest.java index b1c65195dfb..30d575359cc 100644 --- a/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/CrashUploaderTest.java +++ b/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/CrashUploaderTest.java @@ -1,16 +1,19 @@ package datadog.crashtracking; +import static datadog.crashtracking.CrashUploader.HEADER_DD_EVP_SUBDOMAIN; +import static datadog.crashtracking.CrashUploader.HEADER_DD_TELEMETRY_API_VERSION; +import static datadog.crashtracking.CrashUploader.TELEMETRY_API_VERSION; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.when; +import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import datadog.common.version.VersionInfo; import datadog.crashtracking.dto.CrashLog; import datadog.environment.SystemProperties; import datadog.trace.api.Config; -import datadog.trace.bootstrap.config.provider.ConfigProvider; import datadog.trace.util.TraceUtils; import java.io.BufferedReader; import java.io.ByteArrayOutputStream; @@ -50,16 +53,8 @@ public class CrashUploaderTest { // TODO: Add a test to verify overall request timeout rather than IO timeout private final Duration REQUEST_TIMEOUT = Duration.ofSeconds(10); - private final Duration REQUEST_IO_OPERATION_TIMEOUT = Duration.ofSeconds(5); - - // Termination timeout has to be longer than request timeout to make sure that all callbacks are - // called before the termination. - private final Duration TERMINATION_TIMEOUT = REQUEST_TIMEOUT.plus(Duration.ofSeconds(5)); - - private final Duration FOREVER_REQUEST_TIMEOUT = Duration.ofSeconds(1000); private Config config = spy(Config.get()); - private ConfigProvider configProvider; private final MockWebServer server = new MockWebServer(); private HttpUrl url; @@ -77,6 +72,8 @@ public void setup() throws IOException { when(config.getServiceName()).thenReturn(SERVICE); when(config.getVersion()).thenReturn(VERSION); when(config.getFinalCrashTrackingTelemetryUrl()).thenReturn(server.url(URL_PATH).toString()); + when(config.getFinalCrashTrackingErrorTrackingUrl()) + .thenReturn(server.url(URL_PATH).toString()); when(config.isCrashTrackingAgentless()).thenReturn(false); when(config.getApiKey()).thenReturn(null); } @@ -136,9 +133,9 @@ private Path getResourcePath(String resourceName) throws Exception { } @Test - public void testCrashPing() throws Exception { + public void testTelemetryCrashPing() throws Exception { // Given - final String expected = readFileAsString("golden/sample-ping-for-telemetry.txt"); + final String expected = readFileAsString("golden/telemetry/sample-ping-for-telemetry.txt"); ConfigManager.StoredConfig crashConfig = new ConfigManager.StoredConfig.Builder(config) .reportUUID(SAMPLE_UUID) @@ -148,7 +145,7 @@ public void testCrashPing() throws Exception { // When uploader = new CrashUploader(config, crashConfig); server.enqueue(new MockResponse().setResponseCode(200)); - uploader.notifyCrashStarted(null); + uploader.sendPingToTelemetry(null); final RecordedRequest recordedRequest = server.takeRequest(5, TimeUnit.SECONDS); @@ -175,6 +172,71 @@ public void testCrashPing() throws Exception { assertCommonPayload(event); } + @Test + public void testErrorTrackingCrashPing() throws Exception { + // Given + final ObjectMapper mapper = new ObjectMapper(); + final Map expected = + mapper.readValue(readFileAsString("golden/errortracking/sample-ping.txt"), Map.class); + // remove ddtags and osinfo if present from the expected (they will be checked apart) + expected.remove("ddtags"); + expected.remove("os_info"); + expected.remove("timestamp"); + ConfigManager.StoredConfig crashConfig = + new ConfigManager.StoredConfig.Builder(config) + .reportUUID(SAMPLE_UUID) + .processTags("a:b") + .runtimeId("1234") + .tags(ConfigManager.getMergedTagsForSerialization(Config.get())) // take the real ones + .build(); + // When + uploader = new CrashUploader(config, crashConfig); + server.enqueue(new MockResponse().setResponseCode(200)); + + uploader.sendPingToErrorTracking(null); + + final RecordedRequest recordedRequest = server.takeRequest(5, TimeUnit.SECONDS); + + // Then + assertEquals(url, recordedRequest.getRequestUrl()); + final String requestBody = recordedRequest.getBody().readUtf8(); + final Map extracted = mapper.readValue(requestBody, Map.class); + + assertNotNull(extracted.remove("timestamp")); + + // assert osInfo + final Map osInfo = (Map) extracted.remove("os_info"); + + assertNotNull(osInfo); + assertNotNull(osInfo.get("architecture")); + assertNotNull(osInfo.get("version")); + assertNotNull(osInfo.get("os_type")); + assertDoesNotThrow(() -> Long.parseLong((String) osInfo.get("bitness"))); // 32 or 64 typically + + // assert ddtags + String ddtags = (String) extracted.remove("ddtags"); + assertNotNull(ddtags); + assertTrue(ddtags.contains(crashConfig.tags)); // includes the tags + assertTrue(ddtags.contains("service:")); // has the service + assertTrue(ddtags.contains("uuid:" + crashConfig.reportUUID)); + assertTrue(ddtags.contains("is_crash_ping:true")); + + // assert platform independent equality + assertEquals( + expected, + extracted, + () -> { + try { + return "Expected: " + + mapper.writeValueAsString(expected) + + "\nbut got: " + + mapper.writeValueAsString(extracted); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }); + } + @ParameterizedTest @ValueSource( strings = { @@ -184,7 +246,7 @@ public void testCrashPing() throws Exception { }) public void testTelemetryHappyPath(String log) throws Exception { // Given - CrashLog expected = CrashLog.fromJson(readFileAsString("golden/" + log)); + CrashLog expected = CrashLog.fromJson(readFileAsString("golden/telemetry/" + log)); ConfigManager.StoredConfig crashConfig = new ConfigManager.StoredConfig.Builder(config) .reportUUID(SAMPLE_UUID) @@ -194,7 +256,7 @@ public void testTelemetryHappyPath(String log) throws Exception { // When uploader = new CrashUploader(config, crashConfig); server.enqueue(new MockResponse().setResponseCode(200)); - uploader.uploadToTelemetry(getResourcePath(log), SAMPLE_UUID); + uploader.remoteUpload(readFileAsString(log), true, false); final RecordedRequest recordedRequest = server.takeRequest(5, TimeUnit.SECONDS); @@ -222,8 +284,75 @@ public void testTelemetryHappyPath(String log) throws Exception { assertCommonPayload(event); } + @ParameterizedTest + @ValueSource( + strings = { + "sample-crash-for-telemetry.txt", + "sample-crash-for-telemetry-2.txt", + "sample-crash-for-telemetry-3.txt" + }) + public void testErrorTrackingHappyPath(String log) throws Exception { + // Given + final ObjectMapper mapper = new ObjectMapper(); + final Map expected = + mapper.readValue(readFileAsString("golden/errortracking/" + log), Map.class); + // remove ddtags and osinfo if present from the expected (they will be checked apart) + expected.remove("ddtags"); + expected.remove("os_info"); + ConfigManager.StoredConfig crashConfig = + new ConfigManager.StoredConfig.Builder(config) + .reportUUID(SAMPLE_UUID) + .processTags("a:b") + .runtimeId("1234") + .tags(ConfigManager.getMergedTagsForSerialization(Config.get())) // take the real ones + .build(); + // When + uploader = new CrashUploader(config, crashConfig); + server.enqueue(new MockResponse().setResponseCode(200)); + uploader.remoteUpload(readFileAsString(log), false, true); + + final RecordedRequest recordedRequest = server.takeRequest(5, TimeUnit.SECONDS); + + // Then + assertEquals(url, recordedRequest.getRequestUrl()); + final String requestBody = recordedRequest.getBody().readUtf8(); + final Map extracted = mapper.readValue(requestBody, Map.class); + + // assert osInfo + final Map osInfo = (Map) extracted.remove("os_info"); + + assertNotNull(osInfo); + assertNotNull(osInfo.get("architecture")); + assertNotNull(osInfo.get("version")); + assertNotNull(osInfo.get("os_type")); + assertDoesNotThrow(() -> Long.parseLong((String) osInfo.get("bitness"))); // 32 or 64 typically + + // assert ddtags + String ddtags = (String) extracted.remove("ddtags"); + assertNotNull(ddtags); + assertTrue(ddtags.contains(crashConfig.tags)); // includes the tags + assertTrue(ddtags.contains("service:")); // has the service + assertTrue(ddtags.contains("uuid:" + crashConfig.reportUUID)); + assertTrue(ddtags.contains("is_crash:true")); + + // assert platform independent equality + assertEquals( + expected, + extracted, + () -> { + try { + return "Expected: " + + mapper.writeValueAsString(expected) + + "\nbut got: " + + mapper.writeValueAsString(extracted); + } catch (JsonProcessingException e) { + throw new RuntimeException(e); + } + }); + } + private void assertCommonHeader(JsonNode event) { - assertEquals(CrashUploader.TELEMETRY_API_VERSION, event.get("api_version").asText()); + assertEquals(TELEMETRY_API_VERSION, event.get("api_version").asText()); assertEquals("logs", event.get("request_type").asText()); assertEquals("crashtracker", event.get("origin").asText()); assertEquals("1234", event.get("runtime_id").asText()); @@ -252,22 +381,33 @@ public void testTelemetryUnrecognizedFile() throws Exception { // When uploader = new CrashUploader(config, crashConfig); server.enqueue(new MockResponse().setResponseCode(200)); - assertFalse(uploader.uploadToTelemetry(getResourcePath("no-crash.txt"), null)); + uploader.upload(getResourcePath("no-crash.txt")); + assertNotNull(server.takeRequest(5, TimeUnit.SECONDS)); // still sending incomplete report } - @Test - public void testAgentlessRequest() throws Exception { + @ParameterizedTest + @ValueSource(booleans = {true, false}) + public void testAgentlessRequest(boolean telemetryOrErrorTracking) throws Exception { when(config.getApiKey()).thenReturn(API_KEY_VALUE); when(config.isCrashTrackingAgentless()).thenReturn(true); ConfigManager.StoredConfig crashConfig = new ConfigManager.StoredConfig.Builder(config).build(); uploader = new CrashUploader(config, crashConfig); server.enqueue(new MockResponse().setResponseCode(200)); - uploader.upload(getResourcePath("sample-crash.txt")); + uploader.remoteUpload( + readFileAsString("sample-crash.txt"), telemetryOrErrorTracking, !telemetryOrErrorTracking); - final RecordedRequest recordedRequest = server.takeRequest(5, TimeUnit.SECONDS); + RecordedRequest recordedRequest = server.takeRequest(5, TimeUnit.SECONDS); + // common assertNotNull(recordedRequest); assertEquals(API_KEY_VALUE, recordedRequest.getHeader("DD-API-KEY")); + if (telemetryOrErrorTracking) { + // telemetry + assertEquals( + TELEMETRY_API_VERSION, recordedRequest.getHeader(HEADER_DD_TELEMETRY_API_VERSION)); + } else { + assertNull(recordedRequest.getHeader(HEADER_DD_EVP_SUBDOMAIN)); + } } @Test @@ -286,6 +426,34 @@ public void test404() throws Exception { // it would be nice if the test asserted the log line was written out, but it's not essential } + @Test + public void testParallelSendResilience() throws Exception { + try (MockWebServer errorTrackingServer = new MockWebServer()) { + errorTrackingServer.start(); + // Given + when(config.getFinalCrashTrackingErrorTrackingUrl()) + .thenReturn(errorTrackingServer.url(URL_PATH).toString()); + final CrashUploader uploader = + new CrashUploader(config, new ConfigManager.StoredConfig.Builder(config).build()); + + // When + server.enqueue(new MockResponse().setResponseCode(200)); + // simulate a timeout from error tracking + errorTrackingServer.enqueue( + new MockResponse().setHeadersDelay(4, TimeUnit.SECONDS).setResponseCode(200)); + long start = System.nanoTime(); + uploader.remoteUpload(readFileAsString("sample-crash.txt"), true, true); + long elapsed = System.nanoTime() - start; + + // Then + // assert both backends have been contacted + assertNotNull(server.takeRequest(1, TimeUnit.SECONDS)); + assertNotNull(errorTrackingServer.takeRequest(1, TimeUnit.SECONDS)); + // assert that we waited + assertTrue(TimeUnit.NANOSECONDS.toMillis(elapsed) > 2000); + } + } + @Test public void test404Agentless() throws Exception { // test added to get the coverage checks to pass since we log conditionally in this case diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-2.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-2.txt new file mode 100644 index 00000000000..7d9d569cd49 --- /dev/null +++ b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-2.txt @@ -0,0 +1 @@ +{"timestamp":"2024-09-20T13:19:06Z","ddsource":"crashtracker","error":{"is_crash":true,"kind":"SIGSEGV","message":"Process terminated by signal SIGSEGV","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"file":"libpthread.so.0","function":"__pthread_clockjoin_ex"}]}},"sig_info":{"si_addr":"0x00007f011ab1ccd5 (sent by kill)","si_signo_human_readable":"SIGSEGV","si_signo":11},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":"15.7.1"},"ddtags":"host:COMP-LTD4YC2T2N,runtime-id:edbdef5c-7038-4286-97f8-e76bb65b281f,language:jvm,env:,version:,service:worker.org.gradle.process.internal.worker.GradleWorkerMain,is_crash:true,data_schema_version:1.0,language_name:jvm,language_version:1.8.0_345,tracer_version:1.56.0-snapshot_13a0b2e4c7,uuid:a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3"} diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-3.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-3.txt new file mode 100644 index 00000000000..d27f52c4ef9 --- /dev/null +++ b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-3.txt @@ -0,0 +1 @@ +{"ddsource":"crashtracker","error":{"is_crash":true,"kind":"INVALID","message":"Process terminated by signal INVALID","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"file":"libjvm.dylib","function":"VMError::report_and_die()"},{"file":"libjvm.dylib","function":"report_vm_error(char const*, int, char const*, char const*)"},{"file":"libjvm.dylib","function":"report_java_out_of_memory(char const*)"},{"file":"libjvm.dylib","function":"CollectedHeap::common_mem_allocate_noinit(KlassHandle, unsigned long, Thread*)"},{"file":"libjvm.dylib","function":"TypeArrayKlass::allocate_common(int, bool, Thread*)"},{"file":"libjvm.dylib","function":"InterpreterRuntime::newarray(JavaThread*, BasicType, int)"},{"function":"datadog.smoketest.crashtracking.CrashtrackingTestApplication.main([Ljava/lang/String;)V","line":105},{"function":" ~StubRoutines::call_stub"},{"file":"libjvm.dylib","function":"JavaCalls::call_helper(JavaValue*, methodHandle*, JavaCallArguments*, Thread*)"},{"file":"libjvm.dylib","function":"jni_invoke_static(JNIEnv_*, JavaValue*, _jobject*, JNICallType, _jmethodID*, JNI_ArgumentPusher*, Thread*)"},{"file":"libjvm.dylib","function":"jni_CallStaticVoidMethod"},{"file":"java","function":"JavaMain"},{"file":"libsystem_pthread.dylib","function":"_pthread_start"},{"file":"libsystem_pthread.dylib","function":"thread_start"}]}},"sig_info":{"si_addr":"0x0000000000000000","si_signo_human_readable":"INVALID","si_signo":0},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":"15.7.1"},"ddtags":"host:COMP-LTD4YC2T2N,runtime-id:edbdef5c-7038-4286-97f8-e76bb65b281f,language:jvm,env:,version:,service:worker.org.gradle.process.internal.worker.GradleWorkerMain,is_crash:true,data_schema_version:1.0,language_name:jvm,language_version:1.8.0_345,tracer_version:1.56.0-snapshot_13a0b2e4c7,uuid:a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3"} diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry.txt new file mode 100644 index 00000000000..9ee1e8c7e3c --- /dev/null +++ b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry.txt @@ -0,0 +1 @@ +{"timestamp":"2023-10-17T20:25:14+08:00","ddsource":"crashtracker","error":{"is_crash":true,"kind":"SIGSEGV","message":"Process terminated by signal SIGSEGV","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"file":"libjvm.so","function":"vframeStreamForte::forte_next()"},{"file":"libjvm.so","function":"forte_fill_call_trace_given_top(JavaThread*, ASGCT_CallTrace*, int, frame) [clone .isra.22]"},{"file":"libjvm.so","function":"AsyncGetCallTrace"},{"file":"libjavaProfiler586350205236920700.so","function":"Profiler::getJavaTraceAsync(void*, ASGCT_CallFrame*, int, StackContext*, bool*) [clone .isra.531]"},{"file":"libjavaProfiler586350205236920700.so","function":"Profiler::recordSample(void*, unsigned long long, int, int, Event*)"},{"file":"libjavaProfiler586350205236920700.so","function":"WallClock::sharedSignalHandler(int, siginfo_t*, void*)"},{"file":"libpthread.so.0","function":"libpthread.so.0+0x12cf0"},{"file":"libjvm.so","function":"JfrStackTrace::record_safe(JavaThread*, int)"},{"file":"libjvm.so","function":"JfrStackTraceRepository::record_for_leak_profiler(JavaThread*, int)"},{"file":"libjvm.so","function":"ObjectSampler::sample(HeapWordImpl**, unsigned long, JavaThread*)"},{"file":"libjvm.so","function":"JfrAllocationTracer::JfrAllocationTracer(Klass const*, HeapWordImpl**, unsigned long, bool, JavaThread*)"},{"file":"libjvm.so","function":"AllocTracer::send_allocation_in_new_tlab(Klass*, HeapWordImpl**, unsigned long, unsigned long, JavaThread*)"},{"file":"libjvm.so","function":"MemAllocator::allocate() const"},{"file":"libjvm.so","function":"InstanceKlass::allocate_objArray(int, int, JavaThread*)"},{"file":"libjvm.so","function":"OptoRuntime::new_array_C(Klass*, int, JavaThread*)"}]}},"sig_info":{"si_addr":"0x00007f37a18bc187","si_signo_human_readable":"SIGSEGV","si_signo":11},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":"15.7.1"},"ddtags":"host:COMP-LTD4YC2T2N,runtime-id:edbdef5c-7038-4286-97f8-e76bb65b281f,language:jvm,env:,version:,service:worker.org.gradle.process.internal.worker.GradleWorkerMain,is_crash:true,data_schema_version:1.0,language_name:jvm,language_version:1.8.0_345,tracer_version:1.56.0-snapshot_13a0b2e4c7,uuid:a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3"} diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-ping.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-ping.txt new file mode 100644 index 00000000000..7e53a9ca4f5 --- /dev/null +++ b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-ping.txt @@ -0,0 +1 @@ +{"timestamp":"2025-11-19T13:55:23.532+01:00","ddsource":"crashtracker","error":{"is_crash":false,"message":"Crashtracker crash ping: crash processing started","source_type":"crashtracking"},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":"15.7.1"},"ddtags":"host:COMP-LTD4YC2T2N,runtime-id:3a52cf96-691d-4705-a911-7f89f4d4d9c3,language:jvm,env:,version:,service:worker.org.gradle.process.internal.worker.GradleWorkerMain,is_crash_ping:true,data_schema_version:1.0,language_name:jvm,language_version:1.8.0_345,tracer_version:1.56.0-snapshot_13a0b2e4c7,uuid:a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3"} diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/sample-crash-for-telemetry-2.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry-2.txt similarity index 100% rename from dd-java-agent/agent-crashtracking/src/test/resources/golden/sample-crash-for-telemetry-2.txt rename to dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry-2.txt diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/sample-crash-for-telemetry-3.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry-3.txt similarity index 100% rename from dd-java-agent/agent-crashtracking/src/test/resources/golden/sample-crash-for-telemetry-3.txt rename to dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry-3.txt diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/sample-crash-for-telemetry.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry.txt similarity index 100% rename from dd-java-agent/agent-crashtracking/src/test/resources/golden/sample-crash-for-telemetry.txt rename to dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry.txt diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/sample-ping-for-telemetry.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-ping-for-telemetry.txt similarity index 100% rename from dd-java-agent/agent-crashtracking/src/test/resources/golden/sample-ping-for-telemetry.txt rename to dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-ping-for-telemetry.txt diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 6d6bd4d1d46..c9fa8b8f757 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -826,6 +826,7 @@ public static String getHostName() { private final String agentUnixDomainSocket; private final String agentNamedPipe; private final int agentTimeout; + /** Should be set to {@code true} when running in agentless mode in a JVM without TLS */ private final boolean forceClearTextHttpForIntakeClient; @@ -4863,7 +4864,7 @@ public Map getMergedCrashTrackingTags() { + crashTrackingTags.size() + jmxTags.size() + runtimeTags.size() - + 3 /* for serviceName and host and language */); + + 5 /* for serviceName and host and language and env and version */); result.put(HOST_TAG, host); // Host goes first to allow to override it result.putAll(getGlobalTags()); result.putAll(jmxTags); @@ -4873,6 +4874,8 @@ public Map getMergedCrashTrackingTags() { // and may chose to override it. result.put(SERVICE_TAG, serviceName); result.put(LANGUAGE_TAG_KEY, LANGUAGE_TAG_VALUE); + result.put(VERSION, getVersion()); + result.put(ENV, getEnv()); return Collections.unmodifiableMap(result); } @@ -5085,6 +5088,16 @@ public String getFinalCrashTrackingTelemetryUrl() { } } + public String getFinalCrashTrackingErrorTrackingUrl() { + if (crashTrackingAgentless) { + // when agentless crashTracking is turned on we send directly to our intake + return "https://error-tracking-intake." + site + "/api/v2/errorsintake"; + } else { + // when agentless are not set we send to the dd trace agent running locally + return "http://" + agentHost + ":" + agentPort + "/evp_proxy/v4/api/v2/errorsintake"; + } + } + public boolean isJmxFetchIntegrationEnabled( final Iterable integrationNames, final boolean defaultEnabled) { return configProvider.isEnabled(integrationNames, "jmxfetch.", ".enabled", defaultEnabled); From 0882768757e33effc025769cf6605a304805e330 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Fri, 21 Nov 2025 15:55:36 +0100 Subject: [PATCH 2/5] Fix error tracking payload --- .../datadog/crashtracking/CrashUploader.java | 26 +++++++++++++++---- .../datadog/crashtracking/dto/ErrorData.java | 18 +------------ .../datadog/crashtracking/dto/StackTrace.java | 10 +++++++ .../sample-crash-for-telemetry-2.txt | 2 +- .../sample-crash-for-telemetry-3.txt | 2 +- .../sample-crash-for-telemetry.txt | 2 +- .../golden/errortracking/sample-ping.txt | 2 +- 7 files changed, 36 insertions(+), 26 deletions(-) diff --git a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java index 8b57575db59..d830b92a97a 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java @@ -155,7 +155,7 @@ public CrashUploader(@Nonnull final ConfigManager.StoredConfig storedConfig) { dispatcher, /* dispatcher */ telemetryUrl, // will be overridden in each request true, /* retryOnConnectionFailure */ - 4, /* maxRunningRequests */ // allows 1 blocked ping and two crash request + 4, /* maxRunningRequests */ // not having one request blocking the others configProvider.getString(CRASH_TRACKING_PROXY_HOST), configProvider.getInteger(CRASH_TRACKING_PROXY_PORT), configProvider.getString(CRASH_TRACKING_PROXY_USERNAME), @@ -200,8 +200,7 @@ void sendPingToErrorTracking(String error) { null, "Crashtracker crash ping: " + (error != null ? error : "crash processing started"), - null, - false), + null), null, OSInfo.current(), null, @@ -293,6 +292,7 @@ void uploadToLogs(@Nonnull String message, @Nonnull PrintStream out) throws IOEx } // @VisibleForTesting + @SuppressForbidden static String extractErrorKind(String fileContent) { Matcher matcher = ERROR_MESSAGE_PATTERN.matcher(fileContent); if (!matcher.find()) { @@ -324,6 +324,7 @@ static String extractErrorKind(String fileContent) { Pattern.DOTALL | Pattern.MULTILINE); // @VisibleForTesting + @SuppressForbidden static String extractErrorMessage(String fileContent) { Matcher matcher = ERROR_MESSAGE_PATTERN.matcher(fileContent); if (!matcher.find()) { @@ -491,8 +492,23 @@ private RequestBody makeErrorTrackingRequestBody(@Nonnull CrashLog payload, bool writer.name("timestamp").value(payload.timestamp); writer.name("ddsource").value("crashtracker"); // error payload - writer.name("error"); - payload.error.writeAsJson(writer); // flat write an already serialized json object + if (payload.error != null) { + writer.name("error"); + writer.beginObject(); + writer.name("source_type").value("crashtracking"); + if (!isPing) { + writer.name("is_crash").value(true); + } + writer.name("type").value(payload.error.kind); + writer.name("message").value(payload.error.message); + if (payload.error.stack != null) { + writer.name("stack"); + // payload.error.message + payload.error.stack.writeAsJson(writer); + // flat write an already serialized json object + } + writer.endObject(); + } // signal info if (payload.sigInfo != null) { writer.name("sig_info"); diff --git a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/ErrorData.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/ErrorData.java index 6d654571a29..9ccb9a96e0d 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/ErrorData.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/ErrorData.java @@ -1,18 +1,11 @@ package datadog.crashtracking.dto; import com.squareup.moshi.Json; -import com.squareup.moshi.JsonAdapter; -import com.squareup.moshi.JsonWriter; -import com.squareup.moshi.Moshi; -import java.io.IOException; import java.util.Objects; public final class ErrorData { - private static final JsonAdapter ADAPTER = - new Moshi.Builder().build().adapter(ErrorData.class); - @Json(name = "is_crash") - public final boolean isCrash; + public final boolean isCrash = true; public final String kind; public final String message; @@ -23,14 +16,9 @@ public final class ErrorData { public final StackTrace stack; public ErrorData(String kind, String message, StackTrace stack) { - this(kind, message, stack, true); - } - - public ErrorData(String kind, String message, StackTrace stack, boolean isCrash) { this.kind = kind; this.message = message; this.stack = stack; - this.isCrash = isCrash; } @Override @@ -53,8 +41,4 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(isCrash, kind, message, sourceType, stack); } - - public void writeAsJson(final JsonWriter writer) throws IOException { - ADAPTER.toJson(writer, this); - } } diff --git a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/StackTrace.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/StackTrace.java index 23a12ca2c69..9467e01fcba 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/StackTrace.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/StackTrace.java @@ -1,9 +1,15 @@ package datadog.crashtracking.dto; +import com.squareup.moshi.JsonAdapter; +import com.squareup.moshi.JsonWriter; +import com.squareup.moshi.Moshi; +import java.io.IOException; import java.util.Arrays; import java.util.Objects; public final class StackTrace { + private static final JsonAdapter ADAPTER = + new Moshi.Builder().build().adapter(StackTrace.class); private static final String FORMAT = "CrashTrackerV1"; public final String format = FORMAT; @@ -29,4 +35,8 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(format, Arrays.hashCode(frames)); } + + public void writeAsJson(final JsonWriter writer) throws IOException { + ADAPTER.toJson(writer, this); + } } diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-2.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-2.txt index 7d9d569cd49..6020ddd1ee0 100644 --- a/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-2.txt +++ b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-2.txt @@ -1 +1 @@ -{"timestamp":"2024-09-20T13:19:06Z","ddsource":"crashtracker","error":{"is_crash":true,"kind":"SIGSEGV","message":"Process terminated by signal SIGSEGV","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"file":"libpthread.so.0","function":"__pthread_clockjoin_ex"}]}},"sig_info":{"si_addr":"0x00007f011ab1ccd5 (sent by kill)","si_signo_human_readable":"SIGSEGV","si_signo":11},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":"15.7.1"},"ddtags":"host:COMP-LTD4YC2T2N,runtime-id:edbdef5c-7038-4286-97f8-e76bb65b281f,language:jvm,env:,version:,service:worker.org.gradle.process.internal.worker.GradleWorkerMain,is_crash:true,data_schema_version:1.0,language_name:jvm,language_version:1.8.0_345,tracer_version:1.56.0-snapshot_13a0b2e4c7,uuid:a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3"} +{"timestamp":"2024-09-20T13:19:06Z","ddsource":"crashtracker","error":{"source_type":"crashtracking","is_crash":true,"type":"SIGSEGV","message":"Process terminated by signal SIGSEGV","stack":{"format":"CrashTrackerV1","frames":[{"file":"libpthread.so.0","function":"__pthread_clockjoin_ex"}]}},"sig_info":{"si_addr":"0x00007f011ab1ccd5 (sent by kill)","si_signo_human_readable":"SIGSEGV","si_signo":11}} diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-3.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-3.txt index d27f52c4ef9..c7da3a9dbde 100644 --- a/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-3.txt +++ b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry-3.txt @@ -1 +1 @@ -{"ddsource":"crashtracker","error":{"is_crash":true,"kind":"INVALID","message":"Process terminated by signal INVALID","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"file":"libjvm.dylib","function":"VMError::report_and_die()"},{"file":"libjvm.dylib","function":"report_vm_error(char const*, int, char const*, char const*)"},{"file":"libjvm.dylib","function":"report_java_out_of_memory(char const*)"},{"file":"libjvm.dylib","function":"CollectedHeap::common_mem_allocate_noinit(KlassHandle, unsigned long, Thread*)"},{"file":"libjvm.dylib","function":"TypeArrayKlass::allocate_common(int, bool, Thread*)"},{"file":"libjvm.dylib","function":"InterpreterRuntime::newarray(JavaThread*, BasicType, int)"},{"function":"datadog.smoketest.crashtracking.CrashtrackingTestApplication.main([Ljava/lang/String;)V","line":105},{"function":" ~StubRoutines::call_stub"},{"file":"libjvm.dylib","function":"JavaCalls::call_helper(JavaValue*, methodHandle*, JavaCallArguments*, Thread*)"},{"file":"libjvm.dylib","function":"jni_invoke_static(JNIEnv_*, JavaValue*, _jobject*, JNICallType, _jmethodID*, JNI_ArgumentPusher*, Thread*)"},{"file":"libjvm.dylib","function":"jni_CallStaticVoidMethod"},{"file":"java","function":"JavaMain"},{"file":"libsystem_pthread.dylib","function":"_pthread_start"},{"file":"libsystem_pthread.dylib","function":"thread_start"}]}},"sig_info":{"si_addr":"0x0000000000000000","si_signo_human_readable":"INVALID","si_signo":0},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":"15.7.1"},"ddtags":"host:COMP-LTD4YC2T2N,runtime-id:edbdef5c-7038-4286-97f8-e76bb65b281f,language:jvm,env:,version:,service:worker.org.gradle.process.internal.worker.GradleWorkerMain,is_crash:true,data_schema_version:1.0,language_name:jvm,language_version:1.8.0_345,tracer_version:1.56.0-snapshot_13a0b2e4c7,uuid:a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3"} +{"ddsource":"crashtracker","error":{"source_type":"crashtracking","is_crash":true,"type":"INVALID","message":"Process terminated by signal INVALID","stack":{"format":"CrashTrackerV1","frames":[{"file":"libjvm.dylib","function":"VMError::report_and_die()"},{"file":"libjvm.dylib","function":"report_vm_error(char const*, int, char const*, char const*)"},{"file":"libjvm.dylib","function":"report_java_out_of_memory(char const*)"},{"file":"libjvm.dylib","function":"CollectedHeap::common_mem_allocate_noinit(KlassHandle, unsigned long, Thread*)"},{"file":"libjvm.dylib","function":"TypeArrayKlass::allocate_common(int, bool, Thread*)"},{"file":"libjvm.dylib","function":"InterpreterRuntime::newarray(JavaThread*, BasicType, int)"},{"function":"datadog.smoketest.crashtracking.CrashtrackingTestApplication.main([Ljava/lang/String;)V","line":105},{"function":" ~StubRoutines::call_stub"},{"file":"libjvm.dylib","function":"JavaCalls::call_helper(JavaValue*, methodHandle*, JavaCallArguments*, Thread*)"},{"file":"libjvm.dylib","function":"jni_invoke_static(JNIEnv_*, JavaValue*, _jobject*, JNICallType, _jmethodID*, JNI_ArgumentPusher*, Thread*)"},{"file":"libjvm.dylib","function":"jni_CallStaticVoidMethod"},{"file":"java","function":"JavaMain"},{"file":"libsystem_pthread.dylib","function":"_pthread_start"},{"file":"libsystem_pthread.dylib","function":"thread_start"}]}},"sig_info":{"si_addr":"0x0000000000000000","si_signo_human_readable":"INVALID","si_signo":0}} diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry.txt index 9ee1e8c7e3c..e60c327ab73 100644 --- a/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry.txt +++ b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-crash-for-telemetry.txt @@ -1 +1 @@ -{"timestamp":"2023-10-17T20:25:14+08:00","ddsource":"crashtracker","error":{"is_crash":true,"kind":"SIGSEGV","message":"Process terminated by signal SIGSEGV","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"file":"libjvm.so","function":"vframeStreamForte::forte_next()"},{"file":"libjvm.so","function":"forte_fill_call_trace_given_top(JavaThread*, ASGCT_CallTrace*, int, frame) [clone .isra.22]"},{"file":"libjvm.so","function":"AsyncGetCallTrace"},{"file":"libjavaProfiler586350205236920700.so","function":"Profiler::getJavaTraceAsync(void*, ASGCT_CallFrame*, int, StackContext*, bool*) [clone .isra.531]"},{"file":"libjavaProfiler586350205236920700.so","function":"Profiler::recordSample(void*, unsigned long long, int, int, Event*)"},{"file":"libjavaProfiler586350205236920700.so","function":"WallClock::sharedSignalHandler(int, siginfo_t*, void*)"},{"file":"libpthread.so.0","function":"libpthread.so.0+0x12cf0"},{"file":"libjvm.so","function":"JfrStackTrace::record_safe(JavaThread*, int)"},{"file":"libjvm.so","function":"JfrStackTraceRepository::record_for_leak_profiler(JavaThread*, int)"},{"file":"libjvm.so","function":"ObjectSampler::sample(HeapWordImpl**, unsigned long, JavaThread*)"},{"file":"libjvm.so","function":"JfrAllocationTracer::JfrAllocationTracer(Klass const*, HeapWordImpl**, unsigned long, bool, JavaThread*)"},{"file":"libjvm.so","function":"AllocTracer::send_allocation_in_new_tlab(Klass*, HeapWordImpl**, unsigned long, unsigned long, JavaThread*)"},{"file":"libjvm.so","function":"MemAllocator::allocate() const"},{"file":"libjvm.so","function":"InstanceKlass::allocate_objArray(int, int, JavaThread*)"},{"file":"libjvm.so","function":"OptoRuntime::new_array_C(Klass*, int, JavaThread*)"}]}},"sig_info":{"si_addr":"0x00007f37a18bc187","si_signo_human_readable":"SIGSEGV","si_signo":11},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":"15.7.1"},"ddtags":"host:COMP-LTD4YC2T2N,runtime-id:edbdef5c-7038-4286-97f8-e76bb65b281f,language:jvm,env:,version:,service:worker.org.gradle.process.internal.worker.GradleWorkerMain,is_crash:true,data_schema_version:1.0,language_name:jvm,language_version:1.8.0_345,tracer_version:1.56.0-snapshot_13a0b2e4c7,uuid:a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3"} +{"timestamp":"2023-10-17T20:25:14+08:00","ddsource":"crashtracker","error":{"source_type":"crashtracking","is_crash":true,"type":"SIGSEGV","message":"Process terminated by signal SIGSEGV","stack":{"format":"CrashTrackerV1","frames":[{"file":"libjvm.so","function":"vframeStreamForte::forte_next()"},{"file":"libjvm.so","function":"forte_fill_call_trace_given_top(JavaThread*, ASGCT_CallTrace*, int, frame) [clone .isra.22]"},{"file":"libjvm.so","function":"AsyncGetCallTrace"},{"file":"libjavaProfiler586350205236920700.so","function":"Profiler::getJavaTraceAsync(void*, ASGCT_CallFrame*, int, StackContext*, bool*) [clone .isra.531]"},{"file":"libjavaProfiler586350205236920700.so","function":"Profiler::recordSample(void*, unsigned long long, int, int, Event*)"},{"file":"libjavaProfiler586350205236920700.so","function":"WallClock::sharedSignalHandler(int, siginfo_t*, void*)"},{"file":"libpthread.so.0","function":"libpthread.so.0+0x12cf0"},{"file":"libjvm.so","function":"JfrStackTrace::record_safe(JavaThread*, int)"},{"file":"libjvm.so","function":"JfrStackTraceRepository::record_for_leak_profiler(JavaThread*, int)"},{"file":"libjvm.so","function":"ObjectSampler::sample(HeapWordImpl**, unsigned long, JavaThread*)"},{"file":"libjvm.so","function":"JfrAllocationTracer::JfrAllocationTracer(Klass const*, HeapWordImpl**, unsigned long, bool, JavaThread*)"},{"file":"libjvm.so","function":"AllocTracer::send_allocation_in_new_tlab(Klass*, HeapWordImpl**, unsigned long, unsigned long, JavaThread*)"},{"file":"libjvm.so","function":"MemAllocator::allocate() const"},{"file":"libjvm.so","function":"InstanceKlass::allocate_objArray(int, int, JavaThread*)"},{"file":"libjvm.so","function":"OptoRuntime::new_array_C(Klass*, int, JavaThread*)"}]}},"sig_info":{"si_addr":"0x00007f37a18bc187","si_signo_human_readable":"SIGSEGV","si_signo":11}} diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-ping.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-ping.txt index 7e53a9ca4f5..5b0fa90599b 100644 --- a/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-ping.txt +++ b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample-ping.txt @@ -1 +1 @@ -{"timestamp":"2025-11-19T13:55:23.532+01:00","ddsource":"crashtracker","error":{"is_crash":false,"message":"Crashtracker crash ping: crash processing started","source_type":"crashtracking"},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":"15.7.1"},"ddtags":"host:COMP-LTD4YC2T2N,runtime-id:3a52cf96-691d-4705-a911-7f89f4d4d9c3,language:jvm,env:,version:,service:worker.org.gradle.process.internal.worker.GradleWorkerMain,is_crash_ping:true,data_schema_version:1.0,language_name:jvm,language_version:1.8.0_345,tracer_version:1.56.0-snapshot_13a0b2e4c7,uuid:a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3"} +{"timestamp":"2025-11-19T13:55:23.532+01:00","ddsource":"crashtracker","error":{"message":"Crashtracker crash ping: crash processing started","source_type":"crashtracking"},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":"15.7.1"},"ddtags":"host:COMP-LTD4YC2T2N,runtime-id:3a52cf96-691d-4705-a911-7f89f4d4d9c3,language:jvm,env:,version:,service:worker.org.gradle.process.internal.worker.GradleWorkerMain,is_crash_ping:true,data_schema_version:1.0,language_name:jvm,language_version:1.8.0_345,tracer_version:1.56.0-snapshot_13a0b2e4c7,uuid:a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3"} From 2dca2703cb5881db543c7174bd70d58d1d14e9eb Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Mon, 24 Nov 2025 10:14:24 +0100 Subject: [PATCH 3/5] Handle OOM and clean exit --- .../datadog/crashtracking/CrashUploader.java | 40 ++- .../datadog/crashtracking/dto/CrashLog.java | 5 +- .../parsers/HotspotCrashLogParser.java | 33 ++- .../crashtracking/CrashUploaderTest.java | 6 +- .../golden/errortracking/sample_oom.txt | 1 + .../sample-crash-for-telemetry-2.txt | 2 +- .../sample-crash-for-telemetry-3.txt | 2 +- .../telemetry/sample-crash-for-telemetry.txt | 2 +- .../resources/golden/telemetry/sample_oom.txt | 1 + .../src/test/resources/sample_oom.txt | 269 ++++++++++++++++++ .../trace/util/AgentThreadFactory.java | 2 + 11 files changed, 338 insertions(+), 25 deletions(-) create mode 100644 dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample_oom.txt create mode 100644 dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample_oom.txt create mode 100644 dd-java-agent/agent-crashtracking/src/test/resources/sample_oom.txt diff --git a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java index d830b92a97a..ec5d3f575bc 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java @@ -7,6 +7,7 @@ import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_UPLOAD_TIMEOUT; import static datadog.trace.api.config.CrashTrackingConfig.CRASH_TRACKING_UPLOAD_TIMEOUT_DEFAULT; import static datadog.trace.api.telemetry.LogCollector.SEND_TELEMETRY; +import static datadog.trace.util.AgentThreadFactory.AgentThread.CRASHTRACKING_HTTP_DISPATCHER; import static datadog.trace.util.TraceUtils.normalizeServiceName; import static datadog.trace.util.TraceUtils.normalizeTagValue; import static java.time.format.DateTimeFormatter.ISO_OFFSET_DATE_TIME; @@ -24,6 +25,7 @@ import datadog.trace.api.Config; import datadog.trace.api.DDTags; import datadog.trace.bootstrap.config.provider.ConfigProvider; +import datadog.trace.util.AgentThreadFactory; import datadog.trace.util.PidHelper; import de.thetaphi.forbiddenapis.SuppressForbidden; import edu.umd.cs.findbugs.annotations.NonNull; @@ -37,6 +39,10 @@ import java.util.HashMap; import java.util.Map; import java.util.Scanner; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -50,6 +56,7 @@ import okhttp3.RequestBody; import okhttp3.Response; import okio.Buffer; +import okio.ByteString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -112,6 +119,7 @@ public void onResponse(Call call, Response response) throws IOException { private final HttpUrl errorTrackingUrl; private final OkHttpClient uploadClient; private final Dispatcher dispatcher; + private final ExecutorService executor; private final boolean agentless; private final String tags; private final long timeout; @@ -127,8 +135,16 @@ public CrashUploader(@Nonnull final ConfigManager.StoredConfig storedConfig) { this.telemetryUrl = HttpUrl.get(config.getFinalCrashTrackingTelemetryUrl()); this.errorTrackingUrl = HttpUrl.get(config.getFinalCrashTrackingErrorTrackingUrl()); this.agentless = config.isCrashTrackingAgentless(); - this.dispatcher = new Dispatcher(); - dispatcher.setMaxRequests(4); + // This is the same thing OkHttp Dispatcher is doing except thread naming and daemonization + this.executor = + new ThreadPoolExecutor( + 0, + 4, + 60, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + new AgentThreadFactory(CRASHTRACKING_HTTP_DISPATCHER)); + this.dispatcher = new Dispatcher(executor); final StringBuilder tagsBuilder = new StringBuilder(storedConfig.tags != null ? storedConfig.tags : ""); @@ -227,7 +243,11 @@ public void upload(@Nonnull Path file) { } catch (Throwable t) { log.error("Unable to print the error crash as a log message", t); } - remoteUpload(fileContent, true, true); + try { + remoteUpload(fileContent, true, true); + } finally { + uploadClient.dispatcher().cancelAll(); + } } // @VisibleForTesting @@ -258,6 +278,7 @@ void remoteUpload( } if (remaining > 0) { dispatcher.cancelAll(); + uploadClient.connectionPool().evictAll(); log.error( SEND_TELEMETRY, "Failed to fully send the crash report with UUID {}. Still {} calls remaining", @@ -491,21 +512,22 @@ private RequestBody makeErrorTrackingRequestBody(@Nonnull CrashLog payload, bool writer.beginObject(); writer.name("timestamp").value(payload.timestamp); writer.name("ddsource").value("crashtracker"); + // tags + writer.name("ddtags").value(tagsForErrorTracking(payload.uuid, isPing, payload.incomplete)); // error payload if (payload.error != null) { writer.name("error"); writer.beginObject(); - writer.name("source_type").value("crashtracking"); if (!isPing) { writer.name("is_crash").value(true); } writer.name("type").value(payload.error.kind); writer.name("message").value(payload.error.message); + writer.name("source_type").value("crashtracking"); if (payload.error.stack != null) { writer.name("stack"); - // payload.error.message - payload.error.stack.writeAsJson(writer); // flat write an already serialized json object + payload.error.stack.writeAsJson(writer); } writer.endObject(); } @@ -537,11 +559,11 @@ private RequestBody makeErrorTrackingRequestBody(@Nonnull CrashLog payload, bool "os.version")); // this has been restructured under OsInfo so taking raw here writer.endObject(); } - // tags - writer.name("ddtags").value(tagsForErrorTracking(payload.uuid, isPing, payload.incomplete)); writer.endObject(); } - return RequestBody.create(APPLICATION_JSON, buf.readByteString()); + final ByteString tmp = buf.readByteString(); + System.err.println(tmp.utf8()); + return RequestBody.create(APPLICATION_JSON, tmp); } } diff --git a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/CrashLog.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/CrashLog.java index c1254052023..aa3a672ac9c 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/CrashLog.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/dto/CrashLog.java @@ -36,8 +36,8 @@ public final class CrashLog { @Json(name = "version_id") public final int version = VERSION; - // not serialized - public final transient SigInfo sigInfo; + @Json(name = "sig_info") + public final SigInfo sigInfo; public CrashLog( String uuid, @@ -118,6 +118,7 @@ public boolean equalsForTest(Object o) { && Objects.equals(timestamp, crashLog.timestamp) && Objects.equals(error, crashLog.error) && Objects.equals(procInfo, crashLog.procInfo) + && Objects.equals(sigInfo, crashLog.sigInfo) && Objects.equals(dataSchemaVersion, crashLog.dataSchemaVersion); } } diff --git a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java index 15952d9951b..6eef77dbb95 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/parsers/HotspotCrashLogParser.java @@ -26,6 +26,7 @@ public final class HotspotCrashLogParser { DateTimeFormatter.ofPattern("EEE MMM ppd HH:mm:ss yyyy zzz", Locale.getDefault()); private static final DateTimeFormatter OFFSET_DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern("EEE MMM ppd HH:mm:ss yyyy X", Locale.getDefault()); + private static final String OOM_MARKER = "OutOfMemory encountered: "; enum State { NEW, @@ -122,6 +123,7 @@ public CrashLog parse(String uuid, String crashLog) { List frames = new ArrayList<>(); String datetime = null; boolean incomplete = false; + String oomMessage = null; String[] lines = NEWLINE_SPLITTER.split(crashLog); outer: @@ -137,8 +139,11 @@ public CrashLog parse(String uuid, String crashLog) { if (line.toLowerCase().contains("core dump")) { // break out of the message block state = State.HEADER; - } else if (!"#".equals(line)) { - if (sigInfo == null) { + } else if (!"#".equals(line) && (sigInfo == null && oomMessage == null)) { + final int oomIdx = line.indexOf(OOM_MARKER); + if (oomIdx > 0) { + oomMessage = line.substring(oomIdx + OOM_MARKER.length()); + } else { String name = null, address = null; int number = 0; // first non-empty line after the message is the signal @@ -157,7 +162,9 @@ public CrashLog parse(String uuid, String crashLog) { int endIdx = line.indexOf(',', pcIdx); address = line.substring(pcIdx + 3, endIdx); } - sigInfo = new SigInfo(number, name, address); + if (name != null) { + sigInfo = new SigInfo(number, name, address); + } int pidIdx = line.indexOf("pid="); if (pidIdx > -1) { @@ -193,7 +200,10 @@ public CrashLog parse(String uuid, String crashLog) { state = State.DONE; } else { // Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code) - frames.add(parseLine(line)); + final StackFrame frame = parseLine(line); + if (frame != null) { + frames.add(frame); + } } break; case DONE: @@ -209,13 +219,18 @@ public CrashLog parse(String uuid, String crashLog) { // incomplete crash log incomplete = true; } - String message = "Process terminated by signal " + (sigInfo != null ? sigInfo.name : "UNKNOWN"); + final String kind; + final String message; + if (oomMessage != null) { + kind = "OutOfMemory"; + message = oomMessage; + } else { + kind = sigInfo != null && sigInfo.name != null ? sigInfo.name : "UNKNOWN"; + message = "Process terminated by signal " + kind; + } ErrorData error = - new ErrorData( - sigInfo != null ? sigInfo.name : null, - message, - new StackTrace(frames.toArray(new StackFrame[0]))); + new ErrorData(kind, message, new StackTrace(frames.toArray(new StackFrame[0]))); // We can not really extract the full metadata and os info from the crash log // This code assumes the parser is run on the same machine as the crash happened Metadata metadata = new Metadata("dd-trace-java", VersionInfo.VERSION, "java", null); diff --git a/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/CrashUploaderTest.java b/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/CrashUploaderTest.java index 30d575359cc..c8ace5f87c8 100644 --- a/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/CrashUploaderTest.java +++ b/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/CrashUploaderTest.java @@ -242,7 +242,8 @@ public void testErrorTrackingCrashPing() throws Exception { strings = { "sample-crash-for-telemetry.txt", "sample-crash-for-telemetry-2.txt", - "sample-crash-for-telemetry-3.txt" + "sample-crash-for-telemetry-3.txt", + "sample_oom.txt" }) public void testTelemetryHappyPath(String log) throws Exception { // Given @@ -289,7 +290,8 @@ public void testTelemetryHappyPath(String log) throws Exception { strings = { "sample-crash-for-telemetry.txt", "sample-crash-for-telemetry-2.txt", - "sample-crash-for-telemetry-3.txt" + "sample-crash-for-telemetry-3.txt", + "sample_oom.txt" }) public void testErrorTrackingHappyPath(String log) throws Exception { // Given diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample_oom.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample_oom.txt new file mode 100644 index 00000000000..aec5691b6af --- /dev/null +++ b/dd-java-agent/agent-crashtracking/src/test/resources/golden/errortracking/sample_oom.txt @@ -0,0 +1 @@ +{"timestamp":"2025-11-24T09:43:29+01:00","ddsource":"crashtracker","error":{"is_crash":true,"type":"OutOfMemory","message":"Java heap space","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"file":"libjvm.dylib","function":"VMError::report_and_die(int, char const*, char const*, char*, Thread*, unsigned char*, void*, void*, char const*, int, unsigned long)"},{"file":"libjvm.dylib","function":"report_fatal(VMErrorType, char const*, int, char const*, ...)"},{"file":"libjvm.dylib","function":"report_java_out_of_memory(char const*)"},{"file":"libjvm.dylib","function":"MemAllocator::Allocation::check_out_of_memory()"},{"file":"libjvm.dylib","function":"MemAllocator::allocate() const"},{"file":"libjvm.dylib","function":"CollectedHeap::array_allocate(Klass*, unsigned long, int, bool, JavaThread*)"},{"file":"libjvm.dylib","function":"OptoRuntime::new_array_C(Klass*, int, JavaThread*)"},{"function":"Java"},{"function":" ~RuntimeStub::_new_array_Java 0x00000001124cb638"},{"function":"java.nio.ByteBuffer.allocate(I)Ljava/nio/ByteBuffer;"},{"function":"datadog.communication.serialization.FlushingBuffer.(ILdatadog/communication/serialization/ByteBufferConsumer;)V","line":6},{"function":"datadog.trace.agent.common.writer.PayloadDispatcherImpl.selectMapper()V","line":126},{"function":"datadog.trace.agent.common.writer.PayloadDispatcherImpl.addTrace(Ljava/util/List;)V","line":1},{"function":"datadog.trace.agent.common.writer.TraceProcessingWorker$TraceSerializingHandler.onEvent(Ljava/lang/Object;)V","line":22},{"function":"datadog.trace.agent.common.writer.TraceProcessingWorker$TraceSerializingHandler.consumeFromPrimaryQueue()V","line":21},{"function":"datadog.trace.agent.common.writer.TraceProcessingWorker$TraceSerializingHandler.runDutyCycle()V","line":12},{"function":"datadog.trace.agent.common.writer.TraceProcessingWorker$TraceSerializingHandler.run()V","line":1},{"function":"java.lang.Thread.runWith(Ljava/lang/Object;Ljava/lang/Runnable;)V"},{"function":"java.lang.Thread.run()V"},{"function":" ~StubRoutines::call_stub 0x00000001123b0140"}]}}} diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry-2.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry-2.txt index d679a8960b4..4e8611fc812 100644 --- a/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry-2.txt +++ b/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry-2.txt @@ -1 +1 @@ -{"data_schema_version":"1.0","error":{"is_crash":true,"kind":"SIGSEGV","message":"Process terminated by signal SIGSEGV","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"file":"libpthread.so.0","function":"__pthread_clockjoin_ex"}]}},"incomplete":false,"metadata":{"family":"java","library_name":"dd-trace-java","library_version":"1.56.0-SNAPSHOT~37a6360670"},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":{"Semantic":[15,7,1]}},"proc_info":{"pid":"576034"},"timestamp":"2024-09-20T13:19:06Z","uuid":"a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3","version_id":0} +{"data_schema_version":"1.0","error":{"is_crash":true,"kind":"SIGSEGV","message":"Process terminated by signal SIGSEGV","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"file":"libpthread.so.0","function":"__pthread_clockjoin_ex"}]}},"incomplete":false,"metadata":{"family":"java","library_name":"dd-trace-java","library_version":"1.57.0-SNAPSHOT~0882768757"},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":{"Semantic":[15,7,1]}},"proc_info":{"pid":"576034"},"sig_info":{"address":"0x00007f011ab1ccd5 (sent by kill)","name":"SIGSEGV","number":11},"timestamp":"2024-09-20T13:19:06Z","uuid":"a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3","version_id":0} diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry-3.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry-3.txt index 0ea79a66e8f..3315ac5d08e 100644 --- a/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry-3.txt +++ b/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry-3.txt @@ -1 +1 @@ -{"data_schema_version":"1.0","error":{"is_crash":true,"kind":"INVALID","message":"Process terminated by signal INVALID","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"file":"libjvm.dylib","function":"VMError::report_and_die()"},{"file":"libjvm.dylib","function":"report_vm_error(char const*, int, char const*, char const*)"},{"file":"libjvm.dylib","function":"report_java_out_of_memory(char const*)"},{"file":"libjvm.dylib","function":"CollectedHeap::common_mem_allocate_noinit(KlassHandle, unsigned long, Thread*)"},{"file":"libjvm.dylib","function":"TypeArrayKlass::allocate_common(int, bool, Thread*)"},{"file":"libjvm.dylib","function":"InterpreterRuntime::newarray(JavaThread*, BasicType, int)"},{"function":"datadog.smoketest.crashtracking.CrashtrackingTestApplication.main([Ljava/lang/String;)V","line":105},{"function":" ~StubRoutines::call_stub"},{"file":"libjvm.dylib","function":"JavaCalls::call_helper(JavaValue*, methodHandle*, JavaCallArguments*, Thread*)"},{"file":"libjvm.dylib","function":"jni_invoke_static(JNIEnv_*, JavaValue*, _jobject*, JNICallType, _jmethodID*, JNI_ArgumentPusher*, Thread*)"},{"file":"libjvm.dylib","function":"jni_CallStaticVoidMethod"},{"file":"java","function":"JavaMain"},{"file":"libsystem_pthread.dylib","function":"_pthread_start"},{"file":"libsystem_pthread.dylib","function":"thread_start"}]}},"incomplete":false,"metadata":{"family":"java","library_name":"dd-trace-java","library_version":"1.56.0-SNAPSHOT~37a6360670"},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":{"Semantic":[15,7,1]}},"proc_info":{"pid":"96267"},"uuid":"a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3","version_id":0} +{"data_schema_version":"1.0","error":{"is_crash":true,"kind":"INVALID","message":"Process terminated by signal INVALID","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"file":"libjvm.dylib","function":"VMError::report_and_die()"},{"file":"libjvm.dylib","function":"report_vm_error(char const*, int, char const*, char const*)"},{"file":"libjvm.dylib","function":"report_java_out_of_memory(char const*)"},{"file":"libjvm.dylib","function":"CollectedHeap::common_mem_allocate_noinit(KlassHandle, unsigned long, Thread*)"},{"file":"libjvm.dylib","function":"TypeArrayKlass::allocate_common(int, bool, Thread*)"},{"file":"libjvm.dylib","function":"InterpreterRuntime::newarray(JavaThread*, BasicType, int)"},{"function":"datadog.smoketest.crashtracking.CrashtrackingTestApplication.main([Ljava/lang/String;)V","line":105},{"function":" ~StubRoutines::call_stub"},{"file":"libjvm.dylib","function":"JavaCalls::call_helper(JavaValue*, methodHandle*, JavaCallArguments*, Thread*)"},{"file":"libjvm.dylib","function":"jni_invoke_static(JNIEnv_*, JavaValue*, _jobject*, JNICallType, _jmethodID*, JNI_ArgumentPusher*, Thread*)"},{"file":"libjvm.dylib","function":"jni_CallStaticVoidMethod"},{"file":"java","function":"JavaMain"},{"file":"libsystem_pthread.dylib","function":"_pthread_start"},{"file":"libsystem_pthread.dylib","function":"thread_start"}]}},"incomplete":false,"metadata":{"family":"java","library_name":"dd-trace-java","library_version":"1.57.0-SNAPSHOT~0882768757"},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":{"Semantic":[15,7,1]}},"proc_info":{"pid":"96267"},"sig_info":{"address":"0x0000000000000000","name":"INVALID","number":0},"uuid":"a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3","version_id":0} diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry.txt index e46021c6f53..467c1cb5fdc 100644 --- a/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry.txt +++ b/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample-crash-for-telemetry.txt @@ -1 +1 @@ - {"data_schema_version":"1.0","error":{"is_crash":true,"kind":"SIGSEGV","message":"Process terminated by signal SIGSEGV","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"file":"libjvm.so","function":"vframeStreamForte::forte_next()"},{"file":"libjvm.so","function":"forte_fill_call_trace_given_top(JavaThread*, ASGCT_CallTrace*, int, frame) [clone .isra.22]"},{"file":"libjvm.so","function":"AsyncGetCallTrace"},{"file":"libjavaProfiler586350205236920700.so","function":"Profiler::getJavaTraceAsync(void*, ASGCT_CallFrame*, int, StackContext*, bool*) [clone .isra.531]"},{"file":"libjavaProfiler586350205236920700.so","function":"Profiler::recordSample(void*, unsigned long long, int, int, Event*)"},{"file":"libjavaProfiler586350205236920700.so","function":"WallClock::sharedSignalHandler(int, siginfo_t*, void*)"},{"file":"libpthread.so.0","function":"libpthread.so.0+0x12cf0"},{"file":"libjvm.so","function":"JfrStackTrace::record_safe(JavaThread*, int)"},{"file":"libjvm.so","function":"JfrStackTraceRepository::record_for_leak_profiler(JavaThread*, int)"},{"file":"libjvm.so","function":"ObjectSampler::sample(HeapWordImpl**, unsigned long, JavaThread*)"},{"file":"libjvm.so","function":"JfrAllocationTracer::JfrAllocationTracer(Klass const*, HeapWordImpl**, unsigned long, bool, JavaThread*)"},{"file":"libjvm.so","function":"AllocTracer::send_allocation_in_new_tlab(Klass*, HeapWordImpl**, unsigned long, unsigned long, JavaThread*)"},{"file":"libjvm.so","function":"MemAllocator::allocate() const"},{"file":"libjvm.so","function":"InstanceKlass::allocate_objArray(int, int, JavaThread*)"},{"file":"libjvm.so","function":"OptoRuntime::new_array_C(Klass*, int, JavaThread*)"}]}},"incomplete":false,"metadata":{"family":"java","library_name":"dd-trace-java","library_version":"1.56.0-SNAPSHOT~37a6360670"},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":{"Semantic":[15,7,1]}},"proc_info":{"pid":"161958"},"timestamp":"2023-10-17T20:25:14+08:00","uuid":"a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3","version_id":0} +{"data_schema_version":"1.0","error":{"is_crash":true,"kind":"SIGSEGV","message":"Process terminated by signal SIGSEGV","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"file":"libjvm.so","function":"vframeStreamForte::forte_next()"},{"file":"libjvm.so","function":"forte_fill_call_trace_given_top(JavaThread*, ASGCT_CallTrace*, int, frame) [clone .isra.22]"},{"file":"libjvm.so","function":"AsyncGetCallTrace"},{"file":"libjavaProfiler586350205236920700.so","function":"Profiler::getJavaTraceAsync(void*, ASGCT_CallFrame*, int, StackContext*, bool*) [clone .isra.531]"},{"file":"libjavaProfiler586350205236920700.so","function":"Profiler::recordSample(void*, unsigned long long, int, int, Event*)"},{"file":"libjavaProfiler586350205236920700.so","function":"WallClock::sharedSignalHandler(int, siginfo_t*, void*)"},{"file":"libpthread.so.0","function":"libpthread.so.0+0x12cf0"},{"file":"libjvm.so","function":"JfrStackTrace::record_safe(JavaThread*, int)"},{"file":"libjvm.so","function":"JfrStackTraceRepository::record_for_leak_profiler(JavaThread*, int)"},{"file":"libjvm.so","function":"ObjectSampler::sample(HeapWordImpl**, unsigned long, JavaThread*)"},{"file":"libjvm.so","function":"JfrAllocationTracer::JfrAllocationTracer(Klass const*, HeapWordImpl**, unsigned long, bool, JavaThread*)"},{"file":"libjvm.so","function":"AllocTracer::send_allocation_in_new_tlab(Klass*, HeapWordImpl**, unsigned long, unsigned long, JavaThread*)"},{"file":"libjvm.so","function":"MemAllocator::allocate() const"},{"file":"libjvm.so","function":"InstanceKlass::allocate_objArray(int, int, JavaThread*)"},{"file":"libjvm.so","function":"OptoRuntime::new_array_C(Klass*, int, JavaThread*)"}]}},"incomplete":false,"metadata":{"family":"java","library_name":"dd-trace-java","library_version":"1.57.0-SNAPSHOT~0882768757"},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":{"Semantic":[15,7,1]}},"proc_info":{"pid":"161958"},"sig_info":{"address":"0x00007f37a18bc187","name":"SIGSEGV","number":11},"timestamp":"2023-10-17T20:25:14+08:00","uuid":"a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3","version_id":0} diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample_oom.txt b/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample_oom.txt new file mode 100644 index 00000000000..3f9791eff17 --- /dev/null +++ b/dd-java-agent/agent-crashtracking/src/test/resources/golden/telemetry/sample_oom.txt @@ -0,0 +1 @@ +{"data_schema_version":"1.0","error":{"is_crash":true,"kind":"OutOfMemory","message":"Java heap space","source_type":"crashtracking","stack":{"format":"CrashTrackerV1","frames":[{"file":"libjvm.dylib","function":"VMError::report_and_die(int, char const*, char const*, char*, Thread*, unsigned char*, void*, void*, char const*, int, unsigned long)"},{"file":"libjvm.dylib","function":"report_fatal(VMErrorType, char const*, int, char const*, ...)"},{"file":"libjvm.dylib","function":"report_java_out_of_memory(char const*)"},{"file":"libjvm.dylib","function":"MemAllocator::Allocation::check_out_of_memory()"},{"file":"libjvm.dylib","function":"MemAllocator::allocate() const"},{"file":"libjvm.dylib","function":"CollectedHeap::array_allocate(Klass*, unsigned long, int, bool, JavaThread*)"},{"file":"libjvm.dylib","function":"OptoRuntime::new_array_C(Klass*, int, JavaThread*)"},{"function":"Java"},{"function":" ~RuntimeStub::_new_array_Java 0x00000001124cb638"},{"function":"java.nio.ByteBuffer.allocate(I)Ljava/nio/ByteBuffer;"},{"function":"datadog.communication.serialization.FlushingBuffer.(ILdatadog/communication/serialization/ByteBufferConsumer;)V","line":6},{"function":"datadog.trace.agent.common.writer.PayloadDispatcherImpl.selectMapper()V","line":126},{"function":"datadog.trace.agent.common.writer.PayloadDispatcherImpl.addTrace(Ljava/util/List;)V","line":1},{"function":"datadog.trace.agent.common.writer.TraceProcessingWorker$TraceSerializingHandler.onEvent(Ljava/lang/Object;)V","line":22},{"function":"datadog.trace.agent.common.writer.TraceProcessingWorker$TraceSerializingHandler.consumeFromPrimaryQueue()V","line":21},{"function":"datadog.trace.agent.common.writer.TraceProcessingWorker$TraceSerializingHandler.runDutyCycle()V","line":12},{"function":"datadog.trace.agent.common.writer.TraceProcessingWorker$TraceSerializingHandler.run()V","line":1},{"function":"java.lang.Thread.runWith(Ljava/lang/Object;Ljava/lang/Runnable;)V"},{"function":"java.lang.Thread.run()V"},{"function":" ~StubRoutines::call_stub 0x00000001123b0140"}]}},"incomplete":false,"metadata":{"family":"java","library_name":"dd-trace-java","library_version":"1.57.0-SNAPSHOT~0882768757"},"os_info":{"architecture":"aarch64","bitness":"64","os_type":"Mac OS X","version":{"Semantic":[15,7,1]}},"proc_info":{"pid":"74045"},"timestamp":"2025-11-24T09:43:29+01:00","uuid":"a4194cd6-8cb3-45fd-9bd9-3af83e0a3ad3","version_id":0} diff --git a/dd-java-agent/agent-crashtracking/src/test/resources/sample_oom.txt b/dd-java-agent/agent-crashtracking/src/test/resources/sample_oom.txt new file mode 100644 index 00000000000..1f7b1e0cc79 --- /dev/null +++ b/dd-java-agent/agent-crashtracking/src/test/resources/sample_oom.txt @@ -0,0 +1,269 @@ +# +# A fatal error has been detected by the Java Runtime Environment: +# +# Internal Error (debug.cpp:271), pid=74045, tid=34819 +# fatal error: OutOfMemory encountered: Java heap space +# +# JRE version: OpenJDK Runtime Environment Zulu21.30+15-CA (21.0.1+12) (build 21.0.1+12-LTS) +# Java VM: OpenJDK 64-Bit Server VM Zulu21.30+15-CA (21.0.1+12-LTS, mixed mode, sharing, tiered, compressed oops, compressed class ptrs, g1 gc, bsd-aarch64) +# No core dump will be written. Core dumps have been disabled. To enable core dumping, try "ulimit -c unlimited" before starting Java again +# + +--------------- S U M M A R Y ------------ + +Command Line: -Ddd.service=12andrea -Ddd.env=andrea -Ddd.dogstatsd.start-delay=0 -XX:OnError=/var/folders/rr/hft9w7dd6zx84_8p_vszl_ym0000gp/T/dd_crash_uploader.sh -javaagent:./dd-java-agent.jar -Xmx26m -Xms26m -XX:+CrashOnOutOfMemoryError -XX:ErrorFile=/var/folders/rr/hft9w7dd6zx84_8p_vszl_ym0000gp/T/hs_err_pid.log build/libs/spring-petclinic-3.5.0.jar + +Host: "MacBookPro18,4" arm64, 10 cores, 64G, Darwin 24.6.0, macOS 15.7.1 (24G231) +Time: Mon Nov 24 09:43:29 2025 CET elapsed time: 11.468691 seconds (0d 0h 0m 11s) + +--------------- T H R E A D --------------- + +Current thread (0x000000011c35a200): JavaThread "dd-trace-processor" daemon [_thread_in_vm, id=34819, stack(0x00000001744fc000,0x00000001746ff000) (2060K)] + +Stack: [0x00000001744fc000,0x00000001746ff000], sp=0x00000001746fe440, free space=2057k +Native frames: (J=compiled Java code, j=interpreted, Vv=VM code, C=native code) +V [libjvm.dylib+0xaa5f24] VMError::report_and_die(int, char const*, char const*, char*, Thread*, unsigned char*, void*, void*, char const*, int, unsigned long)+0x480 (debug.cpp:271) +V [libjvm.dylib+0x314c74] report_fatal(VMErrorType, char const*, int, char const*, ...)+0x88 +V [libjvm.dylib+0x315248] report_java_out_of_memory(char const*)+0xd8 +V [libjvm.dylib+0x7d9764] MemAllocator::Allocation::check_out_of_memory()+0x8c +V [libjvm.dylib+0x7da2f4] MemAllocator::allocate() const+0xf4 +V [libjvm.dylib+0x2c3de4] CollectedHeap::array_allocate(Klass*, unsigned long, int, bool, JavaThread*)+0x34 +V [libjvm.dylib+0x8ea9ec] OptoRuntime::new_array_C(Klass*, int, JavaThread*)+0x68 +Java frames: (J=compiled Java code, j=interpreted, Vv=VM code) +v ~RuntimeStub::_new_array_Java 0x00000001124cb638 +J 3896 c2 java.nio.ByteBuffer.allocate(I)Ljava/nio/ByteBuffer; java.base@21.0.1 (20 bytes) @ 0x0000000112ad51e8 [0x0000000112ad4fc0+0x0000000000000228] +j datadog.communication.serialization.FlushingBuffer.(ILdatadog/communication/serialization/ByteBufferConsumer;)V+6 +j datadog.trace.agent.common.writer.PayloadDispatcherImpl.selectMapper()V+126 +j datadog.trace.agent.common.writer.PayloadDispatcherImpl.addTrace(Ljava/util/List;)V+1 +j datadog.trace.agent.common.writer.TraceProcessingWorker$TraceSerializingHandler.onEvent(Ljava/lang/Object;)V+22 +j datadog.trace.agent.common.writer.TraceProcessingWorker$TraceSerializingHandler.consumeFromPrimaryQueue()V+21 +j datadog.trace.agent.common.writer.TraceProcessingWorker$TraceSerializingHandler.runDutyCycle()V+12 +j datadog.trace.agent.common.writer.TraceProcessingWorker$TraceSerializingHandler.run()V+1 +j java.lang.Thread.runWith(Ljava/lang/Object;Ljava/lang/Runnable;)V+5 java.base@21.0.1 +j java.lang.Thread.run()V+19 java.base@21.0.1 +v ~StubRoutines::call_stub 0x00000001123b0140 +-------------------------------------------------------------------------------- +Decoding CodeBlob, name: _new_array_Java, at [0x00000001124cb600, 0x00000001124cb688] 136 bytes +[MachCode] + 0x00000001124cb600: 1f20 03d5 | ff43 00d1 | fd7b 00a9 | e903 0091 | 89cf 01f9 | e003 01aa | e103 022a | e203 1caa + 0x00000001124cb620: c900 0010 | 8830 95d2 | 483f a0f2 | 2800 c0f2 | ff27 bfa9 | 0001 3fd6 | 1f20 03d5 | 1f00 80f2 + 0x00000001124cb640: 1f00 80f2 | ff43 0091 | 9fcf 01f9 | 9fd3 01f9 | 80fb 41f9 | 9ffb 01f9 | 8a07 40f9 | 8a00 00b5 + 0x00000001124cb660: fd7b 40a9 | ff43 0091 | c003 5fd6 | ec03 1faa | 2af7 ffb0 | 4a01 0291 | fd7b 40a9 | ff43 0091 + 0x00000001124cb680: 4001 1fd6 | 0000 0000 +[/MachCode] +-------------------------------------------------------------------------------- + + +Compiled method (c2) 11471 3896 4 java.nio.ByteBuffer::allocate (20 bytes) + total in heap [0x0000000112ad4e10,0x0000000112ad53d0] = 1472 + relocation [0x0000000112ad4f60,0x0000000112ad4f98] = 56 + main code [0x0000000112ad4fc0,0x0000000112ad5258] = 664 + stub code [0x0000000112ad5258,0x0000000112ad5268] = 16 + metadata [0x0000000112ad5268,0x0000000112ad52e8] = 128 + scopes data [0x0000000112ad52e8,0x0000000112ad5320] = 56 + scopes pcs [0x0000000112ad5320,0x0000000112ad5370] = 80 + dependencies [0x0000000112ad5370,0x0000000112ad53a0] = 48 + handler table [0x0000000112ad53a0,0x0000000112ad53d0] = 48 + +[Constant Pool (empty)] + +[MachCode] +[Verified Entry Point] + # {method} {0x00000070000c3e90} 'allocate' '(I)Ljava/nio/ByteBuffer;' in 'java/nio/ByteBuffer' + # parm0: c_rarg1 = int + # [sp+0x30] (sp of caller) + 0x0000000112ad4fc0: 1f20 03d5 | e953 40d1 | 3f01 00f9 | ffc3 00d1 | fd7b 02a9 | e813 0018 | 8923 40b9 | 1f01 09eb + 0x0000000112ad4fe0: e112 0054 | ed03 012a | c110 f837 | 93df 40f9 | 8ae7 40f9 | 6be2 0091 | fd03 40b2 | 7f01 0aeb + 0x0000000112ad5000: 020d 0054 | 8bdf 00f9 + + 0x0000000112ad5008: ; {metadata('java/nio/HeapByteBuffer')} + 0x0000000112ad5008: 6a03 a0d2 | 0a7c 9ef2 | 7d02 00f9 | 6a0a 00b9 | 70c1 80f9 | 7f0e 00b9 | 6a42 0091 | 5f01 00f9 + 0x0000000112ad5028: 5ffd 00a9 | 5ffd 01a9 | bf3a 03d5 | aa7d 4093 | bf01 4471 | 2800 0054 | bf01 4471 | 480c 0054 + 0x0000000112ad5048: 80df 40f9 | 4a5d 0091 | 4bf1 7d92 | 8ce7 40f9 | 0b00 0b8b | 7f01 0ceb | 620b 0054 | 8bdf 00f9 + 0x0000000112ad5068: 70c1 80f9 | 1d00 00f9 + + 0x0000000112ad5070: ; {metadata({type array byte})} + 0x0000000112ad5070: 0e00 a0d2 | 0e58 84f2 | 7001 81f9 | 0e08 00b9 | 4cfd 43d3 | 7041 81f9 | 0a40 0091 | 0d0c 00b9 + 0x0000000112ad5090: 8b09 00d1 | 6821 00f1 | 4300 0054 + + 0x0000000112ad509c: ; {runtime_call StubRoutines (final stubs)} + 0x0000000112ad509c: 6979 e497 | 6b00 1036 | 5f7d 81a8 | 5f7d 81a8 | 4b00 0836 | 5f7d 81a8 | 4b00 0036 | 5f01 00f9 + 0x0000000112ad50bc: bf3a 03d5 | 6d22 00b9 | 0b00 a012 | 6d1e 00b9 | 6b0e 00b9 | bf3b 03d5 | ea03 13aa | eb03 00aa + 0x0000000112ad50dc: 6b01 0aca | 0cfc 43d3 | ee03 0032 | 6bfd 54d3 | 6c32 00b9 | f403 7cb2 | 6eb6 0039 | 7fba 0039 + 0x0000000112ad50fc: eb00 00b4 | 4afd 49d3 | ab83 bfd2 | 6001 0a8b | 0a00 c039 | 5f09 0071 | 6101 0054 | 7f2a 00b9 + 0x0000000112ad511c: bf3b 03d5 | 740a 00f9 | e003 13aa | fd7b 42a9 | ffc3 0091 + + 0x0000000112ad5130: ; {poll_return} + 0x0000000112ad5130: 882b 42f9 | ff63 28eb | c807 0054 | c003 5fd6 | 8a27 40f9 | 8b2f 40f9 | bf3b 03d5 | 0c00 c039 + 0x0000000112ad5150: 4cfe ff34 | 1f00 0039 | aa01 00b5 | e103 1caa | c900 0010 + + 0x0000000112ad5164: ; {runtime_call G1BarrierSetRuntime::write_ref_field_post_entry(unsigned char volatile*, JavaThread*)} + 0x0000000112ad5164: 0827 85d2 | 2835 a0f2 | 2800 c0f2 | ff27 bfa9 | 0001 3fd6 + + 0x0000000112ad5178: ; {other} + 0x0000000112ad5178: 1f20 03d5 | 1f00 80f2 | 1f00 80f2 | ff43 0091 | e4ff ff17 | 4c21 00d1 | 6a01 0a8b | 4081 1ff8 + 0x0000000112ad5198: 8c27 00f9 | dfff ff17 | e103 00b9 + + 0x0000000112ad51a4: ; {metadata('java/nio/HeapByteBuffer')} + 0x0000000112ad51a4: 017c 9ed2 | 6103 a0f2 | 010e c0f2 + + 0x0000000112ad51b0: ; ImmutableOopMap {} + ;*new {reexecute=0 rethrow=0 return_oop=1} + ; - java.nio.ByteBuffer::allocate@9 (line 391) + ; {runtime_call _new_instance_Java} + 0x0000000112ad51b0: 54e1 e797 + + 0x0000000112ad51b4: ; {other} + 0x0000000112ad51b4: 1f20 03d5 | 9f74 80f2 | 1f00 80f2 | ed03 40b9 | f303 00aa | 9aff ff17 | f303 00f9 | fd03 0d2a + 0x0000000112ad51d4: ; {metadata({type array byte})} + 0x0000000112ad51d4: 0158 84d2 | 0100 a0f2 | 010e c0f2 | e203 0d2a + + 0x0000000112ad51e4: ; ImmutableOopMap {[0]=Oop } + ;*newarray {reexecute=0 rethrow=0 return_oop=1} + ; - java.nio.HeapByteBuffer::@6 (line 71) + ; - java.nio.ByteBuffer::allocate@16 (line 391) + ; {runtime_call _new_array_Java} + 0x0000000112ad51e4: 07d9 e797 + + 0x0000000112ad51e8: ; {other} + 0x0000000112ad51e8: 1f20 03d5 | 1f7b 80f2 | 1f20 80f2 | ed03 1d2a | f303 40f9 | b0ff ff17 | 4117 8012 | ed03 00b9 + 0x0000000112ad5208: ; ImmutableOopMap {} + ;*ifge {reexecute=1 rethrow=0 return_oop=0} + ; - (reexecute) java.nio.ByteBuffer::allocate@1 (line 389) + ; {runtime_call UncommonTrapBlob} + 0x0000000112ad5208: 5e77 e497 + + 0x0000000112ad520c: ; {other} + 0x0000000112ad520c: 1f20 03d5 | 9f7f 80f2 | 1f40 80f2 | e103 00aa | 0200 0014 | e103 00aa | fd7b 42a9 | ffc3 0091 + 0x0000000112ad522c: ; {runtime_call _rethrow_Java} + 0x0000000112ad522c: 75f4 e717 + + 0x0000000112ad5230: ; {internal_word} + 0x0000000112ad5230: 08f8 ff10 | 8837 02f9 + + 0x0000000112ad5238: ; {runtime_call SafepointBlob} + 0x0000000112ad5238: b275 e417 | 0830 8bd2 | e847 a2f2 | 2800 c0f2 | 0001 3fd6 | 66ff ff17 + + 0x0000000112ad5250: ; {other} + 0x0000000112ad5250: f801 0000 | 0000 0000 +[Exception Handler] + 0x0000000112ad5258: ; {no_reloc} + 0x0000000112ad5258: 4acf e717 +[Deopt Handler Code] + 0x0000000112ad525c: 1e00 0010 + + 0x0000000112ad5260: ; {runtime_call DeoptimizationBlob} + 0x0000000112ad5260: 7876 e417 | 0000 0000 +[/MachCode] + + +--------------- P R O C E S S --------------- + +Threads class SMR info: +_java_thread_list=0x0000600000ea8720, length=35, elements={ +0x000000014280d800, 0x0000000142815200, 0x0000000142815a00, 0x000000011c038600, +0x000000011c03b000, 0x000000011c03b800, 0x000000013201a600, 0x0000000142816200, +0x000000013180a400, 0x000000013189be00, 0x00000001008f1c00, 0x00000001319b5200, +0x00000001319b5a00, 0x00000001319b9600, 0x0000000141a77e00, 0x000000011f80ca00, +0x0000000131b02a00, 0x000000011c35a200, 0x000000011c359600, 0x0000000141a82a00, +0x000000011c35c600, 0x0000000141a7ea00, 0x00000001008e0a00, 0x0000000131ce3200, +0x00000001008e2c00, 0x0000000142bd4a00, 0x0000000131d81200, 0x00000001322fcc00, +0x0000000131db9a00, 0x0000000100f3e400, 0x000000013214d800, 0x0000000131af6200, +0x000000011f982a00, 0x000000011c0e4c00, 0x000000011f983200 +} + +Java Threads: ( => current thread ) + 0x000000014280d800 JavaThread "main" [_thread_in_Java, id=8963, stack(0x000000016fe0c000,0x000000017000f000) (2060K)] + 0x0000000142815200 JavaThread "Reference Handler" daemon [_thread_blocked, id=31747, stack(0x0000000170f84000,0x0000000171187000) (2060K)] + 0x0000000142815a00 JavaThread "Finalizer" daemon [_thread_blocked, id=31491, stack(0x0000000171190000,0x0000000171393000) (2060K)] + 0x000000011c038600 JavaThread "Signal Dispatcher" daemon [_thread_blocked, id=30979, stack(0x000000017139c000,0x000000017159f000) (2060K)] + 0x000000011c03b000 JavaThread "Service Thread" daemon [_thread_in_vm, id=23299, stack(0x00000001715a8000,0x00000001717ab000) (2060K)] + 0x000000011c03b800 JavaThread "Monitor Deflation Thread" daemon [_thread_blocked, id=23555, stack(0x00000001717b4000,0x00000001719b7000) (2060K)] + 0x000000013201a600 JavaThread "C2 CompilerThread0" daemon [_thread_in_native, id=24067, stack(0x00000001719c0000,0x0000000171bc3000) (2060K)] + 0x0000000142816200 JavaThread "C1 CompilerThread0" daemon [_thread_blocked, id=24323, stack(0x0000000171bcc000,0x0000000171dcf000) (2060K)] + 0x000000013180a400 JavaThread "Common-Cleaner" daemon [_thread_blocked, id=29955, stack(0x0000000171dd8000,0x0000000171fdb000) (2060K)] + 0x000000013189be00 JavaThread "dd-task-scheduler" daemon [_thread_blocked, id=28931, stack(0x00000001723fc000,0x00000001725ff000) (2060K)] + 0x00000001008f1c00 JavaThread "statsd-aggregator-thread" daemon [_thread_blocked, id=43011, stack(0x0000000173668000,0x000000017386b000) (2060K)] + 0x00000001319b5200 JavaThread "StatsD-Processor-1" daemon [_thread_blocked, id=33027, stack(0x0000000173874000,0x0000000173a77000) (2060K)] + 0x00000001319b5a00 JavaThread "StatsD-Sender-1" daemon [_thread_blocked, id=33283, stack(0x0000000173a80000,0x0000000173c83000) (2060K)] + 0x00000001319b9600 JavaThread "dd-jmx-collector" daemon [_thread_blocked, id=33539, stack(0x0000000173c8c000,0x0000000173e8f000) (2060K)] + 0x0000000141a77e00 JavaThread "process reaper" daemon [_thread_blocked, id=41731, stack(0x0000000173e98000,0x0000000173ecf000) (220K)] + 0x000000011f80ca00 JavaThread "ForkJoinPool.commonPool-worker-1" daemon [_thread_blocked, id=33795, stack(0x0000000173ed8000,0x00000001740db000) (2060K)] + 0x0000000131b02a00 JavaThread "dd-trace-monitor" daemon [_thread_blocked, id=34307, stack(0x00000001742f0000,0x00000001744f3000) (2060K)] +=>0x000000011c35a200 JavaThread "dd-trace-processor" daemon [_thread_in_vm, id=34819, stack(0x00000001744fc000,0x00000001746ff000) (2060K)] + 0x000000011c359600 JavaThread "dd-data-streams-monitor" daemon [_thread_blocked, id=40451, stack(0x0000000174708000,0x000000017490b000) (2060K)] + 0x0000000141a82a00 JavaThread "OkHttp ConnectionPool" daemon [_thread_blocked, id=40195, stack(0x0000000174914000,0x0000000174b17000) (2060K)] + 0x000000011c35c600 JavaThread "Okio Watchdog" daemon [_thread_blocked, id=35587, stack(0x0000000174b20000,0x0000000174d23000) (2060K)] + 0x0000000141a7ea00 JavaThread "dd-debugger-snapshot-serializer" daemon [_thread_blocked, id=39683, stack(0x0000000174d2c000,0x0000000174f2f000) (2060K)] + 0x00000001008e0a00 JavaThread "dd-remote-config" daemon [_thread_blocked, id=39171, stack(0x0000000174f38000,0x000000017513b000) (2060K)] + 0x0000000131ce3200 JavaThread "dd-telemetry" daemon [_thread_blocked, id=38659, stack(0x0000000175144000,0x0000000175347000) (2060K)] + 0x00000001008e2c00 JavaThread "Notification Thread" daemon [_thread_blocked, id=24583, stack(0x0000000171fe4000,0x00000001721e7000) (2060K)] + 0x0000000142bd4a00 JavaThread "Cleaner-0" daemon [_thread_blocked, id=38147, stack(0x0000000175350000,0x0000000175553000) (2060K)] + 0x0000000131d81200 JavaThread "C2 CompilerThread1" daemon [_thread_blocked, id=24843, stack(0x00000001721f0000,0x00000001723f3000) (2060K)] + 0x00000001322fcc00 JavaThread "C2 CompilerThread2" daemon [_thread_in_native, id=36103, stack(0x000000017555c000,0x000000017575f000) (2060K)] + 0x0000000131db9a00 JavaThread "Catalina-utility-1" [_thread_blocked, id=32519, stack(0x0000000175768000,0x000000017596b000) (2060K)] + 0x0000000100f3e400 JavaThread "Catalina-utility-2" [_thread_blocked, id=37895, stack(0x0000000175974000,0x0000000175b77000) (2060K)] + 0x000000013214d800 JavaThread "container-0" [_thread_blocked, id=36871, stack(0x0000000175b80000,0x0000000175d83000) (2060K)] + 0x0000000131af6200 JavaThread "mysql-cj-abandoned-connection-cleanup" daemon [_thread_blocked, id=37123, stack(0x0000000175d8c000,0x0000000175f8f000) (2060K)] + 0x000000011f982a00 JavaThread "dd-metrics-aggregator" daemon [_thread_blocked, id=65283, stack(0x0000000175f98000,0x000000017619b000) (2060K)] + 0x000000011c0e4c00 JavaThread "HikariPool-1:housekeeper" daemon [_thread_blocked, id=64771, stack(0x00000001761a4000,0x00000001763a7000) (2060K)] + 0x000000011f983200 JavaThread "HikariPool-1:connection-adder" daemon [_thread_blocked, id=43779, stack(0x00000001763b0000,0x00000001765b3000) (2060K)] +Total: 35 + +Other Threads: + 0x0000000131607380 VMThread "VM Thread" [id=19971, stack(0x0000000170c60000,0x0000000170e63000) (2060K)] + 0x0000000131706180 WatcherThread "VM Periodic Task Thread" [id=17411, stack(0x0000000170a54000,0x0000000170c57000) (2060K)] + 0x00000001417082d0 WorkerThread "GC Thread#0" [id=12803, stack(0x0000000170018000,0x000000017021b000) (2060K)] + 0x000000011bf27ef0 WorkerThread "GC Thread#1" [id=25091, stack(0x0000000172608000,0x000000017280b000) (2060K)] + 0x00000001316478c0 WorkerThread "GC Thread#2" [id=25347, stack(0x0000000172814000,0x0000000172a17000) (2060K)] + 0x0000000131647e30 WorkerThread "GC Thread#3" [id=25859, stack(0x0000000172a20000,0x0000000172c23000) (2060K)] + 0x0000000142011fc0 WorkerThread "GC Thread#4" [id=28163, stack(0x0000000172c2c000,0x0000000172e2f000) (2060K)] + 0x0000000142012530 WorkerThread "GC Thread#5" [id=26371, stack(0x0000000172e38000,0x000000017303b000) (2060K)] + 0x0000000142012aa0 WorkerThread "GC Thread#6" [id=26627, stack(0x0000000173044000,0x0000000173247000) (2060K)] + 0x000000013165be50 WorkerThread "GC Thread#7" [id=26883, stack(0x0000000173250000,0x0000000173453000) (2060K)] + 0x000000013165c3c0 WorkerThread "GC Thread#8" [id=32771, stack(0x000000017345c000,0x000000017365f000) (2060K)] + 0x0000000141708c70 ConcurrentGCThread "G1 Main Marker" [id=13315, stack(0x0000000170224000,0x0000000170427000) (2060K)] + 0x0000000142004100 WorkerThread "G1 Conc#0" [id=14083, stack(0x0000000170430000,0x0000000170633000) (2060K)] + 0x0000000131693590 WorkerThread "G1 Conc#1" [id=34051, stack(0x00000001740e4000,0x00000001742e7000) (2060K)] + 0x000000011c038200 ConcurrentGCThread "G1 Refine#0" [id=16387, stack(0x000000017063c000,0x000000017083f000) (2060K)] + 0x00000001316045c0 ConcurrentGCThread "G1 Service" [id=16899, stack(0x0000000170848000,0x0000000170a4b000) (2060K)] +Total: 16 + +Threads with active compile tasks: +C2 CompilerThread0 11475 8959 4 java.lang.invoke.InvokerBytecodeGenerator::emitLoadInsn (21 bytes) +C1 CompilerThread0 11475 8965 3 datadog.trace.agent.core.DDSpan::getTag (7 bytes) +C2 CompilerThread1 11475 8962 4 java.lang.invoke.InvokerBytecodeGenerator::emitPushArgument (150 bytes) +C2 CompilerThread2 11475 8885 4 net.bytebuddy.jar.asm.ClassReader::readCode (5117 bytes) +Total: 4 + +VM state: not at safepoint (normal execution) + +VM Mutex/Monitor currently owned by a thread: None + +Heap address: 0x00000007fe000000, size: 32 MB, Compressed Oops mode: Zero based, Oop shift amount: 3 + +CDS archive(s) mapped at: [0x0000007000000000-0x0000007000ca4000-0x0000007000ca4000), size 13254656, SharedBaseAddress: 0x0000007000000000, ArchiveRelocationMode: 1. +Compressed class space mapped at: 0x0000007001000000-0x0000007041000000, reserved size: 1073741824 +Narrow klass base: 0x0000007000000000, Narrow klass shift: 0, Narrow klass range: 0x100000000 + +GC Precious Log: + CardTable entry size: 512 + Card Set container configuration: InlinePtr #cards 5 size 8 Array Of Cards #cards 12 size 40 Howl #buckets 4 coarsen threshold 1843 Howl Bitmap #cards 512 size 80 coarsen threshold 460 Card regions per heap region 1 cards per card region 2048 + CPUs: 10 total, 10 available + Memory: 65536M + Large Page Support: Disabled + NUMA Support: Disabled + Compressed Oops: Enabled (Zero based) + Heap Region Size: 1M + Heap Min Capacity: 32M + Heap Initial Capacity: 32M + Heap Max Capacity: 32M + Pre-touch: Disabled + Parallel Workers: 9 + Concurrent Workers: 2 + Concurrent Refinement Workers: 9 + Periodic GC: Disabled + +vm_info: OpenJDK 64-Bit Server VM (21.0.1+12-LTS) for bsd-aarch64 JRE (21.0.1+12-LTS) (Zulu21.30+15-CA), built on 2023-10-11T21:40:40Z by "zulu_re" with clang Apple LLVM 12.0.0 (clang-1200.0.32.28) + +END. diff --git a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java index b2573d6b81f..4aa40e06522 100644 --- a/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java +++ b/internal-api/src/main/java/datadog/trace/util/AgentThreadFactory.java @@ -31,6 +31,8 @@ public enum AgentThread { APPSEC_HTTP_DISPATCHER("dd-appsec-http-dispatcher"), + CRASHTRACKING_HTTP_DISPATCHER("dd-crashtracking-http-dispatcher"), + TELEMETRY("dd-telemetry"), FLEET_MANAGEMENT_POLLER("dd-fleet-management-poller"), From 24715acb005da84292c039611902b24552ee5fcb Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Mon, 24 Nov 2025 10:47:28 +0100 Subject: [PATCH 4/5] leftover --- .../src/main/java/datadog/crashtracking/CrashUploader.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java index ec5d3f575bc..e150deb24f7 100644 --- a/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java +++ b/dd-java-agent/agent-crashtracking/src/main/java/datadog/crashtracking/CrashUploader.java @@ -56,7 +56,6 @@ import okhttp3.RequestBody; import okhttp3.Response; import okio.Buffer; -import okio.ByteString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -561,9 +560,7 @@ private RequestBody makeErrorTrackingRequestBody(@Nonnull CrashLog payload, bool } writer.endObject(); } - final ByteString tmp = buf.readByteString(); - System.err.println(tmp.utf8()); - return RequestBody.create(APPLICATION_JSON, tmp); + return RequestBody.create(APPLICATION_JSON, buf.readByteString()); } } From 399f8cc599c1fc248e412bd423bbef77c6c6faa9 Mon Sep 17 00:00:00 2001 From: Andrea Marziali Date: Mon, 24 Nov 2025 11:05:48 +0100 Subject: [PATCH 5/5] adjust smoke test --- .../src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java b/dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java index ecd00fd7f13..d832e5ef382 100644 --- a/dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java +++ b/dd-smoke-tests/crashtracking/src/test/java/datadog/smoketest/CrashtrackingSmokeTest.java @@ -338,7 +338,7 @@ private String assertCrashPing() throws InterruptedException, IOException { private void assertCrashData(String uuid) throws InterruptedException, IOException { CrashTelemetryData crashData = crashEvents.poll(DATA_TIMEOUT_MS, TimeUnit.MILLISECONDS); assertNotNull(crashData, "Crash data not uploaded"); - assertTrue(crashData.payload.get(0).message.contains("Process terminated by signal")); + assertTrue(crashData.payload.get(0).message.contains("Java heap space")); assertTrue(crashData.payload.get(0).tags.contains("severity:crash")); final Map map = moshi.adapter(Map.class).fromJson(crashData.payload.get(0).message); final Object receivedUuid = map.get("uuid");