From 1e3f48cd653c75ae11505fd16d01ec0c13175915 Mon Sep 17 00:00:00 2001 From: Jakob Stolze <45308098+jarinox@users.noreply.github.com> Date: Wed, 19 Mar 2025 11:57:55 +0100 Subject: [PATCH 1/4] feat: add ProgressBarLogger to graphhopper --- core/pom.xml | 30 +++++ .../graphhopper/util/ProgressBarLogger.java | 125 ++++++++++++++++++ 2 files changed, 155 insertions(+) create mode 100644 core/src/main/java/com/graphhopper/util/ProgressBarLogger.java diff --git a/core/pom.xml b/core/pom.xml index 39a82777679..619f29a803a 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -25,6 +25,7 @@ --> yyyy-MM-dd'T'HH:mm:ss'Z' ${maven.build.timestamp} + 2.22.1 @@ -136,6 +137,35 @@ test 4.13.2 + + me.tongfei + progressbar + 0.10.0 + + + org.apache.logging.log4j + log4j + pom + ${log4j.version} + + + + org.apache.logging.log4j + log4j-1.2-api + ${log4j.version} + + + + org.apache.logging.log4j + log4j-api + ${log4j.version} + + + + org.apache.logging.log4j + log4j-core + ${log4j.version} + diff --git a/core/src/main/java/com/graphhopper/util/ProgressBarLogger.java b/core/src/main/java/com/graphhopper/util/ProgressBarLogger.java new file mode 100644 index 00000000000..d3c6ab1461e --- /dev/null +++ b/core/src/main/java/com/graphhopper/util/ProgressBarLogger.java @@ -0,0 +1,125 @@ +package com.graphhopper.util; + +import me.tongfei.progressbar.DelegatingProgressBarConsumer; +import me.tongfei.progressbar.ProgressBar; +import me.tongfei.progressbar.ProgressBarBuilder; +import me.tongfei.progressbar.ProgressBarStyle; +import org.apache.log4j.Logger; +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.LoggerContext; +import org.apache.logging.log4j.core.appender.ConsoleAppender; +import org.apache.logging.log4j.core.config.AppenderRef; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.LoggerConfig; +import org.apache.logging.log4j.core.layout.PatternLayout; + + +public class ProgressBarLogger { + private static String LOGGER_NAME = "ProgressBarLogger"; + private static final Logger CLASS_LOGGER = Logger.getLogger(ProgressBarLogger.class); + + + private static Logger initializeLogger() { + LoggerContext context = LoggerContext.getContext(false); + Configuration config = context.getConfiguration(); + if (Logger.getRootLogger().getAppender(LOGGER_NAME) == null) { + String originalConsolePattern; + try { + originalConsolePattern = config.getAppender("Console").getLayout().getContentFormat().get("format"); + } catch (Exception e) { + originalConsolePattern = "%d{yyyy-MM-dd HH:mm:ss} %highlight{%-7p} %style{%50t}{Cyan} %style{[ %-40.40c{1.} ]}{Bright Cyan} %m"; + } + + // Remove new line characters from the end of the pattern + if (originalConsolePattern.endsWith("%n")) { + originalConsolePattern = originalConsolePattern.substring(0, originalConsolePattern.length() - 2); + } + + // Remove new line characters from the beginning of the pattern + if (originalConsolePattern.startsWith("%n")) { + originalConsolePattern = originalConsolePattern.substring(2); + } + + final PatternLayout layout = PatternLayout.newBuilder() + .withPattern(originalConsolePattern + "\r") + .withConfiguration(config) + .build(); + + Appender consoleAppender = ConsoleAppender.newBuilder() + .setConfiguration(config) + .setName(LOGGER_NAME) + .setLayout(layout) + .setFilter(null) + .setTarget(ConsoleAppender.Target.SYSTEM_OUT) + .setName(LOGGER_NAME) + .setFollow(false) + .build(); + consoleAppender.start(); + + // Create the new appender reference + AppenderRef ref = AppenderRef.createAppenderRef(LOGGER_NAME, null, null); + AppenderRef[] refs = new AppenderRef[]{ref}; + LoggerConfig loggerConfig = LoggerConfig.newBuilder() + .withAdditivity(false) + .withLevel(CLASS_LOGGER.getLevel().getVersion2Level()) + .withLoggerName(LOGGER_NAME) + .withIncludeLocation("true") + .withRefs(refs) + .withProperties(null) + .withConfig(config) + .withtFilter(null) + .build(); + + config.addAppender(consoleAppender); + loggerConfig.addAppender(consoleAppender, null, null); + config.addLogger(LOGGER_NAME, loggerConfig); + context.updateLoggers(); + } + return Logger.getLogger(LOGGER_NAME); + } + + public static void setLoggerName(String loggerName) { + LOGGER_NAME = loggerName; + } + + public static String getLoggerName() { + return LOGGER_NAME; + } + + public static Logger getLogger() { + return initializeLogger(); + } + + /** + * Creates and returns a ProgressListener that receives percentages and updates a progress bar with the given task name. + * + * @param taskName the name of the task to be displayed in the progress bar + * @return a ProgressListener that updates the progress bar + */ + public static ProgressListener getProgressListener(String taskName) { + Logger logger = initializeLogger(); + return new ProgressListener() { + final ProgressBarBuilder progressBarBuilder = new ProgressBarBuilder() + .setStyle(ProgressBarStyle.COLORFUL_UNICODE_BAR) + .setUpdateIntervalMillis(1000) // slow update for better visualization and less IO. Avoids % calculation for each element. + .setInitialMax(100) + .setTaskName(taskName) + .setConsumer(new DelegatingProgressBarConsumer(logger::info)); + + ProgressBar progressBar; + + int lastPercentage = 0; + @Override + public void update(long val) { + if (progressBar == null) { + progressBar = progressBarBuilder.build(); + } + + if (val > lastPercentage) { + progressBar.stepTo(val); + lastPercentage = (int) val; + } + } + }; + } +} From 74225edf0c5e381edd767719782369ba1b0465dd Mon Sep 17 00:00:00 2001 From: Jakob Stolze <45308098+jarinox@users.noreply.github.com> Date: Wed, 19 Mar 2025 12:00:53 +0100 Subject: [PATCH 2/4] feat: add progress bar to elevation download --- .../dem/AbstractSRTMElevationProvider.java | 8 +++++++- .../dem/AbstractTiffElevationProvider.java | 8 +++++++- .../java/com/graphhopper/util/Downloader.java | 17 ++++++++++++++++- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java index d6b599a2d8f..a39d89b6a44 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java @@ -21,6 +21,8 @@ import com.graphhopper.storage.DataAccess; import com.graphhopper.util.BitUtil; import com.graphhopper.util.Downloader; +import com.graphhopper.util.ProgressBarLogger; +import com.graphhopper.util.ProgressListener; import java.io.File; import java.io.FileNotFoundException; @@ -49,7 +51,11 @@ public abstract class AbstractSRTMElevationProvider extends TileBasedElevationPr public AbstractSRTMElevationProvider(String baseUrl, String cacheDir, String downloaderName, int minLat, int maxLat, int defaultWidth) { super(cacheDir); this.baseUrl = baseUrl; - downloader = new Downloader(downloaderName).setTimeout(10000); + + ProgressBarLogger.setLoggerName(AbstractTiffElevationProvider.class.getName()); + ProgressListener progressListener = ProgressBarLogger.getProgressListener("Download elevation data"); + + this.downloader = new Downloader(downloaderName, progressListener).setTimeout(10000); this.DEFAULT_WIDTH = defaultWidth; this.MIN_LAT = minLat; this.MAX_LAT = maxLat; diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java index 0383d2af57b..79de216afd4 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java @@ -19,6 +19,8 @@ import com.graphhopper.storage.DataAccess; import com.graphhopper.util.Downloader; +import com.graphhopper.util.ProgressBarLogger; +import com.graphhopper.util.ProgressListener; import java.awt.image.Raster; import java.io.File; @@ -47,7 +49,11 @@ public abstract class AbstractTiffElevationProvider extends TileBasedElevationPr public AbstractTiffElevationProvider(String baseUrl, String cacheDir, String downloaderName, int width, int height, int latDegree, int lonDegree) { super(cacheDir); this.baseUrl = baseUrl; - this.downloader = new Downloader(downloaderName).setTimeout(10000); + + ProgressBarLogger.setLoggerName(AbstractTiffElevationProvider.class.getName()); + ProgressListener progressListener = ProgressBarLogger.getProgressListener("Download elevation data"); + + this.downloader = new Downloader(downloaderName, progressListener).setTimeout(10000); this.WIDTH = width; this.HEIGHT = height; this.LAT_DEGREE = latDegree; diff --git a/core/src/main/java/com/graphhopper/util/Downloader.java b/core/src/main/java/com/graphhopper/util/Downloader.java index b01d2bf69e9..b9e9f1b0c7c 100644 --- a/core/src/main/java/com/graphhopper/util/Downloader.java +++ b/core/src/main/java/com/graphhopper/util/Downloader.java @@ -32,9 +32,15 @@ public class Downloader { private String referrer = "http://graphhopper.com"; private String acceptEncoding = "gzip, deflate"; private int timeout = 4000; + private final ProgressListener progressListener; - public Downloader(String userAgent) { + public Downloader(String userAgent, ProgressListener progressListener) { this.userAgent = userAgent; + this.progressListener = progressListener; + } + + public Downloader(String userAgent) { + this(userAgent, null); } public static void main(String[] args) throws IOException { @@ -113,13 +119,22 @@ public void downloadFile(String url, String toFile) throws IOException { HttpURLConnection conn = createConnection(url); InputStream iStream = fetch(conn, false); int size = 8 * 1024; + int contentLength = conn.getContentLength(); BufferedOutputStream writer = new BufferedOutputStream(new FileOutputStream(toFile), size); InputStream in = new BufferedInputStream(iStream, size); try { byte[] buffer = new byte[size]; int numRead; + int i = 0; while ((numRead = in.read(buffer)) != -1) { writer.write(buffer, 0, numRead); + i += numRead; + if (progressListener != null) + progressListener.update((int) (100 * i / (double) contentLength)); + } + + if (progressListener != null) { + progressListener.update(100); } } finally { Helper.close(iStream); From 91a9e6b45bddf8411862f8404c2cbcb17e3a73a2 Mon Sep 17 00:00:00 2001 From: Jakob Stolze <45308098+jarinox@users.noreply.github.com> Date: Wed, 19 Mar 2025 12:14:18 +0100 Subject: [PATCH 3/4] fix: remove deprecated method for setting logging level --- core/src/main/java/com/graphhopper/util/ProgressBarLogger.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/graphhopper/util/ProgressBarLogger.java b/core/src/main/java/com/graphhopper/util/ProgressBarLogger.java index d3c6ab1461e..e4c53eecd07 100644 --- a/core/src/main/java/com/graphhopper/util/ProgressBarLogger.java +++ b/core/src/main/java/com/graphhopper/util/ProgressBarLogger.java @@ -5,6 +5,7 @@ import me.tongfei.progressbar.ProgressBarBuilder; import me.tongfei.progressbar.ProgressBarStyle; import org.apache.log4j.Logger; +import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.Appender; import org.apache.logging.log4j.core.LoggerContext; import org.apache.logging.log4j.core.appender.ConsoleAppender; @@ -61,7 +62,7 @@ private static Logger initializeLogger() { AppenderRef[] refs = new AppenderRef[]{ref}; LoggerConfig loggerConfig = LoggerConfig.newBuilder() .withAdditivity(false) - .withLevel(CLASS_LOGGER.getLevel().getVersion2Level()) + .withLevel(Level.INFO) .withLoggerName(LOGGER_NAME) .withIncludeLocation("true") .withRefs(refs) From 5965ee29c7567f168d4b31ebe6160d8941c986cb Mon Sep 17 00:00:00 2001 From: Jakob Stolze Date: Tue, 15 Apr 2025 10:49:39 +0200 Subject: [PATCH 4/4] fix: rewrite downloadFile for consistent download progress HttpURLConnection behaves strange when downloading big files. For a consistent download progress calculation the method was rewritten to using more robust Apache HttpClient 5. --- core/pom.xml | 10 +++ .../java/com/graphhopper/util/Downloader.java | 83 ++++++++++++++----- 2 files changed, 70 insertions(+), 23 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index 619f29a803a..07f570039e4 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -166,6 +166,16 @@ log4j-core ${log4j.version} + + org.apache.httpcomponents.core5 + httpcore5 + 5.3.1 + + + org.apache.httpcomponents.client5 + httpclient5 + 5.4.1 + diff --git a/core/src/main/java/com/graphhopper/util/Downloader.java b/core/src/main/java/com/graphhopper/util/Downloader.java index b9e9f1b0c7c..f53d851fd88 100644 --- a/core/src/main/java/com/graphhopper/util/Downloader.java +++ b/core/src/main/java/com/graphhopper/util/Downloader.java @@ -23,6 +23,13 @@ import java.util.zip.GZIPInputStream; import java.util.zip.Inflater; import java.util.zip.InflaterInputStream; +import org.apache.hc.client5.http.classic.methods.HttpGet; +import org.apache.hc.client5.http.impl.classic.CloseableHttpClient; +import org.apache.hc.client5.http.impl.classic.CloseableHttpResponse; +import org.apache.hc.client5.http.impl.classic.HttpClients; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.apache.hc.core5.http.Header; /** * @author Peter Karich @@ -116,30 +123,60 @@ public HttpURLConnection createConnection(String urlStr) throws IOException { } public void downloadFile(String url, String toFile) throws IOException { - HttpURLConnection conn = createConnection(url); - InputStream iStream = fetch(conn, false); - int size = 8 * 1024; - int contentLength = conn.getContentLength(); - BufferedOutputStream writer = new BufferedOutputStream(new FileOutputStream(toFile), size); - InputStream in = new BufferedInputStream(iStream, size); - try { - byte[] buffer = new byte[size]; - int numRead; - int i = 0; - while ((numRead = in.read(buffer)) != -1) { - writer.write(buffer, 0, numRead); - i += numRead; - if (progressListener != null) - progressListener.update((int) (100 * i / (double) contentLength)); - } - - if (progressListener != null) { - progressListener.update(100); + final int bufferSize = 4096; // 4kB buffer + try (CloseableHttpClient httpClient = HttpClients.createDefault()) { + HttpGet httpGet = new HttpGet(url); + + try (CloseableHttpResponse response = httpClient.execute(httpGet)) { + int statusCode = response.getCode(); + if (statusCode < 200 || statusCode >= 300) { + throw new IOException("Server returned HTTP response code: " + statusCode + " for URL: " + url); + } + + HttpEntity entity = response.getEntity(); + if (entity == null) { + throw new IOException("No entity found in response for URL: " + url); + } + + long contentLength = entity.getContentLength(); // Länge von der Entity holen + if (contentLength <= 0) { + Header lengthHeader = response.getFirstHeader("Content-Length"); + if(lengthHeader != null) { + try { + contentLength = Long.parseLong(lengthHeader.getValue()); + } catch (NumberFormatException e) { + contentLength = -1; + } + } + } + + try (InputStream inputStream = entity.getContent(); // InputStream ist AutoCloseable + BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(toFile))) { + + byte[] buffer = new byte[bufferSize]; + int numRead; + long totalBytesRead = 0; + int lastPercentage = -1; + + while ((numRead = inputStream.read(buffer)) != -1) { + outputStream.write(buffer, 0, numRead); + totalBytesRead += numRead; + + if (progressListener != null && contentLength > 0) { + int currentPercentage = (int) (100 * totalBytesRead / (double) contentLength); + if (currentPercentage > lastPercentage) { + progressListener.update(currentPercentage); + lastPercentage = currentPercentage; + } + } + } + if (progressListener != null && contentLength > 0 && lastPercentage < 100) { + progressListener.update(100); + } + } finally { + EntityUtils.consume(entity); + } } - } finally { - Helper.close(iStream); - Helper.close(writer); - Helper.close(in); } }