From 613a13d5c3e282207f17911165cf8fd8f0ebba32 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Thu, 24 Apr 2025 17:22:36 +0200 Subject: [PATCH 1/2] feat: update Downloader to latest version of graphhopper --- .../java/com/graphhopper/util/Downloader.java | 37 ++++++------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/core/src/main/java/com/graphhopper/util/Downloader.java b/core/src/main/java/com/graphhopper/util/Downloader.java index b01d2bf69e9..43cb612bc16 100644 --- a/core/src/main/java/com/graphhopper/util/Downloader.java +++ b/core/src/main/java/com/graphhopper/util/Downloader.java @@ -20,6 +20,7 @@ import java.io.*; import java.net.HttpURLConnection; import java.net.URL; +import java.util.function.LongConsumer; import java.util.zip.GZIPInputStream; import java.util.zip.Inflater; import java.util.zip.InflaterInputStream; @@ -28,6 +29,7 @@ * @author Peter Karich */ public class Downloader { + private static final int BUFFER_SIZE = 8 * 1024; private final String userAgent; private String referrer = "http://graphhopper.com"; private String acceptEncoding = "gzip, deflate"; @@ -39,12 +41,7 @@ public Downloader(String userAgent) { public static void main(String[] args) throws IOException { new Downloader("GraphHopper Downloader").downloadAndUnzip("http://graphhopper.com/public/maps/0.1/europe_germany_berlin.ghz", "somefolder", - new ProgressListener() { - @Override - public void update(long val) { - System.out.println("progress:" + val); - } - }); + val -> System.out.println("progress:" + val)); } public Downloader setTimeout(int timeout) { @@ -80,9 +77,9 @@ public InputStream fetch(HttpURLConnection connection, boolean readErrorStreamNo try { String encoding = connection.getContentEncoding(); if (encoding != null && encoding.equalsIgnoreCase("gzip")) - is = new GZIPInputStream(is); + is = new GZIPInputStream(is, BUFFER_SIZE); else if (encoding != null && encoding.equalsIgnoreCase("deflate")) - is = new InflaterInputStream(is, new Inflater(true)); + is = new InflaterInputStream(is, new Inflater(true), BUFFER_SIZE); } catch (IOException ex) { } @@ -90,7 +87,7 @@ else if (encoding != null && encoding.equalsIgnoreCase("deflate")) } public InputStream fetch(String url) throws IOException { - return fetch((HttpURLConnection) createConnection(url), false); + return fetch(createConnection(url), false); } public HttpURLConnection createConnection(String urlStr) throws IOException { @@ -112,15 +109,10 @@ public HttpURLConnection createConnection(String urlStr) throws IOException { public void downloadFile(String url, String toFile) throws IOException { HttpURLConnection conn = createConnection(url); InputStream iStream = fetch(conn, false); - int size = 8 * 1024; - BufferedOutputStream writer = new BufferedOutputStream(new FileOutputStream(toFile), size); - InputStream in = new BufferedInputStream(iStream, size); + BufferedOutputStream writer = new BufferedOutputStream(new FileOutputStream(toFile), BUFFER_SIZE); + InputStream in = new BufferedInputStream(iStream, BUFFER_SIZE); try { - byte[] buffer = new byte[size]; - int numRead; - while ((numRead = in.read(buffer)) != -1) { - writer.write(buffer, 0, numRead); - } + in.transferTo(writer); } finally { Helper.close(iStream); Helper.close(writer); @@ -128,20 +120,15 @@ public void downloadFile(String url, String toFile) throws IOException { } } - public void downloadAndUnzip(String url, String toFolder, final ProgressListener progressListener) throws IOException { + public void downloadAndUnzip(String url, String toFolder, final LongConsumer progressListener) throws IOException { HttpURLConnection conn = createConnection(url); final int length = conn.getContentLength(); InputStream iStream = fetch(conn, false); - new Unzipper().unzip(iStream, new File(toFolder), new ProgressListener() { - @Override - public void update(long sumBytes) { - progressListener.update((int) (100 * sumBytes / length)); - } - }); + new Unzipper().unzip(iStream, new File(toFolder), sumBytes -> progressListener.accept((int) (100 * sumBytes / length))); } public String downloadAsString(String url, boolean readErrorStreamNoException) throws IOException { - return Helper.isToString(fetch((HttpURLConnection) createConnection(url), readErrorStreamNoException)); + return Helper.isToString(fetch(createConnection(url), readErrorStreamNoException)); } } From 992f73d62f05587761ce4589dda951d33ed33c00 Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Thu, 24 Apr 2025 17:30:41 +0200 Subject: [PATCH 2/2] feat: add progress logger to Downloader.downloadFile feat: prevent divide by zero feat: prevent divide by zero --- .../java/com/graphhopper/util/Downloader.java | 11 ++- .../util/ProgressOutputStream.java | 85 +++++++++++++++++++ 2 files changed, 94 insertions(+), 2 deletions(-) create mode 100644 core/src/main/java/com/graphhopper/util/ProgressOutputStream.java diff --git a/core/src/main/java/com/graphhopper/util/Downloader.java b/core/src/main/java/com/graphhopper/util/Downloader.java index 43cb612bc16..e3060d3a86e 100644 --- a/core/src/main/java/com/graphhopper/util/Downloader.java +++ b/core/src/main/java/com/graphhopper/util/Downloader.java @@ -17,9 +17,13 @@ */ package com.graphhopper.util; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.*; import java.net.HttpURLConnection; import java.net.URL; +import java.nio.file.Path; import java.util.function.LongConsumer; import java.util.zip.GZIPInputStream; import java.util.zip.Inflater; @@ -29,6 +33,7 @@ * @author Peter Karich */ public class Downloader { + private static final Logger logger = LoggerFactory.getLogger(Downloader.class); private static final int BUFFER_SIZE = 8 * 1024; private final String userAgent; private String referrer = "http://graphhopper.com"; @@ -111,8 +116,10 @@ public void downloadFile(String url, String toFile) throws IOException { InputStream iStream = fetch(conn, false); BufferedOutputStream writer = new BufferedOutputStream(new FileOutputStream(toFile), BUFFER_SIZE); InputStream in = new BufferedInputStream(iStream, BUFFER_SIZE); - try { - in.transferTo(writer); + String file = Path.of(toFile).getFileName().toString(); + logger.info("Downloading {} [{}]", url, conn.getContentLengthLong()); + try (OutputStream progress = new ProgressOutputStream(writer, "downloading %s ..".formatted(file), conn.getContentLengthLong(), logger::info)) { + in.transferTo(progress); } finally { Helper.close(iStream); Helper.close(writer); diff --git a/core/src/main/java/com/graphhopper/util/ProgressOutputStream.java b/core/src/main/java/com/graphhopper/util/ProgressOutputStream.java new file mode 100644 index 00000000000..b08b93b8fd0 --- /dev/null +++ b/core/src/main/java/com/graphhopper/util/ProgressOutputStream.java @@ -0,0 +1,85 @@ +package com.graphhopper.util; + +import org.apache.commons.lang3.time.DurationFormatUtils; +import org.apache.commons.lang3.time.StopWatch; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.function.Consumer; + +class ProgressOutputStream extends OutputStream { + private final OutputStream out; + private final long max; + private final Consumer log; + private final String progressRenderString; + + private long current = 0; + private long speed = 0; + + private final StopWatch time; + private final StopWatch totalTime; + + + public ProgressOutputStream(OutputStream out, String task, long max, Consumer log) { + this.out = out; + this.max = max; + this.log = log; + if (max > 0) { + this.progressRenderString = "%s %%%dd/%d (%%3d%%%%) [%%.2f MB/s / %%.2f MB/s] %%s" + .formatted(task, Long.toString(max).length(), max); + } else { + this.progressRenderString = "%s %%d/? [%%.2f Mbps / %%.2f Mbps] %%s".formatted(task); + } + + this.time = StopWatch.createStarted(); + this.totalTime = StopWatch.createStarted(); + + } + + private void progress(long n) { + current += n; + speed += n; + if (time.getTime() >= 1000) { + triggerLog(); + time.reset(); + time.start(); + speed = 0; + } + } + + private void triggerLog() { + log.accept(render((speed) / (Math.max(time.getTime(), 1) * 1000.0), (current) / (Math.max(totalTime.getTime(), 1) * 1000.0))); + } + + private String render(double mbits, double mbitsTotal) { + if (max > 0) { + return progressRenderString + .formatted(current, (int) ((current / (double) max) * 100), + mbits, mbitsTotal, + DurationFormatUtils.formatDuration(totalTime.getTime(), "mm:ss.SSS")); + } + return progressRenderString + .formatted(current, + mbits, mbitsTotal, + DurationFormatUtils.formatDuration(totalTime.getTime(), "mm:ss.SSS")); + + } + + @Override + public void write(byte[] b, int off, int len) throws IOException { + out.write(b, off, len); + progress(len); + } + + @Override + public void write(int b) throws IOException { + out.write(b); + progress(1); + } + + @Override + public void close() throws IOException { + out.close(); + triggerLog(); + } +} \ No newline at end of file