From 0ef7dfe82c7c8f977c74f4137e55efd5fd0aedac Mon Sep 17 00:00:00 2001 From: Festerdam Date: Tue, 22 Aug 2023 15:07:21 +0100 Subject: [PATCH 1/6] Add MirrorSwitchButton --- .../remote_editor_download/remote_editor_download.tscn | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/components/editors/remote/remote_editor_download/remote_editor_download.tscn b/src/components/editors/remote/remote_editor_download/remote_editor_download.tscn index 5451d925..cab4be42 100644 --- a/src/components/editors/remote/remote_editor_download/remote_editor_download.tscn +++ b/src/components/editors/remote/remote_editor_download/remote_editor_download.tscn @@ -68,6 +68,11 @@ layout_mode = 2 layout_mode = 2 size_flags_horizontal = 3 +[node name="MirrorSwitchButton" type="Button" parent="MarginContainer/HBoxContainer/VBoxContainer/HBoxContainer2"] +unique_name_in_owner = true +visible = false +layout_mode = 2 + [node name="RetryButton" type="Button" parent="MarginContainer/HBoxContainer/VBoxContainer/HBoxContainer2"] unique_name_in_owner = true layout_mode = 2 From 034222e7e7aeef809aa511ddaebe3a451e643cbd Mon Sep 17 00:00:00 2001 From: Festerdam Date: Tue, 22 Aug 2023 17:23:31 +0100 Subject: [PATCH 2/6] Rework mirror switching --- .../remote_editor_download.gd | 38 ++++++++++++++----- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/src/components/editors/remote/remote_editor_download/remote_editor_download.gd b/src/components/editors/remote/remote_editor_download/remote_editor_download.gd index e35203bd..9ee324ea 100644 --- a/src/components/editors/remote/remote_editor_download/remote_editor_download.gd +++ b/src/components/editors/remote/remote_editor_download/remote_editor_download.gd @@ -5,6 +5,20 @@ const uuid = preload("res://addons/uuid.gd") signal downloaded(abs_zip_path: String) var _retry_callback +var _url: String +var _fallback_url: String = "": + set(new_fallback): + if "tuxfamily" in new_fallback: + _mirror_switch_button.text = "Switch to TuxFamily" + elif "github" in new_fallback: + _mirror_switch_button.text = "Switch to GitHub" + elif new_fallback == "": + pass + else: + assert(false, "unknown fallback") + _fallback_url = new_fallback +var _target_abs_dir: String +var _file_name: String @onready var _progress_bar: ProgressBar = get_node("%ProgressBar") @onready var _status: Label = get_node("%Status") @@ -13,6 +27,7 @@ var _retry_callback @onready var _title_label: Label = %TitleLabel @onready var _install_button: Button = %InstallButton @onready var _retry_button: Button = %RetryButton +@onready var _mirror_switch_button: Button = %MirrorSwitchButton func _ready() -> void: @@ -31,9 +46,14 @@ func _ready() -> void: _install_button.pressed.connect(func(): downloaded.emit(_download.download_file) ) + + _mirror_switch_button.pressed.connect(func(): + assert(_fallback_url) + start(_fallback_url, _target_abs_dir, _file_name, _url) + ) -func start(url, target_abs_dir, file_name, tux_fallback = ""): +func start(url, target_abs_dir, file_name, fallback_url = ""): var download_completed_callback = func(result: int, response_code: int, headers, body, download_completed_callback: Callable): # https://github.com/godotengine/godot/blob/a7583881af5477cd73110cc859fecf7ceaf39bd7/editor/plugins/asset_library_editor_plugin.cpp#L316 @@ -41,13 +61,6 @@ func start(url, target_abs_dir, file_name, tux_fallback = ""): var error_text = null var status = "" - if ((result != HTTPRequest.RESULT_SUCCESS or response_code != 200) - and "github.com" in url and tux_fallback): - print("Failure! Falling back to TuxFamily.") - _download.request_completed.disconnect(download_completed_callback) - start(tux_fallback, target_abs_dir, file_name, "") - return - match result: HTTPRequest.RESULT_CHUNKED_BODY_SIZE_MISMATCH, HTTPRequest.RESULT_CONNECTION_ERROR, HTTPRequest.RESULT_BODY_SIZE_LIMIT_EXCEEDED: error_text = "Connection error, prease try again." @@ -84,6 +97,8 @@ func start(url, target_abs_dir, file_name, tux_fallback = ""): $AcceptErrorDialog.dialog_text = "Download error:" + "\n" + error_text $AcceptErrorDialog.popup_centered() _retry_button.show() + if _fallback_url: + _mirror_switch_button.show() _status.text = status else: _install_button.disabled = false @@ -93,9 +108,14 @@ func start(url, target_abs_dir, file_name, tux_fallback = ""): assert(target_abs_dir.ends_with("/")) print("Downloading " + url) - _retry_callback = func(): start(url, target_abs_dir, file_name) + _url = url + _target_abs_dir = target_abs_dir + _file_name = file_name + _fallback_url = fallback_url + _retry_callback = func(): start(url, target_abs_dir, file_name, fallback_url) _retry_button.hide() + _mirror_switch_button.hide() _install_button.disabled = true _progress_bar.modulate = Color(1, 1, 1, 1) _title_label.text = file_name From 6859dcef4eee5598bb709506fc744f751cad507a Mon Sep 17 00:00:00 2001 From: Festerdam Date: Tue, 22 Aug 2023 20:03:19 +0100 Subject: [PATCH 3/6] Implement checksum checking --- .../remote_editor_download.gd | 66 ++++++++++++++++++- 1 file changed, 63 insertions(+), 3 deletions(-) diff --git a/src/components/editors/remote/remote_editor_download/remote_editor_download.gd b/src/components/editors/remote/remote_editor_download/remote_editor_download.gd index 9ee324ea..fb47b8fb 100644 --- a/src/components/editors/remote/remote_editor_download/remote_editor_download.gd +++ b/src/components/editors/remote/remote_editor_download/remote_editor_download.gd @@ -4,6 +4,8 @@ const uuid = preload("res://addons/uuid.gd") signal downloaded(abs_zip_path: String) +const _CHECKSUM_FILENAME = "SHA512-SUMS.txt" + var _retry_callback var _url: String var _fallback_url: String = "": @@ -100,10 +102,21 @@ func start(url, target_abs_dir, file_name, fallback_url = ""): if _fallback_url: _mirror_switch_button.show() _status.text = status + return else: - _install_button.disabled = false - _status.text = "Ready to install" - downloaded.emit(_download.download_file) + _status.text = "Checking file integrity..." + if not await _check_file_integrity(): + $AcceptErrorDialog.dialog_text = "Integrity check failed!\n" + \ + "Retry or use another mirror." + $AcceptErrorDialog.popup_centered() + _retry_button.show() + if _fallback_url: + _mirror_switch_button.show() + return + + _install_button.disabled = false + _status.text = "Ready to install" + downloaded.emit(_download.download_file) assert(target_abs_dir.ends_with("/")) print("Downloading " + url) @@ -171,8 +184,55 @@ func _notification(what: int) -> void: _remove_downloaded_file() +## Checks integrity of the downloaded file by veryfing that the SHA512 +## checksum is correct. Downloads SHA512-SUMS.txt to do so.[br] +## +## The authenticity of that checksum cannot be verified, however.[br] +## +## Failing to download SHA512-SUMS.txt is treated as success, because, again, +## this method does not verify authenticity of the release.[br] +## +## Returns true on success and false on failure. +func _check_file_integrity() -> bool: + var checksum_url: String = _url.get_base_dir().path_join(_CHECKSUM_FILENAME) + var checksum_downloader := HTTPRequest.new() + checksum_downloader.download_file = _target_abs_dir.get_base_dir() \ + .path_join(_CHECKSUM_FILENAME) + checksum_downloader.timeout = 10 + checksum_downloader.download_chunk_size = 2048 + print("Downloading ", checksum_url) + add_child(checksum_downloader) + checksum_downloader.request(checksum_url) + var sig_result = await checksum_downloader.request_completed + var result = sig_result[0] + var response_code = sig_result[1] + if result != HTTPRequest.RESULT_SUCCESS or response_code != 200: + return true + + var globalized_directory_path = ProjectSettings.globalize_path(_target_abs_dir) + match OS.get_name(): + "Linux", "OpenBSD", "FreeBSD", "NetBSD", "BSD": + var status = OS.execute("bash", ["-c", "cd " + globalized_directory_path + + " && sha512sum -c --ignore-missing --status " + + _CHECKSUM_FILENAME]) + return status == 0 or status == 127 + "Windows": + return true #TODO + "macOS": + return true #TODO + _: + return true + + checksum_downloader.queue_free() + + func _remove_downloaded_file(): if _download.download_file: DirAccess.remove_absolute( ProjectSettings.globalize_path(_download.download_file) ) + var sum_file_path = _target_abs_dir.path_join(_CHECKSUM_FILENAME) + if FileAccess.file_exists(sum_file_path): + DirAccess.remove_absolute( + ProjectSettings.globalize_path(sum_file_path) + ) From 6038ee09d2934f0029d602332769e06cf0b01950 Mon Sep 17 00:00:00 2001 From: Festerdam Date: Wed, 23 Aug 2023 01:56:28 +0100 Subject: [PATCH 4/6] Implement integrity check for windows --- .../remote_editor_download.gd | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) mode change 100644 => 100755 src/components/editors/remote/remote_editor_download/remote_editor_download.gd diff --git a/src/components/editors/remote/remote_editor_download/remote_editor_download.gd b/src/components/editors/remote/remote_editor_download/remote_editor_download.gd old mode 100644 new mode 100755 index fb47b8fb..9e2ad4d7 --- a/src/components/editors/remote/remote_editor_download/remote_editor_download.gd +++ b/src/components/editors/remote/remote_editor_download/remote_editor_download.gd @@ -217,7 +217,25 @@ func _check_file_integrity() -> bool: + _CHECKSUM_FILENAME]) return status == 0 or status == 127 "Windows": - return true #TODO + var certutil_output = [] + OS.execute("certutil", ["-hashfile", + globalized_directory_path.path_join(_file_name), + "SHA512"], certutil_output) + var output_lines = certutil_output[0].split("\n") + if len(output_lines) <= 2: + return true + + var obtained_sum = output_lines[1].strip_edges() + if not obtained_sum.is_valid_hex_number(): + return true + + var checksum_file_contents = FileAccess.open(_target_abs_dir.path_join( + _CHECKSUM_FILENAME), FileAccess.READ).get_as_text() + + if obtained_sum + " " + _file_name in checksum_file_contents: + return true + else: + return false "macOS": return true #TODO _: From c954d6600aa7aa9f33815918d6a4bf5165c83228 Mon Sep 17 00:00:00 2001 From: Festerdam Date: Thu, 24 Aug 2023 17:31:36 +0100 Subject: [PATCH 5/6] Remove unneeded callback --- .../remote/remote_editor_download/remote_editor_download.gd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/components/editors/remote/remote_editor_download/remote_editor_download.gd b/src/components/editors/remote/remote_editor_download/remote_editor_download.gd index 9e2ad4d7..4ac900ae 100755 --- a/src/components/editors/remote/remote_editor_download/remote_editor_download.gd +++ b/src/components/editors/remote/remote_editor_download/remote_editor_download.gd @@ -57,7 +57,7 @@ func _ready() -> void: func start(url, target_abs_dir, file_name, fallback_url = ""): var download_completed_callback = func(result: int, response_code: int, - headers, body, download_completed_callback: Callable): + headers, body): # https://github.com/godotengine/godot/blob/a7583881af5477cd73110cc859fecf7ceaf39bd7/editor/plugins/asset_library_editor_plugin.cpp#L316 var host = url var error_text = null @@ -147,7 +147,7 @@ func start(url, target_abs_dir, file_name, fallback_url = ""): _status.text = "Something went wrong." return - _download.request_completed.connect(download_completed_callback.bind(download_completed_callback)) + _download.request_completed.connect(download_completed_callback) #TODO handle deadlock while _download.get_http_client_status() != HTTPClient.STATUS_DISCONNECTED: From 34b09c6c19e5898234a12649ed169d04620e51a3 Mon Sep 17 00:00:00 2001 From: Festerdam Date: Thu, 24 Aug 2023 20:55:47 +0100 Subject: [PATCH 6/6] Avoid hanging main thread --- .../remote_editor_download.gd | 101 +++++++++++------- 1 file changed, 60 insertions(+), 41 deletions(-) diff --git a/src/components/editors/remote/remote_editor_download/remote_editor_download.gd b/src/components/editors/remote/remote_editor_download/remote_editor_download.gd index 4ac900ae..291f485b 100755 --- a/src/components/editors/remote/remote_editor_download/remote_editor_download.gd +++ b/src/components/editors/remote/remote_editor_download/remote_editor_download.gd @@ -3,6 +3,7 @@ extends PanelContainer const uuid = preload("res://addons/uuid.gd") signal downloaded(abs_zip_path: String) +signal integrity_check_completed(passed: bool) const _CHECKSUM_FILENAME = "SHA512-SUMS.txt" @@ -21,6 +22,7 @@ var _fallback_url: String = "": _fallback_url = new_fallback var _target_abs_dir: String var _file_name: String +var _integrity_check_thread: Thread @onready var _progress_bar: ProgressBar = get_node("%ProgressBar") @onready var _status: Label = get_node("%Status") @@ -41,6 +43,7 @@ func _ready() -> void: _retry_button.pressed.connect(func(): _remove_downloaded_file() + _integrity_check_thread = null if _retry_callback: _retry_callback.call() ) @@ -53,6 +56,8 @@ func _ready() -> void: assert(_fallback_url) start(_fallback_url, _target_abs_dir, _file_name, _url) ) + + integrity_check_completed.connect(_on_integrity_check_completed) func start(url, target_abs_dir, file_name, fallback_url = ""): @@ -102,21 +107,29 @@ func start(url, target_abs_dir, file_name, fallback_url = ""): if _fallback_url: _mirror_switch_button.show() _status.text = status - return else: _status.text = "Checking file integrity..." - if not await _check_file_integrity(): - $AcceptErrorDialog.dialog_text = "Integrity check failed!\n" + \ - "Retry or use another mirror." - $AcceptErrorDialog.popup_centered() - _retry_button.show() - if _fallback_url: - _mirror_switch_button.show() - return - - _install_button.disabled = false - _status.text = "Ready to install" - downloaded.emit(_download.download_file) + var checksum_url: String = _url.get_base_dir().path_join(_CHECKSUM_FILENAME) + var checksum_downloader = HTTPRequest.new() + checksum_downloader.timeout = 10 + checksum_downloader.download_chunk_size = 2048 + checksum_downloader.download_file = _target_abs_dir.get_base_dir() \ + .path_join(_CHECKSUM_FILENAME) + add_child(checksum_downloader) + print("Downloading ", checksum_url) + checksum_downloader.request(checksum_url) + var sig_result = await checksum_downloader.request_completed + var request_result = sig_result[0] + var request_response_code = sig_result[1] + checksum_downloader.queue_free() + if request_result != HTTPRequest.RESULT_SUCCESS \ + or request_response_code != 200: + integrity_check_completed.emit(true, false) + else: + _dismiss_button.disabled = true + _dismiss_button.hide() + _integrity_check_thread = Thread.new() + _integrity_check_thread.start(_check_file_integrity) assert(target_abs_dir.ends_with("/")) print("Downloading " + url) @@ -192,30 +205,19 @@ func _notification(what: int) -> void: ## Failing to download SHA512-SUMS.txt is treated as success, because, again, ## this method does not verify authenticity of the release.[br] ## -## Returns true on success and false on failure. -func _check_file_integrity() -> bool: - var checksum_url: String = _url.get_base_dir().path_join(_CHECKSUM_FILENAME) - var checksum_downloader := HTTPRequest.new() - checksum_downloader.download_file = _target_abs_dir.get_base_dir() \ - .path_join(_CHECKSUM_FILENAME) - checksum_downloader.timeout = 10 - checksum_downloader.download_chunk_size = 2048 - print("Downloading ", checksum_url) - add_child(checksum_downloader) - checksum_downloader.request(checksum_url) - var sig_result = await checksum_downloader.request_completed - var result = sig_result[0] - var response_code = sig_result[1] - if result != HTTPRequest.RESULT_SUCCESS or response_code != 200: - return true - +## This is supposed to be run by a separate thread to avoid freezing the main +## thread. Emits [signal integrity_check_completed] when done, with +## [code]passed[/code] set to [code]true[/code] if successful, +## [code]false[/code] otherwise. +func _check_file_integrity() -> void: var globalized_directory_path = ProjectSettings.globalize_path(_target_abs_dir) match OS.get_name(): "Linux", "OpenBSD", "FreeBSD", "NetBSD", "BSD": var status = OS.execute("bash", ["-c", "cd " + globalized_directory_path + " && sha512sum -c --ignore-missing --status " + _CHECKSUM_FILENAME]) - return status == 0 or status == 127 + call_deferred("emit_signal", "integrity_check_completed", + status == 0 or status == 127) "Windows": var certutil_output = [] OS.execute("certutil", ["-hashfile", @@ -223,25 +225,42 @@ func _check_file_integrity() -> bool: "SHA512"], certutil_output) var output_lines = certutil_output[0].split("\n") if len(output_lines) <= 2: - return true + call_deferred("emit_signal", "integrity_check_completed", true) var obtained_sum = output_lines[1].strip_edges() if not obtained_sum.is_valid_hex_number(): - return true + call_deferred("emit_signal", "integrity_check_completed", true) var checksum_file_contents = FileAccess.open(_target_abs_dir.path_join( _CHECKSUM_FILENAME), FileAccess.READ).get_as_text() - if obtained_sum + " " + _file_name in checksum_file_contents: - return true - else: - return false + call_deferred("emit_signal", "integrity_check_completed", + obtained_sum + " " + _file_name in checksum_file_contents) "macOS": - return true #TODO + call_deferred("emit_signal", "integrity_check_completed", true) _: - return true - - checksum_downloader.queue_free() + call_deferred("emit_signal", "integrity_check_completed", true) + + +func _on_integrity_check_completed(passed: bool, from_thread: bool = true) -> void: + assert(_integrity_check_thread == null or not _integrity_check_thread.is_alive()) + _dismiss_button.show() + _dismiss_button.disabled = false + if _integrity_check_thread == null: + return + if passed: + _install_button.disabled = false + _status.text = "Ready to install" + downloaded.emit(_download.download_file) + else: + $AcceptErrorDialog.dialog_text = "Integrity check failed!\n" + \ + "Retry or use another mirror." + $AcceptErrorDialog.popup_centered() + _retry_button.show() + if _fallback_url: + _mirror_switch_button.show() + _integrity_check_thread.wait_to_finish() + _integrity_check_thread = null func _remove_downloaded_file():