-
Notifications
You must be signed in to change notification settings - Fork 567
feat(storage): setting default checksum #30878
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
eb4c130
a5bb385
9000bdb
fae8845
0147623
13275c4
a89ca60
cbcbe63
6158a74
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -101,6 +101,40 @@ | |
| _(bucket_complete.autoclass_enabled).must_equal bucket_autoclass_enabled | ||
| _(bucket_complete.autoclass_terminal_storage_class).must_equal bucket_autoclass_terminal_storage_class | ||
| end | ||
|
|
||
| it "creates a file with checksum: :crc32c by default" do | ||
| new_file_name = random_file_path | ||
|
|
||
| Tempfile.open ["google-cloud", ".txt"] do |tmpfile| | ||
| tmpfile.write "Hello world!" | ||
| tmpfile.rewind | ||
|
|
||
| crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile | ||
|
|
||
| mock = Minitest::Mock.new | ||
| mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), | ||
| [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) | ||
|
|
||
| bucket.service.mocked_service = mock | ||
| bucket.create_file tmpfile, new_file_name | ||
|
|
||
| mock.verify | ||
| end | ||
| end | ||
|
|
||
| it "creates a file with a StringIO and checksum: :crc32c by default" do | ||
| new_file_name = random_file_path | ||
| new_file_contents = StringIO.new "Hello world" | ||
| mock = Minitest::Mock.new | ||
| mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. how are we verifying the checksum for the new String IO object here, also use a slightly different string in different tests for robustness |
||
| [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: new_file_contents, options: {retries: 0}) | ||
|
|
||
| bucket.service.mocked_service = mock | ||
|
|
||
| bucket.create_file new_file_contents, new_file_name | ||
|
|
||
| mock.verify | ||
| end | ||
|
|
||
| it "returns frozen cors" do | ||
| bucket_complete.cors.each do |cors| | ||
|
|
@@ -595,9 +629,11 @@ | |
| new_file_name = random_file_path | ||
|
|
||
| Tempfile.create ["google-cloud", ".txt"] do |tmpfile| | ||
|
|
||
| crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile | ||
| mock = Minitest::Mock.new | ||
| mock.expect :insert_object, create_file_gapi(bucket_user_project.name, new_file_name), | ||
| [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0}) | ||
| [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0}) | ||
|
|
||
| bucket_user_project.service.mocked_service = mock | ||
|
|
||
|
|
@@ -610,7 +646,7 @@ | |
|
|
||
| it "creates an file with a StringIO" do | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. nit: a |
||
| new_file_name = random_file_path | ||
| new_file_contents = StringIO.new | ||
| new_file_contents = StringIO.new("Hello world") | ||
|
|
||
| mock = Minitest::Mock.new | ||
| mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), | ||
|
|
@@ -1417,6 +1453,12 @@ def empty_file_gapi cache_control: nil, content_disposition: nil, | |
| content_type: nil, crc32c: nil, md5: nil, metadata: nil, | ||
| storage_class: nil, temporary_hold: nil, | ||
| event_based_hold: nil | ||
|
|
||
| # Set crc32c if both md5 and crc32c are not provided | ||
| if md5.nil? && crc32c.nil? | ||
| crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for(StringIO.new("Hello world")) | ||
| end | ||
|
|
||
| params = { | ||
| cache_control: cache_control, content_type: content_type, | ||
| content_disposition: content_disposition, md5_hash: md5, | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -244,6 +244,25 @@ | |
| mock.verify | ||
| end | ||
| end | ||
|
|
||
| it "creates a file with checksum: :crc32c by default" do | ||
| new_file_name = random_file_path | ||
|
|
||
| Tempfile.open ["google-cloud", ".txt"] do |tmpfile| | ||
| tmpfile.write "Hello world!" | ||
| tmpfile.rewind | ||
|
|
||
| crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile | ||
| mock = Minitest::Mock.new | ||
| mock.expect :insert_object, create_file_gapi(bucket.name, new_file_name), | ||
| [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, options: {retries: 0}) | ||
|
|
||
| bucket.service.mocked_service = mock | ||
| bucket.create_file tmpfile, new_file_name | ||
|
|
||
| mock.verify | ||
| end | ||
| end | ||
|
|
||
| it "creates a file with attributes" do | ||
| new_file_name = random_file_path | ||
|
|
@@ -340,9 +359,10 @@ | |
| new_file_name = random_file_path | ||
|
|
||
| Tempfile.create ["google-cloud", ".txt"] do |tmpfile| | ||
| crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for tmpfile | ||
| mock = Minitest::Mock.new | ||
| mock.expect :insert_object, create_file_gapi(bucket_user_project.name, new_file_name), | ||
| [bucket.name, empty_file_gapi], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0}) | ||
| [bucket.name, empty_file_gapi(crc32c: crc32c)], **insert_object_args(name: new_file_name, upload_source: tmpfile, user_project: "test", options: {retries: 0}) | ||
|
|
||
| bucket_user_project.service.mocked_service = mock | ||
|
|
||
|
|
@@ -1091,6 +1111,10 @@ def empty_file_gapi cache_control: nil, content_disposition: nil, | |
| content_encoding: nil, content_language: nil, | ||
| content_type: nil, crc32c: nil, md5: nil, metadata: nil, | ||
| storage_class: nil | ||
| # Set crc32c if both md5 and crc32c are not provided | ||
| if md5.nil? && crc32c.nil? | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This section is repeated multiple times across test files, do we have a common test helper where it could be moved. |
||
| crc32c = Google::Cloud::Storage::File::Verifier.crc32c_for(StringIO.new("Hello world")) | ||
| end | ||
| params = { | ||
| cache_control: cache_control, content_type: content_type, | ||
| content_disposition: content_disposition, md5_hash: md5, | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We should document this clearly in an appropriate, that Crc32c will be used by default for integrity checks