Skip to content

Commit 203ba9e

Browse files
authored
Fix bug when S3Request is GC'd before download completes (#506)
**Issue:** I discovered that, if you drop the reference to an S3Request before the operation completes and it's downloading using `recv_filepath`, then everything looks like it succeeds but you end up with a 0 byte file on disk. **Description of changes:** Ensure the file isn't closed until the operation is complete. Most of our code was careful to keep things alive until the operation was complete. But for whatever reason we put the file-closing code in the wrong place.
1 parent cf9b73f commit 203ba9e

File tree

2 files changed

+14
-5
lines changed

2 files changed

+14
-5
lines changed

source/s3_meta_request.c

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ struct aws_s3_meta_request *aws_py_get_s3_meta_request(PyObject *meta_request) {
4444
}
4545

4646
static void s_destroy(struct s3_meta_request_binding *meta_request) {
47+
if (meta_request->recv_file) {
48+
fclose(meta_request->recv_file);
49+
}
4750
if (meta_request->copied_message) {
4851
aws_http_message_release(meta_request->copied_message);
4952
}
@@ -297,14 +300,13 @@ static void s_s3_request_on_finish(
297300
/*************** GIL RELEASE ***************/
298301
}
299302

300-
/* Invoked when the python object get cleaned up */
303+
/* Invoked when S3Request._binding gets cleaned up.
304+
* DO NOT destroy the C binding struct or anything inside it yet.
305+
* The user might have let S3Request get GC'd,
306+
* but the s3_meta_request_binding* must outlive the native aws_s3_meta_request* */
301307
static void s_s3_meta_request_capsule_destructor(PyObject *capsule) {
302308
struct s3_meta_request_binding *meta_request = PyCapsule_GetPointer(capsule, s_capsule_name_s3_meta_request);
303309

304-
if (meta_request->recv_file) {
305-
fclose(meta_request->recv_file);
306-
meta_request->recv_file = NULL;
307-
}
308310
if (meta_request->native) {
309311
aws_s3_meta_request_release(meta_request->native);
310312
} else {

test/test_s3.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,13 @@ def test_get_object_file_object(self):
270270
on_headers=self._on_request_headers,
271271
on_progress=self._on_progress)
272272
finished_future = s3_request.finished_future
273+
274+
# Regression test: Let S3Request get GC'd early.
275+
# The download should continue without problems.
276+
# We once had a bug where the file would get closed too early:
277+
# https://github.com/awslabs/aws-crt-python/pull/506
278+
del s3_request
279+
273280
finished_future.result(self.timeout)
274281

275282
# Result check

0 commit comments

Comments
 (0)