diff --git a/README.md b/README.md index acdf5c7..a96100e 100644 --- a/README.md +++ b/README.md @@ -17,11 +17,12 @@ Click on the name of a role to view that content's documentation: ### Roles Name | Description --- | --- +[cloud.gcp_ops.move_objects_between_storage_buckets](https://github.com/redhat-cop/cloud.gcp_ops/blob/main/roles/move_objects_between_storage_buckets/README.md)|A role to move objects between GCP Storage buckets. ### Playbooks Name | Description --- | --- - +cloud.gcp_ops.move_objects_between_storage_buckets](https://github.com/redhat-cop/cloud.gcp_ops/blob/main/playbooks/MOVE_OBJECTS_FROM_STORAGE_BUCKETS.md)|A playbook to move objects between GCP Storage buckets. ## Installation and Usage diff --git a/changelogs/fragments/20230817-move-objects-between-storage-buckets.yaml b/changelogs/fragments/20230817-move-objects-between-storage-buckets.yaml new file mode 100644 index 0000000..552f47d --- /dev/null +++ b/changelogs/fragments/20230817-move-objects-between-storage-buckets.yaml @@ -0,0 +1,3 @@ +--- +minor_changes: + - add new role and playbook to move objects between GCP Storage buckets (https://github.com/redhat-cop/cloud.gcp_ops/pull/4). diff --git a/playbooks/MOVE_OBJECTS_BETWEEN_STORAGE_BUCKETS.md b/playbooks/MOVE_OBJECTS_BETWEEN_STORAGE_BUCKETS.md new file mode 100644 index 0000000..6eaccbc --- /dev/null +++ b/playbooks/MOVE_OBJECTS_BETWEEN_STORAGE_BUCKETS.md @@ -0,0 +1,12 @@ +## cloud.gcp_ops.move_objects_between_storage_buckets + +A playbook to move objects between GCP Storage buckets. + +Variables +-------------- + +* **move_objects_between_storage_buckets_source_bucket**: The name of the GCP storage bucket to retrieve objects from. **Required** +* **move_objects_between_storage_buckets_dest_bucket**: The name of the GCP storage bucket to download objects to. **Required** +* **move_objects_between_storage_buckets_objects**: A list of existing objects from the source bucket. **Required** + +See [cloud.gcp_ops.gcp_setup_credentials](https://github.com/redhat-cop/cloud.gcp_ops/blob/main/roles/gcp_setup_credentials/README.md) for required credentials variables. \ No newline at end of file diff --git a/playbooks/move_objects_between_storage_buckets.yaml b/playbooks/move_objects_between_storage_buckets.yaml new file mode 100644 index 0000000..ede14e2 --- /dev/null +++ b/playbooks/move_objects_between_storage_buckets.yaml @@ -0,0 +1,7 @@ +--- +- name: Move objects between Storage buckets + hosts: localhost + gather_facts: false + + roles: + - role: cloud.gcp_ops.move_objects_between_storage_buckets diff --git a/roles/move_objects_between_storage_buckets/README.md b/roles/move_objects_between_storage_buckets/README.md new file mode 100644 index 0000000..4f75720 --- /dev/null +++ b/roles/move_objects_between_storage_buckets/README.md @@ -0,0 +1,50 @@ +move_objects_between_storage_buckets +================== + +A role to move an object from one GCP storage bucket to another. + +Requirements +------------ + +GCP credentials with permission to upload, download and delete objects from GCP storage bucket. + + +Role Variables +-------------- + +* **move_objects_between_storage_buckets_source_bucket**: The name of the GCP storage bucket to retrieve objects from. **Required** +* **move_objects_between_storage_buckets_dest_bucket**: The name of the GCP storage bucket to download objects to. **Required** +* **move_objects_between_storage_buckets_objects**: A list of existing objects from the source bucket. **Required** + +Dependencies +------------ + +- role: [gcp_setup_credentials](../gcp_setup_credentials/README.md) + +## Example: +``` +--- +- name: Playbook for moving one object from one GCP storage bucket into another. + hosts: localhost + gather_facts: false + + roles: + - role: cloud.gcp_ops.move_objects_between_storage_buckets + move_objects_between_storage_buckets_source_bucket: my-src-storage + move_objects_between_storage_buckets_dest_bucket: my-dest-storage + move_objects_between_storage_buckets_objects: + - object-1 + - object-2 +``` + +License +------- + +GNU General Public License v3.0 or later + +See [LICENCE](https://github.com/redhat-cop/cloud.gcp_ops/blob/main/LICENSE) to see the full text. + +Author Information +------------------ + +- Ansible Cloud Content Team diff --git a/roles/move_objects_between_storage_buckets/handlers/main.yml b/roles/move_objects_between_storage_buckets/handlers/main.yml new file mode 100644 index 0000000..a623040 --- /dev/null +++ b/roles/move_objects_between_storage_buckets/handlers/main.yml @@ -0,0 +1,5 @@ +--- +- name: Delete temporary directory + ansible.builtin.file: + state: absent + path: "{{ move_objects_between_storage_buckets__tmpdir.path }}" diff --git a/roles/move_objects_between_storage_buckets/meta/main.yml b/roles/move_objects_between_storage_buckets/meta/main.yml new file mode 100644 index 0000000..c498e42 --- /dev/null +++ b/roles/move_objects_between_storage_buckets/meta/main.yml @@ -0,0 +1,3 @@ +--- +dependencies: + - role: cloud.gcp_ops.gcp_setup_credentials diff --git a/roles/move_objects_between_storage_buckets/tasks/main.yml b/roles/move_objects_between_storage_buckets/tasks/main.yml new file mode 100644 index 0000000..5059377 --- /dev/null +++ b/roles/move_objects_between_storage_buckets/tasks/main.yml @@ -0,0 +1,27 @@ +--- +- name: Fail when 'move_objects_between_storage_buckets_source_bucket' is undefined + ansible.builtin.fail: + msg: GCP storage bucket source should be defined as move_objects_between_storage_buckets_source_bucket + when: move_objects_between_storage_buckets_source_bucket is undefined + +- name: Fail when 'move_objects_between_storage_buckets_dest_bucket' is undefined + ansible.builtin.fail: + msg: GCP storage bucket destination should be defined as move_objects_between_storage_buckets_dest_bucket + when: move_objects_between_storage_buckets_dest_bucket is undefined + +- name: Fail when 'move_objects_between_storage_buckets_objects' is undefined + ansible.builtin.fail: + msg: Objects to move should be defined as move_objects_between_storage_buckets_objects + when: move_objects_between_storage_buckets_objects is undefined + +- name: Create temporary directory to download objects in + ansible.builtin.tempfile: + state: directory + suffix: .storage + register: move_objects_between_storage_buckets__tmpdir + notify: + - 'Delete temporary directory' + +- name: Include tasks 'move_object.yml' + ansible.builtin.include_tasks: move_object.yml + with_items: "{{ move_objects_between_storage_buckets_objects }}" diff --git a/roles/move_objects_between_storage_buckets/tasks/move_object.yml b/roles/move_objects_between_storage_buckets/tasks/move_object.yml new file mode 100644 index 0000000..d7aa244 --- /dev/null +++ b/roles/move_objects_between_storage_buckets/tasks/move_object.yml @@ -0,0 +1,24 @@ +--- +- name: 'Move single object from source bucket into destination bucket' + module_defaults: + group/gcp: "{{ gcp_setup_credentials__output }}" + block: + - name: "Download object from bucket source bucket" + google.cloud.gcp_storage_object: + action: download + src: "{{ item }}" + dest: "{{ move_objects_between_storage_buckets__tmpdir.path }}/{{ item }}" + bucket: "{{ move_objects_between_storage_buckets_source_bucket }}" + + - name: "Updload object into destination bucket" + google.cloud.gcp_storage_object: + action: upload + src: "{{ move_objects_between_storage_buckets__tmpdir.path }}/{{ item }}" + dest: "{{ item }}" + bucket: "{{ move_objects_between_storage_buckets_dest_bucket }}" + + - name: "Delete object from source bucket" + google.cloud.gcp_storage_object: + action: delete + src: "{{ item }}" + bucket: "{{ move_objects_between_storage_buckets_source_bucket }}" diff --git a/tests/integration/targets/test_move_objects_between_storage_buckets/aliases b/tests/integration/targets/test_move_objects_between_storage_buckets/aliases new file mode 100644 index 0000000..d4e7aea --- /dev/null +++ b/tests/integration/targets/test_move_objects_between_storage_buckets/aliases @@ -0,0 +1,3 @@ +cloud/gcp +role/move_objects_between_storage_buckets +time=10s diff --git a/tests/integration/targets/test_move_objects_between_storage_buckets/defaults/main.yml b/tests/integration/targets/test_move_objects_between_storage_buckets/defaults/main.yml new file mode 100644 index 0000000..0932974 --- /dev/null +++ b/tests/integration/targets/test_move_objects_between_storage_buckets/defaults/main.yml @@ -0,0 +1,8 @@ +--- +test_source_storage_bucket: "{{ resource_prefix }}-bucket-src" +test_dest_storage_bucket: "{{ resource_prefix }}-bucket-dest" +test_bucket_objects: + - name: "{{ resource_prefix }}-obj-1" + value: "This has been created using Ansible Seeded content Role" + - name: "{{ resource_prefix }}-obj-2" + value: "Ansible roles for managing GCP resources" diff --git a/tests/integration/targets/test_move_objects_between_storage_buckets/tasks/create_buckets.yml b/tests/integration/targets/test_move_objects_between_storage_buckets/tasks/create_buckets.yml new file mode 100644 index 0000000..f462b38 --- /dev/null +++ b/tests/integration/targets/test_move_objects_between_storage_buckets/tasks/create_buckets.yml @@ -0,0 +1,42 @@ +--- +- name: Create GCP Storage bucket + google.cloud.gcp_storage_bucket: + name: "{{ item }}" + auth_kind: "{{ gcp_auth_kind }}" + service_account_file: "{{ gcp_cred_file }}" + project: "{{ gcp_project }}" + with_items: + - "{{ test_source_storage_bucket }}" + - "{{ test_dest_storage_bucket }}" + +- name: Create temporary directory to store data + ansible.builtin.tempfile: + state: directory + suffix: .upload + register: _tmpdir + +- name: Upload objects into source bucket + block: + - name: Copy content into files + ansible.builtin.copy: + dest: "{{ _tmpdir.path }}/{{ item.name }}.txt" + content: "{{ item.value }}" + mode: '0755' + with_items: "{{ test_bucket_objects }}" + + - name: Upload object into source bucket + google.cloud.gcp_storage_object: + action: upload + bucket: "{{ test_source_storage_bucket }}" + src: "{{ _tmpdir.path }}/{{ item.name }}.txt" + dest: "{{ item.name }}" + auth_kind: "{{ gcp_auth_kind }}" + service_account_file: "{{ gcp_cred_file }}" + project: "{{ gcp_project }}" + with_items: "{{ test_bucket_objects }}" + + always: + - name: Delete temporary directory + ansible.builtin.file: + state: absent + path: "{{ _tmpdir.path }}" diff --git a/tests/integration/targets/test_move_objects_between_storage_buckets/tasks/delete_buckets.yml b/tests/integration/targets/test_move_objects_between_storage_buckets/tasks/delete_buckets.yml new file mode 100644 index 0000000..0afa969 --- /dev/null +++ b/tests/integration/targets/test_move_objects_between_storage_buckets/tasks/delete_buckets.yml @@ -0,0 +1,35 @@ +--- +- name: Delete objects from source buckets + google.cloud.gcp_storage_object: + action: delete + bucket: "{{ test_source_storage_bucket }}" + src: "{{ item.name }}" + auth_kind: "{{ gcp_auth_kind }}" + service_account_file: "{{ gcp_cred_file }}" + project: "{{ gcp_project }}" + with_items: "{{ test_bucket_objects }}" + register: delete_result + failed_when: (delete_result is failed) and (delete_result.msg != "File does not exist in bucket") + +- name: Delete objects from destination buckets + google.cloud.gcp_storage_object: + action: delete + bucket: "{{ test_dest_storage_bucket }}" + src: "{{ item.name }}" + auth_kind: "{{ gcp_auth_kind }}" + service_account_file: "{{ gcp_cred_file }}" + project: "{{ gcp_project }}" + with_items: "{{ test_bucket_objects }}" + register: delete_result + failed_when: (delete_result is failed) and (delete_result.msg != "File does not exist in bucket") + +- name: Delete GCP Storage bucket + google.cloud.gcp_storage_bucket: + name: "{{ item }}" + auth_kind: "{{ gcp_auth_kind }}" + service_account_file: "{{ gcp_cred_file }}" + project: "{{ gcp_project }}" + state: absent + with_items: + - "{{ test_source_storage_bucket }}" + - "{{ test_dest_storage_bucket }}" diff --git a/tests/integration/targets/test_move_objects_between_storage_buckets/tasks/main.yml b/tests/integration/targets/test_move_objects_between_storage_buckets/tasks/main.yml new file mode 100644 index 0000000..5732d7f --- /dev/null +++ b/tests/integration/targets/test_move_objects_between_storage_buckets/tasks/main.yml @@ -0,0 +1,36 @@ +--- +- name: Test Role move_objects_between_storage_buckets + block: + - name: Create buckets + ansible.builtin.include_tasks: create_buckets.yml + + - name: Move objects from source to destination storage bucket + ansible.builtin.include_role: + name: cloud.gcp_ops.move_objects_between_storage_buckets + vars: + move_objects_between_storage_buckets_source_bucket: "{{ test_source_storage_bucket }}" + move_objects_between_storage_buckets_dest_bucket: "{{ test_dest_storage_bucket }}" + move_objects_between_storage_buckets_objects: "{{ test_bucket_objects | map(attribute='name') | list }}" + + # Validate that objects were deleted from Source bucket + - name: Validate that objects have been removed from Source bucket + google.cloud.gcp_storage_object: + action: download + bucket: "{{ test_dest_storage_bucket }}" + src: "{{ item }}" + dest: "{{ item }}.txt" + auth_kind: "{{ gcp_auth_kind }}" + service_account_file: "{{ gcp_cred_file }}" + project: "{{ gcp_project }}" + with_items: "{{ test_bucket_objects }}" + register: _download + failed_when: (_download is not failed) or (_download.msg != "File does not exist in bucket") + + # Validate that objects from destination are stored as expected + - name: Validate objects move + ansible.builtin.include_tasks: validate_objects_from_bucket.yml + with_items: "{{ test_bucket_objects }}" + + always: + - name: Delete buckets + ansible.builtin.include_tasks: delete_buckets.yml diff --git a/tests/integration/targets/test_move_objects_between_storage_buckets/tasks/validate_objects_from_bucket.yml b/tests/integration/targets/test_move_objects_between_storage_buckets/tasks/validate_objects_from_bucket.yml new file mode 100644 index 0000000..1f8e2c9 --- /dev/null +++ b/tests/integration/targets/test_move_objects_between_storage_buckets/tasks/validate_objects_from_bucket.yml @@ -0,0 +1,30 @@ +--- +- name: Download content from bucket + block: + - name: Create temporary file + ansible.builtin.tempfile: + suffix: .object + register: _tmpfile + + - name: Download objects from destination bucket + google.cloud.gcp_storage_object: + action: download + bucket: "{{ test_dest_storage_bucket }}" + src: "{{ item.name }}" + dest: "{{ _tmpfile.path }}" + auth_kind: "{{ gcp_auth_kind }}" + service_account_file: "{{ gcp_cred_file }}" + project: "{{ gcp_project }}" + + - name: Assert that object value from Storage bucket is as expected + ansible.builtin.assert: + that: + - item.value == object_data + vars: + object_data: "{{ lookup('file', _tmpfile.path) }}" + + always: + - name: Delete temporary file + ansible.builtin.file: + state: absent + path: "{{ _tmpfile.path }}"