Playbook and Role to move objects from one storage bucket into another

.yamllint

@@ -0,0 +1,20 @@
+---
+extends: default
+
+rules:
+  braces:
+    max-spaces-inside: 1
+    level: error
+  brackets:
+    max-spaces-inside: 1
+    level: error
+  document-start: disable
+  line-length: disable
+  truthy: disable
+  indentation:
+    spaces: 2
+    indent-sequences: consistent
+ignore: |
+  .cache
+  .tox
+  tests/output

README.md

@@ -17,11 +17,13 @@ Click on the name of a role to view that content's documentation:
 ### Roles
 Name | Description
 --- | ---
+[cloud.gcp_ops.gcp_setup_credentials](https://github.com/redhat-cop/cloud.gcp_ops/blob/main/roles/gcp_setup_credentials/README.md)|A role to define credentials for google.cloud modules.
+[cloud.gcp_ops.move_objects_between_storage_buckets](https://github.com/redhat-cop/cloud.gcp_ops/blob/main/roles/move_objects_between_storage_buckets/README.md)|A role to move objects between GCP Storage buckets.
 
 ### Playbooks
 Name | Description
 --- | ---
-<!--end collection content-->
+cloud.gcp_ops.move_objects_between_storage_buckets](https://github.com/redhat-cop/cloud.gcp_ops/blob/main/playbooks/MOVE_OBJECTS_FROM_STORAGE_BUCKETS.md)|A playbook to move objects between GCP Storage buckets.
 
 ## Installation and Usage
 

changelogs/changelog.yaml

@@ -0,0 +1 @@
+releases: {}

changelogs/config.yaml

@@ -0,0 +1,30 @@
+---
+changelog_filename_template: ../CHANGELOG.rst
+changelog_filename_version_depth: 0
+changes_file: changelog.yaml
+changes_format: combined
+keep_fragments: false
+mention_ancestor: true
+new_plugins_after_name: removed_features
+notesdir: fragments
+prelude_section_name: release_summary
+prelude_section_title: Release Summary
+sections:
+  - - major_changes
+    - Major Changes
+  - - minor_changes
+    - Minor Changes
+  - - breaking_changes
+    - Breaking Changes / Porting Guide
+  - - deprecated_features
+    - Deprecated Features
+  - - removed_features
+    - Removed Features (previously deprecated)
+  - - security_fixes
+    - Security Fixes
+  - - bugfixes
+    - Bugfixes
+  - - known_issues
+    - Known Issues
+title: CHANGE THIS IN changelogs/config.yaml!
+trivial_section_name: trivial

changelogs/fragments/20230817-move-objects-between-storage-buckets.yaml

@@ -0,0 +1,3 @@
+---
+minor_changes:
+  - add new role and playbook to move objects between GCP Storage buckets (https://github.com/redhat-cop/cloud.gcp_ops/pull/4).

playbooks/MOVE_OBJECTS_BETWEEN_STORAGE_BUCKETS.md

@@ -0,0 +1,12 @@
+## cloud.gcp_ops.move_objects_between_storage_buckets
+
+A playbook to move objects between GCP Storage buckets.
+
+Variables
+--------------
+
+* **move_objects_between_storage_buckets_source_bucket**: The name of the GCP storage bucket to retrieve objects from. **Required**
+* **move_objects_between_storage_buckets_dest_bucket**: The name of the GCP storage bucket to download objects to. **Required**
+* **move_objects_between_storage_buckets_objects**: A list of existing objects from the source bucket. **Required**
+
+See [cloud.gcp_ops.gcp_setup_credentials](https://github.com/redhat-cop/cloud.gcp_ops/blob/main/roles/gcp_setup_credentials/README.md) for required credentials variables.

playbooks/move_objects_between_storage_buckets.yaml

@@ -0,0 +1,7 @@
+---
+- name: Move objects between Storage buckets
+  hosts: localhost
+  gather_facts: false
+
+  roles:
+    - role: cloud.gcp_ops.move_objects_between_storage_buckets

roles/gcp_setup_credentials/README.md

@@ -0,0 +1,52 @@
+# aws_setup_credentials
+
+A role to define credentials for google.cloud modules. The role defines a variable named **gcp_setup_credentials\_\_output** which contains GCP credentials for google.cloud modules based on user input.
+
+## Requirements
+
+N/A
+
+## Role Variables
+
+- **gcp_auth_kind**: The type of credential to use. Choices: ``application``, ``machineaccount``, ``serviceaccount``, ``accesstoken``. Default to ``serviceaccount``
+- **gcp_service_account_content**: The contents of a Service Account JSON file, either in a dictionary or as a JSON string.
+- **gcp_service_account_file**: The path of a Service Account JSON file.
+- **gcp_service_account_email**: An optional service account email address if ``gcp_auth_kind`` is set to ``machineaccount``and the user does not wish to use the default email.
+- **gcp_access_token**: An OAuth2 access token if ``gcp_auth_kind`` is set to ``accesstoken``.
+- **gcp_project**: The GCP project to use..
+- **gcp_scopes**: A List of GCP scopes to be used.
+
+
+## Dependencies
+
+- NA
+
+## Example Playbook
+
+    - hosts: localhost
+      gather_facts: false
+
+      roles:
+        - role: cloud.gcp_ops.gcp_setup_credentials
+          gcp_service_account: path_to_my_service_account_file
+          gcp_project: my-sample-project
+
+      tasks:
+        - name: Create a GCP storage bucket
+          module_defaults:
+            group/gcp:
+              '{{ gcp_setup_credentials__output }}'
+          block:
+            - name: Create a GCP Storage bucket
+              google.cloud.gcp_storage_bucket:
+                name: my-bucket-test
+
+## License
+
+GNU General Public License v3.0 or later
+
+See [LICENCE](https://github.com/ansible-collections/cloud.aws_troubleshooting/blob/main/LICENSE) to see the full text.
+
+## Author Information
+
+- Ansible Cloud Content Team

roles/gcp_setup_credentials/defaults/main.yml

@@ -0,0 +1,7 @@
+---
+gcp_auth_kind: serviceaccount
+gcp_auth_kind_choices:
+  - application
+  - machineaccount
+  - serviceaccount
+  - accesstoken

roles/gcp_setup_credentials/tasks/main.yml

@@ -0,0 +1,52 @@
+---
+- name: Validate auth_kind value
+  ansible.builtin.fail:
+    msg: "{{ gcp_auth_kind }} variable should be one of {{ gcp_auth_kind_choices }}"
+  when: gcp_auth_kind not in gcp_auth_kind_choices
+
+- name: Define intial value for credentials output
+  ansible.builtin.set_fact:
+    gcp_setup_credentials__output: {}
+
+- name: Set GCP auth kind
+  ansible.builtin.set_fact:
+    gcp_setup_credentials__output: "{{ gcp_setup_credentials__output | combine({'auth_kind': gcp_auth_kind}) }}"
+  no_log: true
+
+- name: Set GCP project
+  ansible.builtin.set_fact:
+    gcp_setup_credentials__output: "{{ gcp_setup_credentials__output | combine({'project': gcp_project}) }}"
+  when: gcp_project is defined
+  no_log: true
+
+- name: Set GCP scopes
+  ansible.builtin.set_fact:
+    gcp_setup_credentials__output: "{{ gcp_setup_credentials__output | combine({'scopes': gcp_scopes}) }}"
+  when: gcp_scopes is defined
+  no_log: true
+
+- name: Set GCP service account email
+  ansible.builtin.set_fact:
+    gcp_setup_credentials__output: "{{ gcp_setup_credentials__output | combine({'service_account_email': gcp_service_account_email}) }}"
+  when: gcp_service_account_email is defined
+  no_log: true
+
+- name: Set GCP access token
+  ansible.builtin.set_fact:
+    gcp_setup_credentials__output: "{{ gcp_setup_credentials__output | combine({'access_token': gcp_access_token}) }}"
+  when:
+    - gcp_access_token is defined
+    - gcp_auth_kind == "accesstoken"
+  no_log: true
+
+- name: Set GCP service account content
+  ansible.builtin.set_fact:
+    gcp_setup_credentials__output: "{{ gcp_setup_credentials__output | combine({'service_account_contents': gcp_service_account_content}) }}"
+  when: gcp_service_account_content is defined
+  no_log: true
+
+- name: Set GCP service account file
+  ansible.builtin.set_fact:
+    gcp_setup_credentials__output: "{{ gcp_setup_credentials__output | combine({'service_account_file': gcp_service_account_file}) }}"
+  when: gcp_service_account_file is defined
+  no_log: true

roles/move_objects_between_storage_buckets/README.md

@@ -0,0 +1,50 @@
+move_objects_between_storage_buckets
+==================
+
+A role to move an object from one GCP storage bucket to another.
+
+Requirements
+------------
+
+GCP credentials with permission to upload, download and delete objects from GCP storage bucket.
+
+
+Role Variables
+--------------
+
+* **move_objects_between_storage_buckets_source_bucket**: The name of the GCP storage bucket to retrieve objects from. **Required**
+* **move_objects_between_storage_buckets_dest_bucket**: The name of the GCP storage bucket to download objects to. **Required**
+* **move_objects_between_storage_buckets_objects**: A list of existing objects from the source bucket. **Required**
+
+Dependencies
+------------
+
+- role: [gcp_setup_credentials](../gcp_setup_credentials/README.md)
+
+## Example:
+```
+---
+- name: Playbook for moving one object from one GCP storage bucket into another.
+  hosts: localhost
+  gather_facts: false
+
+  roles:
+    - role: cloud.gcp_ops.move_objects_between_storage_buckets
+      move_objects_between_storage_buckets_source_bucket: my-src-storage
+      move_objects_between_storage_buckets_dest_bucket: my-dest-storage
+      move_objects_between_storage_buckets_objects:
+        - object-1
+        - object-2
+```
+
+License
+-------
+
+GNU General Public License v3.0 or later
+
+See [LICENCE](https://github.com/redhat-cop/cloud.gcp_ops/blob/main/LICENSE) to see the full text.
+
+Author Information
+------------------
+
+- Ansible Cloud Content Team

roles/move_objects_between_storage_buckets/handlers/main.yml

@@ -0,0 +1,5 @@
+---
+- name: Delete temporary directory
+  ansible.builtin.file:
+    state: absent
+    path: "{{ move_objects_between_storage_buckets__tmpdir.path }}"

roles/move_objects_between_storage_buckets/meta/main.yml

@@ -0,0 +1,3 @@
+---
+dependencies:
+  - role: cloud.gcp_ops.gcp_setup_credentials

roles/move_objects_between_storage_buckets/tasks/main.yml

@@ -0,0 +1,27 @@
+---
+- name: Fail when 'move_objects_between_storage_buckets_source_bucket' is undefined
+  ansible.builtin.fail:
+    msg: GCP storage bucket source should be defined as move_objects_between_storage_buckets_source_bucket
+  when: move_objects_between_storage_buckets_source_bucket is undefined
+
+- name: Fail when 'move_objects_between_storage_buckets_dest_bucket' is undefined
+  ansible.builtin.fail:
+    msg: GCP storage bucket destination should be defined as move_objects_between_storage_buckets_dest_bucket
+  when: move_objects_between_storage_buckets_dest_bucket is undefined
+
+- name: Fail when 'move_objects_between_storage_buckets_objects' is undefined
+  ansible.builtin.fail:
+    msg: Objects to move should be defined as move_objects_between_storage_buckets_objects
+  when: move_objects_between_storage_buckets_objects is undefined
+
+- name: Create temporary directory to download objects in
+  ansible.builtin.tempfile:
+    state: directory
+    suffix: .storage
+  register: move_objects_between_storage_buckets__tmpdir
+  notify:
+    - 'Delete temporary directory'
+
+- name: Include tasks 'move_object.yml'
+  ansible.builtin.include_tasks: move_object.yml
+  with_items: "{{ move_objects_between_storage_buckets_objects }}"

roles/move_objects_between_storage_buckets/tasks/move_object.yml

@@ -0,0 +1,24 @@
+---
+- name: 'Move single object from source bucket into destination bucket'
+  module_defaults:
+    group/gcp: "{{ gcp_setup_credentials__output }}"
+  block:
+    - name: "Download object from bucket source bucket"
+      google.cloud.gcp_storage_object:
+        action: download
+        src: "{{ item }}"
+        dest: "{{ move_objects_between_storage_buckets__tmpdir.path }}/{{ item }}"
+        bucket: "{{ move_objects_between_storage_buckets_source_bucket }}"
+
+    - name: "Updload object into destination bucket"
+      google.cloud.gcp_storage_object:
+        action: upload
+        src: "{{ move_objects_between_storage_buckets__tmpdir.path }}/{{ item }}"
+        dest: "{{ item }}"
+        bucket: "{{ move_objects_between_storage_buckets_dest_bucket }}"
+
+    - name: "Delete object from source bucket"
+      google.cloud.gcp_storage_object:
+        action: delete
+        src: "{{ item }}"
+        bucket: "{{ move_objects_between_storage_buckets_source_bucket }}"

tests/integration/targets/test_gcp_setup_credentials/aliases

@@ -0,0 +1,3 @@
+cloud/gcp
+role/gcp_setup_credentials
+time=10s

tests/integration/targets/test_gcp_setup_credentials/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+gcp_storage_bucket_name: "{{ resource_prefix }}-bucket"
+gcp_service_account_file: "{{ gcp_cred_file }}"
+gcp_auth_kind: "{{ gcp_cred_kind }}"

tests/integration/targets/test_gcp_setup_credentials/tasks/main.yml

@@ -0,0 +1,21 @@
+---
+- name: Include 'gcp_setup_credentials' roles
+  ansible.builtin.include_role:
+    name: cloud.gcp_ops.gcp_setup_credentials
+
+- name: Trying calling module using generating credentials
+  module_defaults:
+    group/gcp:
+      "{{ gcp_setup_credentials__output }}"
+  block:
+
+    - name: Create GCP storage bucket
+      google.cloud.gcp_storage_bucket:
+        name: "{{ gcp_storage_bucket_name }}"
+        state: present
+
+  always:
+    - name: Delete GCP storage bucket
+      google.cloud.gcp_storage_bucket:
+        name: "{{ gcp_storage_bucket_name }}"
+        state: absent

tests/integration/targets/test_move_objects_between_storage_buckets/aliases

@@ -0,0 +1,3 @@
+cloud/gcp
+role/move_objects_between_storage_buckets
+time=10s

tests/integration/targets/test_move_objects_between_storage_buckets/defaults/main.yml

@@ -0,0 +1,8 @@
+---
+test_source_storage_bucket: "{{ resource_prefix }}-bucket-src"
+test_dest_storage_bucket: "{{ resource_prefix }}-bucket-dest"
+test_bucket_objects:
+  - name: "{{ resource_prefix }}-obj-1"
+    value: "This has been created using Ansible Seeded content Role"
+  - name: "{{ resource_prefix }}-obj-2"
+    value: "Ansible roles for managing GCP resources"