Created
April 24, 2024 14:23
-
-
Save dmzoneill/497746f38c5786c96e8859f1131667af to your computer and use it in GitHub Desktop.
Revisions
-
dmzoneill created this gist
Apr 24, 2024 .There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode charactersOriginal file line number Diff line number Diff line change @@ -0,0 +1,1161 @@ daoneill@redhat:~/src/awx-operator$ git diff 2.15.0 2.16.0 diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 41724d2..90f7cfd 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -17,6 +17,7 @@ jobs: - -t replicas env: DOCKER_API_VERSION: "1.41" + DEBUG_OUTPUT_DIR: /tmp/awx_operator_molecule_test steps: - uses: actions/checkout@v3 @@ -37,10 +38,18 @@ jobs: MOLECULE_VERBOSITY: 3 PY_COLORS: '1' ANSIBLE_FORCE_COLOR: '1' + STORE_DEBUG_OUTPUT: true run: | sudo rm -f $(which kustomize) make kustomize KUSTOMIZE_PATH=$(readlink -f bin/kustomize) molecule test -s kind -- ${{ matrix.ansible_args }} + + - name: Upload artifacts for failed tests if Run Molecule fails + if: failure() + uses: actions/upload-artifact@v2 + with: + name: awx_operator_molecule_test + path: ${{ env.DEBUG_OUTPUT_DIR }} helm: runs-on: ubuntu-latest name: helm diff --git a/config/crd/bases/awx.ansible.com_awxs.yaml b/config/crd/bases/awx.ansible.com_awxs.yaml index 5137f5f..34e441d 100644 --- a/config/crd/bases/awx.ansible.com_awxs.yaml +++ b/config/crd/bases/awx.ansible.com_awxs.yaml @@ -1940,26 +1940,36 @@ spec: description: Metrics-Utility Image PullPolicy type: string metrics_utility_configmap: - description: Metrics-Utlity ConfigMap + description: Metrics-Utility ConfigMap + type: string + metrics_utility_secret: + description: Metrics-Utility Secret type: string metrics_utility_cronjob_gather_schedule: - description: Metrics-Utlity Gather Data CronJob Schedule + description: Metrics-Utility Gather Data CronJob Schedule type: string default: '@hourly' metrics_utility_cronjob_report_schedule: - description: Metrics-Utlity Report CronJob Schedule + description: Metrics-Utility Report CronJob Schedule type: string default: '@monthly' + metrics_utility_ship_target: + description: Metrics-Utility Ship Target + type: string metrics_utility_pvc_claim: - description: Metrics-Utlity PVC Claim + description: Metrics-Utility PVC Claim type: string metrics_utility_pvc_claim_size: - description: Metrics-Utlity PVC Claim Size + description: Metrics-Utility PVC Claim Size type: string default: 5Gi metrics_utility_pvc_claim_storage_class: - description: Metrics-Utlity PVC Claim Storage Class + description: Metrics-Utility PVC Claim Storage Class type: string + metrics_utility_console_enabled: + description: Enable metrics utility shipping to Red Hat Hybrid Cloud Console + type: boolean + default: false type: object status: properties: diff --git a/config/manifests/bases/awx-operator.clusterserviceversion.yaml b/config/manifests/bases/awx-operator.clusterserviceversion.yaml index 679585c..d7c9fa2 100644 --- a/config/manifests/bases/awx-operator.clusterserviceversion.yaml +++ b/config/manifests/bases/awx-operator.clusterserviceversion.yaml @@ -1039,7 +1039,7 @@ spec: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:text - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true - - displayName: Metrics-Utlity Image Version + - displayName: Metrics-Utility Image Version path: metrics_utility_image_version x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced @@ -1051,42 +1051,60 @@ spec: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:imagePullPolicy - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true - - displayName: Metrics-Utlity ConfigMap + - displayName: Metrics-Utility ConfigMap path: metrics_utility_configmap x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:io.kubernetes:ConfigMap - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true - - displayName: Metrics-Utlity Gather Data CronJob Schedule + - displayName: Metrics-Utility Secret + path: metrics_utility_secret + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:io.kubernetes:Secret + - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true + - displayName: Metrics-Utility Gather Data CronJob Schedule path: metrics_utility_cronjob_gather_schedule x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:text - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true - - displayName: Metrics-Utlity Report CronJob Schedule + - displayName: Metrics-Utility Report CronJob Schedule path: metrics_utility_cronjob_report_schedule x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:text - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true - - displayName: Metrics-Utlity PVC Claim + - displayName: Metrics-Utility Ship Target + path: metrics_utility_ship_target + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:text + - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true + - displayName: Metrics-Utility PVC Claim path: metrics_utility_pvc_claim x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:text - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true - - displayName: Metrics-Utlity PVC Claim Size + - displayName: Metrics-Utility PVC Claim Size path: metrics_utility_pvc_claim_size x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:com.tectonic.ui:text - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true - - displayName: Metrics-Utlity PVC Claim Storage Class + - displayName: Metrics-Utility PVC Claim Storage Class path: metrics_utility_pvc_claim_storage_class x-descriptors: - urn:alm:descriptor:com.tectonic.ui:advanced - urn:alm:descriptor:io.kubernetes:StorageClass - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true + - displayName: Metrics-Utility Enabled Shipping to Red Hat Hybrid Cloud Console + path: metrics_utility_console_enabled + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced + - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - urn:alm:descriptor:com.tectonic.ui:fieldDependency:metrics_utility_enabled:true statusDescriptors: - description: Route to access the instance deployed displayName: URL diff --git a/docs/README.md b/docs/README.md index acae2a7..dd37160 100644 --- a/docs/README.md +++ b/docs/README.md @@ -2,9 +2,15 @@ To build the AWX Operator docs locally: -1. Clone the AWX operator repository. -2. From the root directory: - a. pip install --user -r docs/requirements.txt - b. mkdocs build +1. Clone the AWX operator repository. +1. Preferrably, create a virtual environment for installing the dependencies. + a. `python3 -m venv venv` + b. `source venv/bin/activate` +1. From the root directory: + a. `pip install -r docs/requirements.txt` + b. `mkdocs build` +1. View the docs in your browser: + a. `mkdocs serve` + b. Open your browser and navigate to `http://127.0.0.1:8000/` -This will create a new directory called `site/` in the root of your clone containing the index.html and static files. To view the docs in your browser, navigate there in your file explorer and double-click on the `index.html` file. This should open the docs site in your browser. \ No newline at end of file +This will create a new directory called `site/` in the root of your clone containing the index.html and static files. diff --git a/docs/requirements.txt b/docs/requirements.txt index 30dc0b8..9b424e3 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -50,6 +50,7 @@ idna==3.6 jinja2==3.1.3 # via # mkdocs + # mkdocs-macros-plugin # mkdocs-material # mkdocstrings jsmin==3.0.1 @@ -88,18 +89,21 @@ mkdocs==1.5.3 # mkdocs-autorefs # mkdocs-gen-files # mkdocs-htmlproofer-plugin + # mkdocs-macros-plugin # mkdocs-material # mkdocs-minify-plugin # mkdocs-monorepo-plugin # mkdocstrings -mkdocs-ansible==24.2.1 - # via -r docs/requirements.in +mkdocs-ansible==24.3.0 + # via -r requirements.in mkdocs-autorefs==0.5.0 # via mkdocstrings mkdocs-gen-files==0.5.0 # via mkdocs-ansible mkdocs-htmlproofer-plugin==1.0.0 # via mkdocs-ansible +mkdocs-macros-plugin==1.0.5 + # via mkdocs-ansible mkdocs-material==9.2.6 # via mkdocs-ansible mkdocs-material-extensions==1.3.1 @@ -145,12 +149,15 @@ pymdown-extensions==10.0.1 pyquery==2.0.0 # via readtime python-dateutil==2.8.2 - # via ghp-import + # via + # ghp-import + # mkdocs-macros-plugin python-slugify==8.0.4 # via mkdocs-monorepo-plugin pyyaml==6.0.1 # via # mkdocs + # mkdocs-macros-plugin # pymdown-extensions # pyyaml-env-tag pyyaml-env-tag==0.1 @@ -168,6 +175,8 @@ six==1.16.0 # via python-dateutil soupsieve==2.5 # via beautifulsoup4 +termcolor==2.4.0 + # via mkdocs-macros-plugin text-unidecode==1.3 # via python-slugify tinycss2==1.2.1 diff --git a/docs/upgrade/upgrading.md b/docs/upgrade/upgrading.md index 0bf61c8..cde0e79 100644 --- a/docs/upgrade/upgrading.md +++ b/docs/upgrade/upgrading.md @@ -1,6 +1,7 @@ ### Upgrading -To upgrade AWX, it is recommended to upgrade the awx-operator to the version that maps to the desired version of AWX. To find the version of AWX that will be installed by the awx-operator by default, check the version specified in the `DEFAULT_AWX_VERSION` variable for that particular release. You can do so by running the following command +To upgrade AWX, it is recommended to upgrade the awx-operator to the version that maps to the desired version of AWX. To find the version of AWX that will be installed by the awx-operator by default, check the version specified in the `DEFAULT_AWX_VERSION` variable for that particular release. You can do so by running the following command + ```shell AWX_OPERATOR_VERSION=2.8.0 docker run --entrypoint="" quay.io/ansible/awx-operator:$AWX_OPERATOR_VERSION bash -c "env | grep DEFAULT_AWX_VERSION" @@ -12,26 +13,23 @@ Apply the awx-operator.yml for that release to upgrade the operator, and in turn The first part of any upgrade should be a backup. Note, there are secrets in the pod which work in conjunction with the database. Having just a database backup without the required secrets will not be sufficient for recovering from an issue when upgrading to a new version. See the [backup role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/backup) for information on how to backup your database and secrets. -In the event you need to recover the backup see the [restore role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/restore). *Before Restoring from a backup*, be sure to: -* delete the old existing AWX CR -* delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-15-<deployment-name>-postgres-15-0` +In the event you need to recover the backup see the [restore role documentation](https://github.com/ansible/awx-operator/tree/devel/roles/restore). _Before Restoring from a backup_, be sure to: -**Note**: Do not delete the namespace/project, as that will delete the backup and the backup's PVC as well. +- delete the old existing AWX CR +- delete the persistent volume claim (PVC) for the database from the old deployment, which has a name like `postgres-15-<deployment-name>-postgres-15-0` +**Note**: Do not delete the namespace/project, as that will delete the backup and the backup's PVC as well. #### PostgreSQL Upgrade Considerations If there is a PostgreSQL major version upgrade, after the data directory on the PVC is migrated to the new version, the old PVC is kept by default. -This provides the ability to roll back if needed, but can take up extra storage space in your cluster unnecessarily. You can configure it to be deleted automatically -after a successful upgrade by setting the following variable on the AWX spec. - +This provides the ability to roll back if needed, but can take up extra storage space in your cluster unnecessarily. You can configure it to be deleted automatically after a successful upgrade by setting the following variable on the AWX spec. ```yaml - spec: - postgres_keep_pvc_after_upgrade: False +spec: + postgres_keep_pvc_after_upgrade: False ``` - #### v0.14.0 ##### Cluster-scope to Namespace-scope considerations diff --git a/docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md b/docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md index af6025c..95eb817 100644 --- a/docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md +++ b/docs/user-guide/advanced-configuration/custom-volume-and-volume-mount-options.md @@ -13,7 +13,8 @@ In a scenario where custom volumes and volume mounts are required to either over | init_container_extra_commands | Specify additional commands for Init container | '' | -> :warning: The `ee_extra_volume_mounts` and `extra_volumes` will only take effect to the globally available Execution Environments. For custom `ee`, please [customize the Pod spec](https://docs.ansible.com/ansible-tower/latest/html/administration/external_execution_envs.html#customize-the-pod-spec). +!!! warning + The `ee_extra_volume_mounts` and `extra_volumes` will only take effect to the globally available Execution Environments. For custom `ee`, please [customize the Pod spec](https://docs.ansible.com/ansible-tower/latest/html/administration/external_execution_envs.html#customize-the-pod-spec). Example configuration for ConfigMap @@ -26,62 +27,68 @@ metadata: namespace: <target namespace> data: ansible.cfg: | - [defaults] - remote_tmp = /tmp - [ssh_connection] - ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s + [defaults] + remote_tmp = /tmp + [ssh_connection] + ssh_args = -C -o ControlMaster=auto -o ControlPersist=60s custom.py: | - INSIGHTS_URL_BASE = "example.org" - AWX_CLEANUP_PATHS = True + INSIGHTS_URL_BASE = "example.org" + AWX_CLEANUP_PATHS = True ``` Example spec file for volumes and volume mounts ```yaml --- - spec: - ... - extra_volumes: | - - name: ansible-cfg - configMap: - defaultMode: 420 - items: - - key: ansible.cfg - path: ansible.cfg - name: <resourcename>-extra-config - - name: custom-py - configMap: - defaultMode: 420 - items: - - key: custom.py - path: custom.py - name: <resourcename>-extra-config - - name: shared-volume - persistentVolumeClaim: - claimName: my-external-volume-claim - - init_container_extra_volume_mounts: | - - name: shared-volume - mountPath: /shared - - init_container_extra_commands: | - # set proper permissions (rwx) for the awx user - chmod 775 /shared - chgrp 1000 /shared - - ee_extra_volume_mounts: | - - name: ansible-cfg - mountPath: /etc/ansible/ansible.cfg - subPath: ansible.cfg - - task_extra_volume_mounts: | - - name: custom-py - mountPath: /etc/tower/conf.d/custom.py - subPath: custom.py - - name: shared-volume - mountPath: /shared +spec: + ... + extra_volumes: | + - name: ansible-cfg + configMap: + defaultMode: 420 + items: + - key: ansible.cfg + path: ansible.cfg + name: <resourcename>-extra-config + - name: custom-py + configMap: + defaultMode: 420 + items: + - key: custom.py + path: custom.py + name: <resourcename>-extra-config + - name: shared-volume + persistentVolumeClaim: + claimName: my-external-volume-claim + + init_container_extra_volume_mounts: | + - name: shared-volume + mountPath: /shared + + init_container_extra_commands: | + # set proper permissions (rwx) for the awx user + chmod 775 /shared + chgrp 1000 /shared + + ee_extra_volume_mounts: | + - name: ansible-cfg + mountPath: /etc/ansible/ansible.cfg + subPath: ansible.cfg + + web_extra_volume_mounts: | + - name: custom-py + mountPath: /etc/tower/conf.d/custom.py + subPath: custom.py + + task_extra_volume_mounts: | + - name: custom-py + mountPath: /etc/tower/conf.d/custom.py + subPath: custom.py + - name: shared-volume + mountPath: /shared ``` -> :warning: **Volume and VolumeMount names cannot contain underscores(_)** +!!! warning + **Volume and VolumeMount names cannot contain underscores(_)** ##### Custom UWSGI Configuration We allow the customization of two UWSGI parameters: @@ -143,7 +150,9 @@ $ oc create configmap favicon-configmap --from-file favicon.ico Then specify the extra_volume and web_extra_volume_mounts on your AWX CR spec ```yaml +--- spec: + ... extra_volumes: | - name: favicon configMap: diff --git a/docs/user-guide/advanced-configuration/extra-settings.md b/docs/user-guide/advanced-configuration/extra-settings.md index f798cb9..319cdfd 100644 --- a/docs/user-guide/advanced-configuration/extra-settings.md +++ b/docs/user-guide/advanced-configuration/extra-settings.md @@ -24,3 +24,7 @@ Example configuration of `extra_settings` parameter ``` Note for some settings, such as `LOG_AGGREGATOR_LEVEL`, the value may need double quotes. + +!!! tip + Alternatively, you can pass any additional settings by mounting ConfigMaps or Secrets of the python files (`*.py`) that contain custom settings to under `/etc/tower/conf.d/` in the web and task pods. + See the example of `custom.py` in the [Custom Volume and Volume Mount Options](custom-volume-and-volume-mount-options.md) section. diff --git a/molecule/default/molecule.yml b/molecule/default/molecule.yml index 58f50ee..9a1898f 100644 --- a/molecule/default/molecule.yml +++ b/molecule/default/molecule.yml @@ -21,6 +21,7 @@ provisioner: namespace: ${TEST_OPERATOR_NAMESPACE:-osdk-test} host_vars: localhost: + awx_ee_image: ${AWX_EE_TEST_IMAGE:-""} awx_image: ${AWX_TEST_IMAGE:-""} awx_version: ${AWX_TEST_VERSION:-""} default_awx_version: "{{ lookup('url', 'https://api.github.com/repos/ansible/awx/releases/latest') | from_json | json_query('tag_name') }}" @@ -30,6 +31,8 @@ provisioner: operator_image: ${OPERATOR_IMAGE:-""} operator_pull_policy: ${OPERATOR_PULL_POLICY:-"Always"} kustomize: ${KUSTOMIZE_PATH:-kustomize} + store_debug_output: ${STORE_DEBUG_OUTPUT:-false} + debug_output_dir: ${DEBUG_OUTPUT_DIR:-"/tmp/awx_operator_molecule_test"} env: K8S_AUTH_KUBECONFIG: ${KUBECONFIG:-"~/.kube/config"} verifier: diff --git a/molecule/default/tasks/apply_awx_spec.yml b/molecule/default/tasks/apply_awx_spec.yml index abd5708..01aff07 100644 --- a/molecule/default/tasks/apply_awx_spec.yml +++ b/molecule/default/tasks/apply_awx_spec.yml @@ -1,5 +1,5 @@ --- -- name: Create or update the awx.ansible.com/v1alpha1.AWX +- name: Create or update the awx.ansible.com/v1beta1.AWX k8s: state: present namespace: '{{ namespace }}' diff --git a/molecule/default/tasks/awx_test.yml b/molecule/default/tasks/awx_test.yml index f6f3757..aaac0b8 100644 --- a/molecule/default/tasks/awx_test.yml +++ b/molecule/default/tasks/awx_test.yml @@ -1,14 +1,16 @@ --- - include_tasks: apply_awx_spec.yml -- name: Obtain generated admin password - k8s_info: - namespace: '{{ namespace }}' - kind: Secret - name: example-awx-admin-password - register: admin_pw_secret +- name: Validate AWX deployment + block: + - name: Look up details for this AWX instance + k8s_info: + namespace: "{{ namespace }}" + api_version: "awx.ansible.com/v1beta1" + kind: AWX + name: example-awx + register: this_awx -- block: - name: Get web pod details k8s_info: namespace: '{{ namespace }}' @@ -16,7 +18,6 @@ label_selectors: - app.kubernetes.io/name = example-awx-web register: awx_web_pod - when: not awx_version - name: Get task pod details k8s_info: @@ -25,33 +26,109 @@ label_selectors: - app.kubernetes.io/name = example-awx-task register: awx_task_pod - when: not awx_version - - name: Extract tags from images from web pod - set_fact: - web_image_tags: | - {{ awx_web_pod.resources[0].spec.containers | - map(attribute='image') | - map('regex_search', default_awx_version) }} + - name: Validate DEFAULT_AWX_VERSION + block: + - name: Extract tags from images from web pod + set_fact: + web_image_tags: | + {{ awx_web_pod.resources[0].spec.containers | + map(attribute='image') | + map('regex_search', default_awx_version) }} + - name: Extract tags from images from task pod + set_fact: + task_image_tags: | + {{ awx_task_pod.resources[0].spec.containers | + map(attribute='image') | + map('regex_search', default_awx_version) }} + - fail: + msg: | + It looks like you may have broken the DEFAULT_AWX_VERSION functionality. + This is an environment variable that is set via build arg when releasing awx-operator. + when: + - default_awx_version not in web_image_tags + - default_awx_version not in task_image_tags when: not awx_version - - name: Extract tags from images from task pod - set_fact: - task_image_tags: | - {{ awx_task_pod.resources[0].spec.containers | - map(attribute='image') | - map('regex_search', default_awx_version) }} - when: not awx_version + - name: Validate additional_labels + block: + - name: Extract additional_labels from AWX spec + set_fact: + awx_additional_labels: >- + {{ this_awx.resources[0].metadata.labels + | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels) + | list + }} + + - name: Extract additional_labels from AWX web Pod + set_fact: + awx_web_pod_additional_labels: >- + {{ awx_web_pod.resources[0].metadata.labels + | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels) + | list + }} + + - name: Extract additional_labels from AWX task Pod + set_fact: + awx_task_pod_additional_labels: >- + {{ awx_task_pod.resources[0].metadata.labels + | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels) + | list + }} + + - name: Assert AWX web Pod contains additional_labels + ansible.builtin.assert: + that: + - awx_web_pod_additional_labels == awx_additional_labels + + - name: Assert AWX task Pod contains additional_labels + ansible.builtin.assert: + that: + - awx_task_pod_additional_labels == awx_additional_labels + + - name: Extract web Pod labels which shouldn't have been propagated to it from AWX + set_fact: + awx_web_pod_extra_labels: >- + {{ awx_web_pod.resources[0].metadata.labels + | dict2items | selectattr('key', 'in', ["my/do-not-inherit"]) + | list + }} + + - name: AWX web Pod doesn't contain AWX labels not in additional_labels + ansible.builtin.assert: + that: + - awx_web_pod_extra_labels == [] + + - name: Extract task Pod labels which shouldn't have been propagated to it from AWX + set_fact: + awx_task_pod_extra_labels: >- + {{ awx_task_pod.resources[0].metadata.labels + | dict2items | selectattr('key', 'in', ["my/do-not-inherit"]) + | list + }} + + - name: AWX task Pod doesn't contain AWX labels not in additional_labels + ansible.builtin.assert: + that: + - awx_task_pod_extra_labels == [] - - fail: - msg: | - It looks like you may have broken the DEFAULT_AWX_VERSION functionality. - This is an environment variable that is set via build arg when releasing awx-operator. - when: - - not awx_version - - default_awx_version not in web_image_tags - - default_awx_version not in task_image_tags + rescue: + - name: Re-emit failure + vars: + failed_task: + result: '{{ ansible_failed_result }}' + fail: + msg: '{{ failed_task }}' + +- name: Obtain generated admin password + k8s_info: + namespace: '{{ namespace }}' + kind: Secret + name: example-awx-admin-password + register: admin_pw_secret +- name: Validate demo job launch + block: - name: Launch Demo Job Template awx.awx.job_launch: name: Demo Job Template @@ -60,6 +137,7 @@ controller_host: localhost/awx/ controller_username: admin controller_password: "{{ admin_pw_secret.resources[0].data.password | b64decode }}" + rescue: - name: Get list of project updates and jobs uri: @@ -74,6 +152,12 @@ loop_control: loop_var: resource + - name: Store job_lists debug output + copy: + content: "{{ job_lists | to_nice_json }}" + dest: "{{ debug_output_dir }}/job_lists.json" + when: store_debug_output | default(false) + - name: Get all job and project details uri: url: "http://localhost{{ endpoint }}" @@ -84,6 +168,23 @@ {{ job_lists.results | map(attribute='json') | map(attribute='results') | flatten | map(attribute='url') }} loop_control: loop_var: endpoint + register: job_details + + - name: Store job_details debug output + copy: + content: "{{ job_details | to_nice_json }}" + dest: "{{ debug_output_dir }}/job_details.json" + when: store_debug_output | default(false) + + ## TODO: figure out why this doesn't work + # - name: Store debug outputs + # copy: + # content: '{{ item }}' + # dest: "{{ debug_output_dir }}/{{ item }}.json" + # loop: + # - job_lists + # - job_details + # when: store_debug_output | default(false) - name: Re-emit failure vars: @@ -91,96 +192,3 @@ result: '{{ ansible_failed_result }}' fail: msg: '{{ failed_task }}' - -- block: - - name: Look up details for this AWX instance - k8s_info: - namespace: "{{ namespace }}" - api_version: "awx.ansible.com/v1beta1" - kind: AWX - name: example-awx - register: this_awx - - - name: Get web pod details - k8s_info: - namespace: '{{ namespace }}' - kind: Pod - label_selectors: - - app.kubernetes.io/name = example-awx-web - register: awx_web_pod - - - name: Get task pod details - k8s_info: - namespace: '{{ namespace }}' - kind: Pod - label_selectors: - - app.kubernetes.io/name = example-awx-task - register: awx_task_pod - - - name: Extract additional_labels from AWX spec - set_fact: - awx_additional_labels: >- - {{ this_awx.resources[0].metadata.labels - | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels) - | list - }} - - - name: Extract additional_labels from AWX web Pod - set_fact: - awx_web_pod_additional_labels: >- - {{ awx_web_pod.resources[0].metadata.labels - | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels) - | list - }} - - - name: Extract additional_labels from AWX task Pod - set_fact: - awx_task_pod_additional_labels: >- - {{ awx_task_pod.resources[0].metadata.labels - | dict2items | selectattr('key', 'in', this_awx.resources[0].spec.additional_labels) - | list - }} - - - name: Assert AWX web Pod contains additional_labels - ansible.builtin.assert: - that: - - awx_web_pod_additional_labels == awx_additional_labels - - - name: Assert AWX task Pod contains additional_labels - ansible.builtin.assert: - that: - - awx_task_pod_additional_labels == awx_additional_labels - - - name: Extract web Pod labels which shouldn't have been propagated to it from AWX - set_fact: - awx_web_pod_extra_labels: >- - {{ awx_web_pod.resources[0].metadata.labels - | dict2items | selectattr('key', 'in', ["my/do-not-inherit"]) - | list - }} - - - name: AWX web Pod doesn't contain AWX labels not in additional_labels - ansible.builtin.assert: - that: - - awx_web_pod_extra_labels == [] - - - name: Extract task Pod labels which shouldn't have been propagated to it from AWX - set_fact: - awx_task_pod_extra_labels: >- - {{ awx_task_pod.resources[0].metadata.labels - | dict2items | selectattr('key', 'in', ["my/do-not-inherit"]) - | list - }} - - - name: AWX task Pod doesn't contain AWX labels not in additional_labels - ansible.builtin.assert: - that: - - awx_task_pod_extra_labels == [] - - rescue: - - name: Re-emit failure - vars: - failed_task: - result: '{{ ansible_failed_result }}' - fail: - msg: '{{ failed_task }}' diff --git a/molecule/default/templates/awx_cr_molecule.yml.j2 b/molecule/default/templates/awx_cr_molecule.yml.j2 index f581ecb..16b6837 100644 --- a/molecule/default/templates/awx_cr_molecule.yml.j2 +++ b/molecule/default/templates/awx_cr_molecule.yml.j2 @@ -13,6 +13,12 @@ spec: {% endif %} {% if awx_version %} image_version: {{ awx_version }} +{% endif %} +{% if awx_ee_image %} + control_plane_ee_image: {{ awx_ee_image }} + ee_images: + - image: {{ awx_ee_image }} + name: AWX EE {% endif %} ingress_type: ingress ingress_path: /awx diff --git a/molecule/default/utils/output_all_container_logs_for_pod.yml b/molecule/default/utils/output_all_container_logs_for_pod.yml new file mode 100644 index 0000000..d4e3246 --- /dev/null +++ b/molecule/default/utils/output_all_container_logs_for_pod.yml @@ -0,0 +1,15 @@ +--- +- name: Get all container log in pod + kubernetes.core.k8s_log: + namespace: '{{ namespace }}' + name: '{{ item.metadata.name }}' + all_containers: true + register: all_container_logs + +- name: Store logs in file + ansible.builtin.copy: + content: "{{ all_container_logs.log_lines | join('\n') }}" + dest: '{{ debug_output_dir }}/{{ item.metadata.name }}.log' + +# TODO: all_containser option dump all of the output in a single output make it hard to read we probably should iterate through each of the container to get specific logs +# also we should probably investigate toolings to do OpenShift style sosreport/must-gather for kind cluster or switch to microshift where sosreport is supported diff --git a/molecule/default/utils/output_k8s_resources.yml b/molecule/default/utils/output_k8s_resources.yml new file mode 100644 index 0000000..71fb137 --- /dev/null +++ b/molecule/default/utils/output_k8s_resources.yml @@ -0,0 +1,29 @@ +--- +- name: Retrieve relevant k8s resources + kubernetes.core.k8s_info: + api_version: '{{ item.api_version }}' + kind: '{{ item.kind }}' + namespace: '{{ namespace }}' + loop: + - api_version: v1 + kind: Pod + - api_version: apps/v1 + kind: Deployment + - api_version: v1 + kind: Secret + - api_version: v1 + kind: ConfigMap + - api_version: "awx.ansible.com/v1beta1" + kind: AWX + register: debug_resources + +- name: debug print item.kind and item.metadata.name + debug: + msg: '{{ item.kind }}-{{ item.metadata.name }}' + loop: "{{ debug_resources.results | map(attribute='resources') | flatten }}" + +- name: Output gathered resource to files + ansible.builtin.copy: + content: '{{ item | to_nice_json }}' + dest: '{{ debug_output_dir }}/{{ item.kind }}-{{ item.metadata.name }}.json' + loop: "{{ debug_resources.results | map(attribute='resources') | flatten }}" diff --git a/molecule/default/verify.yml b/molecule/default/verify.yml index 78733c0..31b95d3 100644 --- a/molecule/default/verify.yml +++ b/molecule/default/verify.yml @@ -10,53 +10,41 @@ ctrl_label: control-plane=controller-manager tasks: - - block: + - name: Perform awx tests + block: - name: Import all test files from tasks/ - include_tasks: '{{ item }}' + ansible.builtin.include_tasks: '{{ item }}' with_fileglob: - tasks/awx_test.yml - tasks/awx_replicas_test.yml tags: - always rescue: - - name: Retrieve relevant resources - k8s_info: - api_version: '{{ item.api_version }}' - kind: '{{ item.kind }}' - namespace: '{{ namespace }}' - loop: - - api_version: v1 - kind: Pod - - api_version: apps/v1 - kind: Deployment - - api_version: v1 - kind: Secret - - api_version: v1 - kind: ConfigMap - register: debug_resources + - name: Create debug output directory + ansible.builtin.file: + path: '{{ debug_output_dir }}' + state: directory tags: - always - - name: Retrieve Pod logs - k8s_log: - name: '{{ item.metadata.name }}' - namespace: '{{ namespace }}' - container: awx-manager - loop: "{{ q('k8s', api_version='v1', kind='Pod', namespace=namespace, label_selector=ctrl_label) }}" - register: debug_logs + - name: Gather and output K8s resources + ansible.builtin.include_tasks: utils/output_k8s_resources.yml tags: - always - - name: Output gathered resources - debug: - var: debug_resources + - name: Get all pods + kubernetes.core.k8s_info: + api_version: v1 + kind: Pod + namespace: '{{ namespace }}' + register: all_pods tags: - always - - name: Output gathered logs - debug: - var: item.log_lines - loop: '{{ debug_logs.results }}' + - name: Get all container logs for all pods + ansible.builtin.include_tasks: utils/output_all_container_logs_for_pod.yml + loop: '{{ all_pods.resources }}' + ignore_errors: yes tags: - always @@ -64,7 +52,7 @@ vars: failed_task: result: '{{ ansible_failed_result }}' - fail: + ansible.builtin.fail: msg: '{{ failed_task }}' tags: - always diff --git a/molecule/kind/molecule.yml b/molecule/kind/molecule.yml index aa7f277..e382312 100644 --- a/molecule/kind/molecule.yml +++ b/molecule/kind/molecule.yml @@ -23,6 +23,7 @@ provisioner: namespace: ${TEST_OPERATOR_NAMESPACE:-osdk-test} host_vars: localhost: + awx_ee_image: ${AWX_EE_TEST_IMAGE:-""} awx_image: ${AWX_TEST_IMAGE:-""} awx_version: ${AWX_TEST_VERSION:-""} ansible_python_interpreter: '{{ ansible_playbook_python }}' @@ -34,6 +35,8 @@ provisioner: operator_pull_policy: "Never" kubeconfig: "{{ lookup('env', 'KUBECONFIG') }}" kustomize: ${KUSTOMIZE_PATH:-kustomize} + store_debug_output: ${STORE_DEBUG_OUTPUT:-false} + debug_output_dir: ${DEBUG_OUTPUT_DIR:-"/tmp/awx_operator_molecule_test"} env: K8S_AUTH_KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig KUBECONFIG: ${MOLECULE_EPHEMERAL_DIRECTORY}/kubeconfig diff --git a/roles/backup/tasks/secrets.yml b/roles/backup/tasks/secrets.yml index fa38238..134698f 100644 --- a/roles/backup/tasks/secrets.yml +++ b/roles/backup/tasks/secrets.yml @@ -20,9 +20,7 @@ - name: Dump ingress tls secret names from awx spec and data into file include_tasks: dump_ingress_tls_secrets.yml - with_items: - - "{{ awx_spec.spec['ingress_hosts'] | default('') | map(attribute='tls_secret', default='') | select() | list }}" - when: awx_spec.spec['ingress_hosts'] | default('') | map(attribute='tls_secret', default='') | select() | list | length + with_items: "{{ awx_spec.spec['ingress_hosts'] | default([]) | selectattr('tls_secret', 'defined') | map(attribute='tls_secret') | list }}" - name: Dump receptor secret names and data into file include_tasks: dump_receptor_secrets.yml diff --git a/roles/installer/defaults/main.yml b/roles/installer/defaults/main.yml index 0651316..db9d586 100644 --- a/roles/installer/defaults/main.yml +++ b/roles/installer/defaults/main.yml @@ -494,9 +494,11 @@ nginx_listen_queue_size: "{{ uwsgi_listen_queue_size }}" # metrics-utility (github.com/ansible/metrics-utility) _metrics_utility_enabled: "{{ metrics_utility_enabled | default(false) }}" _metrics_utility_configmap: "{{ metrics_utility_configmap | default(deployment_type + '-metrics-utility-configmap') }}" +_metrics_utility_console_enabled: "{{ metrics_utility_console_enabled | default(false) }}" _metrics_utility_image: "{{ metrics_utility_image | default(_image) }}" _metrics_utility_image_version: "{{ metrics_utility_image_version | default(_image_version) }}" _metrics_utility_image_pull_policy: "{{ metrics_utility_image_pull_policy | default('IfNotPresent') }}" +_metrics_utility_ship_target: "{{ metrics_utility_ship_target | default('directory') }}" _metrics_utility_pvc_claim: "{{ metrics_utility_pvc_claim | default(deployment_type + '-metrics-utility') }}" _metrics_utility_pvc_claim_size: "{{ metrics_utility_pvc_claim_size | default('5Gi') }}" _metrics_utility_cronjob_gather_schedule: "{{ metrics_utility_cronjob_gather_schedule | default('@hourly') }}" diff --git a/roles/installer/tasks/enable_metrics_utility.yml b/roles/installer/tasks/enable_metrics_utility.yml index d9e72a1..ffbb983 100644 --- a/roles/installer/tasks/enable_metrics_utility.yml +++ b/roles/installer/tasks/enable_metrics_utility.yml @@ -1,23 +1,42 @@ --- -# Check to make sure provided pvc exists, error loudly if not. Otherwise, the management pod will just stay in pending state forever. -- name: Check provided PVC claim exists - kubernetes.core.k8s_info: - name: "{{ _metrics_utility_pvc_claim }}" - kind: PersistentVolumeClaim - namespace: "{{ ansible_operator_meta.namespace }}" - when: - - _metrics_utility_pvc_claim | length +- name: Setup PVC if using directory ship target + block: -- name: Create PVC for metrics-utility + # Check to make sure provided pvc exists + - name: Check provided PVC claim exists + kubernetes.core.k8s_info: + name: "{{ _metrics_utility_pvc_claim }}" + kind: PersistentVolumeClaim + namespace: "{{ ansible_operator_meta.namespace }}" + when: + - _metrics_utility_pvc_claim | length + + - name: Create PVC for metrics-utility + kubernetes.core.k8s: + kind: PersistentVolumeClaim + definition: "{{ lookup('template', 'storage/metrics-utility.yaml.j2') }}" + + when: _metrics_utility_ship_target == "directory" + +- name: Create default metrics-utility Kubernetes CronJobs kubernetes.core.k8s: - kind: PersistentVolumeClaim - definition: "{{ lookup('template', 'storage/metrics-utility.yaml.j2') }}" + definition: "{{ lookup('template', item.template) }}" + apply: true + wait: true + vars: + cronjob_name: "{{ item.name }}" + loop: + - {name: 'metrics-utility-gather', template: 'cronjobs/metrics-utility-gather.yaml.j2'} + - {name: 'metrics-utility-report', template: 'cronjobs/metrics-utility-report.yaml.j2'} -- name: Create Kubernetes CronJobs for metrics-utility +- name: Create metrics-utility Kubernetes CronJob for Red Hat Hybrid Cloud Console kubernetes.core.k8s: - definition: "{{ lookup('template', item) }}" + definition: "{{ lookup('template', item.template) }}" apply: true wait: true + vars: + cronjob_name: "{{ item.name }}" + metrics_utility_ship_target: crc # TODO - Update to console when changed loop: - - cronjobs/metrics-utility-gather.yaml.j2 - - cronjobs/metrics-utility-report.yaml.j2 + - {name: 'metrics-utility-gather-console', template: 'cronjobs/metrics-utility-gather.yaml.j2'} + when: _metrics_utility_console_enabled diff --git a/roles/installer/templates/cronjobs/metrics-utility-gather.yaml.j2 b/roles/installer/templates/cronjobs/metrics-utility-gather.yaml.j2 index 35a5cbb..4e717a4 100644 --- a/roles/installer/templates/cronjobs/metrics-utility-gather.yaml.j2 +++ b/roles/installer/templates/cronjobs/metrics-utility-gather.yaml.j2 @@ -2,10 +2,10 @@ apiVersion: batch/v1 kind: CronJob metadata: - name: {{ ansible_operator_meta.name }}-metrics-utility-gather + name: {{ ansible_operator_meta.name }}-{{ cronjob_name }} namespace: '{{ ansible_operator_meta.namespace }}' labels: - app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-gather' + app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-{{ cronjob_name }}' {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }} {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }} spec: @@ -16,7 +16,7 @@ spec: template: metadata: labels: - app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-gather' + app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-{{ cronjob_name }}' {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=12) | trim }} {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=12) | trim }} spec: @@ -30,7 +30,7 @@ spec: {% endfor %} {% endif %} containers: - - name: {{ ansible_operator_meta.name }}-metrics-utility-gather + - name: {{ ansible_operator_meta.name }}-{{ cronjob_name }} image: "{{ _metrics_utility_image }}" imagePullPolicy: "{{ image_pull_policy }}" resources: @@ -41,9 +41,16 @@ spec: - /bin/sh - -c - metrics-utility gather_automation_controller_billing_data --ship --until=10m + env: + - name: METRICS_UTILITY_SHIP_TARGET + value: "{{ _metrics_utility_ship_target }}" envFrom: - configMapRef: name: {{ _metrics_utility_configmap }} +{% if _metrics_utility_secret is defined %} + - secretRef: + name: {{ _metrics_utility_secret }} +{% endif %} volumeMounts: - name: {{ ansible_operator_meta.name }}-metrics-utility mountPath: /metrics-utility diff --git a/roles/installer/templates/cronjobs/metrics-utility-report.yaml.j2 b/roles/installer/templates/cronjobs/metrics-utility-report.yaml.j2 index 2a2a766..c1e7aed 100644 --- a/roles/installer/templates/cronjobs/metrics-utility-report.yaml.j2 +++ b/roles/installer/templates/cronjobs/metrics-utility-report.yaml.j2 @@ -2,10 +2,10 @@ apiVersion: batch/v1 kind: CronJob metadata: - name: {{ ansible_operator_meta.name }}-metrics-utility-report + name: {{ ansible_operator_meta.name }}-{{ cronjob_name }} namespace: '{{ ansible_operator_meta.namespace }}' labels: - app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-report' + app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-{{ cronjob_name }}' {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=4) | trim }} {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=4) | trim }} spec: @@ -16,7 +16,7 @@ spec: template: metadata: labels: - app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-metrics-utility-report' + app.kubernetes.io/name: '{{ ansible_operator_meta.name }}-{{ cronjob_name }}' {{ lookup("template", "../common/templates/labels/common.yaml.j2") | indent(width=12) | trim }} {{ lookup("template", "../common/templates/labels/version.yaml.j2") | indent(width=12) | trim }} spec: @@ -30,7 +30,7 @@ spec: {% endfor %} {% endif %} containers: - - name: {{ ansible_operator_meta.name }}-metrics-utility-report + - name: {{ ansible_operator_meta.name }}-{{ cronjob_name }} image: "{{ _metrics_utility_image }}" imagePullPolicy: "{{ image_pull_policy }}" resources: @@ -44,6 +44,10 @@ spec: envFrom: - configMapRef: name: {{ _metrics_utility_configmap }} +{% if _metrics_utility_secret is defined %} + - secretRef: + name: {{ _metrics_utility_secret }} +{% endif %} volumeMounts: - name: {{ ansible_operator_meta.name }}-metrics-utility mountPath: /metrics-utility daoneill@redhat:~/src/awx-operator$