# resource_properties_data

created_at: 2025-11-30 07:22:12.239312
data:
  {"type": "json", "value": {"barbican": {"endpoints": {"public": "http://172.21.0.2:9311", "internal": "http://172.17.0.2:9311", "admin": "http://172.17.0.2:9311"}, "users": {"barbican": {"password": "G5XhguvbNw19nfLNKira8K2xr"}}, "region": "regionOne", "service": "key-manager", "roles": ["key-manager:service-admin", "creator", "observer", "audit"]}, "cinder": {"users": {"cinder": {"password": "Olf2iTUMR6eJ33irnMaVcSJQX", "roles": ["admin", "service"]}}}, "cinderv3": {"endpoints": {"public": "http://172.21.0.2:8776/v3/%(tenant_id)s", "internal": "http://172.17.0.2:8776/v3/%(tenant_id)s", "admin": "http://172.17.0.2:8776/v3/%(tenant_id)s"}, "users": {"cinderv3": {"password": "Olf2iTUMR6eJ33irnMaVcSJQX", "roles": ["admin", "service"]}}, "region": "regionOne", "service": "volumev3"}, "glance": {"endpoints": {"public": "http://172.21.0.2:9292", "internal": "http://172.17.0.2:9293", "admin": "http://172.17.0.2:9293"}, "users": {"glance": {"password": "PmWCSVDW3B2uUjMQtJIVdd5mn"}}, "region": "regionOne", "service": "image"}, "heat": {"endpoints": {"public": "http://172.21.0.2:8004/v1/%(tenant_id)s", "internal": "http://172.17.0.2:8004/v1/%(tenant_id)s", "admin": "http://172.17.0.2:8004/v1/%(tenant_id)s"}, "users": {"heat": {"password": "zoQ75Cm0pXTer2ifVDLy2roE1"}, "heat_stack_domain_admin": {"password": "n4UsoSw5TVdj2qe8wfVPQKKbC", "roles": ["admin"], "domain": "heat_stack"}}, "region": "regionOne", "service": "orchestration", "roles": ["heat_stack_user"], "domains": ["heat_stack"]}, "heat-cfn": {"endpoints": {"public": "http://172.21.0.2:8000/v1", "internal": "http://172.17.0.2:8000/v1", "admin": "http://172.17.0.2:8000/v1"}, "users": {"heat-cfn": {"password": "zoQ75Cm0pXTer2ifVDLy2roE1"}}, "region": "regionOne", "service": "cloudformation"}, "manila": {"endpoints": {"public": "http://172.21.0.2:8786/v1/%(tenant_id)s", "internal": "http://172.17.0.2:8786/v1/%(tenant_id)s", "admin": "http://172.17.0.2:8786/v1/%(tenant_id)s"}, "users": {"manila": {"password": "2R4FJ91KHJyj5ppSY0XKBqLJg"}}, "region": "regionOne", "service": "share"}, "manilav2": {"endpoints": {"public": "http://172.21.0.2:8786/v2", "internal": "http://172.17.0.2:8786/v2", "admin": "http://172.17.0.2:8786/v2"}, "users": {"manilav2": {"password": "2R4FJ91KHJyj5ppSY0XKBqLJg"}}, "region": "regionOne", "service": "sharev2"}, "neutron": {"endpoints": {"public": "http://172.21.0.2:9696", "internal": "http://172.17.0.2:9696", "admin": "http://172.17.0.2:9696"}, "users": {"neutron": {"password": "Y7cJplS04DjBDFMUWgbx3kdiC"}}, "region": "regionOne", "service": "network"}, "nova": {"endpoints": {"public": "http://172.21.0.2:8774/v2.1", "internal": "http://172.17.0.2:8774/v2.1", "admin": "http://172.17.0.2:8774/v2.1"}, "users": {"nova": {"roles": ["admin", "service"], "password": "nad1A8ZvKDsT60yawIjN1cH4K"}}, "region": "regionOne", "service": "compute"}, "placement": {"endpoints": {"public": "http://172.21.0.2:8778/placement", "internal": "http://172.17.0.2:8778/placement", "admin": "http://172.17.0.2:8778/placement"}, "users": {"placement": {"password": "P6qVEsGa91uUJyG6cL2sqYpFU"}}, "region": "regionOne", "service": "placement"}, "swift": {"endpoints": {"public": "http://172.21.0.2:8080/v1/AUTH_%(tenant_id)s", "internal": "http://172.18.0.2:8080/v1/AUTH_%(tenant_id)s", "admin": "http://172.18.0.2:8080"}, "users": {"swift": {"password": "8dzPI93pJLWng5a7IqjhGTq4v"}}, "region": "regionOne", "service": "object-store", "roles": ["swiftoperator", "ResellerAdmin"]}}}


created_at: 2025-11-30 07:22:12.603531
data:
  {"type": "comma_delimited_list", "value": [{"block": [{"name": "is additonal Cell?", "set_fact": {"nova_additional_cell": false}}, {"name": "check if discover hosts is required", "when": ["not nova_additional_cell|bool", "nova_cellv2_discovery_done is not defined"], "block": [{"name": "discover via nova_manager?", "set_fact": {"nova_cellv2_discovery_delegate_host": "{{ groups['nova_manager'][0] }}", "nova_cellv2_discovery_container": "nova_manager"}, "when": ["groups['nova_manager'] is defined and (groups['nova_manager']|length>0)"]}, {"name": "discover via nova_api?", "set_fact": {"nova_cellv2_discovery_delegate_host": "{{ groups['nova_api'][0] }}", "nova_cellv2_discovery_container": "nova_api"}, "when": ["nova_cellv2_discovery_delegate_host is not defined", "groups['nova_api'] is defined and (groups['nova_api']|length>0)"]}, {"name": "Warn if no discovery host available", "fail": {"msg": "No hosts available to run nova cell_v2 host discovery."}, "ignore_errors": true, "when": ["nova_cellv2_discovery_delegate_host is not defined"]}, {"name": "Discovering nova hosts", "command": "{{ container_cli }} exec {{ nova_cellv2_discovery_container }} nova-manage cell_v2 discover_hosts --by-service", "become": true, "changed_when": false, "delegate_to": "{{ nova_cellv2_discovery_delegate_host }}", "when": ["nova_cellv2_discovery_delegate_host is defined"]}, {"set_fact": {"nova_cellv2_discovery_done": true}}]}]}]}


created_at: 2025-11-30 07:22:12.787252
data:
  {"type": "json", "value": {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "nova_wait_for_api_service.py": {"mode": "0755", "content": "#!/usr/bin/python3\n#\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\nimport argparse\nimport logging\nimport os\nimport six\nimport sys\nimport time\n\n\nfrom keystoneauth1 import loading\nfrom keystoneauth1 import session\n\nfrom novaclient import client\nfrom novaclient.exceptions import ClientException\n\n# In python3 SafeConfigParser was renamed to ConfigParser and the default\n# for duplicate options default to true. In case of nova it is valid to\n# have duplicate option lines, e.g. passthrough_whitelist which leads to\n# issues reading the nova.conf\n# https://bugs.launchpad.net/tripleo/+bug/1827775\nif six.PY3:\n    from six.moves.configparser import ConfigParser\n    config = ConfigParser(strict=False)\nelse:\n    from six.moves.configparser import SafeConfigParser\n    config = SafeConfigParser()\n\ndebug = os.getenv('__OS_DEBUG', 'false')\n\nif debug.lower() == 'true':\n    loglevel = logging.DEBUG\nelse:\n    loglevel = logging.INFO\n\nlogging.basicConfig(stream=sys.stdout, level=loglevel)\nLOG = logging.getLogger('nova_wait_for_api_service')\n\niterations = 60\ntimeout = 10\nnova_cfg = '/etc/nova/nova.conf'\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser(usage='%(prog)s [options]')\n    parser.add_argument('-k', '--insecure',\n                        action=\"store_false\",\n                        dest='insecure',\n                        default=True,\n                        help='Allow insecure connection when using SSL')\n\n    args = parser.parse_args()\n    LOG.debug('Running with parameter insecure = %s',\n              args.insecure)\n\n    if os.path.isfile(nova_cfg):\n        try:\n            config.read(nova_cfg)\n        except Exception:\n            LOG.exception('Error while reading nova.conf:')\n    else:\n        LOG.error('Nova configuration file %s does not exist', nova_cfg)\n        sys.exit(1)\n\n    loader = loading.get_plugin_loader('password')\n    auth = loader.load_from_options(\n        auth_url=config.get('neutron',\n                            'auth_url'),\n        username=config.get('neutron',\n                            'username'),\n        password=config.get('neutron',\n                            'password'),\n        project_name=config.get('neutron',\n                                'project_name'),\n        project_domain_name=config.get('neutron',\n                                       'project_domain_name'),\n        user_domain_name=config.get('neutron',\n                                    'user_domain_name'))\n    sess = session.Session(auth=auth, verify=args.insecure)\n\n    # Wait until this host is listed in the service list\n    for i in range(iterations):\n        try:\n            nova = client.Client('2.11', session=sess, endpoint_type='internal')\n            nova.versions.list()\n            LOG.info('Nova-api service active')\n            sys.exit(0)\n        except ClientException:\n            LOG.info('Waiting for nova-api service')\n        except Exception:\n            LOG.exception(\n                'Error while waiting for nova-api service')\n        time.sleep(timeout)\nsys.exit(1)\n\n# vim: set et ts=4 sw=4 :\n"}, "nova_api_ensure_default_cells.py": {"mode": "0755", "content": "#!/usr/bin/python3\n#\n# Copyright 2022 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\nfrom configparser import ConfigParser\nimport logging\nimport os\nimport subprocess\nimport sys\nfrom urllib import parse as urlparse\n\nconfig = ConfigParser(strict=False)\n\ndebug = os.getenv('__OS_DEBUG', 'false')\n\nif debug.lower() == 'true':\n    loglevel = logging.DEBUG\nelse:\n    loglevel = logging.INFO\n\nlogging.basicConfig(stream=sys.stdout, level=loglevel)\nLOG = logging.getLogger('nova_api_ensure_default_cells')\n\nNOVA_CFG = '/etc/nova/nova.conf'\nCELL0_ID = '00000000-0000-0000-0000-000000000000'\nDEFAULT_CELL_NAME = 'default'\n\n\ndef template_netloc_credentials(netloc, index=None):\n    if '@' in netloc:\n        userpass, hostport = netloc.split('@', 1)\n        has_pass = ':' in userpass\n        if index is None:\n            cred_template = '{username}'\n            if has_pass:\n                cred_template += ':{password}'\n        else:\n            cred_template = '{{username{index}}}'.format(index=index)\n            if has_pass:\n                cred_template += ':{{password{index}}}'.format(index=index)\n        return '@'.join((cred_template, hostport))\n    else:\n        return netloc\n\n\ndef template_url(url):\n    parsed = urlparse.urlparse(url)\n    if ',' in parsed.netloc:\n        orig_netlocs = parsed.netloc.split(',')\n        templ_netlocs = []\n        index = 0\n        for netloc in orig_netlocs:\n            index += 1\n            templ_netlocs.append(template_netloc_credentials(netloc, index))\n        new_netloc = ','.join(templ_netlocs)\n    else:\n        new_netloc = template_netloc_credentials(parsed.netloc)\n    return parsed._replace(netloc=new_netloc).geturl()\n\n\ndef parse_list_cells(list_cells_output):\n    list_cells_lines = list_cells_output.split('\\n')\n    if len(list_cells_lines) < 5:\n        raise ValueError('Invalid nova-manage cell_v2 list_cells output')\n\n    data_rows = list_cells_lines[3:-2]\n    by_name = {}\n    by_uuid = {}\n\n    for row in data_rows:\n        parts = row.split('|')\n        entry = {\n            'name': parts[1].strip(),\n            'uuid': parts[2].strip(),\n            'transport_url': parts[3].strip(),\n            'database_connection': parts[4].strip(),\n        }\n        by_name[entry['name']] = entry\n        by_uuid[entry['uuid']] = entry\n\n    return by_name, by_uuid\n\n\ndef create_or_update_default_cells(cell0_db, default_db, default_transport_url):\n    list_cells_cmd = ['/usr/bin/nova-manage', 'cell_v2', 'list_cells', '--verbose']\n    list_cells_output = subprocess.check_output(list_cells_cmd, encoding='utf-8')\n    cells_by_name, cells_by_uuid = parse_list_cells(list_cells_output)\n\n    if CELL0_ID in cells_by_uuid:\n        LOG.info('Setting cell0 database connection to \\'{}\\''.format(cell0_db))\n        cmd = [\n            '/usr/bin/nova-manage', 'cell_v2', 'update_cell',\n            '--cell_uuid', CELL0_ID,\n            '--database_connection', cell0_db,\n            '--transport-url', 'none:///'\n        ]\n    else:\n        LOG.info('Creating cell0 with database connection \\'{}\\''.format(cell0_db))\n        cmd = [\n            '/usr/bin/nova-manage', 'cell_v2', 'map_cell0',\n             '--database_connection', cell0_db\n        ]\n    subprocess.check_call(cmd)\n\n    if DEFAULT_CELL_NAME in cells_by_name:\n        LOG.info('Setting default cell database connection to \\'{}\\' and transport url to \\'{}\\''.format(\n            default_db, default_transport_url))\n        cmd = [\n            '/usr/bin/nova-manage', 'cell_v2', 'update_cell',\n            '--cell_uuid', cells_by_name[DEFAULT_CELL_NAME]['uuid'],\n            '--database_connection', default_db,\n            '--transport-url', default_transport_url\n        ]\n    else:\n        LOG.info('Creating default cell with database connection \\'{}\\' and transport url \\'{}\\''.format(\n            default_db, default_transport_url))\n        cmd = [\n            '/usr/bin/nova-manage', 'cell_v2', 'create_cell',\n            '--name', DEFAULT_CELL_NAME,\n             '--database_connection', default_db,\n            '--transport-url', default_transport_url\n        ]\n    subprocess.check_call(cmd)\n\n\ndef replace_db_name(db_url, db_name):\n    return urlparse.urlparse(db_url)._replace(path=db_name).geturl()\n\n\nif __name__ == '__main__':\n    if os.path.isfile(NOVA_CFG):\n        try:\n            config.read(NOVA_CFG)\n        except Exception:\n            LOG.exception('Error while reading nova.conf:')\n            sys.exit(1)\n    else:\n        LOG.error('Nova configuration file %s does not exist', NOVA_CFG)\n        sys.exit(1)\n\n    default_database_connection = config.get('database', 'connection')\n    cell0_database_connection = replace_db_name(default_database_connection, 'nova_cell0')\n    default_transport_url = config.get('DEFAULT', 'transport_url')\n\n    create_or_update_default_cells(\n        template_url(cell0_database_connection),\n        template_url(default_database_connection),\n        template_url(default_transport_url)\n    )\n"}, "nova_statedir_ownership.py": {"mode": "0700", "content": "#!/usr/bin/python3\n#\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\nimport logging\nimport os\nimport pwd\nimport selinux\nimport stat\nimport sys\n\ndebug = os.getenv('__OS_DEBUG', 'false')\n\nif debug.lower() == 'true':\n    loglevel = logging.DEBUG\nelse:\n    loglevel = logging.INFO\n\nlogging.basicConfig(stream=sys.stdout, level=loglevel)\nLOG = logging.getLogger('nova_statedir')\n\n\nclass PathManager(object):\n    \"\"\"Helper class to manipulate ownership of a given path\"\"\"\n    def __init__(self, path):\n        self.path = path\n        self.uid = None\n        self.gid = None\n        self.is_dir = None\n        self.secontext = None\n        self._update()\n\n    def _update(self):\n        try:\n            statinfo = os.stat(self.path)\n            self.is_dir = stat.S_ISDIR(statinfo.st_mode)\n            self.uid = statinfo.st_uid\n            self.gid = statinfo.st_gid\n            self.secontext = selinux.lgetfilecon(self.path)[1]\n        except Exception:\n            LOG.exception('Could not update metadata for %s', self.path)\n            raise\n\n    def __str__(self):\n        return \"uid: {} gid: {} path: {}{}\".format(\n            self.uid,\n            self.gid,\n            self.path,\n            '/' if self.is_dir else ''\n        )\n\n    def has_owner(self, uid, gid):\n        return self.uid == uid and self.gid == gid\n\n    def has_either(self, uid, gid):\n        return self.uid == uid or self.gid == gid\n\n    def chown(self, uid, gid):\n        target_uid = -1\n        target_gid = -1\n        if self.uid != uid:\n            target_uid = uid\n        if self.gid != gid:\n            target_gid = gid\n        if (target_uid, target_gid) != (-1, -1):\n            LOG.info('Changing ownership of %s from %d:%d to %d:%d',\n                     self.path,\n                     self.uid,\n                     self.gid,\n                     self.uid if target_uid == -1 else target_uid,\n                     self.gid if target_gid == -1 else target_gid)\n            try:\n                os.chown(self.path, target_uid, target_gid)\n                self._update()\n            except Exception:\n                LOG.exception('Could not change ownership of %s: ',\n                              self.path)\n                raise\n        else:\n            LOG.info('Ownership of %s already %d:%d',\n                     self.path,\n                     uid,\n                     gid)\n\n    def chcon(self, context):\n        # If dir returns whether to recusively set context\n        try:\n            try:\n                selinux.lsetfilecon(self.path, context)\n                LOG.info('Setting selinux context of %s to %s',\n                     self.path, context)\n                return True\n            except OSError as e:\n                if self.is_dir and e.errno == 95:\n                    # Operation not supported, assume NFS mount and skip\n                    LOG.info('Setting selinux context not supported for %s',\n                             self.path)\n                    return False\n                else:\n                    raise\n        except Exception:\n            LOG.exception('Could not set selinux context of %s to %s:',\n                          self.path, context)\n            raise\n\n\nclass NovaStatedirOwnershipManager(object):\n    \"\"\"Class to manipulate the ownership of the nova statedir (/var/lib/nova).\n\n       The nova uid/gid differ on the host and container images. An upgrade\n       that switches from host systemd services to docker requires a change in\n       ownership. Previously this was a naive recursive chown, however this\n       causes issues if nova instance are shared via an NFS mount: any open\n       filehandles in qemu/libvirt fail with an I/O error (LP1778465).\n\n       Instead the upgrade/FFU ansible tasks now lay down a marker file when\n       stopping and disabling the host systemd services. We use this file to\n       determine the host nova uid/gid. We then walk the tree and update any\n       files that have the host uid/gid to the docker nova uid/gid. As files\n       owned by root/qemu etc... are ignored this avoids the issues with open\n       filehandles. The marker is removed once the tree has been walked.\n\n       For subsequent runs, or for a new deployment, we simply ensure that the\n       docker nova user/group owns all directories. This is required as the\n       directories are created with root ownership in host_prep_tasks (the\n       docker nova uid/gid is not known in this context).\n    \"\"\"\n    def __init__(self, statedir, upgrade_marker='upgrade_marker',\n                 nova_user='nova', secontext_marker='../_nova_secontext',\n                 exclude_paths=None):\n        self.statedir = statedir\n        self.nova_user = nova_user\n\n        self.upgrade_marker_path = os.path.join(statedir, upgrade_marker)\n        self.secontext_marker_path = os.path.normpath(os.path.join(statedir, secontext_marker))\n        self.upgrade = os.path.exists(self.upgrade_marker_path)\n\n        self.exclude_paths = [self.upgrade_marker_path]\n        if exclude_paths is not None:\n            for p in exclude_paths:\n                if not p.startswith(os.path.sep):\n                    p = os.path.join(self.statedir, p)\n                self.exclude_paths.append(p)\n\n        self.target_uid, self.target_gid = self._get_nova_ids()\n        self.previous_uid, self.previous_gid = self._get_previous_nova_ids()\n        self.id_change = (self.target_uid, self.target_gid) != \\\n            (self.previous_uid, self.previous_gid)\n        self.target_secontext = self._get_secontext()\n\n    def _get_nova_ids(self):\n        nova_uid, nova_gid = pwd.getpwnam(self.nova_user)[2:4]\n        return nova_uid, nova_gid\n\n    def _get_previous_nova_ids(self):\n        if self.upgrade:\n            statinfo = os.stat(self.upgrade_marker_path)\n            return statinfo.st_uid, statinfo.st_gid\n        else:\n            return self._get_nova_ids()\n\n    def _get_secontext(self):\n        if os.path.exists(self.secontext_marker_path):\n            return selinux.lgetfilecon(self.secontext_marker_path)[1]\n        else:\n            return None\n\n    def _walk(self, top, chcon=True):\n        for f in os.listdir(top):\n            pathname = os.path.join(top, f)\n\n            if pathname in self.exclude_paths:\n                continue\n\n            try:\n                pathinfo = PathManager(pathname)\n                LOG.info(\"Checking %s\", pathinfo)\n                if pathinfo.is_dir:\n                    # Always chown the directories\n                    pathinfo.chown(self.target_uid, self.target_gid)\n                    chcon_r = chcon\n                    if chcon:\n                        chcon_r = pathinfo.chcon(self.target_secontext)\n                    self._walk(pathname, chcon_r)\n                elif self.id_change:\n                    # Only chown files if it's an upgrade and the file is owned by\n                    # the host nova uid/gid\n                    pathinfo.chown(\n                        self.target_uid if pathinfo.uid == self.previous_uid\n                        else pathinfo.uid,\n                        self.target_gid if pathinfo.gid == self.previous_gid\n                        else pathinfo.gid\n                    )\n                    if chcon:\n                        pathinfo.chcon(self.target_secontext)\n            except Exception:\n                # Likely to have been caused by external systems\n                # interacting with this directory tree,\n                # especially on NFS e.g snapshot dirs.\n                # Just ignore it and continue on to the next entry\n                continue\n\n    def run(self):\n        LOG.info('Applying nova statedir ownership')\n        LOG.info('Target ownership for %s: %d:%d',\n                 self.statedir,\n                 self.target_uid,\n                 self.target_gid)\n\n        pathinfo = PathManager(self.statedir)\n        LOG.info(\"Checking %s\", pathinfo)\n        pathinfo.chown(self.target_uid, self.target_gid)\n        chcon = self.target_secontext is not None\n\n        if chcon:\n            pathinfo.chcon(self.target_secontext)\n\n        self._walk(self.statedir, chcon)\n\n        if self.upgrade:\n            LOG.info('Removing upgrade_marker %s',\n                     self.upgrade_marker_path)\n            os.unlink(self.upgrade_marker_path)\n\n        LOG.info('Nova statedir ownership complete')\n\n\ndef get_exclude_paths():\n    exclude_paths = os.environ.get('NOVA_STATEDIR_OWNERSHIP_SKIP')\n    if exclude_paths is not None:\n        exclude_paths = exclude_paths.split(os.pathsep)\n    return exclude_paths\n\n\nif __name__ == '__main__':\n    NovaStatedirOwnershipManager('/var/lib/nova', exclude_paths=get_exclude_paths()).run()\n"}, "nova_wait_for_compute_service.py": {"mode": "0755", "content": "#!/usr/bin/python3\n#\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\nimport argparse\nimport logging\nimport os\nimport six\nimport socket\nimport sys\nimport time\n\n\nfrom keystoneauth1 import loading\nfrom keystoneauth1 import session\n\nfrom novaclient import client\n\n# In python3 SafeConfigParser was renamed to ConfigParser and the default\n# for duplicate options default to true. In case of nova it is valid to\n# have duplicate option lines, e.g. passthrough_whitelist which leads to\n# issues reading the nova.conf\n# https://bugs.launchpad.net/tripleo/+bug/1827775\nif six.PY3:\n    from six.moves.configparser import ConfigParser\n    config = ConfigParser(strict=False)\nelse:\n    from six.moves.configparser import SafeConfigParser\n    config = SafeConfigParser()\n\ndebug = os.getenv('__OS_DEBUG', 'false')\n\nif debug.lower() == 'true':\n    loglevel = logging.DEBUG\nelse:\n    loglevel = logging.INFO\n\nlogging.basicConfig(stream=sys.stdout, level=loglevel)\nLOG = logging.getLogger('nova_wait_for_compute_service')\n\niterations = 60\ntimeout = 10\nnova_cfg = '/etc/nova/nova.conf'\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser(usage='%(prog)s [options]')\n    parser.add_argument('-k', '--insecure',\n                        action=\"store_false\",\n                        dest='insecure',\n                        default=True,\n                        help='Allow insecure connection when using SSL')\n\n    args = parser.parse_args()\n    LOG.debug('Running with parameter insecure = %s',\n              args.insecure)\n\n    if os.path.isfile(nova_cfg):\n        try:\n            config.read(nova_cfg)\n        except Exception:\n            LOG.exception('Error while reading nova.conf:')\n    else:\n        LOG.error('Nova configuration file %s does not exist', nova_cfg)\n        sys.exit(1)\n\n    my_host = config.get('DEFAULT', 'host')\n    if not my_host:\n        # If host isn't set nova defaults to this\n        my_host = socket.gethostname()\n\n    loader = loading.get_plugin_loader('password')\n    auth = loader.load_from_options(\n        auth_url=config.get('neutron',\n                            'auth_url'),\n        username=config.get('neutron',\n                            'username'),\n        password=config.get('neutron',\n                            'password'),\n        project_name=config.get('neutron',\n                                'project_name'),\n        project_domain_name=config.get('neutron',\n                                       'project_domain_name'),\n        user_domain_name=config.get('neutron',\n                                    'user_domain_name'))\n    sess = session.Session(auth=auth, verify=args.insecure)\n    nova = client.Client('2.11', session=sess, endpoint_type='internal',\n                         region_name=config.get('neutron', 'region_name'))\n\n    # Wait until this host is listed in the service list\n    for i in range(iterations):\n        try:\n            service_list = nova.services.list(binary='nova-compute')\n            for entry in service_list:\n                host = getattr(entry, 'host', '')\n                zone = getattr(entry, 'zone', '')\n                if host == my_host and zone != 'internal':\n                    LOG.info('Nova-compute service registered')\n                    sys.exit(0)\n            LOG.info('Waiting for nova-compute service to register')\n        except Exception:\n            LOG.exception(\n                'Error while waiting for nova-compute service to register')\n        time.sleep(timeout)\nsys.exit(1)\n\n# vim: set et ts=4 sw=4 :\n"}, "nova_libvirt_init_secret.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -e\n\nCEPH_INFO=($*)\n\nif [ -z \"$CEPH_INFO\" ]; then\n    echo \"error: At least one CLUSTER:CLIENT tuple must be specified\"\n    exit 1\nfi\n\necho \"------------------------------------------------\"\necho \"Initializing virsh secrets for: ${CEPH_INFO[@]}\"\n\nfor INFO in ${CEPH_INFO[@]}; do\n    IFS=: read CLUSTER CLIENT <<< $INFO\n    if [ ! -f /etc/ceph/${CLUSTER}.conf ]; then\n        echo \"Error: /etc/ceph/${CLUSTER}.conf was not found\"\n        echo \"Path to nova_libvirt_init_secret was ${CEPH_INFO}\"\n        exit 1\n    fi\n    FSID=$(awk '$1 == \"fsid\" {print $3}' /etc/ceph/${CLUSTER}.conf)\n    if [ -z \"${FSID}\" ]; then\n        echo \"Error: /etc/ceph/${CLUSTER}.conf contained an empty fsid definition\"\n        echo \"Check your ceph configuration\"\n        exit 1\n    fi\n\n    echo \"--------\"\n    echo \"Initializing the virsh secret for '$CLUSTER' cluster ($FSID) '$CLIENT' client\"\n\n    # Ensure the secret XML file exists. Puppet should have created a secret.xml\n    # file for the first cluster's secret, so detect when to use that file.\n    if grep -q $FSID /etc/nova/secret.xml; then\n        SECRET_FILE=\"/etc/nova/secret.xml\"\n        SECRET_NAME=\"client.${CLIENT} secret\"\n    else\n        SECRET_FILE=\"/etc/nova/${CLUSTER}-secret.xml\"\n        SECRET_NAME=\"${CLUSTER}.client.${CLIENT} secret\"\n    fi\n\n    if [ ! -f $SECRET_FILE ]; then\n        echo \"Creating $SECRET_FILE\"\n        cat <<EOF > $SECRET_FILE\n<secret ephemeral='no' private='no'>\n  <usage type='ceph'>\n    <name>${SECRET_NAME}</name>\n  </usage>\n  <uuid>${FSID}</uuid>\n</secret>\nEOF\n    else\n        echo \"The $SECRET_FILE file already exists\"\n    fi\n\n    # Ensure the libvirt secret is defined\n    if /usr/bin/virsh secret-list | grep -q $FSID; then\n        echo \"The virsh secret for $FSID has already been defined\"\n    else\n        /usr/bin/virsh secret-define --file $SECRET_FILE\n    fi\n\n    # Fetch the key from the keyring and ensure the secret is set\n    KEY=$(awk '$1 == \"key\" {print $3}' /etc/ceph/${CLUSTER}.client.${CLIENT}.keyring)\n    if /usr/bin/virsh secret-get-value $FSID 2>/dev/null | grep -q $KEY; then\n        echo \"The virsh secret for $FSID has already been set\"\n    else\n        /usr/bin/virsh secret-set-value --secret $FSID --base64 $KEY\n    fi\ndone\n"}, "placement_wait_for_service.py": {"mode": "0755", "content": "#!/usr/bin/python3\n#\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\n# shell script to check if placement API is up after X attempts.\n# Default max is 60 iterations with 10s (default) timeout in between.\n\nimport logging\nimport os\nimport re\nimport six\nimport sys\nimport time\n\nfrom keystoneauth1.identity import v3\nfrom keystoneauth1 import session\nfrom keystoneclient.v3 import client\nimport requests\n\n# In python3 SafeConfigParser was renamed to ConfigParser and the default\n# for duplicate options default to true. In case of nova it is valid to\n# have duplicate option lines, e.g. passthrough_whitelist which leads to\n# issues reading the nova.conf\n# https://bugs.launchpad.net/tripleo/+bug/1827775\nif six.PY3:\n    from six.moves.configparser import ConfigParser\n    config = ConfigParser(strict=False)\nelse:\n    from six.moves.configparser import SafeConfigParser\n    config = SafeConfigParser()\n\n\ndebug = os.getenv('__OS_DEBUG', 'false')\n\nif debug.lower() == 'true':\n    loglevel = logging.DEBUG\nelse:\n    loglevel = logging.INFO\n\nlogging.basicConfig(stream=sys.stdout, level=loglevel)\nLOG = logging.getLogger('placement_wait_for_service')\n\niterations = 60\ntimeout = 10\nplacement_cfg = '/etc/placement/placement.conf'\n\nif __name__ == '__main__':\n    if os.path.isfile(placement_cfg):\n        try:\n            config.read(placement_cfg)\n        except Exception:\n            LOG.exception('Error while reading placement.conf:')\n    else:\n        LOG.error('Placement configuration file %s does not exist',\n                  placement_cfg)\n        sys.exit(1)\n\n    # get keystone client with details from [keystone_authtoken] section\n    auth = v3.Password(\n        user_domain_name=config.get('keystone_authtoken', 'user_domain_name'),\n        username=config.get('keystone_authtoken', 'username'),\n        password=config.get('keystone_authtoken', 'password'),\n        project_name=config.get('keystone_authtoken', 'project_name'),\n        project_domain_name=config.get('keystone_authtoken',\n                                       'project_domain_name'),\n        auth_url=config.get('keystone_authtoken', 'auth_url') + '/v3')\n    sess = session.Session(auth=auth, verify=False)\n    keystone = client.Client(session=sess, interface='internal')\n\n    iterations_endpoint = iterations\n    placement_endpoint_url = None\n    while iterations_endpoint > 1:\n        iterations_endpoint -= 1\n        try:\n            # get placement service id\n            placement_service_id = keystone.services.list(\n                name='placement')[0].id\n\n            # get placement endpoint\n            # Note: puppet-placement does not support setting the interface\n            #       until we have https://review.opendev.org/688862.\n            #       Lets hard code 'internal' for now.\n            placement_endpoint_url = keystone.endpoints.list(\n                service=placement_service_id,\n                region=config.get('keystone_authtoken', 'region_name'),\n                interface='internal')[0].url\n            if not placement_endpoint_url:\n                LOG.error('Failed to get placement service endpoint!')\n            else:\n                break\n        except Exception:\n            LOG.exception('Retry - Failed to get placement service endpoint:')\n        time.sleep(timeout)\n\n    if not placement_endpoint_url:\n        LOG.error('Failed to get placement service endpoint!')\n        sys.exit(1)\n\n    # we should have CURRENT in the request response from placement:\n    # {\"versions\": [{\"status\": \"CURRENT\", \"min_version\": \"1.0\", \"max_version\":\n    # \"1.29\", \"id\": \"v1.0\", \"links\": [{\"href\": \"\", \"rel\": \"self\"}]}]}\n    response_reg = re.compile('.*CURRENT,*')\n\n    while iterations > 1:\n        iterations -= 1\n        try:\n            r = requests.get(placement_endpoint_url + '/', verify=False)\n            if r.status_code == 200 and response_reg.match(r.text):\n                LOG.info('Placement service up! - %s', r.text)\n                sys.exit(0)\n                break\n            else:\n                LOG.info('response - %r', r)\n                LOG.info('Placement service not up - %s, %s',\n                         r.status_code,\n                         r.text)\n        except Exception:\n            LOG.exception('Error query the placement endpoint:')\n        time.sleep(timeout)\n\n    sys.exit(1)\n\n# vim: set et ts=4 sw=4 :\n"}, "create_swift_secret.sh": {"mode": "0700", "content": "#!/bin/bash\nexport OS_PROJECT_DOMAIN_ID=$(crudini --get /etc/swift/keymaster.conf kms_keymaster project_domain_id)\nexport OS_USER_DOMAIN_ID=$(crudini --get /etc/swift/keymaster.conf kms_keymaster user_domain_id)\nexport OS_PROJECT_NAME=$(crudini --get /etc/swift/keymaster.conf kms_keymaster project_name)\nexport OS_USERNAME=$(crudini --get /etc/swift/keymaster.conf kms_keymaster username)\nexport OS_PASSWORD=$(crudini --get /etc/swift/keymaster.conf kms_keymaster password)\nexport OS_AUTH_URL=$(crudini --get /etc/swift/keymaster.conf kms_keymaster auth_endpoint)\nexport OS_AUTH_TYPE=password\nexport OS_IDENTITY_API_VERSION=3\n\necho \"Check if secret already exists\"\nsecret_href=$(openstack secret list --name swift_root_secret_uuid)\nrc=$?\nif [[ $rc != 0 ]]; then\n  echo \"Failed to check secrets, check if Barbican in enabled and responding properly\"\n  exit $rc;\nfi\nif [ -z \"$secret_href\" ]; then\n  echo \"Create new secret\"\n  order_href=$(openstack secret order create --name swift_root_secret_uuid --payload-content-type=\"application/octet-stream\" --algorithm aes --bit-length 256 --mode ctr key -f value -c \"Order href\")\nfi\n"}, "set_swift_keymaster_key_id.sh": {"mode": "0700", "content": "#!/bin/bash\nexport OS_PROJECT_DOMAIN_ID=$(crudini --get /etc/swift/keymaster.conf kms_keymaster project_domain_id)\nexport OS_USER_DOMAIN_ID=$(crudini --get /etc/swift/keymaster.conf kms_keymaster user_domain_id)\nexport OS_PROJECT_NAME=$(crudini --get /etc/swift/keymaster.conf kms_keymaster project_name)\nexport OS_USERNAME=$(crudini --get /etc/swift/keymaster.conf kms_keymaster username)\nexport OS_PASSWORD=$(crudini --get /etc/swift/keymaster.conf kms_keymaster password)\nexport OS_AUTH_URL=$(crudini --get /etc/swift/keymaster.conf kms_keymaster auth_endpoint)\nexport OS_AUTH_TYPE=password\nexport OS_IDENTITY_API_VERSION=3\necho \"retrieve key_id\"\nloop_wait=2\nfor i in {0..5}; do\n  #TODO update uuid from mistral here too\n  secret_href=$(openstack secret list --name swift_root_secret_uuid)\n  if [ \"$secret_href\" ]; then\n    echo \"set key_id in keymaster.conf\"\n    secret_href=$(openstack secret list --name swift_root_secret_uuid -f value -c \"Secret href\")\n    crudini --set /etc/swift/keymaster.conf kms_keymaster key_id ${secret_href##*/}\n    exit 0\n  else\n    echo \"no key, wait for $loop_wait and check again\"\n    sleep $loop_wait\n    ((loop_wait++))\n  fi\ndone\necho \"Failed to set secret in keymaster.conf, check if Barbican is enabled and responding properly\"\nexit 1\n"}}}


created_at: 2025-11-30 07:22:13.194401
data:
  {"type": "json", "value": {"frontend": {"100 barbican_haproxy_frontend": {"dport": [9311]}, "100 cinder_haproxy_frontend": {"dport": [8776]}, "100 docker_registry_haproxy_frontend": {"dport": [8787]}, "100 glance_api_haproxy_frontend": {"dport": [9292]}, "100 glance_api_internal_haproxy_frontend": {"dport": ["9293"]}, "100 heat_api_haproxy_frontend": {"dport": [8004]}, "100 heat_api_cfn_haproxy_frontend": {"dport": [8000]}, "100 horizon_haproxy_frontend": {"dport": [80]}, "100 keystone_public_haproxy_frontend": {"dport": [5000]}, "100 keystone_admin_haproxy_frontend": {"dport": ["35357"]}, "100 manila_haproxy_frontend": {"dport": [8786]}, "100 mysql_haproxy": {"dport": [3306]}, "100 neutron_haproxy_frontend": {"dport": [9696]}, "100 nova_osapi_haproxy_frontend": {"dport": [8774]}, "100 nova_metadatahaproxy_frontend": {"dport": [8775]}, "100 nova_vncproxy_haproxy_frontend": {"dport": [6080]}, "100 placement_haproxy_frontend": {"dport": [8778]}, "100 swift_proxy_server_haproxy_frontend": {"dport": [8080]}}, "ssl_frontend": {"100 barbican_haproxy_frontend_ssl": {"dport": [13311]}, "100 cinder_haproxy_frontend_ssl": {"dport": [13776]}, "100 docker_registry_haproxy_frontend_ssl": {"dport": [13787]}, "100 glance_api_haproxy_frontend_ssl": {"dport": [13292]}, "100 heat_api_haproxy_frontend_ssl": {"dport": [13004]}, "100 heat_api_cfn_haproxy_frontend_ssl": {"dport": [13005]}, "100 horizon_haproxy_frontend_ssl": {"dport": [443]}, "100 keystone_public_haproxy_frontend_ssl": {"dport": [13000]}, "100 manila_haproxy_frontend_ssl": {"dport": [13786]}, "100 neutron_haproxy_frontend_ssl": {"dport": [13696]}, "100 nova_osapi_haproxy_frontend_ssl": {"dport": [13774]}, "100 nova_metadatahaproxy_frontend_ssl": {"dport": [13775]}, "100 nova_vncproxy_haproxy_frontend_ssl": {"dport": [13080]}, "100 placement_haproxy_frontend_ssl": {"dport": [13778]}, "100 swift_proxy_server_haproxy_frontend_ssl": {"dport": [13808]}}, "edge_frontend": {}, "edge_sslfrontend": {}}}


created_at: 2025-11-30 07:22:13.486485
data:
  {"type": "comma_delimited_list", "value": [{"name": "Configure tuned before reboot", "when": "step|int == 0", "include_role": {"name": "tuned"}, "vars": {"tuned_profile": "throughput-performance", "tuned_isolated_cores": ""}}, {"name": "Configure kernel args and reboot", "when": "step|int == 0", "include_role": {"name": "tripleo_kernel", "tasks_from": "kernelargs.yml"}, "vars": {"tripleo_kernel_args": "", "tripleo_kernel_hugepages": {}, "tripleo_kernel_hugepages_remove": false, "tripleo_kernel_defer_reboot": false, "tripleo_kernel_reboot_timeout": 900}}, {"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"name": "Create dirs for certificates and keys", "file": {"path": "{{ item }}", "state": "directory", "serole": "object_r", "setype": "cert_t", "seuser": "system_u"}, "with_items": ["/etc/pki/tls/certs/httpd", "/etc/pki/tls/private/httpd"]}, {"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "httpd-ctlplane", "dns": "{{ fqdn_ctlplane }}", "principal": "HTTP/{{ fqdn_ctlplane }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-ctlplane.crt /etc/pki/tls/certs/httpd/httpd-ctlplane.crt\ncp /etc/pki/tls/private/httpd-ctlplane.key /etc/pki/tls/private/httpd/httpd-ctlplane.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage", "dns": "{{ fqdn_storage }}", "principal": "HTTP/{{ fqdn_storage }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage.crt /etc/pki/tls/certs/httpd/httpd-storage.crt\ncp /etc/pki/tls/private/httpd-storage.key /etc/pki/tls/private/httpd/httpd-storage.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage_mgmt", "dns": "{{ fqdn_storage_mgmt }}", "principal": "HTTP/{{ fqdn_storage_mgmt }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage_mgmt.crt /etc/pki/tls/certs/httpd/httpd-storage_mgmt.crt\ncp /etc/pki/tls/private/httpd-storage_mgmt.key /etc/pki/tls/private/httpd/httpd-storage_mgmt.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-internal_api", "dns": "{{ fqdn_internal_api }}", "principal": "HTTP/{{ fqdn_internal_api }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-internal_api.crt /etc/pki/tls/certs/httpd/httpd-internal_api.crt\ncp /etc/pki/tls/private/httpd-internal_api.key /etc/pki/tls/private/httpd/httpd-internal_api.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-external", "dns": "{{ fqdn_external }}", "principal": "HTTP/{{ fqdn_external }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-external.crt /etc/pki/tls/certs/httpd/httpd-external.crt\ncp /etc/pki/tls/private/httpd-external.key /etc/pki/tls/private/httpd/httpd-external.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}]}}]}, {"name": "Cinder Backup tag container image for pacemaker", "when": "step|int == 1", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-backup:17.1", "container_image_latest": "cluster.common.tag/cinder-backup:pcmklatest"}}, {"name": "Cinder Backup HA Wrappers Step", "when": "step|int == 5", "block": [{"name": "Cinder backup puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "cinder_backup", "tripleo_ha_wrapper_resource_name": "openstack-cinder-backup", "tripleo_ha_wrapper_bundle_name": "openstack-cinder-backup", "tripleo_ha_wrapper_resource_state": "_ Started", "tripleo_ha_wrapper_puppet_config_volume": "cinder", "tripleo_ha_wrapper_puppet_execute": "include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::cinder::backup_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ip,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".cinder_backup_previous_run"}}]}, {"name": "Cinder Volume tag container image for pacemaker", "when": "step|int == 1", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-volume:17.1", "container_image_latest": "cluster.common.tag/cinder-volume:pcmklatest"}}, {"name": "Cinder Volume HA Wrappers Step", "when": "step|int == 5", "block": [{"name": "Cinder volume puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "cinder_volume", "tripleo_ha_wrapper_resource_name": "openstack-cinder-volume", "tripleo_ha_wrapper_bundle_name": "openstack-cinder-volume", "tripleo_ha_wrapper_resource_state": "_ Started", "tripleo_ha_wrapper_puppet_config_volume": "cinder", "tripleo_ha_wrapper_puppet_execute": "include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::cinder::volume_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ip,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".cinder_volume_previous_run"}}]}, {"name": "configure tmpwatch on the host", "when": "step|int == 2", "block": [{"name": "Push script", "copy": {"dest": "/usr/local/sbin/containers-tmpwatch", "owner": "root", "group": "root", "mode": 493, "content": "#!/bin/sh\ntmpwatch --nodirs \\\n  -X \"/var/log/containers/*/*log\" \\\n  -X \"/var/log/containers/*/*/*log\" \\\n  -X \"/var/log/containers/*/*err\" \\\n  {{ LogrotatePurgeAfterDays|int +1 }}d \\\n  /var/log/containers/ 2>&1 | logger -t container-tmpwatch\n"}, "vars": {"LogrotatePurgeAfterDays": "14"}}, {"name": "Insert cronjob in root crontab", "cron": {"name": "Remove old logs", "special_time": "daily", "user": "root", "job": "/usr/local/sbin/containers-tmpwatch"}}]}, {"name": "Configure rsyslog for HAproxy container managed by Pacemaker", "when": "step|int == 1", "block": [{"name": "Check if rsyslog exists", "shell": "systemctl is-active rsyslog", "register": "rsyslog_config"}, {"when": ["rsyslog_config is changed", "rsyslog_config.rc == 0"], "block": [{"name": "Forward logging to haproxy.log file", "blockinfile": {"content": "if $syslogfacility-text == '{{facility}}' and $programname == 'haproxy' then -/var/log/containers/haproxy/haproxy.log\n& stop\n", "create": true, "path": "/etc/rsyslog.d/openstack-haproxy.conf"}, "vars": {"facility": "local0"}, "register": "logconfig"}, {"name": "restart rsyslog service after logging conf change", "service": {"name": "rsyslog", "state": "restarted"}, "when": "logconfig is changed"}]}]}, {"name": "HAproxy tag container image for pacemaker", "when": "step|int == 1", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "registry.redhat.io/rhosp-rhel9/openstack-haproxy:17.1", "container_image_latest": "cluster.common.tag/haproxy:pcmklatest"}}, {"name": "HAproxy HA Wrappers Step", "when": "step|int == 2", "block": [{"name": "HAproxy puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "haproxy", "tripleo_ha_wrapper_resource_name": "haproxy-bundle", "tripleo_ha_wrapper_bundle_name": "haproxy-bundle", "tripleo_ha_wrapper_resource_state": "Started", "tripleo_ha_wrapper_puppet_config_volume": "haproxy", "tripleo_ha_wrapper_puppet_execute": "include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::haproxy_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ip,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".previous_run"}}]}, {"name": "Run lvmfilter role", "include_role": {"name": "tripleo_lvmfilter"}, "when": ["step|int == 1"]}, {"name": "validate keystone container state", "containers.podman.podman_container_info": {"name": "keystone"}, "register": "keystone_infos", "failed_when": ["keystone_infos.containers.0.Healthcheck.Status is defined", "'healthy' not in keystone_infos.containers.0.Healthcheck.Status"], "retries": 10, "delay": 30, "tags": ["opendev-validation", "opendev-validation-keystone"], "when": ["container_cli == 'podman'", "not container_healthcheck_disabled", "step|int == 4"]}, {"name": "Manila Share tag container image for pacemaker", "when": "step|int == 1", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "registry.redhat.io/rhosp-rhel9/openstack-manila-share:17.1", "container_image_latest": "cluster.common.tag/manila-share:pcmklatest"}}, {"name": "Manila-Share HA Wrappers Step", "when": "step|int == 5", "block": [{"name": "Manila-Share puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "manila_share", "tripleo_ha_wrapper_resource_name": "openstack-manila-share", "tripleo_ha_wrapper_bundle_name": "openstack-manila-share", "tripleo_ha_wrapper_resource_state": "_ Started", "tripleo_ha_wrapper_puppet_config_volume": "manila", "tripleo_ha_wrapper_puppet_execute": "include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::manila::share_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ip,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".previous_run"}}]}, {"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "mysql", "dns": ["{{fqdn_internal_api}}", "{{cloud_names.cloud_name_internal_api}}"], "principal": "mysql/{{fqdn_internal_api}}@{{idm_realm}}", "key_size": "2048", "ca": "ipa"}]}}]}, {"name": "MySQL tag container image for pacemaker", "when": "step|int == 1", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "registry.redhat.io/rhosp-rhel9/openstack-mariadb:17.1", "container_image_latest": "cluster.common.tag/mariadb:pcmklatest"}}, {"name": "MySQL HA Wrappers Step", "when": "step|int == 2", "block": [{"name": "Mysql puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "mysql", "tripleo_ha_wrapper_resource_name": "galera", "tripleo_ha_wrapper_bundle_name": "galera-bundle", "tripleo_ha_wrapper_resource_state": "Master", "tripleo_ha_wrapper_puppet_config_volume": "mysql", "tripleo_ha_wrapper_puppet_execute": "[\"Mysql_datadir\", \"Mysql_user\", \"Mysql_database\", \"Mysql_grant\", \"Mysql_plugin\"].each |String $val| { noop_resource($val) }; include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::database::mysql_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".previous_run"}}]}, {"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "neutron", "dns": "{{fqdn_internal_api}}", "principal": "neutron/{{fqdn_internal_api}}@{{idm_realm}}", "run_after": "container_name=$({{container_cli}} ps --format=\\{\\{.Names\\}\\} | grep neutron_dhcp)\n# The certificate is also installed on the computes, but neutron_dhcp is only\n# present on the controllers, so we exit if the container could not be found.\n[[ -z $container_name ]] && exit 0\n\nservice_crt=\"/etc/pki/tls/certs/neutron.crt\"\nservice_key=\"/etc/pki/tls/private/neutron.key\"\n# Copy the new cert from the mount-point to the real path\n{{container_cli}} exec -u root \"$container_name\" cp \"/var/lib/kolla/config_files/src-tls$service_crt\" \"$service_crt\"\n# Copy the new key from the mount-point to the real path\n{{container_cli}} exec -u root \"$container_name\" cp \"/var/lib/kolla/config_files/src-tls$service_key \"$service_key\"\n# No need to trigger a reload for neutron dhcpd since the cert is not cached\n", "key_size": "2048", "ca": "ipa"}]}}]}, {"name": "validate nova-api container state", "containers.podman.podman_container_info": {"name": "nova_api"}, "register": "nova_api_infos", "failed_when": ["nova_api_infos.containers.0.Healthcheck.Status is defined", "'healthy' not in nova_api_infos.containers.0.Healthcheck.Status"], "retries": 10, "delay": 30, "tags": ["opendev-validation", "opendev-validation-nova"], "when": ["container_cli == 'podman'", "not container_healthcheck_disabled", "step|int == 4"]}, {"name": "validate nova-conductor container state", "containers.podman.podman_container_info": {"name": "nova_conductor"}, "register": "nova_conductor_infos", "failed_when": ["nova_conductor_infos.containers.0.Healthcheck.Status is defined", "'healthy' not in nova_conductor_infos.containers.0.Healthcheck.Status"], "retries": 10, "delay": 30, "tags": ["opendev-validation", "opendev-validation-nova"], "when": ["container_cli == 'podman'", "not container_healthcheck_disabled", "step|int == 5"]}, {"name": "validate nova-metadata container state", "containers.podman.podman_container_info": {"name": "nova_metadata"}, "register": "nova_metadata_infos", "failed_when": ["nova_metadata_infos.containers.0.Healthcheck.Status is defined", "'healthy' not in nova_metadata_infos.containers.0.Healthcheck.Status"], "retries": 10, "delay": 30, "tags": ["opendev-validation", "opendev-validation-nova"], "when": ["container_cli == 'podman'", "not container_healthcheck_disabled", "step|int == 5"]}, {"name": "validate nova-scheduler container state", "containers.podman.podman_container_info": {"name": "nova_scheduler"}, "register": "nova_scheduler_infos", "failed_when": ["nova_scheduler_infos.containers.0.Healthcheck.Status is defined", "'healthy' not in nova_scheduler_infos.containers.0.Healthcheck.Status"], "retries": 10, "delay": 30, "tags": ["opendev-validation", "opendev-validation-nova"], "when": ["container_cli == 'podman'", "not container_healthcheck_disabled", "step|int == 5"]}, {"name": "install systemd-container for a greenfield", "when": ["step|int == 1", "not check_nova_instances_result.stat.exists"], "package": {"name": "systemd-container", "state": "present"}}, {"name": "validate nova-compute container state", "containers.podman.podman_container_info": {"name": "nova_compute"}, "register": "nova_compute_infos", "failed_when": ["nova_compute_infos.containers.0.Healthcheck.Status is defined", "'healthy' not in nova_compute_infos.containers.0.Healthcheck.Status"], "retries": 10, "delay": 30, "tags": ["opendev-validation", "opendev-validation-nova"], "when": ["container_cli == 'podman'", "not container_healthcheck_disabled", "step|int == 6"]}, {"name": "manage PMEM namespaces for vPMEM", "include_role": {"name": "tripleo_nvdimm"}, "vars": {"tripleo_nvdimm_pmem_namespaces": ""}, "when": ["step|int == 1", "tripleo_nvdimm_pmem_namespaces != ''"]}, {"name": "manage OS version 9 specific sysctls", "when": ["step|int == 1", "ansible_facts['os_family'] == 'RedHat'", "ansible_facts['distribution_major_version'] is version('9', '==')"], "block": [{"name": "Enable post-copy by setting unprivileged_userfaultfd", "sysctl": {"name": "vm.unprivileged_userfaultfd", "value": 1, "sysctl_set": true, "sysctl_file": "/etc/sysctl.d/99-tripleo-postcopy.conf", "state": "present", "reload": true}}]}, {"name": "enable/disable ksm", "when": ["step|int == 1"], "block": [{"name": "is KSM enabled", "set_fact": {"compute_ksm_enabled": false}}, {"name": "disable KSM on compute", "when": "not compute_ksm_enabled|bool", "block": [{"name": "Check for ksm", "shell": "systemctl is-active ksm.service || systemctl is-enabled ksm.service", "become": true, "failed_when": false, "register": "ksm_service_check"}, {"name": "disable KSM services", "service": {"name": "{{ item }}", "state": "stopped", "enabled": false}, "with_items": ["ksm.service", "ksmtuned.service"], "when": ["ksm_service_check.rc is defined", "ksm_service_check.rc == 0"], "register": "ksmdisabled"}, {"name": "delete PageKSM after disable ksm on compute", "command": "echo 2 >/sys/kernel/mm/ksm/run", "when": ["ksm_service_check.rc is defined", "ksm_service_check.rc == 0", "ksmdisabled is changed"]}]}, {"name": "enable KSM on compute", "when": "compute_ksm_enabled|bool", "block": [{"name": "make sure package providing ksmtuned is installed (RHEL8 or CentOS8)", "package": {"name": "qemu-kvm-common", "state": "present"}, "when": ["ansible_facts['distribution_major_version'] is version('8', '==')"]}, {"name": "make sure package providing ksmtuned is installed (RHEL9 or CentOS9)", "package": {"name": "ksmtuned", "state": "present"}, "when": ["ansible_facts['distribution_major_version'] is version('9', '==')"]}, {"name": "enable ksmtunded", "service": {"name": "{{ item }}", "state": "started", "enabled": true}, "with_items": ["ksm.service", "ksmtuned.service"]}]}]}, {"name": "Ensure recovery of containerized virtqemud", "when": "step|int == 4", "become": true, "block": [{"name": "Create virtqemud recovery script", "copy": {"dest": "/usr/libexec/recover_tripleo_nova_virtqemud.sh", "mode": "0755", "content": "#!/bin/bash\npgrep -f /sbin/virtqemud\n[ $? -eq 0 ] && exit 0\nkill -9 $(cat /run/nova_virtqemud.pid)\n"}}, {"name": "Create virtqemud recovery trigger service", "copy": {"dest": "/etc/systemd/system/tripleo_nova_virtqemud_recover.service", "mode": "0644", "content": "[Unit]\nDescription=Check and recover tripleo_nova_virtqemud\nAfter=tripleo_nova_virtqemud.service\nRequisite=tripleo_nova_virtqemud.service\n[Service]\nType=oneshot\nExecStart=bash /usr/libexec/recover_tripleo_nova_virtqemud.sh\nSyslogIdentifier=recover_tripleo_nova_virtqemud\n[Install]\nWantedBy=multi-user.target\n"}}, {"name": "Create virtqemud recovery trigger timer", "copy": {"dest": "/etc/systemd/system/tripleo_nova_virtqemud_recover.timer", "mode": "0644", "content": "[Unit]\nDescription=Check and recover tripleo_nova_virtqemud every 10m\nPartOf=tripleo_nova_virtqemud.service\n[Timer]\nOnCalendar=*:5/10\nOnActiveSec=120\nOnUnitActiveSec=60\nRandomizedDelaySec=1m\n[Install]\nWantedBy=timers.target\n"}}, {"name": "Enable virtqemud recovery trigger service", "systemd": {"name": "tripleo_nova_virtqemud_recover.service", "enabled": true, "daemon_reload": true}}, {"name": "Enable virtqemud recovery trigger timer", "systemd": {"name": "tripleo_nova_virtqemud_recover.timer", "enabled": true, "state": "restarted", "daemon_reload": true}, "register": "virtqemud_recover_timer_result"}, {"name": "Really enable virtqemud recovery trigger timer", "shell": {"cmd": "systemctl enable --now tripleo_nova_virtqemud_recover.timer"}, "when": "virtqemud_recover_timer_result.status.UnitFileState != \"enabled\""}]}, {"name": "Set up systemd target for libvirt services", "when": "step|int == 4", "block": [{"name": "Create systemd file", "copy": {"dest": "/etc/systemd/system/tripleo_nova_libvirt.target", "mode": "0644", "owner": "root", "group": "root", "content": "[Unit]\nWants=tripleo_nova_virtsecretd.service\nWants=tripleo_nova_virtnodedevd.service\nWants=tripleo_nova_virtstoraged.service\nWants=tripleo_nova_virtproxyd.service\nWants=tripleo_nova_virtqemud.service\nAfter=tripleo_nova_virtsecretd.service\nAfter=tripleo_nova_virtnodedevd.service\nAfter=tripleo_nova_virtstoraged.service\nAfter=tripleo_nova_virtproxyd.service\nAfter=tripleo_nova_virtqemud.service\n"}, "become": true, "register": "libvirt_target_result"}, {"name": "Reload systemd", "systemd": {"name": "tripleo_nova_libvirt.target", "enabled": true, "state": "restarted", "daemon_reload": true}, "become": true, "when": "libvirt_target_result.changed"}]}, {"name": "validate nova-migration-target container state", "containers.podman.podman_container_info": {"name": "nova_migration_target"}, "register": "nova_migration_target_infos", "failed_when": ["nova_migration_target_infos.containers.0.Healthcheck.Status is defined", "'healthy' not in nova_migration_target_infos.containers.0.Healthcheck.Status"], "retries": 10, "delay": 30, "tags": ["opendev-validation", "opendev-validation-nova"], "when": ["container_cli == 'podman'", "not container_healthcheck_disabled", "step|int == 5"]}, {"name": "validate nova-vnc-proxy container state", "containers.podman.podman_container_info": {"name": "nova_vnc_proxy"}, "register": "nova_vnc_proxy_infos", "failed_when": ["nova_vnc_proxy_infos.containers.0.Healthcheck.Status is defined", "'healthy' not in nova_vnc_proxy_infos.containers.0.Healthcheck.Status"], "retries": 10, "delay": 30, "tags": ["opendev-validation", "opendev-validation-nova"], "when": ["container_cli == 'podman'", "not container_healthcheck_disabled", "step|int == 5"]}, {"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "ovn_controller", "dns": "{{fqdn_internal_api}}", "principal": "ovn_controller/{{fqdn_internal_api}}@{{idm_realm}}", "key_size": "2048", "ca": "ipa"}]}}]}, {"name": "set is_ovn_dbs_bootstrap_node fact", "set_fact": "is_ovn_dbs_bootstrap_node={{ovn_dbs_short_bootstrap_node_name|lower == ansible_facts['hostname']|lower}}", "when": ["step|int == 1"]}, {"name": "Populate ansible service facts so we can check for pacemaker service", "ansible.builtin.service_facts": null, "when": ["is_ovn_dbs_bootstrap_node | bool", "step|int == 1"]}, {"name": "Remove OVNDBs from Pacemaker", "when": ["step|int == 1", "is_ovn_dbs_bootstrap_node | bool", "ansible_facts['services']['pacemaker.service']['state']  | default('unknown') == 'running'"], "block": [{"name": "Fetch ovn-dbs-bundle", "shell": "pcs status |grep ovn-dbs-bundle ||true\n", "register": "ovn_dbs"}, {"name": "Fetch ovn VIP", "shell": "pcs constraint config |grep \"with ovn-dbs-bundle\" |awk '{print $1}'\n", "register": "ovn_vip"}, {"name": "Remove OVNDBs from pacemaker", "pacemaker_resource": {"resource": "ovn-dbs-bundle", "state": "delete"}, "when": "ovn_dbs.stdout |length  > 0"}, {"name": "Remove OVNDBs VIP from pacemaker", "pacemaker_resource": {"resource": "{{ ovn_vip.stdout }}", "state": "delete"}, "when": "ovn_vip.stdout |length > 0"}, {"name": "Clean up pacemaker remote nodes cache", "shell": "pcs cluster node clear ovn-dbs-bundle-0\npcs cluster node clear ovn-dbs-bundle-1\npcs cluster node clear ovn-dbs-bundle-2\ncrm_attribute --name OVN_REPL_INFO --delete\n"}, {"name": "Remove pacemaker attributes", "shell": "pcs node attribute \"{{ item }}\" ovn-dbs-role= || true\n", "loop": "{{ ovn_dbs_short_node_names }}"}]}, {"name": "Prepare OVN cluster", "when": "step|int == 1", "block": [{"name": "Certificate generation", "when": "enable_internal_tls | bool", "block": [{"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "ovn_dbs", "dns": "{{fqdn_internal_api}}", "principal": "ovn_dbs/{{fqdn_internal_api}}@{{idm_realm}}", "key_size": "2048", "ca": "ipa"}]}}]}, {"block": [{"name": "Configure OVN DBs and northd", "include_role": {"name": "tripleo_ovn_cluster"}, "vars": {"tripleo_ovn_cluster_dbs_protocol": "{{ enable_internal_tls | ternary('ssl', 'tcp', 'tcp') }}", "tripleo_ovn_cluster_network": "internal_api", "tripleo_ovn_cluster_nb_db_port": 6641, "tripleo_ovn_cluster_sb_db_port": 6642, "tripleo_ovn_cluster_nb_local_port": 6643, "tripleo_ovn_cluster_nb_remote_port": 6643, "tripleo_ovn_cluster_sb_local_port": 6644, "tripleo_ovn_cluster_sb_remote_port": 6644, "tripleo_ovn_cluster_northd_ssl_ca_cert": "/etc/ipa/ca.crt", "tripleo_ovn_cluster_sb_ssl_ca_cert": "/etc/ipa/ca.crt", "tripleo_ovn_cluster_nb_ssl_ca_cert": "/etc/ipa/ca.crt"}}]}]}, {"name": "Start OVN DBs and northd containers (bootstrap node)", "when": ["step|int == 3", "is_ovn_dbs_bootstrap_node | bool"], "block": [{"block": [{"name": "Start OVN container", "include_role": {"name": "tripleo_container_manage"}, "vars": {"tripleo_container_manage_config": "/var/lib/tripleo-config/container-startup-config/step_0", "tripleo_container_manage_config_id": "{{ ovn_container }}", "tripleo_container_manage_config_patterns": "{{ ovn_container }}.json"}, "loop": ["ovn_cluster_north_db_server", "ovn_cluster_south_db_server", "ovn_cluster_northd"], "loop_control": {"loop_var": "ovn_container"}}]}, {"name": "Set NB connection", "become": true, "shell": "podman exec ovn_cluster_north_db_server bash -c \"ovn-nbctl --no-leader-only --inactivity-probe={{ tripleo_ovn_cluster_probe_interval }} set-connection p{{ tripleo_ovn_cluster_dbs_protocol }}:{{ tripleo_ovn_cluster_nb_db_port }}:{{ tripleo_ovn_cluster_dbs_addr }}\"\n", "register": "nb_result", "retries": 20, "delay": 5, "until": "nb_result.rc == 0", "when": ["is_ovn_dbs_bootstrap_node | bool"], "vars": {"tripleo_ovn_cluster_network": "internal_api", "tripleo_ovn_cluster_nb_db_port": 6641, "tripleo_ovn_cluster_dbs_protocol": "{{ enable_internal_tls | ternary('ssl', 'tcp', 'tcp') }}", "tripleo_ovn_cluster_dbs_addr": "0.0.0.0", "tripleo_ovn_cluster_probe_interval": 60000}}, {"name": "Set SB connection", "become": true, "shell": "podman exec ovn_cluster_south_db_server bash -c \"ovn-sbctl --no-leader-only --inactivity-probe={{ tripleo_ovn_cluster_probe_interval }} set-connection p{{ tripleo_ovn_cluster_dbs_protocol }}:{{ tripleo_ovn_cluster_sb_db_port }}:{{ tripleo_ovn_cluster_dbs_addr }}\"\n", "register": "sb_result", "retries": 20, "delay": 5, "until": "sb_result.rc == 0", "when": ["is_ovn_dbs_bootstrap_node | bool"], "vars": {"tripleo_ovn_cluster_network": "internal_api", "tripleo_ovn_cluster_sb_db_port": 6642, "tripleo_ovn_cluster_dbs_protocol": "{{ enable_internal_tls | ternary('ssl', 'tcp', 'tcp') }}", "tripleo_ovn_cluster_dbs_addr": "0.0.0.0", "tripleo_ovn_cluster_probe_interval": 60000}}]}, {"name": "Start OVN DBs and northd containers (non-bootstrap nodes)", "when": ["step|int == 4", "not is_ovn_dbs_bootstrap_node | bool"], "block": [{"block": [{"name": "Start OVN container", "include_role": {"name": "tripleo_container_manage"}, "vars": {"tripleo_container_manage_config": "/var/lib/tripleo-config/container-startup-config/step_0", "tripleo_container_manage_config_id": "{{ ovn_container }}", "tripleo_container_manage_config_patterns": "{{ ovn_container }}.json"}, "loop": ["ovn_cluster_north_db_server", "ovn_cluster_south_db_server", "ovn_cluster_northd"], "loop_control": {"loop_var": "ovn_container"}}]}, {"name": "Set NB connection", "become": true, "shell": "podman exec ovn_cluster_north_db_server bash -c \"ovn-nbctl --no-leader-only --inactivity-probe={{ tripleo_ovn_cluster_probe_interval }} set-connection p{{ tripleo_ovn_cluster_dbs_protocol }}:{{ tripleo_ovn_cluster_nb_db_port }}:{{ tripleo_ovn_cluster_dbs_addr }}\"\n", "register": "nb_result", "retries": 20, "delay": 5, "until": "nb_result.rc == 0", "when": ["is_ovn_dbs_bootstrap_node | bool"], "vars": {"tripleo_ovn_cluster_network": "internal_api", "tripleo_ovn_cluster_nb_db_port": 6641, "tripleo_ovn_cluster_dbs_protocol": "{{ enable_internal_tls | ternary('ssl', 'tcp', 'tcp') }}", "tripleo_ovn_cluster_dbs_addr": "0.0.0.0", "tripleo_ovn_cluster_probe_interval": 60000}}, {"name": "Set SB connection", "become": true, "shell": "podman exec ovn_cluster_south_db_server bash -c \"ovn-sbctl --no-leader-only --inactivity-probe={{ tripleo_ovn_cluster_probe_interval }} set-connection p{{ tripleo_ovn_cluster_dbs_protocol }}:{{ tripleo_ovn_cluster_sb_db_port }}:{{ tripleo_ovn_cluster_dbs_addr }}\"\n", "register": "sb_result", "retries": 20, "delay": 5, "until": "sb_result.rc == 0", "when": ["is_ovn_dbs_bootstrap_node | bool"], "vars": {"tripleo_ovn_cluster_network": "internal_api", "tripleo_ovn_cluster_sb_db_port": 6642, "tripleo_ovn_cluster_dbs_protocol": "{{ enable_internal_tls | ternary('ssl', 'tcp', 'tcp') }}", "tripleo_ovn_cluster_dbs_addr": "0.0.0.0", "tripleo_ovn_cluster_probe_interval": 60000}}]}, {"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "ovn_metadata", "dns": "{{fqdn_internal_api}}", "principal": "ovn_metadata/{{fqdn_internal_api}}@{{idm_realm}}", "key_size": "2048", "ca": "ipa"}]}}]}, {"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "rabbitmq", "dns": "{{fqdn_internal_api}}", "principal": "rabbitmq/{{fqdn_internal_api}}@{{idm_realm}}", "run_after": "container_name=$({{container_cli}} ps --format=\\{\\{.Names\\}\\} | grep -w -E 'rabbitmq(-bundle-.*-[0-9]+)?')\nservice_crt=\"/etc/pki/tls/certs/rabbitmq.crt\"\nservice_key=\"/etc/pki/tls/private/rabbitmq.key\"\nif echo \"$container_name\" | grep -q \"^rabbitmq-bundle\"; then\n  # lp#1917868: Do not use podman cp with HA containers as they get\n  # frozen temporarily and that can make pacemaker operation fail.\n  tar -c \"$service_crt\" \"$service_key\" | {{container_cli}} exec -i \"$container_name\" tar -C / -xv\n  # no need to update the mount point, because pacemaker\n  # recreates the container when it's restarted\nelse\n  # Refresh the cert at the mount-point\n  {{container_cli}} cp $service_crt \"$container_name:/var/lib/kolla/config_files/src-tls/$service_crt\"\n  # Refresh the key at the mount-point\n  {{container_cli}} cp $service_key \"$container_name:/var/lib/kolla/config_files/src-tls/$service_key\"\n  # Copy the new cert from the mount-point to the real path\n  {{container_cli}} exec -u root \"$container_name\" cp \"/var/lib/kolla/config_files/src-tls$service_crt\" \"$service_crt\"\n  # Copy the new key from the mount-point to the real path\n  {{container_cli}} exec -u root \"$container_name\" cp \"/var/lib/kolla/config_files/src-tls$service_key\" \"$service_key\"\nfi\n# Set appropriate permissions\n{{container_cli}} exec -u root \"$container_name\" chown rabbitmq:rabbitmq \"$service_crt\"\n{{container_cli}} exec -u root \"$container_name\" chown rabbitmq:rabbitmq \"$service_key\"\n# Trigger a pem cache clear in RabbitMQ to read the new certificates\n{{container_cli}} exec \"$container_name\" rabbitmqctl eval \"ssl:clear_pem_cache().\"\n", "key_size": "2048", "ca": "ipa"}]}}]}, {"name": "RabbitMQ tag container image for pacemaker", "when": "step|int == 1", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.1", "container_image_latest": "cluster.common.tag/rabbitmq:pcmklatest"}}, {"name": "RabbitMQ RPC HA Wrappers Step", "when": "step|int == 2", "block": [{"name": "Rabbitmq rpc puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "oslo_messaging_rpc", "tripleo_ha_wrapper_resource_name": "rabbitmq", "tripleo_ha_wrapper_bundle_name": "rabbitmq-bundle", "tripleo_ha_wrapper_resource_state": "Started", "tripleo_ha_wrapper_puppet_config_volume": "rabbitmq", "tripleo_ha_wrapper_puppet_execute": "[\"Rabbitmq_policy\", \"Rabbitmq_user\"].each |String $val| { noop_resource($val) }; include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::rabbitmq_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ip,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".previous_run"}}]}, {"when": ["(step|int) == 1"], "block": [{"name": "Pre-fetch all the containers", "become": true, "containers.podman.podman_image": {"name": "{{ prefetch_image }}", "validate_certs": false, "force": true}, "register": "result", "until": "result is succeeded", "retries": 5, "delay": 5, "loop_control": {"loop_var": "prefetch_image"}, "loop": "{{ lookup('template', tripleo_role_name + '/docker_config.yaml', errors='ignore') | default('{}', True) | from_yaml | recursive_get_key_from_dict(key='image') | unique }}"}]}, {"name": "Configure rsyslog for swift", "when": ["step|int == 1"], "block": [{"name": "Check if rsyslog exists", "shell": "systemctl is-active rsyslog", "register": "swift_rsyslog_config"}, {"name": "Forward logging to swift.log file", "copy": {"content": "# Fix for https://bugs.launchpad.net/tripleo/+bug/1776180\nlocal2.*                 /var/log/containers/swift/swift.log\n&                        stop\n", "dest": "/etc/rsyslog.d/openstack-swift.conf"}, "register": "swift_logconfig", "when": ["swift_rsyslog_config is defined", "swift_rsyslog_config.rc == 0"]}, {"name": "Restart rsyslogd service after logging conf change", "service": {"name": "rsyslog", "state": "restarted"}, "when": ["swift_logconfig is defined", "swift_logconfig is changed"]}]}, {"name": "Run kolla_set_configs to copy ring files", "when": "step|int == 5", "shell": "{{ container_cli }} exec -u root {{ item }} /usr/local/bin/kolla_set_configs ", "become": true, "register": "kolla_set_configs_result", "failed_when": ["kolla_set_configs_result.rc is defined", "kolla_set_configs_result.rc not in [0, 125]"], "with_items": ["swift_proxy"]}, {"name": "Run kolla_set_configs to copy ring files", "when": "step|int == 5", "shell": "{{ container_cli }} exec -u root {{ item }} /usr/local/bin/kolla_set_configs", "become": true, "register": "kolla_set_configs_result", "failed_when": ["kolla_set_configs_result.rc is defined", "kolla_set_configs_result.rc not in [0, 125]"], "with_items": ["swift_account_auditor", "swift_account_reaper", "swift_account_replicator", "swift_account_server", "swift_container_auditor", "swift_container_replicator", "swift_container_server", "swift_container_updater", "swift_object_auditor", "swift_object_expirer", "swift_object_replicator", "swift_object_server", "swift_object_updater"]}]}


created_at: 2025-11-30 07:22:13.573853
data:
  {"tuple:('role_data', 'config_settings')": [null, {"apache::ip": "%{hiera('internal_api')}", "apache::default_vhost": false, "apache::trace_enable": "Off", "apache::server_signature": "Off", "apache::server_tokens": "Prod", "apache::timeout": 90, "apache::default_mods": false, "apache::mod::prefork::maxrequestworkers": 100, "apache::mod::prefork::serverlimit": 100, "apache::mod::remoteip::proxy_ips": ["172.17.0.0/24"], "apache::mod::alias::icons_options": "None", "barbican::keystone::authtoken::password": "G5XhguvbNw19nfLNKira8K2xr", "barbican::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "barbican::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "barbican::keystone::authtoken::project_name": "service", "barbican::keystone::authtoken::region_name": "regionOne", "barbican::keystone::authtoken::interface": "internal", "barbican::keystone::authtoken::memcache_use_advanced_pool": true, "barbican::keystone::notification::enable_keystone_notification": true, "barbican::keystone::notification::keystone_notification_topic": "barbican_notifications", "barbican::policy::policies": {}, "barbican::api::host_href": "http://172.21.0.2:9311", "barbican::api::db_auto_create": false, "barbican::api::enabled_certificate_plugins": ["simple_certificate"], "barbican::api::enable_queue": true, "barbican::api::logging::debug": true, "barbican::api::notification_driver": ["noop"], "barbican::api::rabbit_heartbeat_in_pthread": false, "barbican::api::service_name": "httpd", "barbican::wsgi::apache::bind_host": "%{hiera('internal_api')}", "barbican::wsgi::apache::ssl": false, "barbican::wsgi::apache::workers": "%{::processorcount}", "barbican::wsgi::apache::servername": "%{hiera('fqdn_internal_api')}", "barbican::db::database_connection": "mysql+pymysql://barbican:G5XhguvbNw19nfLNKira8K2xr@172.17.0.2/barbican?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo"}, {"barbican::plugins::simple_crypto::simple_crypto_plugin_kek": "OYlzhn5Z789Pm8uX1xw6rb9SXqoG3q3o3GIkxbUm8OU=", "barbican::plugins::simple_crypto::global_default": true}, {"tripleo::trusted_cas::ca_map": {}}, {}, null, null, null, null, {"cinder::database_connection": "mysql+pymysql://cinder:Olf2iTUMR6eJ33irnMaVcSJQX@172.17.0.2/cinder?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "cinder::logging::debug": true, "cinder::storage_availability_zone": "nova", "cinder::rabbit_heartbeat_timeout_threshold": 60, "cinder::rpc_response_timeout": 60, "cinder::rabbit_heartbeat_in_pthread": false, "cinder::db::database_db_max_retries": -1, "cinder::db::database_max_retries": -1, "cinder::cron::db_purge::minute": "1", "cinder::cron::db_purge::hour": "0", "cinder::cron::db_purge::monthday": "*", "cinder::cron::db_purge::month": "*", "cinder::cron::db_purge::weekday": "*", "cinder::cron::db_purge::user": "cinder", "cinder::cron::db_purge::age": "30", "cinder::cron::db_purge::destination": "/var/log/cinder/cinder-rowsflush.log", "cinder::cron::db_purge::maxdelay": "3600", "cinder::glance::glance_api_servers": "http://172.17.0.2:9293", "cinder::keystone::service_user::send_service_user_token": true, "cinder::keystone::service_user::project_name": "service", "cinder::keystone::service_user::auth_type": "password", "cinder::keystone::service_user::auth_url": "http://172.17.0.2:5000/", "cinder::keystone::service_user::password": "Olf2iTUMR6eJ33irnMaVcSJQX", "cinder::keystone::service_user::region_name": "regionOne", "cinder::nova::interface": "admin", "cinder::nova::project_name": "service", "cinder::nova::auth_type": "password", "cinder::nova::auth_url": "http://172.17.0.2:5000/", "cinder::nova::password": "nad1A8ZvKDsT60yawIjN1cH4K", "cinder::nova::region_name": "regionOne", "apache::ip": "%{hiera('internal_api')}", "apache::default_vhost": false, "apache::trace_enable": "Off", "apache::server_signature": "Off", "apache::server_tokens": "Prod", "apache::timeout": 90, "apache::default_mods": false, "apache::mod::prefork::maxrequestworkers": 100, "apache::mod::prefork::serverlimit": 100, "apache::mod::remoteip::proxy_ips": ["172.17.0.0/24"], "apache::mod::alias::icons_options": "None", "keystone_resources_managed": false, "cinder::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "cinder::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "cinder::keystone::authtoken::password": "Olf2iTUMR6eJ33irnMaVcSJQX", "cinder::keystone::authtoken::project_name": "service", "cinder::keystone::authtoken::user_domain_name": "Default", "cinder::keystone::authtoken::project_domain_name": "Default", "cinder::keystone::authtoken::region_name": "regionOne", "cinder::keystone::authtoken::interface": "internal", "cinder::keystone::authtoken::memcache_use_advanced_pool": true, "cinder::policy::policies": {}, "cinder::notification_driver": ["noop"], "cinder::api::default_volume_type": "tripleo", "cinder::api::enable_proxy_headers_parsing": true, "tripleo::profile::base::cinder::cinder_enable_db_purge": true, "cinder::api::bind_host": "%{hiera('fqdn_internal_api')}", "cinder::wsgi::apache::ssl": false, "cinder::api::service_name": "httpd", "cinder::wsgi::apache::bind_host": "%{hiera('internal_api')}", "cinder::wsgi::apache::servername": "%{hiera('fqdn_internal_api')}", "cinder::wsgi::apache::vhost_custom_fragment": "Timeout 60", "cinder::wsgi::apache::workers": "1"}, {"cinder::database_connection": "mysql+pymysql://cinder:Olf2iTUMR6eJ33irnMaVcSJQX@172.17.0.2/cinder?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "cinder::logging::debug": true, "cinder::storage_availability_zone": "nova", "cinder::rabbit_heartbeat_timeout_threshold": 60, "cinder::rpc_response_timeout": 60, "cinder::rabbit_heartbeat_in_pthread": false, "cinder::db::database_db_max_retries": -1, "cinder::db::database_max_retries": -1, "cinder::cron::db_purge::minute": "1", "cinder::cron::db_purge::hour": "0", "cinder::cron::db_purge::monthday": "*", "cinder::cron::db_purge::month": "*", "cinder::cron::db_purge::weekday": "*", "cinder::cron::db_purge::user": "cinder", "cinder::cron::db_purge::age": "30", "cinder::cron::db_purge::destination": "/var/log/cinder/cinder-rowsflush.log", "cinder::cron::db_purge::maxdelay": "3600", "cinder::glance::glance_api_servers": "http://172.17.0.2:9293", "cinder::keystone::service_user::send_service_user_token": true, "cinder::keystone::service_user::project_name": "service", "cinder::keystone::service_user::auth_type": "password", "cinder::keystone::service_user::auth_url": "http://172.17.0.2:5000/", "cinder::keystone::service_user::password": "Olf2iTUMR6eJ33irnMaVcSJQX", "cinder::keystone::service_user::region_name": "regionOne", "cinder::nova::interface": "admin", "cinder::nova::project_name": "service", "cinder::nova::auth_type": "password", "cinder::nova::auth_url": "http://172.17.0.2:5000/", "cinder::nova::password": "nad1A8ZvKDsT60yawIjN1cH4K", "cinder::nova::region_name": "regionOne", "cinder::backup::backup_workers": 1, "cinder::backup::backup_max_operations": 15, "cinder::backup::ceph::backup_ceph_user": "openstack", "cinder::backup::ceph::backup_ceph_pool": "backups", "cinder::backup::ceph::backup_ceph_conf": "/etc/ceph/ceph.conf", "tripleo::profile::pacemaker::cinder::backup_bundle::cinder_backup_docker_image": "cluster.common.tag/cinder-backup:pcmklatest", "tripleo::profile::pacemaker::cinder::backup_bundle::docker_volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro", "/var/log/containers/cinder:/var/log/cinder:z", "/var/lib/kolla/config_files/cinder_backup.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/iscsid/etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/dev:/dev", "/run:/run", "/sys:/sys", "/lib/modules:/lib/modules:ro", "/var/lib/cinder:/var/lib/cinder:z", "/var/lib/iscsi:/var/lib/iscsi:z"], "tripleo::profile::pacemaker::cinder::backup_bundle::docker_environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}, "tripleo::profile::pacemaker::cinder::backup_bundle::container_backend": "podman", "tripleo::profile::pacemaker::cinder::backup_bundle::ceph_conf_path": "/var/lib/tripleo-config/ceph", "cinder::backup::manage_service": false, "cinder::backup::enabled": false}, {"cinder::database_connection": "mysql+pymysql://cinder:Olf2iTUMR6eJ33irnMaVcSJQX@172.17.0.2/cinder?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "cinder::logging::debug": true, "cinder::storage_availability_zone": "nova", "cinder::rabbit_heartbeat_timeout_threshold": 60, "cinder::rpc_response_timeout": 60, "cinder::rabbit_heartbeat_in_pthread": false, "cinder::db::database_db_max_retries": -1, "cinder::db::database_max_retries": -1, "cinder::cron::db_purge::minute": "1", "cinder::cron::db_purge::hour": "0", "cinder::cron::db_purge::monthday": "*", "cinder::cron::db_purge::month": "*", "cinder::cron::db_purge::weekday": "*", "cinder::cron::db_purge::user": "cinder", "cinder::cron::db_purge::age": "30", "cinder::cron::db_purge::destination": "/var/log/cinder/cinder-rowsflush.log", "cinder::cron::db_purge::maxdelay": "3600", "cinder::glance::glance_api_servers": "http://172.17.0.2:9293", "cinder::keystone::service_user::send_service_user_token": true, "cinder::keystone::service_user::project_name": "service", "cinder::keystone::service_user::auth_type": "password", "cinder::keystone::service_user::auth_url": "http://172.17.0.2:5000/", "cinder::keystone::service_user::password": "Olf2iTUMR6eJ33irnMaVcSJQX", "cinder::keystone::service_user::region_name": "regionOne", "cinder::nova::interface": "admin", "cinder::nova::project_name": "service", "cinder::nova::auth_type": "password", "cinder::nova::auth_url": "http://172.17.0.2:5000/", "cinder::nova::password": "nad1A8ZvKDsT60yawIjN1cH4K", "cinder::nova::region_name": "regionOne", "cinder::scheduler::scheduler_driver": "cinder.scheduler.filter_scheduler.FilterScheduler"}, {"cinder::database_connection": "mysql+pymysql://cinder:Olf2iTUMR6eJ33irnMaVcSJQX@172.17.0.2/cinder?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "cinder::logging::debug": true, "cinder::storage_availability_zone": "nova", "cinder::rabbit_heartbeat_timeout_threshold": 60, "cinder::rpc_response_timeout": 60, "cinder::rabbit_heartbeat_in_pthread": false, "cinder::db::database_db_max_retries": -1, "cinder::db::database_max_retries": -1, "cinder::cron::db_purge::minute": "1", "cinder::cron::db_purge::hour": "0", "cinder::cron::db_purge::monthday": "*", "cinder::cron::db_purge::month": "*", "cinder::cron::db_purge::weekday": "*", "cinder::cron::db_purge::user": "cinder", "cinder::cron::db_purge::age": "30", "cinder::cron::db_purge::destination": "/var/log/cinder/cinder-rowsflush.log", "cinder::cron::db_purge::maxdelay": "3600", "cinder::glance::glance_api_servers": "http://172.17.0.2:9293", "cinder::keystone::service_user::send_service_user_token": true, "cinder::keystone::service_user::project_name": "service", "cinder::keystone::service_user::auth_type": "password", "cinder::keystone::service_user::auth_url": "http://172.17.0.2:5000/", "cinder::keystone::service_user::password": "Olf2iTUMR6eJ33irnMaVcSJQX", "cinder::keystone::service_user::region_name": "regionOne", "cinder::nova::interface": "admin", "cinder::nova::project_name": "service", "cinder::nova::auth_type": "password", "cinder::nova::auth_url": "http://172.17.0.2:5000/", "cinder::nova::password": "nad1A8ZvKDsT60yawIjN1cH4K", "cinder::nova::region_name": "regionOne", "tripleo::profile::base::lvm::enable_udev": false, "tripleo::profile::base::cinder::volume::cinder_enable_iscsi_backend": false, "tripleo::profile::base::cinder::volume::cinder_enable_nfs_backend": false, "tripleo::profile::base::cinder::volume::cinder_enable_rbd_backend": true, "tripleo::profile::base::cinder::volume::cinder_volume_cluster": "", "tripleo::profile::base::cinder::volume::nfs::cinder_nfs_mount_options": "context=system_u:object_r:container_file_t:s0", "tripleo::profile::base::cinder::volume::nfs::cinder_nfs_servers": [], "tripleo::profile::base::cinder::volume::nfs::cinder_nfs_snapshot_support": false, "tripleo::profile::base::cinder::volume::nfs::cinder_nas_secure_file_operations": "False", "tripleo::profile::base::cinder::volume::nfs::cinder_nas_secure_file_permissions": "False", "tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_helper": "lioadm", "tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_protocol": "iscsi", "tripleo::profile::base::cinder::volume::rbd::backend_name": ["tripleo_ceph"], "tripleo::profile::base::cinder::volume::rbd::cinder_rbd_ceph_conf": "/etc/ceph/ceph.conf", "tripleo::profile::base::cinder::volume::rbd::cinder_rbd_pool_name": "volumes", "tripleo::profile::base::cinder::volume::rbd::cinder_rbd_extra_pools": [], "tripleo::profile::base::cinder::volume::rbd::cinder_rbd_secret_uuid": "55dae5e5-7ec8-55ff-b761-b30fedcc71d6", "tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name": "openstack", "tripleo::profile::base::cinder::volume::rbd::cinder_rbd_flatten_volume_from_snapshot": false, "tripleo::profile::base::cinder::volume::rbd::multi_config": {}, "tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_address": "%{hiera('storage')}", "tripleo::profile::pacemaker::cinder::volume_bundle::cinder_volume_docker_image": "cluster.common.tag/cinder-volume:pcmklatest", "tripleo::profile::pacemaker::cinder::volume_bundle::docker_volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro", "/var/log/containers/cinder:/var/log/cinder:z", "/var/lib/kolla/config_files/cinder_volume.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/iscsid/etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/lib/modules:/lib/modules:ro", "/dev:/dev", "/run:/run", "/sys:/sys", "/var/lib/cinder:/var/lib/cinder:z", "/var/lib/iscsi:/var/lib/iscsi:z"], "tripleo::profile::pacemaker::cinder::volume_bundle::docker_environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}, "tripleo::profile::pacemaker::cinder::volume_bundle::container_backend": "podman", "tripleo::profile::pacemaker::cinder::volume_bundle::ceph_conf_path": "/var/lib/tripleo-config/ceph", "cinder::volume::manage_service": false, "cinder::volume::enabled": false, "cinder::backend_host": "hostgroup"}, {"mysql::server::package_name": "mariadb-galera-server", "mysql::server::manage_config_file": true, "mysql_ipv6": false, "mysql_max_connections": 4096, "mysql::server::root_password": "NhgQDgRVoo", "mysql_clustercheck_password": "sRxOzvWhwndqoiP4taJzvBzao", "enable_galera": true, "mysql_auth_ed25519": false, "mysql_bind_host": "%{hiera('internal_api')}", "tripleo::profile::base::database::mysql::bind_address": "%{hiera('fqdn_internal_api')}", "tripleo::profile::base::database::mysql::client_bind_address": "%{hiera('internal_api')}", "tripleo::profile::base::database::mysql::generate_dropin_file_limit": true, "innodb_buffer_pool_size": "128M", "tripleo::profile::pacemaker::database::mysql::bind_address": "%{hiera('fqdn_internal_api')}", "tripleo::profile::pacemaker::database::mysql::gmcast_listen_addr": "%{hiera('internal_api')}", "tripleo::profile::pacemaker::database::mysql::ca_file": "/etc/ipa/ca.crt", "tripleo::profile::pacemaker::database::mysql_bundle::mysql_docker_image": "cluster.common.tag/mariadb:pcmklatest", "tripleo::profile::pacemaker::database::mysql_bundle::control_port": 3123, "tripleo::profile::pacemaker::database::mysql_bundle::container_backend": "podman", "tripleo::profile::pacemaker::database::mysql_bundle::bind_address": "%{hiera('fqdn_internal_api')}", "tripleo::profile::pacemaker::database::mysql_bundle::sst_method": "rsync", "tripleo::profile::pacemaker::database::mysql_bundle::mariabackup_user": "mariabackup", "tripleo::profile::pacemaker::database::mysql_bundle::mariabackup_password": "3Qi03Zmdc69WCw00okVFEQzzM", "tripleo::profile::pacemaker::database::mysql_bundle::force_ocf": true, "tripleo::profile::pacemaker::database::mysql_bundle::gcomm_cipher": "AES128-SHA256", "tripleo::profile::pacemaker::clustercheck::bind_address": "%{hiera('internal_api')}"}, null, {"tripleo::profile::base::logging::logrotate::maxsize": "10M", "tripleo::profile::base::logging::logrotate::rotation": "daily", "tripleo::profile::base::logging::logrotate::rotate": "14", "tripleo::profile::base::logging::logrotate::purge_after_days": "14", "tripleo::profile::base::logging::logrotate::dateext": false}, null, {"glance::api::db::database_connection": "mysql+pymysql://glance:PmWCSVDW3B2uUjMQtJIVdd5mn@172.17.0.2/glance?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "glance::api::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "glance::api::authtoken::auth_url": "http://172.17.0.2:5000", "glance::api::enable_v1_api": false, "glance::api::enable_v2_api": true, "glance::api::authtoken::password": "PmWCSVDW3B2uUjMQtJIVdd5mn", "glance::api::enable_proxy_headers_parsing": true, "glance::api::logging::debug": true, "glance::policy::policies": {}, "glance::api::authtoken::project_name": "service", "glance::api::authtoken::region_name": "regionOne", "glance::api::authtoken::user_domain_name": "Default", "glance::api::authtoken::project_domain_name": "Default", "glance::api::authtoken::interface": "internal", "glance::api::authtoken::memcache_use_advanced_pool": true, "glance::api::pipeline": "keystone", "glance::api::os_region_name": "regionOne", "glance::api::image_member_quota": 128, "glance::api::enabled_import_methods": ["web-download"], "glance::api::node_staging_uri": "file:///var/lib/glance/staging", "glance::api::image_import_plugins": ["no_op", "image_conversion"], "glance::api::image_conversion_output_format": "raw", "glance::api::inject_metadata_properties": [], "glance::api::ignore_user_roles": ["admin"], "tripleo::profile::base::glance::api::tls_proxy_bind_ip": "%{hiera('internal_api')}", "tripleo::profile::base::glance::api::tls_proxy_fqdn": "%{hiera('fqdn_internal_api')}", "tripleo::profile::base::glance::api::tls_proxy_port": 9292, "glance::api::bind_host": "%{hiera('internal_api')}", "glance_notifier_strategy": "noop", "glance_log_file": "", "glance::backend::rbd::rbd_thin_provisioning": false, "glance::backend::file::filesystem_thin_provisioning": false, "glance::backend::swift::swift_store_auth_address": "http://172.17.0.2:5000/v3", "glance::backend::swift::swift_store_user": "service:glance", "glance::backend::swift::swift_store_key": "PmWCSVDW3B2uUjMQtJIVdd5mn", "glance::backend::swift::swift_store_create_container_on_put": true, "glance::backend::swift::swift_store_auth_version": 3, "glance::backend::rbd::rbd_store_ceph_conf": "/etc/ceph/ceph.conf", "glance::backend::rbd::rbd_store_pool": "images", "glance::backend::rbd::rbd_store_user": "openstack", "glance_backend": "rbd", "tripleo::profile::base::glance::api::glance_backend_id": "default_backend", "tripleo::profile::base::glance::api::glance_store_description": "Default glance store backend.", "tripleo::profile::base::glance::api::multistore_config": {}, "tripleo::profile::base::glance::backend::rbd::glance_rbd_ceph_conf_path": "/var/lib/tripleo-config/ceph", "glance::notify::rabbitmq::notification_driver": ["noop"], "glance::notify::rabbitmq::rabbit_heartbeat_in_pthread": false, "glance::cron::db_purge::minute": "1", "glance::cron::db_purge::hour": "0", "glance::cron::db_purge::monthday": "*", "glance::cron::db_purge::month": "*", "glance::cron::db_purge::weekday": "*", "glance::cron::db_purge::user": "glance", "glance::cron::db_purge::age": "30", "glance::cron::db_purge::max_rows": "100", "glance::cron::db_purge::destination": "/var/log/glance/glance-rowsflush.log", "glance::cron::db_purge::maxdelay": "3600", "glance::api::workers": "1"}, {"glance::api::db::database_connection": "mysql+pymysql://glance:PmWCSVDW3B2uUjMQtJIVdd5mn@172.17.0.2/glance?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "glance::api::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "glance::api::authtoken::auth_url": "http://172.17.0.2:5000", "glance::api::enable_v1_api": false, "glance::api::enable_v2_api": true, "glance::api::authtoken::password": "PmWCSVDW3B2uUjMQtJIVdd5mn", "glance::api::enable_proxy_headers_parsing": true, "glance::api::logging::debug": true, "glance::policy::policies": {}, "glance::api::authtoken::project_name": "service", "glance::api::authtoken::region_name": "regionOne", "glance::api::authtoken::user_domain_name": "Default", "glance::api::authtoken::project_domain_name": "Default", "glance::api::authtoken::interface": "internal", "glance::api::authtoken::memcache_use_advanced_pool": true, "glance::api::pipeline": "keystone", "glance::api::os_region_name": "regionOne", "glance::api::image_member_quota": 128, "glance::api::enabled_import_methods": ["web-download"], "glance::api::node_staging_uri": "file:///var/lib/glance/staging", "glance::api::image_import_plugins": ["no_op", "image_conversion"], "glance::api::image_conversion_output_format": "raw", "glance::api::inject_metadata_properties": [], "glance::api::ignore_user_roles": ["admin"], "tripleo::profile::base::glance::api::tls_proxy_bind_ip": "%{hiera('internal_api')}", "tripleo::profile::base::glance::api::tls_proxy_fqdn": "%{hiera('fqdn_internal_api')}", "tripleo::profile::base::glance::api::tls_proxy_port": 9292, "glance::api::bind_host": "%{hiera('internal_api')}", "glance_notifier_strategy": "noop", "glance_log_file": "", "glance::backend::rbd::rbd_thin_provisioning": false, "glance::backend::file::filesystem_thin_provisioning": false, "glance::backend::swift::swift_store_auth_address": "http://172.17.0.2:5000/v3", "glance::backend::swift::swift_store_user": "service:glance", "glance::backend::swift::swift_store_key": "PmWCSVDW3B2uUjMQtJIVdd5mn", "glance::backend::swift::swift_store_create_container_on_put": true, "glance::backend::swift::swift_store_auth_version": 3, "glance::backend::rbd::rbd_store_ceph_conf": "/etc/ceph/ceph.conf", "glance::backend::rbd::rbd_store_pool": "images", "glance::backend::rbd::rbd_store_user": "openstack", "glance_backend": "rbd", "tripleo::profile::base::glance::api::glance_backend_id": "default_backend", "tripleo::profile::base::glance::api::glance_store_description": "Default glance store backend.", "tripleo::profile::base::glance::api::multistore_config": {}, "tripleo::profile::base::glance::backend::rbd::glance_rbd_ceph_conf_path": "/var/lib/tripleo-config/ceph", "glance::notify::rabbitmq::notification_driver": ["noop"], "glance::notify::rabbitmq::rabbit_heartbeat_in_pthread": false, "glance::cron::db_purge::minute": "1", "glance::cron::db_purge::hour": "0", "glance::cron::db_purge::monthday": "*", "glance::cron::db_purge::month": "*", "glance::cron::db_purge::weekday": "*", "glance::cron::db_purge::user": "glance", "glance::cron::db_purge::age": "30", "glance::cron::db_purge::max_rows": "100", "glance::cron::db_purge::destination": "/var/log/glance/glance-rowsflush.log", "glance::cron::db_purge::maxdelay": "3600", "glance::api::workers": "1"}, {"tripleo::haproxy::haproxy_service_manage": false, "tripleo::haproxy::haproxy_log_address": "/dev/log", "tripleo::haproxy::haproxy_log_facility": "local0", "tripleo::haproxy::haproxy_stats_user": "admin", "tripleo::haproxy::haproxy_stats_password": "1BkjVM1bzatAC1q8QQRKR2F3K", "tripleo::haproxy::haproxy_stats_bind_address": "%{hiera('ctlplane')}", "tripleo::haproxy::redis_password": "SxsXCWFxPxaSD4h7eclNJxxIm", "tripleo::haproxy::crl_file": null, "tripleo::haproxy::haproxy_stats": true, "enable_load_balancer": true, "tripleo::profile::base::haproxy::certificates_specs": {}, "tripleo::haproxy::mysql_clustercheck": true, "haproxy_docker": true, "tripleo::profile::pacemaker::haproxy_bundle::container_backend": "podman", "tripleo::profile::pacemaker::haproxy_bundle::tls_mapping": [], "tripleo::profile::pacemaker::haproxy_bundle::tls_mapping_init_bundle": [], "tripleo::profile::pacemaker::haproxy_bundle::internal_certs_directory": "/etc/pki/tls/certs/haproxy", "tripleo::profile::pacemaker::haproxy_bundle::internal_keys_directory": "/etc/pki/tls/private/haproxy", "tripleo::profile::pacemaker::haproxy_bundle::haproxy_docker_image": "cluster.common.tag/haproxy:pcmklatest"}, {"heat::notification_driver": ["noop"], "heat::logging::debug": true, "heat::policy::policies": {}, "heat::enable_proxy_headers_parsing": true, "heat::rpc_response_timeout": 600, "heat::rabbit_heartbeat_timeout_threshold": 60, "heat::rabbit_heartbeat_in_pthread": false, "heat::region_name": "regionOne", "heat::keystone::authtoken::project_name": "service", "heat::keystone::authtoken::user_domain_name": "Default", "heat::keystone::authtoken::project_domain_name": "Default", "heat::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "heat::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "heat::keystone::authtoken::password": "zoQ75Cm0pXTer2ifVDLy2roE1", "heat::keystone::authtoken::region_name": "regionOne", "heat::keystone::authtoken::interface": "internal", "heat::keystone::authtoken::memcache_use_advanced_pool": true, "heat::keystone::domain::domain_name": "heat_stack", "heat::keystone::domain::domain_admin": "heat_stack_domain_admin", "heat::keystone::domain::domain_admin_email": "heat_stack_domain_admin@localhost", "heat::db::database_db_max_retries": -1, "heat::db::database_max_retries": -1, "heat::yaql_memory_quota": 100000, "heat::yaql_limit_iterators": 1000, "heat::cors::max_age": 3600, "heat::cors::allow_headers": "Content-Type,Cache-Control,Content-Language,Expires,Last-Modified,Pragma,X-Auth-Token", "heat::cors::expose_headers": "Content-Type,Cache-Control,Content-Language,Expires,Last-Modified,Pragma", "heat::cron::purge_deleted::ensure": "present", "heat::cron::purge_deleted::minute": "1", "heat::cron::purge_deleted::hour": "0", "heat::cron::purge_deleted::monthday": "*", "heat::cron::purge_deleted::month": "*", "heat::cron::purge_deleted::weekday": "*", "heat::cron::purge_deleted::maxdelay": "3600", "heat::cron::purge_deleted::user": "heat", "heat::cron::purge_deleted::age": "30", "heat::cron::purge_deleted::age_type": "days", "heat::cron::purge_deleted::destination": "/dev/null", "heat::max_json_body_size": 4194304, "heat::cache::enabled": true, "heat::cache::tls_enabled": false, "heat::cache::resource_finder_caching": false, "heat::cache::backend": "dogpile.cache.memcached", "apache::ip": "%{hiera('internal_api')}", "apache::default_vhost": false, "apache::trace_enable": "Off", "apache::server_signature": "Off", "apache::server_tokens": "Prod", "apache::timeout": 90, "apache::default_mods": false, "apache::mod::prefork::maxrequestworkers": 100, "apache::mod::prefork::serverlimit": 100, "apache::mod::remoteip::proxy_ips": ["172.17.0.0/24"], "apache::mod::alias::icons_options": "None", "heat::api::bind_host": "%{hiera('internal_api')}", "heat::wsgi::apache_api::ssl": false, "heat::wsgi::apache_api::vhost_custom_fragment": "Timeout 600", "heat::api::service_name": "httpd", "heat::wsgi::apache_api::bind_host": "%{hiera('internal_api')}", "heat::wsgi::apache_api::servername": "%{hiera('fqdn_internal_api')}", "heat::wsgi::apache_api::workers": 1}, {"heat::notification_driver": ["noop"], "heat::logging::debug": true, "heat::policy::policies": {}, "heat::enable_proxy_headers_parsing": true, "heat::rpc_response_timeout": 600, "heat::rabbit_heartbeat_timeout_threshold": 60, "heat::rabbit_heartbeat_in_pthread": false, "heat::region_name": "regionOne", "heat::keystone::authtoken::project_name": "service", "heat::keystone::authtoken::user_domain_name": "Default", "heat::keystone::authtoken::project_domain_name": "Default", "heat::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "heat::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "heat::keystone::authtoken::password": "zoQ75Cm0pXTer2ifVDLy2roE1", "heat::keystone::authtoken::region_name": "regionOne", "heat::keystone::authtoken::interface": "internal", "heat::keystone::authtoken::memcache_use_advanced_pool": true, "heat::keystone::domain::domain_name": "heat_stack", "heat::keystone::domain::domain_admin": "heat_stack_domain_admin", "heat::keystone::domain::domain_admin_email": "heat_stack_domain_admin@localhost", "heat::db::database_db_max_retries": -1, "heat::db::database_max_retries": -1, "heat::yaql_memory_quota": 100000, "heat::yaql_limit_iterators": 1000, "heat::cors::max_age": 3600, "heat::cors::allow_headers": "Content-Type,Cache-Control,Content-Language,Expires,Last-Modified,Pragma,X-Auth-Token", "heat::cors::expose_headers": "Content-Type,Cache-Control,Content-Language,Expires,Last-Modified,Pragma", "heat::cron::purge_deleted::ensure": "present", "heat::cron::purge_deleted::minute": "1", "heat::cron::purge_deleted::hour": "0", "heat::cron::purge_deleted::monthday": "*", "heat::cron::purge_deleted::month": "*", "heat::cron::purge_deleted::weekday": "*", "heat::cron::purge_deleted::maxdelay": "3600", "heat::cron::purge_deleted::user": "heat", "heat::cron::purge_deleted::age": "30", "heat::cron::purge_deleted::age_type": "days", "heat::cron::purge_deleted::destination": "/dev/null", "heat::max_json_body_size": 4194304, "heat::cache::enabled": true, "heat::cache::tls_enabled": false, "heat::cache::resource_finder_caching": false, "heat::cache::backend": "dogpile.cache.memcached", "apache::default_vhost": false, "heat::api_cfn::bind_host": "%{hiera('internal_api')}", "heat::wsgi::apache_api_cfn::ssl": false, "heat::api_cfn::service_name": "httpd", "heat::wsgi::apache_api_cfn::bind_host": "%{hiera('internal_api')}", "heat::wsgi::apache_api_cfn::servername": "%{hiera('fqdn_internal_api')}", "heat::wsgi::apache_api_cfn::workers": 1}, {"heat::notification_driver": ["noop"], "heat::logging::debug": true, "heat::policy::policies": {}, "heat::enable_proxy_headers_parsing": true, "heat::rpc_response_timeout": 600, "heat::rabbit_heartbeat_timeout_threshold": 60, "heat::rabbit_heartbeat_in_pthread": false, "heat::region_name": "regionOne", "heat::keystone::authtoken::project_name": "service", "heat::keystone::authtoken::user_domain_name": "Default", "heat::keystone::authtoken::project_domain_name": "Default", "heat::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "heat::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "heat::keystone::authtoken::password": "zoQ75Cm0pXTer2ifVDLy2roE1", "heat::keystone::authtoken::region_name": "regionOne", "heat::keystone::authtoken::interface": "internal", "heat::keystone::authtoken::memcache_use_advanced_pool": true, "heat::keystone::domain::domain_name": "heat_stack", "heat::keystone::domain::domain_admin": "heat_stack_domain_admin", "heat::keystone::domain::domain_admin_email": "heat_stack_domain_admin@localhost", "heat::db::database_db_max_retries": -1, "heat::db::database_max_retries": -1, "heat::yaql_memory_quota": 100000, "heat::yaql_limit_iterators": 1000, "heat::cors::max_age": 3600, "heat::cors::allow_headers": "Content-Type,Cache-Control,Content-Language,Expires,Last-Modified,Pragma,X-Auth-Token", "heat::cors::expose_headers": "Content-Type,Cache-Control,Content-Language,Expires,Last-Modified,Pragma", "heat::cron::purge_deleted::ensure": "present", "heat::cron::purge_deleted::minute": "1", "heat::cron::purge_deleted::hour": "0", "heat::cron::purge_deleted::monthday": "*", "heat::cron::purge_deleted::month": "*", "heat::cron::purge_deleted::weekday": "*", "heat::cron::purge_deleted::maxdelay": "3600", "heat::cron::purge_deleted::user": "heat", "heat::cron::purge_deleted::age": "30", "heat::cron::purge_deleted::age_type": "days", "heat::cron::purge_deleted::destination": "/dev/null", "heat::max_json_body_size": 4194304, "heat::cache::enabled": true, "heat::cache::tls_enabled": false, "heat::cache::resource_finder_caching": false, "heat::cache::backend": "dogpile.cache.memcached", "apache::default_vhost": false, "heat::engine::configure_delegated_roles": false, "heat::engine::trusts_delegated_roles": [], "heat::engine::max_nested_stack_depth": 6, "heat::engine::max_resources_per_stack": 1000, "heat::engine::reauthentication_auth_method": "", "heat::engine::heat_metadata_server_url": "http://172.21.0.2:8000", "heat::engine::heat_waitcondition_server_url": "http://172.21.0.2:8000/v1/waitcondition", "heat::engine::convergence_engine": true, "heat::engine::client_retry_limit": 2, "tripleo::profile::base::heat::manage_db_purge": true, "heat::database_connection": "mysql+pymysql://heat:zoQ75Cm0pXTer2ifVDLy2roE1@172.17.0.2/heat?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "heat::keystone_ec2_uri": "http://172.17.0.2:5000/v3/ec2tokens", "heat::keystone::domain::domain_password": "n4UsoSw5TVdj2qe8wfVPQKKbC", "heat::engine::auth_encryption_key": "7OJ84fd3Qs6InvwvKVASVKB0UZMMnbNC", "heat::engine::plugin_dirs": [], "heat::engine::num_engine_workers": 1}, {"horizon::allowed_hosts": ["*"], "horizon::enable_secure_proxy_ssl_header": true, "horizon::secure_proxy_addr_header": "HTTP_X_FORWARDED_FOR", "horizon::disable_password_reveal": true, "horizon::enforce_password_check": true, "horizon::disallow_iframe_embed": true, "horizon::cache_backend": "django.core.cache.backends.memcached.MemcachedCache", "horizon::django_session_engine": "django.contrib.sessions.backends.cache", "horizon::vhost_extra_params": {"add_listen": true, "priority": 10, "access_log_format": "%a %l %u %t \\\"%r\\\" %>s %b \\\"%%{}{Referer}i\\\" \\\"%%{}{User-Agent}i\\\"", "options": ["FollowSymLinks", "MultiViews"]}, "horizon::bind_address": "%{hiera('internal_api')}", "horizon::keystone_url": "http://172.21.0.2:5000/v3", "horizon::password_validator": "", "horizon::password_validator_help": "", "horizon::secret_key": "NiewE8UjEw", "horizon::secure_cookies": false, "horizon::session_timeout": 1800, "memcached_ipv6": false, "horizon::servername": "%{hiera('fqdn_internal_api')}", "horizon::listen_ssl": false, "horizon::customization_module": "", "horizon::timezone": "UTC", "horizon::file_upload_temp_dir": "/var/tmp", "horizon::help_url": "http://docs.openstack.org", "horizon::enable_user_pass": "UNSET", "horizon::django_debug": true, "horizon::log_level": "DEBUG"}, {"tripleo::profile::base::iscsid::chap_algs": "SHA3-256,SHA256,SHA1,MD5"}, null, {"apache::ip": "%{hiera('internal_api')}", "apache::default_vhost": false, "apache::trace_enable": "Off", "apache::server_signature": "Off", "apache::server_tokens": "Prod", "apache::timeout": 90, "apache::default_mods": false, "apache::mod::prefork::maxrequestworkers": 100, "apache::mod::prefork::serverlimit": 100, "apache::mod::remoteip::proxy_ips": ["172.17.0.0/24"], "apache::mod::alias::icons_options": "None", "keystone::database_connection": "mysql+pymysql://keystone:1atINBzulchyizB7OeFxWH1ed@172.17.0.2/keystone?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "keystone::token_expiration": 3600, "keystone::policy::policies": {}, "keystone_ssl_certificate": "", "keystone_ssl_certificate_key": "", "keystone::token_provider": "fernet", "keystone::enable_fernet_setup": true, "keystone::fernet_max_active_keys": 5, "keystone::enable_proxy_headers_parsing": true, "keystone::enable_credential_setup": true, "keystone::credential_keys": {"/etc/keystone/credential-keys/0": {"content": "vn8Ahvk3PcwUIjPlU2cQ4ssV0LZrx4qHN7sXq0cJe3w="}, "/etc/keystone/credential-keys/1": {"content": "6Q7wVkDLZrhAceec2rpUf5jUKuaRp2ETFnn1S1CcQWQ="}}, "keystone::fernet_keys": {"/etc/keystone/fernet-keys/0": {"content": "puYLH9X_q7AKedRexvfvqE0Kg7bkZG-Eci3u88ckVag="}, "/etc/keystone/fernet-keys/1": {"content": "VmF8KXPGfie7IK3_G6iiqoS_459PwX0nBsoQkN893bM="}}, "keystone::fernet_replace_keys": true, "keystone::logging::debug": true, "keystone::notification_driver": ["messagingv2"], "keystone::notification_format": "basic", "tripleo::profile::base::keystone::extra_notification_topics": [], "keystone_enable_db_purge": true, "keystone::cron::trust_flush::ensure": "present", "keystone::cron::trust_flush::minute": "1", "keystone::cron::trust_flush::hour": "*", "keystone::cron::trust_flush::monthday": "*", "keystone::cron::trust_flush::month": "*", "keystone::cron::trust_flush::weekday": "*", "keystone::cron::trust_flush::maxdelay": 0, "keystone::cron::trust_flush::destination": "/var/log/keystone/keystone-trustflush.log", "keystone::cron::trust_flush::user": "keystone", "keystone::rabbit_heartbeat_timeout_threshold": 60, "keystone::rabbit_heartbeat_in_pthread": false, "keystone::service_name": "httpd", "keystone::enable_ssl": false, "keystone::wsgi::apache::api_port": [5000, "35357"], "keystone::wsgi::apache::ssl": false, "keystone::wsgi::apache::servername": "%{hiera('fqdn_internal_api')}", "keystone::wsgi::apache::workers": "1", "keystone::wsgi::apache::threads": 1, "keystone::db::database_db_max_retries": -1, "keystone::db::database_max_retries": -1, "keystone::wsgi::apache::bind_host": ["%{hiera('internal_api')}", "%{hiera('ctlplane')}"], "keystone::cache::enabled": true, "keystone::cache::backend_argument": ["memcached_expire_time: 3600"], "keystone::cache::tls_enabled": false, "keystone::cache::backend": "dogpile.cache.memcached"}, {"manila::storage_availability_zone": "nova", "manila::notification_driver": ["noop"], "manila::logging::debug": true, "manila::db::database_db_max_retries": -1, "manila::db::database_max_retries": -1, "manila::sql_connection": "mysql+pymysql://manila:2R4FJ91KHJyj5ppSY0XKBqLJg@172.17.0.2/manila?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "manila::network::neutron::network_plugin_ipv6_enabled": false, "manila::network::neutron::network_plugin_ipv4_enabled": true, "manila::rabbit_heartbeat_in_pthread": false, "apache::ip": "%{hiera('internal_api')}", "apache::default_vhost": false, "apache::trace_enable": "Off", "apache::server_signature": "Off", "apache::server_tokens": "Prod", "apache::timeout": 90, "apache::default_mods": false, "apache::mod::prefork::maxrequestworkers": 100, "apache::mod::prefork::serverlimit": 100, "apache::mod::remoteip::proxy_ips": ["172.17.0.0/24"], "apache::mod::alias::icons_options": "None", "manila::keystone::authtoken::password": "2R4FJ91KHJyj5ppSY0XKBqLJg", "manila::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "manila::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "manila::keystone::authtoken::project_name": "service", "manila::keystone::authtoken::user_domain_name": "Default", "manila::keystone::authtoken::project_domain_name": "Default", "manila::keystone::authtoken::region_name": "regionOne", "manila::keystone::authtoken::interface": "internal", "manila::keystone::authtoken::memcache_use_advanced_pool": true, "manila::api::bind_host": "%{hiera('internal_api')}", "manila::wsgi::apache::bind_host": "%{hiera('internal_api')}", "manila::wsgi::apache::ssl": false, "manila::api::service_name": "httpd", "manila::api::enable_proxy_headers_parsing": true, "manila::api::default_share_type": "default", "manila_enabled_share_protocols": [], "manila::policy::policies": {}, "manila::cron::db_purge::minute": "1", "manila::cron::db_purge::hour": "0", "manila::cron::db_purge::monthday": "*", "manila::cron::db_purge::month": "*", "manila::cron::db_purge::weekday": "*", "manila::cron::db_purge::user": "manila", "manila::cron::db_purge::age": "30", "manila::cron::db_purge::destination": "/var/log/manila/manila-rowsflush.log", "manila::cron::db_purge::maxdelay": "3600", "manila::wsgi::apache::servername": "%{hiera('fqdn_internal_api')}", "manila::wsgi::apache::workers": "%{::os_workers}"}, {"manila::backend::cephfs::title": "cephfs", "manila::backend::cephfs::driver_handles_share_servers": false, "manila::backend::cephfs::pool_name": "manila_data", "manila::backend::cephfs::share_backend_name": "cephfs", "manila::backend::cephfs::cephfs_filesystem_name": "cephfs", "manila_cephfs_ceph_conf_path": "/var/lib/tripleo-config/ceph", "manila::backend::cephfs::cephfs_conf_path": "/etc/ceph/ceph.conf", "manila::backend::cephfs::cephfs_auth_id": "manila", "manila::backend::cephfs::cephfs_cluster_name": "ceph", "manila::backend::cephfs::cephfs_volume_mode": "0755", "manila::backend::cephfs::cephfs_protocol_helper_type": "CEPHFS"}, {"manila::storage_availability_zone": "nova", "manila::notification_driver": ["noop"], "manila::logging::debug": true, "manila::db::database_db_max_retries": -1, "manila::db::database_max_retries": -1, "manila::sql_connection": "mysql+pymysql://manila:2R4FJ91KHJyj5ppSY0XKBqLJg@172.17.0.2/manila?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "manila::network::neutron::network_plugin_ipv6_enabled": false, "manila::network::neutron::network_plugin_ipv4_enabled": true, "manila::rabbit_heartbeat_in_pthread": false}, {"manila::storage_availability_zone": "nova", "manila::notification_driver": ["noop"], "manila::logging::debug": true, "manila::db::database_db_max_retries": -1, "manila::db::database_max_retries": -1, "manila::sql_connection": "mysql+pymysql://manila:2R4FJ91KHJyj5ppSY0XKBqLJg@172.17.0.2/manila?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "manila::network::neutron::network_plugin_ipv6_enabled": false, "manila::network::neutron::network_plugin_ipv4_enabled": true, "manila::rabbit_heartbeat_in_pthread": false, "manila::keystone::authtoken::password": "2R4FJ91KHJyj5ppSY0XKBqLJg", "manila::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000/", "manila::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "manila::keystone::authtoken::project_name": "service", "manila::keystone::authtoken::user_domain_name": "Default", "manila::keystone::authtoken::project_domain_name": "Default", "manila::keystone::authtoken::region_name": "regionOne", "manila::keystone::authtoken::interface": "internal", "manila::compute::nova::username": "manila", "manila::compute::nova::password": "2R4FJ91KHJyj5ppSY0XKBqLJg", "manila::compute::nova::auth_type": "password", "manila::compute::nova::project_name": "service", "manila::compute::nova::user_domain_name": "Default", "manila::compute::nova::project_domain_name": "Default", "manila::compute::nova::auth_url": "http://172.17.0.2:5000", "manila::compute::nova::region_name": "regionOne", "manila::network::neutron::username": "manila", "manila::network::neutron::password": "2R4FJ91KHJyj5ppSY0XKBqLJg", "manila::network::neutron::auth_type": "password", "manila::network::neutron::project_name": "service", "manila::network::neutron::user_domain_name": "Default", "manila::network::neutron::project_domain_name": "Default", "manila::network::neutron::auth_url": "http://172.17.0.2:5000", "manila::network::neutron::region_name": "regionOne", "manila::volume::cinder::username": "manila", "manila::volume::cinder::password": "2R4FJ91KHJyj5ppSY0XKBqLJg", "manila::volume::cinder::auth_type": "password", "manila::volume::cinder::project_name": "service", "manila::volume::cinder::user_domain_name": "Default", "manila::volume::cinder::project_domain_name": "Default", "manila::volume::cinder::auth_url": "http://172.17.0.2:5000", "manila::volume::cinder::region_name": "regionOne", "tripleo::profile::pacemaker::manila::share_bundle::manila_share_docker_image": "cluster.common.tag/manila-share:pcmklatest", "tripleo::profile::pacemaker::manila::share_bundle::docker_volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/manila_share.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/manila:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/dev:/dev", "/run:/run", "/sys:/sys", "/lib/modules:/lib/modules:ro", "/var/lib/manila:/var/lib/manila:z", "/var/log/containers/manila:/var/log/manila:z"], "tripleo::profile::pacemaker::manila::share_bundle::docker_environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}, "tripleo::profile::pacemaker::manila::share_bundle::container_backend": "podman", "tripleo::profile::pacemaker::manila::share_bundle::ceph_conf_path": "/var/lib/tripleo-config/ceph", "manila::share::manage_service": false, "manila::share::enabled": false, "manila::host": "hostgroup"}, {"memcached::listen": ["127.0.0.1", "%{hiera('internal_api')}"], "memcached::listen_ip": ["127.0.0.1", "%{hiera('internal_api')}"], "memcached::listen_ip_uri": ["127.0.0.1", "%{hiera('internal_api_uri')}"], "memcached::tcp_port": 11211, "memcached::max_connections": 8192, "memcached::max_memory": "50%", "memcached::udp_port": 0, "memcached::verbosity": "vv", "memcached::disable_cachedump": true, "memcached::logstdout": true, "tripleo::profile::base::memcached::enable_internal_memcached_tls": false}, {"mysql::server::package_name": "mariadb-galera-server", "mysql::server::manage_config_file": true, "mysql_ipv6": false, "mysql_max_connections": 4096, "mysql::server::root_password": "NhgQDgRVoo", "mysql_clustercheck_password": "sRxOzvWhwndqoiP4taJzvBzao", "enable_galera": true, "mysql_auth_ed25519": false, "mysql_bind_host": "%{hiera('internal_api')}", "tripleo::profile::base::database::mysql::bind_address": "%{hiera('fqdn_internal_api')}", "tripleo::profile::base::database::mysql::client_bind_address": "%{hiera('internal_api')}", "tripleo::profile::base::database::mysql::generate_dropin_file_limit": true, "innodb_buffer_pool_size": "128M", "tripleo::profile::pacemaker::database::mysql::bind_address": "%{hiera('fqdn_internal_api')}", "tripleo::profile::pacemaker::database::mysql::gmcast_listen_addr": "%{hiera('internal_api')}", "tripleo::profile::pacemaker::database::mysql::ca_file": "/etc/ipa/ca.crt", "tripleo::profile::pacemaker::database::mysql_bundle::mysql_docker_image": "cluster.common.tag/mariadb:pcmklatest", "tripleo::profile::pacemaker::database::mysql_bundle::control_port": 3123, "tripleo::profile::pacemaker::database::mysql_bundle::container_backend": "podman", "tripleo::profile::pacemaker::database::mysql_bundle::bind_address": "%{hiera('fqdn_internal_api')}", "tripleo::profile::pacemaker::database::mysql_bundle::sst_method": "rsync", "tripleo::profile::pacemaker::database::mysql_bundle::mariabackup_user": "mariabackup", "tripleo::profile::pacemaker::database::mysql_bundle::mariabackup_password": "3Qi03Zmdc69WCw00okVFEQzzM", "tripleo::profile::pacemaker::database::mysql_bundle::force_ocf": true, "tripleo::profile::pacemaker::database::mysql_bundle::gcomm_cipher": "AES128-SHA256"}, {"tripleo::profile::base::database::mysql::client::mysql_client_bind_address": "%{hiera('internal_api')}", "tripleo::profile::base::database::mysql::client::enable_ssl": false, "tripleo::profile::base::database::mysql::client::ssl_ca": "/etc/ipa/ca.crt"}, {"neutron::notification_driver": ["noop"], "neutron::core_plugin": "ml2", "neutron::service_plugins": ["qos", "ovn-router", "trunk", "segments", "port_forwarding", "log"], "neutron::report_interval": 300, "neutron::logging::debug": true, "neutron::purge_config": false, "neutron::allow_overlapping_ips": true, "neutron::dhcp_agent_notification": true, "neutron::dns_domain": "localdomain", "neutron::rabbit_heartbeat_timeout_threshold": 60, "neutron::host": "%{hiera('fqdn_canonical')}", "neutron::db::database_db_max_retries": -1, "neutron::db::database_max_retries": -1, "neutron::db::sync::db_sync_timeout": 900, "neutron::global_physnet_mtu": 1500, "neutron::db::sync::extra_params": "", "neutron::vlan_transparent": false, "neutron::rabbit_heartbeat_in_pthread": false, "neutron::db::database_connection": "mysql+pymysql://neutron:Y7cJplS04DjBDFMUWgbx3kdiC@172.17.0.2/ovs_neutron?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "neutron::policy::policies": {}, "if": [false, {"neutron::policy::enforce_scope": true, "neutron::policy::enforce_new_defaults": true}], "neutron::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "neutron::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "neutron::server::agent_down_time": 600, "neutron::server::allow_automatic_l3agent_failover": "True", "neutron::server::enable_proxy_headers_parsing": true, "neutron::server::igmp_snooping_enable": false, "neutron::keystone::authtoken::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "neutron::server::notifications::nova::auth_url": "http://172.17.0.2:5000", "neutron::server::notifications::nova::project_name": "service", "neutron::server::notifications::nova::user_domain_name": "Default", "neutron::server::notifications::nova::project_domain_name": "Default", "neutron::server::notifications::nova::region_name": "regionOne", "neutron::server::notifications::nova::password": "nad1A8ZvKDsT60yawIjN1cH4K", "neutron::server::notifications::nova::endpoint_type": "internal", "neutron::keystone::authtoken::project_name": "service", "neutron::keystone::authtoken::user_domain_name": "Default", "neutron::keystone::authtoken::project_domain_name": "Default", "neutron::keystone::authtoken::region_name": "regionOne", "neutron::keystone::authtoken::interface": "internal", "neutron::keystone::authtoken::memcache_use_advanced_pool": true, "neutron::quota::quota_port": "500", "neutron::quota::quota_security_group": "10", "neutron::server::placement::password": "nad1A8ZvKDsT60yawIjN1cH4K", "neutron::server::placement::project_domain_name": "Default", "neutron::server::placement::project_name": "service", "neutron::server::placement::user_domain_name": "Default", "neutron::server::placement::username": "nova", "neutron::server::placement::auth_url": "http://172.17.0.2:5000", "neutron::server::placement::auth_type": "password", "neutron::server::placement::region_name": "regionOne", "tripleo::profile::base::neutron::server::tls_proxy_bind_ip": "%{hiera('internal_api')}", "tripleo::profile::base::neutron::server::tls_proxy_fqdn": "%{hiera('fqdn_internal_api')}", "tripleo::profile::base::neutron::server::tls_proxy_port": "9696", "neutron::bind_host": "%{hiera('internal_api')}", "tripleo::profile::base::neutron::server::l3_ha_override": "", "neutron::server::api_workers": "1", "neutron::server::rpc_workers": "1"}, {"neutron::notification_driver": ["noop"], "neutron::core_plugin": "ml2", "neutron::service_plugins": ["qos", "ovn-router", "trunk", "segments", "port_forwarding", "log"], "neutron::report_interval": 300, "neutron::logging::debug": true, "neutron::purge_config": false, "neutron::allow_overlapping_ips": true, "neutron::dhcp_agent_notification": true, "neutron::dns_domain": "localdomain", "neutron::rabbit_heartbeat_timeout_threshold": 60, "neutron::host": "%{hiera('fqdn_canonical')}", "neutron::db::database_db_max_retries": -1, "neutron::db::database_max_retries": -1, "neutron::db::sync::db_sync_timeout": 900, "neutron::global_physnet_mtu": 1500, "neutron::db::sync::extra_params": "", "neutron::vlan_transparent": false, "neutron::rabbit_heartbeat_in_pthread": false, "neutron::plugins::ml2::mechanism_drivers": ["sriovnicswitch", "ovn"], "neutron::plugins::ml2::type_drivers": ["geneve", "vxlan", "vlan", "flat"], "neutron::plugins::ml2::flat_networks": ["datacentre"], "neutron::plugins::ml2::extension_drivers": ["qos", "port_security", "dns_domain_ports"], "neutron::plugins::ml2::network_vlan_ranges": ["datacentre:1:1000"], "neutron::plugins::ml2::tunnel_id_ranges": ["1:4094"], "neutron::plugins::ml2::vni_ranges": ["1:65536"], "neutron::plugins::ml2::tenant_network_types": ["geneve"], "neutron::plugins::ml2::ovs_driver::vnic_type_blacklist": [], "neutron::plugins::ml2::sriov_driver::vnic_type_blacklist": [], "neutron::plugins::ml2::overlay_ip_version": 4, "ovn::southbound::port": 6642, "ovn::northbound::port": 6641, "neutron::plugins::ml2::ovn::ovsdb_connection_timeout": 180, "neutron::plugins::ml2::ovn::ovsdb_probe_interval": 60000, "neutron::plugins::ml2::ovn::neutron_sync_mode": "log", "neutron::plugins::ml2::ovn::ovn_metadata_enabled": true, "neutron::server::qos_notification_drivers": "ovn-qos", "neutron::server::igmp_snooping_enable": false, "neutron::plugins::ml2::max_header_size": 38, "neutron::plugins::ml2::ovn::dns_servers": [], "neutron::plugins::ml2::ovn::ovn_emit_need_to_frag": true, "neutron::plugins::ml2::ovn::dvr_enabled": true, "neutron::plugins::ml2::ovn::network_log_rate_limit": 100, "neutron::plugins::ml2::ovn::network_log_burst_limit": 25}, {"neutron::notification_driver": ["noop"], "neutron::core_plugin": "ml2", "neutron::service_plugins": ["qos", "ovn-router", "trunk", "segments", "port_forwarding", "log"], "neutron::report_interval": 300, "neutron::logging::debug": true, "neutron::purge_config": false, "neutron::allow_overlapping_ips": true, "neutron::dhcp_agent_notification": true, "neutron::dns_domain": "localdomain", "neutron::rabbit_heartbeat_timeout_threshold": 60, "neutron::host": "%{hiera('fqdn_canonical')}", "neutron::db::database_db_max_retries": -1, "neutron::db::database_max_retries": -1, "neutron::db::sync::db_sync_timeout": 900, "neutron::global_physnet_mtu": 1500, "neutron::db::sync::extra_params": "", "neutron::vlan_transparent": false, "neutron::rabbit_heartbeat_in_pthread": false, "neutron::agents::ml2::sriov::physical_device_mappings": ["dummy_sriov_net:dummy-dev"], "neutron::agents::ml2::sriov::exclude_devices": [], "neutron::agents::ml2::sriov::extensions": [], "neutron::agents::ml2::sriov::resource_provider_bandwidths": [], "ContainerNeutronSriovImage": "registry.redhat.io/rhosp-rhel9/openstack-neutron-sriov-agent:17.1", "ContainerNeutronConfigImage": "registry.redhat.io/rhosp-rhel9/openstack-neutron-server:17.1", "neutron::agents::ml2::sriov::resource_provider_default_hypervisor": "%{hiera('fqdn_canonical')}"}, {"neutron::notification_driver": ["noop"], "neutron::core_plugin": "ml2", "neutron::service_plugins": ["qos", "ovn-router", "trunk", "segments", "port_forwarding", "log"], "neutron::report_interval": 300, "neutron::logging::debug": true, "neutron::purge_config": false, "neutron::allow_overlapping_ips": true, "neutron::dhcp_agent_notification": true, "neutron::dns_domain": "localdomain", "neutron::rabbit_heartbeat_timeout_threshold": 60, "neutron::host": "%{hiera('fqdn_canonical')}", "neutron::db::database_db_max_retries": -1, "neutron::db::database_max_retries": -1, "neutron::db::sync::db_sync_timeout": 900, "neutron::global_physnet_mtu": 1500, "neutron::db::sync::extra_params": "", "neutron::vlan_transparent": false, "neutron::rabbit_heartbeat_in_pthread": false, "tripleo::profile::base::neutron::dhcp_agent_wrappers::enable_dnsmasq_wrapper": true, "tripleo::profile::base::neutron::dhcp_agent_wrappers::dnsmasq_process_wrapper": "/var/lib/neutron/dnsmasq_wrapper", "tripleo::profile::base::neutron::dhcp_agent_wrappers::dnsmasq_image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-dhcp-agent:17.1", "tripleo::profile::base::neutron::dhcp_agent_wrappers::enable_haproxy_wrapper": true, "tripleo::profile::base::neutron::dhcp_agent_wrappers::haproxy_process_wrapper": "/var/lib/neutron/dhcp_haproxy_wrapper", "tripleo::profile::base::neutron::dhcp_agent_wrappers::haproxy_image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-dhcp-agent:17.1", "tripleo::profile::base::neutron::dhcp_agent_wrappers::debug": true, "tripleo::profile::base::neutron::container_cli": "podman", "neutron::agents::dhcp::enable_isolated_metadata": false, "neutron::agents::dhcp::enable_force_metadata": true, "neutron::agents::dhcp::enable_metadata_network": false, "neutron::agents::dhcp::dnsmasq_local_resolv": false, "neutron::agents::dhcp::dnsmasq_dns_servers": [], "neutron::agents::dhcp::interface_driver": "neutron.agent.linux.interface.OVSInterfaceDriver", "neutron::agents::dhcp::dhcp_broadcast_reply": false, "neutron::agents::dhcp::dnsmasq_enable_addr6_list": true, "neutron::agents::dhcp::debug": true}, {"nova::my_ip": "%{hiera('internal_api')}", "nova::rabbit_use_ssl": "False", "nova::keystone::service_user::send_service_user_token": true, "nova::keystone::service_user::project_name": "service", "nova::keystone::service_user::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::service_user::auth_url": "http://172.17.0.2:5000", "nova::keystone::service_user::region_name": "regionOne", "nova::placement::project_name": "service", "nova::placement::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "nova::placement::auth_url": "http://172.17.0.2:5000", "nova::placement::region_name": "regionOne", "nova::placement::valid_interfaces": "internal", "nova::os_region_name": "regionOne", "nova::logging::debug": true, "nova::purge_config": false, "nova::network::neutron::project_name": "service", "nova::network::neutron::username": "neutron", "nova::network::neutron::region_name": "regionOne", "nova::dhcp_domain": "", "nova::network::neutron::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "nova::network::neutron::auth_url": "http://172.17.0.2:5000/v3", "nova::network::neutron::valid_interfaces": "internal", "nova::glance::valid_interfaces": "internal", "nova::rabbit_heartbeat_timeout_threshold": 60, "nova::cinder_catalog_info": "volumev3:cinderv3:internalURL", "nova::host": "%{hiera('fqdn_canonical')}", "nova::notify_on_state_change": "vm_and_task_state", "nova::notification_driver": ["noop"], "nova::notification_format": "unversioned", "nova::network::neutron::auth_type": "v3password", "nova::db::database_db_max_retries": -1, "nova::db::database_max_retries": -1, "nova::network::neutron::ovs_bridge": "br-int", "nova::compute::sync_power_state_interval": 600, "nova_is_additional_cell": false, "nova::cross_az_attach": true, "nova::cache::enabled": true, "nova::cache::tls_enabled": false, "nova::cache::backend": "dogpile.cache.memcached", "nova::rabbit_heartbeat_in_pthread": false, "nova::ovsdb_connection": "", "nova::api_database_connection": "mysql+pymysql://nova_api:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova_api?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "nova::database_connection": "mysql+pymysql://nova:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "apache::default_vhost": false, "nova::keystone::authtoken::project_name": "service", "nova::keystone::authtoken::user_domain_name": "Default", "nova::keystone::authtoken::project_domain_name": "Default", "nova::keystone::authtoken::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "nova::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "nova::keystone::authtoken::region_name": "regionOne", "nova::keystone::authtoken::interface": "internal", "nova::keystone::authtoken::memcache_use_advanced_pool": true, "nova::api::max_limit": 1000, "nova::api::enabled": true, "nova::network::neutron::default_floating_pool": "public", "nova::api::enable_proxy_headers_parsing": true, "nova::api::api_bind_address": "%{hiera('fqdn_internal_api')}", "nova::api::service_name": "httpd", "nova::wsgi::apache_api::ssl": false, "nova::wsgi::apache_api::bind_host": "%{hiera('internal_api')}", "nova::wsgi::apache_api::servername": "%{hiera('fqdn_internal_api')}", "nova::api::instance_name_template": "instance-%08x", "nova::policy::policies": {}, "nova::api::allow_resize_to_same_host": false, "nova_enable_db_purge": true, "nova::cron::purge_shadow_tables::minute": "0", "nova::cron::purge_shadow_tables::hour": "5", "nova::cron::purge_shadow_tables::monthday": "*", "nova::cron::purge_shadow_tables::month": "*", "nova::cron::purge_shadow_tables::weekday": "*", "nova::cron::purge_shadow_tables::user": "nova", "nova::cron::purge_shadow_tables::destination": "/var/log/nova/nova-rowspurge.log", "nova::cron::purge_shadow_tables::maxdelay": "3600", "nova::cron::purge_shadow_tables::age": 14, "nova::cron::purge_shadow_tables::verbose": false, "nova::cron::purge_shadow_tables::all_cells": true, "nova_enable_db_archive": true, "nova::cron::archive_deleted_rows::minute": "1", "nova::cron::archive_deleted_rows::hour": "0", "nova::cron::archive_deleted_rows::monthday": "*", "nova::cron::archive_deleted_rows::month": "*", "nova::cron::archive_deleted_rows::weekday": "*", "nova::cron::archive_deleted_rows::max_rows": "1000", "nova::cron::archive_deleted_rows::user": "nova", "nova::cron::archive_deleted_rows::destination": "/var/log/nova/nova-rowsflush.log", "nova::cron::archive_deleted_rows::maxdelay": "3600", "nova::cron::archive_deleted_rows::until_complete": true, "nova::cron::archive_deleted_rows::purge": false, "nova::cron::archive_deleted_rows::all_cells": true, "nova::cron::archive_deleted_rows::age": 90, "nova::policy::enforce_new_defaults": false, "nova::policy::enforce_scope": false, "nova::api::osapi_compute_workers": 1, "nova::wsgi::apache_api::workers": 1}, {"nova::my_ip": "%{hiera('internal_api')}", "nova::rabbit_use_ssl": "False", "nova::keystone::service_user::send_service_user_token": true, "nova::keystone::service_user::project_name": "service", "nova::keystone::service_user::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::service_user::auth_url": "http://172.17.0.2:5000", "nova::keystone::service_user::region_name": "regionOne", "nova::placement::project_name": "service", "nova::placement::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "nova::placement::auth_url": "http://172.17.0.2:5000", "nova::placement::region_name": "regionOne", "nova::placement::valid_interfaces": "internal", "nova::os_region_name": "regionOne", "nova::logging::debug": true, "nova::purge_config": false, "nova::network::neutron::project_name": "service", "nova::network::neutron::username": "neutron", "nova::network::neutron::region_name": "regionOne", "nova::dhcp_domain": "", "nova::network::neutron::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "nova::network::neutron::auth_url": "http://172.17.0.2:5000/v3", "nova::network::neutron::valid_interfaces": "internal", "nova::glance::valid_interfaces": "internal", "nova::rabbit_heartbeat_timeout_threshold": 60, "nova::cinder_catalog_info": "volumev3:cinderv3:internalURL", "nova::host": "%{hiera('fqdn_canonical')}", "nova::notify_on_state_change": "vm_and_task_state", "nova::notification_driver": ["noop"], "nova::notification_format": "unversioned", "nova::network::neutron::auth_type": "v3password", "nova::db::database_db_max_retries": -1, "nova::db::database_max_retries": -1, "nova::network::neutron::ovs_bridge": "br-int", "nova::compute::sync_power_state_interval": 600, "nova_is_additional_cell": false, "nova::cross_az_attach": true, "nova::cache::enabled": true, "nova::cache::tls_enabled": false, "nova::cache::backend": "dogpile.cache.memcached", "nova::rabbit_heartbeat_in_pthread": false, "nova::ovsdb_connection": "", "nova::api_database_connection": "mysql+pymysql://nova_api:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova_api?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "nova::database_connection": "mysql+pymysql://nova:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "nova::conductor::workers": 1}, {"nova::my_ip": "%{hiera('internal_api')}", "nova::rabbit_use_ssl": "False", "nova::keystone::service_user::send_service_user_token": true, "nova::keystone::service_user::project_name": "service", "nova::keystone::service_user::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::service_user::auth_url": "http://172.17.0.2:5000", "nova::keystone::service_user::region_name": "regionOne", "nova::placement::project_name": "service", "nova::placement::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "nova::placement::auth_url": "http://172.17.0.2:5000", "nova::placement::region_name": "regionOne", "nova::placement::valid_interfaces": "internal", "nova::os_region_name": "regionOne", "nova::logging::debug": true, "nova::purge_config": false, "nova::network::neutron::project_name": "service", "nova::network::neutron::username": "neutron", "nova::network::neutron::region_name": "regionOne", "nova::dhcp_domain": "", "nova::network::neutron::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "nova::network::neutron::auth_url": "http://172.17.0.2:5000/v3", "nova::network::neutron::valid_interfaces": "internal", "nova::glance::valid_interfaces": "internal", "nova::rabbit_heartbeat_timeout_threshold": 60, "nova::cinder_catalog_info": "volumev3:cinderv3:internalURL", "nova::host": "%{hiera('fqdn_canonical')}", "nova::notify_on_state_change": "vm_and_task_state", "nova::notification_driver": ["noop"], "nova::notification_format": "unversioned", "nova::network::neutron::auth_type": "v3password", "nova::db::database_db_max_retries": -1, "nova::db::database_max_retries": -1, "nova::network::neutron::ovs_bridge": "br-int", "nova::compute::sync_power_state_interval": 600, "nova_is_additional_cell": false, "nova::cross_az_attach": true, "nova::cache::enabled": true, "nova::cache::tls_enabled": false, "nova::cache::backend": "dogpile.cache.memcached", "nova::rabbit_heartbeat_in_pthread": false, "nova::ovsdb_connection": "", "nova::api_database_connection": "mysql+pymysql://nova_api:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova_api?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "nova::database_connection": "mysql+pymysql://nova:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "apache::ip": "%{hiera('internal_api')}", "apache::default_vhost": false, "apache::trace_enable": "Off", "apache::server_signature": "Off", "apache::server_tokens": "Prod", "apache::timeout": 90, "apache::default_mods": false, "apache::mod::prefork::maxrequestworkers": 100, "apache::mod::prefork::serverlimit": 100, "apache::mod::remoteip::proxy_ips": ["172.17.0.0/24"], "apache::mod::alias::icons_options": "None", "nova::keystone::authtoken::project_name": "service", "nova::keystone::authtoken::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "nova::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "nova::keystone::authtoken::region_name": "regionOne", "nova::keystone::authtoken::interface": "internal", "nova::keystone::authtoken::memcache_use_advanced_pool": true, "nova::wsgi::apache_metadata::api_port": "8775", "nova::wsgi::apache_metadata::ssl": false, "nova::metadata::local_metadata_per_cell": false, "nova::wsgi::apache_metadata::bind_host": "%{hiera('internal_api')}", "nova::wsgi::apache_metadata::servername": "%{hiera('fqdn_internal_api')}", "nova::wsgi::apache_metadata::workers": 1, "nova::metadata::neutron_metadata_proxy_shared_secret": "mfZEPZHNi0vmUaK2wU20kjgr2"}, {"nova::my_ip": "%{hiera('internal_api')}", "nova::rabbit_use_ssl": "False", "nova::keystone::service_user::send_service_user_token": true, "nova::keystone::service_user::project_name": "service", "nova::keystone::service_user::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::service_user::auth_url": "http://172.17.0.2:5000", "nova::keystone::service_user::region_name": "regionOne", "nova::placement::project_name": "service", "nova::placement::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "nova::placement::auth_url": "http://172.17.0.2:5000", "nova::placement::region_name": "regionOne", "nova::placement::valid_interfaces": "internal", "nova::os_region_name": "regionOne", "nova::logging::debug": true, "nova::purge_config": false, "nova::network::neutron::project_name": "service", "nova::network::neutron::username": "neutron", "nova::network::neutron::region_name": "regionOne", "nova::dhcp_domain": "", "nova::network::neutron::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "nova::network::neutron::auth_url": "http://172.17.0.2:5000/v3", "nova::network::neutron::valid_interfaces": "internal", "nova::glance::valid_interfaces": "internal", "nova::rabbit_heartbeat_timeout_threshold": 60, "nova::cinder_catalog_info": "volumev3:cinderv3:internalURL", "nova::host": "%{hiera('fqdn_canonical')}", "nova::notify_on_state_change": "vm_and_task_state", "nova::notification_driver": ["noop"], "nova::notification_format": "unversioned", "nova::network::neutron::auth_type": "v3password", "nova::db::database_db_max_retries": -1, "nova::db::database_max_retries": -1, "nova::network::neutron::ovs_bridge": "br-int", "nova::compute::sync_power_state_interval": 600, "nova_is_additional_cell": false, "nova::cross_az_attach": true, "nova::cache::enabled": true, "nova::cache::tls_enabled": false, "nova::cache::backend": "dogpile.cache.memcached", "nova::rabbit_heartbeat_in_pthread": false, "nova::ovsdb_connection": "", "nova::api_database_connection": "mysql+pymysql://nova_api:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova_api?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "nova::database_connection": "mysql+pymysql://nova:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "nova::scheduler::filter::scheduler_available_filters": ["nova.scheduler.filters.all_filters", "nova.scheduler.filters.pci_passthrough_filter.PciPassthroughFilter"], "nova::scheduler::filter::scheduler_enabled_filters": ["AvailabilityZoneFilter", "ComputeFilter", "ComputeCapabilitiesFilter", "ImagePropertiesFilter", "ServerGroupAntiAffinityFilter", "ServerGroupAffinityFilter", "PciPassthroughFilter"], "nova::scheduler::filter::scheduler_max_attempts": 3, "nova::scheduler::filter::scheduler_host_subset_size": 1, "nova::scheduler::filter::shuffle_best_same_weighed_hosts": false, "nova::scheduler::discover_hosts_in_cells_interval": -1, "nova::scheduler::query_placement_for_image_type_support": true, "nova::scheduler::limit_tenants_to_placement_aggregate": false, "nova::scheduler::placement_aggregate_required_for_tenants": false, "nova::scheduler::enable_isolated_aggregate_filtering": false, "nova::scheduler::query_placement_for_availability_zone": false, "nova::scheduler::query_placement_for_routed_network_aggregates": false, "nova::scheduler::workers": 1}, {"nova::my_ip": "%{hiera('internal_api')}", "nova::rabbit_use_ssl": "False", "nova::keystone::service_user::send_service_user_token": true, "nova::keystone::service_user::project_name": "service", "nova::keystone::service_user::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::service_user::auth_url": "http://172.17.0.2:5000", "nova::keystone::service_user::region_name": "regionOne", "nova::placement::project_name": "service", "nova::placement::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "nova::placement::auth_url": "http://172.17.0.2:5000", "nova::placement::region_name": "regionOne", "nova::placement::valid_interfaces": "internal", "nova::os_region_name": "regionOne", "nova::logging::debug": true, "nova::purge_config": false, "nova::network::neutron::project_name": "service", "nova::network::neutron::username": "neutron", "nova::network::neutron::region_name": "regionOne", "nova::dhcp_domain": "", "nova::network::neutron::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "nova::network::neutron::auth_url": "http://172.17.0.2:5000/v3", "nova::network::neutron::valid_interfaces": "internal", "nova::glance::valid_interfaces": "internal", "nova::rabbit_heartbeat_timeout_threshold": 60, "nova::cinder_catalog_info": "volumev3:cinderv3:internalURL", "nova::host": "%{hiera('fqdn_canonical')}", "nova::notify_on_state_change": "vm_and_task_state", "nova::notification_driver": ["noop"], "nova::notification_format": "unversioned", "nova::network::neutron::auth_type": "v3password", "nova::db::database_db_max_retries": -1, "nova::db::database_max_retries": -1, "nova::network::neutron::ovs_bridge": "br-int", "nova::compute::sync_power_state_interval": 600, "nova_is_additional_cell": false, "nova::cross_az_attach": true, "nova::cache::enabled": true, "nova::cache::tls_enabled": false, "nova::cache::backend": "dogpile.cache.memcached", "nova::rabbit_heartbeat_in_pthread": false, "nova::ovsdb_connection": "", "nova::compute::vcpu_pin_set": [], "nova::compute::cpu_shared_set": [], "nova::compute::cpu_dedicated_set": [], "nova::compute::reserved_host_memory": 512, "nova::compute::reserved_huge_pages": [], "nova::compute::neutron_physnets_numa_nodes_mapping": {}, "nova::compute::neutron_tunnel_numa_nodes": [], "nova::compute::resume_guests_state_on_host_boot": false, "nova::compute::libvirt::rx_queue_size": 512, "nova::compute::libvirt::tx_queue_size": 512, "nova::compute::libvirt::file_backed_memory": 0, "nova::compute::libvirt::volume_use_multipath": false, "nova::compute::libvirt::hw_machine_type": "x86_64=pc-q35-rhel9.0.0", "nova::compute::mkisofs_cmd": "mkisofs", "compute_enable_ksm": false, "nova::compute::rbd::libvirt_images_rbd_pool": "vms", "tripleo::profile::base::nova::compute::nova_nfs_enabled": false, "nfs_backend_enable": false, "nfs_share": "", "nfs_options": "context=system_u:object_r:nfs_t:s0", "nfs_vers": "4.2", "nova::compute::libvirt::num_pcie_ports": 16, "nova::compute::libvirt::mem_stats_period_seconds": 10, "nova::compute::rbd::ephemeral_storage": true, "resume_guests_state_on_host_boot": false, "nova::cpu_allocation_ratio": 0.0, "nova::ram_allocation_ratio": 1.0, "nova::disk_allocation_ratio": 0.0, "nova::compute::max_disk_devices_to_attach": -1, "nova::compute::libvirt::pmem_namespaces": "", "nova_pmem_namespaces": "", "nova::compute::image_cache::remove_unused_original_minimum_age_seconds": 86400, "nova::compute::libvirt::swtpm_enabled": false, "nova::compute::vgpu::vgpu_types_device_addresses_mapping": {}, "compute_disable_irqbalance": false, "nova::compute::libvirt::cpu_mode": "host-model", "nova::compute::libvirt::cpu_models": [], "nova::compute::libvirt::cpu_model_extra_flags": "", "nova_compute_opt_volumes": [], "nova_compute_startup_delay": 0, "nova_compute_opt_env_vars": {}, "nova::workarounds::never_download_image_if_on_rbd": false, "nova::compute::image_type_exclude_list": [], "nova_permit_post_copy": true, "nova_permit_auto_converge": true, "nova_libvirt_max_queues": 0, "nova_glance_enable_rbd_download": false, "nova_glance_rbd_download_multistore_id": "", "nova::compute::force_raw_images": true, "nova::compute::use_cow_images": true, "nova::compute::libvirt::preallocate_images": "none", "nova::compute::image_cache::manager_interval": 2400, "nova::compute::image_cache::remove_unused_base_images": true, "nova::compute::image_cache::remove_unused_resized_minimum_age_seconds": 3600, "nova::compute::image_cache::precache_concurrency": 1, "nova::compute::provider::custom_inventories": {}, "nova::compute::rbd::libvirt_images_rbd_glance_copy_poll_interval": 15, "nova::compute::rbd::libvirt_images_rbd_glance_copy_timeout": 600, "ContainerNovaComputeImage": "registry.redhat.io/rhosp-rhel9/openstack-nova-compute:17.1", "ContainerNovaLibvirtConfigImage": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "nova::compute::libvirt::manage_libvirt_services": false, "nova::compute::pci::passthrough": "[{\"devname\": \"dummy-dev\", \"physical_network\": \"dummy_sriov_net\"}]", "nova::compute::libvirt::migration_support": false, "tripleo::profile::base::nova::migration::client::nova_compute_enabled": true, "tripleo::profile::base::nova::migration::client::ssh_private_key": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEApEW+iSDuQm4pBk/4gct8N21CUktJkYPZh42iMMBRmjvtdesa\nzByskHhSHzsdcZVXB6QMglFiP2qlX3OCmioW5N0jqTmpTbU/69vDerntqnzxNWpY\nJdycEi7LLztxn6UOlIQykb9KiEY+ou9NO4k8SAU72a5PplhZVPN0PTVs303ASP1r\nSV/mwd6ybKKqzEZm7bqeAAY/v34jevkPmPP0b7E31jDLDHMyWlEBrQQc9dzpnfMA\nzm4XJ7FoY/fxKYhzT1BFCpXJidTGvskPlta9yshXNigWBtbcQaJvt1bIEoofl5WD\n/BXa20TrltlD/nQ6VTbBHADz9uCbRQxaHI0e6QIDAQABAoIBADmks6CA6rpq14B8\njXJWSidU8qHYA1/JiYqUe6Zpxfi+MKxhLI0VPsSPRBnrb4CM6/YlbbJJBpXEQXWG\nHf0k6dWzd7EJJ8y53k1QV374QVrC7w1UXMwkZqvGucvAKzXfAIfC/8++9VNuespH\n+Kz8kieflonjLdXsSKSTrWVRGvwcRohAQ6k5YofMeOfke+Tk3/QNDmNTOOofyEN0\npeGouReKeh9N7FV1Uy0QzEi1Bq8odpcK7dHUNXPJM4NlISMXl3hsCu19OOenxLT4\nMPAcQodjIYO795S9hN9HL7tWNTwEReq5n3X6U3ZuDH5nedy5ngEATt9cEIboTjSh\nmNdbXuUCgYEAtobrDEiywQXXOjGTejM8l1G/cteml6aJPS5hc9qUJZr6UzW+HkfH\nBK3eh0ACoFxsiy1pP1wXCG79V1nROJu2josvXWqkkmJ5MUgIsg1NO4qhBppMhlEX\n1qhEbrf0k05ykrsp2ZLkAYkRbQeCyVbSud0flZxPWO7fTJqk5ZxC7Y8CgYEA5mW5\nUSYt3j5T8BhEkdHKj7DacnQE6CWBck1iMZ61W/gbhJgqUlb14v8fvf2guVgSI+nk\nZJwBo7A/O+yWxMCHVkzSiEvn7VQUA04cK8vvyA0sXq5XpR6smlh6kXYd8z2/Ke6U\n8M1iUsipNwV4QBOa1jvAkLPWEy2TvCBN24ytYAcCgYEAoZG0Jl+Du9ELv3t67ETG\ncbvvGB1EXiEQgpO3t/HYCaGxzE5aSDuJM98kh9xldsXZXymHbIp76P82Skg7JzzO\n0EbsWmi33MxGMxVsfQ/BOd9wwoN7Ebpeu6ifStKTMIHUcYYzeWDJa4KFZlmrr8Wn\nKtdNygVfR33LM+f/5y/3HokCgYAp7Rz6DW6PYNw5dtETkA/CcjZuWXqDHEAgIsDZ\nsr3iDJZFXMNm1HhOZIrMMhlLYSF785YiUNljhjEboIQXLcnzBGvTbay43XH+3/n0\nZlouF2eZmzRL0fcq5yr2yX3Nw9DDuqzT3SwlUGqqno4xat/kWF9EddVhDAw0UdSZ\nPVJXhQKBgBJuA1NrpGyERbKfwmTriaQ/4jXXZwcIZf+U0KOKchDtaR9PmY05qQm8\nbwH5USjLpLcfIZCxGRh+KPBusLOnvnKX4PZwv397Zfq9j5y8UcsWKab71RDqVY8V\nZj25fDMEw29QQyaxQ+SRV78MG20YJtsb9TieU1XvJkpFURUpypNX\n-----END RSA PRIVATE KEY-----\n", "tripleo::profile::base::nova::migration::client::ssh_port": 2022, "nova::compute::rbd::libvirt_images_rbd_glance_store_name": "default_backend", "nova::compute::rbd::libvirt_images_rbd_ceph_conf": "/etc/ceph/ceph.conf", "nova::compute::rbd::libvirt_rbd_user": "openstack", "nova::compute::rbd::rbd_keyring": "client.openstack", "tripleo::profile::base::nova::compute::cinder_nfs_backend": false, "rbd_persistent_storage": true, "rbd_disk_cachemodes": ["network=writeback"], "nova::cinder::username": "cinder", "nova::cinder::auth_type": "v3password", "nova::cinder::project_name": "service", "nova::cinder::password": "Olf2iTUMR6eJ33irnMaVcSJQX", "nova::cinder::auth_url": "http://172.17.0.2:5000/v3", "nova::cinder::region_name": "regionOne", "nova::compute::rbd::libvirt_rbd_secret_key": "AQBP7ytpAAAAABAAXQLQTQkMxLJ7uGigDaf94Q==", "nova::compute::rbd::libvirt_rbd_secret_uuid": "55dae5e5-7ec8-55ff-b761-b30fedcc71d6", "nova::compute::instance_usage_audit": true, "nova::compute::instance_usage_audit_period": "hour", "nova::compute::consecutive_build_service_disable_threshold": "10", "nova::compute::live_migration_wait_for_vif_plug": true, "nova::migration::libvirt::live_migration_permit_post_copy": true, "nova::migration::libvirt::live_migration_permit_auto_converge": true, "nova::migration::libvirt::live_migration_tunnelled": false, "nova::compute::vncserver_proxyclient_address": "%{hiera('internal_api')}", "nova::compute::vncproxy_host": "172.21.0.2", "nova::vncproxy::common::vncproxy_protocol": "http", "nova::vncproxy::common::vncproxy_host": "172.21.0.2", "nova::vncproxy::common::vncproxy_port": "6080", "nova::compute::verify_glance_signatures": false, "nova::workarounds::disable_compute_service_check_for_ffu": false}, {"nova::my_ip": "%{hiera('internal_api')}", "nova::rabbit_use_ssl": "False", "nova::keystone::service_user::send_service_user_token": true, "nova::keystone::service_user::project_name": "service", "nova::keystone::service_user::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::service_user::auth_url": "http://172.17.0.2:5000", "nova::keystone::service_user::region_name": "regionOne", "nova::placement::project_name": "service", "nova::placement::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "nova::placement::auth_url": "http://172.17.0.2:5000", "nova::placement::region_name": "regionOne", "nova::placement::valid_interfaces": "internal", "nova::os_region_name": "regionOne", "nova::logging::debug": true, "nova::purge_config": false, "nova::network::neutron::project_name": "service", "nova::network::neutron::username": "neutron", "nova::network::neutron::region_name": "regionOne", "nova::dhcp_domain": "", "nova::network::neutron::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "nova::network::neutron::auth_url": "http://172.17.0.2:5000/v3", "nova::network::neutron::valid_interfaces": "internal", "nova::glance::valid_interfaces": "internal", "nova::rabbit_heartbeat_timeout_threshold": 60, "nova::cinder_catalog_info": "volumev3:cinderv3:internalURL", "nova::host": "%{hiera('fqdn_canonical')}", "nova::notify_on_state_change": "vm_and_task_state", "nova::notification_driver": ["noop"], "nova::notification_format": "unversioned", "nova::network::neutron::auth_type": "v3password", "nova::db::database_db_max_retries": -1, "nova::db::database_max_retries": -1, "nova::network::neutron::ovs_bridge": "br-int", "nova::compute::sync_power_state_interval": 600, "nova_is_additional_cell": false, "nova::cross_az_attach": true, "nova::cache::enabled": true, "nova::cache::tls_enabled": false, "nova::cache::backend": "dogpile.cache.memcached", "nova::rabbit_heartbeat_in_pthread": false, "nova::ovsdb_connection": "", "vhostuser_socket_group": "qemu", "nova_libvirt_opt_volumes": [], "ContainerNovaLibvirtImage": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "ContainerNovaLibvirtConfigImage": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "nova::compute::libvirt::log_outputs": "1:file:/var/log/libvirt/libvirtd.log", "nova::compute::libvirt::virtlogd::log_outputs": "1:file:/var/log/libvirt/virtlogd.log", "nova::compute::libvirt::virtsecretd::log_outputs": "1:file:/var/log/libvirt/virtsecretd.log", "nova::compute::libvirt::virtnodedevd::log_outputs": "1:file:/var/log/libvirt/virtnodedevd.log", "nova::compute::libvirt::virtqemud::log_outputs": "1:file:/var/log/libvirt/virtqemud.log", "nova::compute::libvirt::virtproxyd::log_outputs": "1:file:/var/log/libvirt/virtproxyd.log", "nova::compute::libvirt::virtstoraged::log_outputs": "1:file:/var/log/libvirt/virtstoraged.log", "nova::compute::libvirt::qemu::memory_backing_dir": "", "nova::compute::libvirt::manage_libvirt_services": false, "tripleo::profile::base::nova::virtlogd_wrapper::enable_wrapper": true, "tripleo::profile::base::nova::virtlogd_wrapper::virtlogd_process_wrapper": "/var/lib/container-config-scripts/virtlogd_wrapper", "tripleo::profile::base::nova::virtlogd_wrapper::virtlogd_image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "tripleo::profile::base::nova::virtlogd_wrapper::debug": true, "nova::compute::libvirt::migration_support": false, "nova::compute::rbd::libvirt_images_rbd_ceph_conf": "/etc/ceph/ceph.conf", "nova::compute::rbd::libvirt_rbd_user": "openstack", "tripleo::profile::base::nova::compute_libvirt_shared::nova_rbd_ceph_conf_path": "/var/lib/tripleo-config/ceph", "nova::compute::rbd::rbd_keyring": "client.openstack", "nova::compute::rbd::libvirt_rbd_secret_key": "AQBP7ytpAAAAABAAXQLQTQkMxLJ7uGigDaf94Q==", "nova::compute::rbd::libvirt_rbd_secret_uuid": "55dae5e5-7ec8-55ff-b761-b30fedcc71d6", "tripleo::profile::base::nova::migration::client::libvirt_enabled": true, "tripleo::profile::base::nova::migration::client::ssh_private_key": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEApEW+iSDuQm4pBk/4gct8N21CUktJkYPZh42iMMBRmjvtdesa\nzByskHhSHzsdcZVXB6QMglFiP2qlX3OCmioW5N0jqTmpTbU/69vDerntqnzxNWpY\nJdycEi7LLztxn6UOlIQykb9KiEY+ou9NO4k8SAU72a5PplhZVPN0PTVs303ASP1r\nSV/mwd6ybKKqzEZm7bqeAAY/v34jevkPmPP0b7E31jDLDHMyWlEBrQQc9dzpnfMA\nzm4XJ7FoY/fxKYhzT1BFCpXJidTGvskPlta9yshXNigWBtbcQaJvt1bIEoofl5WD\n/BXa20TrltlD/nQ6VTbBHADz9uCbRQxaHI0e6QIDAQABAoIBADmks6CA6rpq14B8\njXJWSidU8qHYA1/JiYqUe6Zpxfi+MKxhLI0VPsSPRBnrb4CM6/YlbbJJBpXEQXWG\nHf0k6dWzd7EJJ8y53k1QV374QVrC7w1UXMwkZqvGucvAKzXfAIfC/8++9VNuespH\n+Kz8kieflonjLdXsSKSTrWVRGvwcRohAQ6k5YofMeOfke+Tk3/QNDmNTOOofyEN0\npeGouReKeh9N7FV1Uy0QzEi1Bq8odpcK7dHUNXPJM4NlISMXl3hsCu19OOenxLT4\nMPAcQodjIYO795S9hN9HL7tWNTwEReq5n3X6U3ZuDH5nedy5ngEATt9cEIboTjSh\nmNdbXuUCgYEAtobrDEiywQXXOjGTejM8l1G/cteml6aJPS5hc9qUJZr6UzW+HkfH\nBK3eh0ACoFxsiy1pP1wXCG79V1nROJu2josvXWqkkmJ5MUgIsg1NO4qhBppMhlEX\n1qhEbrf0k05ykrsp2ZLkAYkRbQeCyVbSud0flZxPWO7fTJqk5ZxC7Y8CgYEA5mW5\nUSYt3j5T8BhEkdHKj7DacnQE6CWBck1iMZ61W/gbhJgqUlb14v8fvf2guVgSI+nk\nZJwBo7A/O+yWxMCHVkzSiEvn7VQUA04cK8vvyA0sXq5XpR6smlh6kXYd8z2/Ke6U\n8M1iUsipNwV4QBOa1jvAkLPWEy2TvCBN24ytYAcCgYEAoZG0Jl+Du9ELv3t67ETG\ncbvvGB1EXiEQgpO3t/HYCaGxzE5aSDuJM98kh9xldsXZXymHbIp76P82Skg7JzzO\n0EbsWmi33MxGMxVsfQ/BOd9wwoN7Ebpeu6ifStKTMIHUcYYzeWDJa4KFZlmrr8Wn\nKtdNygVfR33LM+f/5y/3HokCgYAp7Rz6DW6PYNw5dtETkA/CcjZuWXqDHEAgIsDZ\nsr3iDJZFXMNm1HhOZIrMMhlLYSF785YiUNljhjEboIQXLcnzBGvTbay43XH+3/n0\nZlouF2eZmzRL0fcq5yr2yX3Nw9DDuqzT3SwlUGqqno4xat/kWF9EddVhDAw0UdSZ\nPVJXhQKBgBJuA1NrpGyERbKfwmTriaQ/4jXXZwcIZf+U0KOKchDtaR9PmY05qQm8\nbwH5USjLpLcfIZCxGRh+KPBusLOnvnKX4PZwv397Zfq9j5y8UcsWKab71RDqVY8V\nZj25fDMEw29QQyaxQ+SRV78MG20YJtsb9TieU1XvJkpFURUpypNX\n-----END RSA PRIVATE KEY-----\n", "tripleo::profile::base::nova::migration::client::ssh_port": 2022, "nova::compute::libvirt::services::libvirt_virt_type": "kvm", "nova::compute::libvirt::virt_type": "kvm", "nova::compute::libvirt::enabled_perf_events": [], "nova::compute::libvirt::qemu::configure_qemu": true, "nova::compute::libvirt::qemu::max_files": 32768, "nova::compute::libvirt::qemu::max_processes": 131072, "nova::migration::qemu::configure_qemu": true, "nova::migration::qemu::migration_port_min": 61152, "nova::migration::qemu::migration_port_max": 61215, "nova::compute::libvirt::vncserver_listen": "%{hiera('internal_api')}", "nova::compute::libvirt::log_filters": "1:libvirt 1:qemu 1:conf 1:security 3:event 3:json 3:file 3:object 1:util", "nova::compute::libvirt::virtlogd::log_filters": "1:logging 4:object 4:json 4:event 1:util", "rbd_persistent_storage": true, "rbd_disk_cachemodes": ["network=writeback"], "nova::compute::libvirt::virtsecretd::log_filters": "1:qemu 1:libvirt 4:object 4:json 4:event 1:util", "nova::compute::libvirt::virtnodedevd::log_filters": "1:qemu 1:libvirt 4:object 4:json 4:event 1:util", "nova::compute::libvirt::virtstoraged::log_filters": "1:qemu 1:libvirt 4:object 4:json 4:event 1:util", "nova::compute::libvirt::virtqemud::log_filters": "1:qemu 1:libvirt 4:object 4:json 4:event 1:util", "nova::compute::libvirt::virtproxyd::log_filters": "1:qemu 1:libvirt 4:object 4:json 4:event 1:util", "tripleo::profile::base::nova::libvirt::modular_libvirt": true, "nova::compute::libvirt::services::modular_libvirt": true, "nova::compute::libvirt::modular_libvirt": true, "nova::migration::libvirt::modular_libvirt": true, "nova::migration::libvirt::live_migration_inbound_addr": "%{hiera('fqdn_internal_api')}"}, {"tripleo::profile::base::sshd::bannertext": "", "tripleo::profile::base::sshd::motd": "", "tripleo::profile::base::sshd::options": {"HostKey": ["/etc/ssh/ssh_host_rsa_key", "/etc/ssh/ssh_host_ecdsa_key", "/etc/ssh/ssh_host_ed25519_key"], "SyslogFacility": "AUTHPRIV", "AuthorizedKeysFile": ".ssh/authorized_keys", "ChallengeResponseAuthentication": "no", "GSSAPIAuthentication": "no", "GSSAPICleanupCredentials": "no", "UsePAM": "yes", "UseDNS": "no", "PrintMotd": "no", "X11Forwarding": "yes", "AcceptEnv": ["LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES", "LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT", "LC_IDENTIFICATION LC_ALL LANGUAGE", "XMODIFIERS"], "Subsystem": "sftp  /usr/libexec/openssh/sftp-server"}, "tripleo::profile::base::sshd::password_authentication": "no", "tripleo::profile::base::nova::migration::target::ssh_authorized_keys": ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCkRb6JIO5CbikGT/iBy3w3bUJSS0mRg9mHjaIwwFGaO+116xrMHKyQeFIfOx1xlVcHpAyCUWI/aqVfc4KaKhbk3SOpOalNtT/r28N6ue2qfPE1algl3JwSLssvO3GfpQ6UhDKRv0qIRj6i7007iTxIBTvZrk+mWFlU83Q9NWzfTcBI/WtJX+bB3rJsoqrMRmbtup4ABj+/fiN6+Q+Y8/RvsTfWMMsMczJaUQGtBBz13Omd8wDObhcnsWhj9/EpiHNPUEUKlcmJ1Ma+yQ+W1r3KyFc2KBYG1txBom+3VsgSih+XlYP8FdrbROuW2UP+dDpVNsEcAPP24JtFDFocjR7p Generated by TripleO"], "tripleo::profile::base::nova::migration::target::ssh_localaddrs": ["%{hiera('cold_migration_ssh_inbound_addr')}", "%{hiera('live_migration_ssh_inbound_addr')}"], "live_migration_ssh_inbound_addr": "%{hiera('internal_api')}", "cold_migration_ssh_inbound_addr": "%{hiera('internal_api')}", "tripleo::profile::base::sshd::port": [22]}, {"nova::my_ip": "%{hiera('internal_api')}", "nova::rabbit_use_ssl": "False", "nova::keystone::service_user::send_service_user_token": true, "nova::keystone::service_user::project_name": "service", "nova::keystone::service_user::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::service_user::auth_url": "http://172.17.0.2:5000", "nova::keystone::service_user::region_name": "regionOne", "nova::placement::project_name": "service", "nova::placement::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "nova::placement::auth_url": "http://172.17.0.2:5000", "nova::placement::region_name": "regionOne", "nova::placement::valid_interfaces": "internal", "nova::os_region_name": "regionOne", "nova::logging::debug": true, "nova::purge_config": false, "nova::network::neutron::project_name": "service", "nova::network::neutron::username": "neutron", "nova::network::neutron::region_name": "regionOne", "nova::dhcp_domain": "", "nova::network::neutron::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "nova::network::neutron::auth_url": "http://172.17.0.2:5000/v3", "nova::network::neutron::valid_interfaces": "internal", "nova::glance::valid_interfaces": "internal", "nova::rabbit_heartbeat_timeout_threshold": 60, "nova::cinder_catalog_info": "volumev3:cinderv3:internalURL", "nova::host": "%{hiera('fqdn_canonical')}", "nova::notify_on_state_change": "vm_and_task_state", "nova::notification_driver": ["noop"], "nova::notification_format": "unversioned", "nova::network::neutron::auth_type": "v3password", "nova::db::database_db_max_retries": -1, "nova::db::database_max_retries": -1, "nova::network::neutron::ovs_bridge": "br-int", "nova::compute::sync_power_state_interval": 600, "nova_is_additional_cell": false, "nova::cross_az_attach": true, "nova::cache::enabled": true, "nova::cache::tls_enabled": false, "nova::cache::backend": "dogpile.cache.memcached", "nova::rabbit_heartbeat_in_pthread": false, "nova::ovsdb_connection": "", "nova::database_connection": "mysql+pymysql://nova:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "nova::vncproxy::enabled": true, "nova::vncproxy::common::vncproxy_protocol": "http", "nova::vncproxy::common::vncproxy_host": "172.21.0.2", "nova::vncproxy::common::vncproxy_port": "6080", "nova::vncproxy::host": "%{hiera('internal_api')}"}, {"ovn::controller::ovn_bridge_mappings": ["datacentre:br-ctlplane"], "ovn::controller::ovn_cms_options": "enable-chassis-as-gw", "ovn::controller::ovn_encap_tos": "0", "vswitch::ovs::enable_hw_offload": false, "container_cpuset_cpus": "", "ContainerOvnControllerImage": "registry.redhat.io/rhosp-rhel9/openstack-ovn-controller:17.1", "ContainerOvnControllerConfigImage": "registry.redhat.io/rhosp-rhel9/openstack-ovn-controller:17.1", "ovn::southbound::port": 6642, "ovn::controller::ovn_encap_ip": "%{hiera('tenant')}", "ovn::controller::ovn_encap_type": "geneve", "ovn::controller::ovn_bridge": "br-int", "ovn::controller::hostname": "%{hiera('fqdn_canonical')}", "ovn::controller::ovn_remote_probe_interval": 60000, "ovn::controller::ovn_openflow_probe_interval": 60, "ovn::controller::ovn_ofctrl_wait_before_clear": 8000, "ovn::controller::garp_max_timeout_sec": 0, "ovn::controller::ovn_monitor_all": true}, null, {"neutron::notification_driver": ["noop"], "neutron::core_plugin": "ml2", "neutron::service_plugins": ["qos", "ovn-router", "trunk", "segments", "port_forwarding", "log"], "neutron::report_interval": 300, "neutron::logging::debug": true, "neutron::purge_config": false, "neutron::allow_overlapping_ips": true, "neutron::dhcp_agent_notification": true, "neutron::dns_domain": "localdomain", "neutron::rabbit_heartbeat_timeout_threshold": 60, "neutron::host": "%{hiera('fqdn_canonical')}", "neutron::db::database_db_max_retries": -1, "neutron::db::database_max_retries": -1, "neutron::db::sync::db_sync_timeout": 900, "neutron::global_physnet_mtu": 1500, "neutron::db::sync::extra_params": "", "neutron::vlan_transparent": false, "neutron::rabbit_heartbeat_in_pthread": false, "tripleo::profile::base::neutron::ovn_metadata_agent_wrappers::enable_haproxy_wrapper": true, "tripleo::profile::base::neutron::ovn_metadata_agent_wrappers::haproxy_process_wrapper": "/var/lib/neutron/ovn_metadata_haproxy_wrapper", "tripleo::profile::base::neutron::ovn_metadata_agent_wrappers::haproxy_image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-metadata-agent-ovn:17.1", "tripleo::profile::base::neutron::ovn_metadata_agent_wrappers::debug": true, "tripleo::profile::base::neutron::container_cli": "podman", "neutron::agents::ovn_metadata::shared_secret": "mfZEPZHNi0vmUaK2wU20kjgr2", "neutron::agents::ovn_metadata::auth_password": "Y7cJplS04DjBDFMUWgbx3kdiC", "neutron::agents::ovn_metadata::auth_url": "http://172.17.0.2:5000", "neutron::agents::ovn_metadata::auth_tenant": "service", "neutron::agents::ovn_metadata::metadata_host": "172.17.0.2", "neutron::agents::ovn_metadata::ovsdb_connection_timeout": 180, "neutron::agents::ovn_metadata::ovsdb_probe_interval": 60000, "neutron::agents::ovn_metadata::hwol_qos_enabled": false, "ovn::northbound::port": 6641, "ovn::southbound::port": 6642, "neutron::agents::ovn_metadata::debug": true, "neutron::agents::ovn_metadata::metadata_protocol": "http", "neutron::agents::ovn_metadata::metadata_workers": "1"}, null, null, {"rabbitmq::file_limit": 65536, "rabbitmq::default_user": "guest", "rabbitmq::default_pass": "orxoJKQXPaAVXVpoTxqAR2c6v", "rabbit_ipv6": false, "rabbitmq::delete_guest_user": false, "rabbitmq::wipe_db_on_cookie_change": true, "rabbitmq::port": 5672, "rabbitmq::loopback_users": [], "rabbitmq::tcp_backlog": 4096, "rabbitmq::repos_ensure": false, "rabbitmq::tcp_keepalive": true, "rabbitmq_environment": {"LANG": "en_US.UTF-8", "LC_ALL": "en_US.UTF-8", "NODE_PORT": "", "NODE_IP_ADDRESS": "", "RABBITMQ_NODENAME": "rabbit@%{hiera('fqdn_internal_api')}", "RABBITMQ_USE_LONGNAME": "true", "RABBITMQ_CTL_DIST_PORT_MIN": "25673", "RABBITMQ_CTL_DIST_PORT_MAX": "25683", "export ERL_EPMD_ADDRESS": "%{hiera('rabbitmq::interface')}"}, "tripleo::profile::base::rabbitmq::additional_erl_args": "'+sbwt none +sbwtdcpu none +sbwtdio none'", "rabbitmq_kernel_variables": {"inet_dist_listen_min": "25672", "inet_dist_listen_max": "25672", "net_ticktime": 15}, "rabbitmq_config_variables": {"cluster_partition_handling": "ignore", "queue_master_locator": "<<\"min-masters\">>"}, "rabbitmq::config_additional_variables": {"lager": "[{crash_log_count, 0}, { crash_log_date, \"\"}, { crash_log_size, 0 }]"}, "rabbitmq::erlang_cookie": "XNOQDfh9VshBYfsmJBif", "rabbitmq::interface": "%{hiera('internal_api')}", "rabbitmq::nr_ha_queues": 0, "rabbitmq::ssl": false, "rabbitmq::ssl_erl_dist": false, "rabbitmq::ssl_port": 5672, "rabbitmq::ssl_depth": 1, "rabbitmq::ssl_only": false, "rabbitmq::ssl_interface": "%{hiera('internal_api')}", "tripleo::profile::base::rabbitmq::enable_internal_tls": false, "tripleo::profile::base::rabbitmq::fips_mode": false, "rabbitmq::collect_statistics_interval": 30000, "rabbitmq::admin_enable": false, "rabbitmq::management_enable": true, "rabbitmq::use_config_file_for_plugins": true, "rabbitmq::management_ip_address": "127.0.0.1", "rabbitmq::management_port": 15672, "rabbitmq::config_management_variables": {"rates_mode": "none"}, "rabbitmq::ssl_cacert": "/etc/ipa/ca.crt", "rabbitmq::ssl_management_cacert": "/etc/ipa/ca.crt", "rabbitmq::service_manage": false, "tripleo::profile::pacemaker::rabbitmq_bundle::rabbitmq_docker_image": "cluster.common.tag/rabbitmq:pcmklatest", "tripleo::profile::pacemaker::rabbitmq_bundle::control_port": 3122, "tripleo::profile::pacemaker::rabbitmq_bundle::container_backend": "podman", "tripleo::profile::pacemaker::rabbitmq_bundle::force_ocf": true}, {"pacemaker::corosync::cluster_name": "tripleo_cluster", "pacemaker::corosync::manage_fw": false, "pacemaker::resource_defaults::defaults": {"resource-stickiness": {"value": "INFINITY"}}, "corosync_token_timeout": 10000, "pacemaker::corosync::settle_tries": 360, "pacemaker::resource::bundle::deep_compare": true, "pacemaker::resource::ip::deep_compare": true, "pacemaker::resource::ocf::deep_compare": true, "corosync_ipv6": false, "tripleo::fencing::config": {}, "tripleo::fencing::deep_compare": true, "enable_fencing": false, "hacluster_pwd": "V7RyZXR24nKo1fUk", "tripleo::profile::base::pacemaker::remote_authkey": "[encoded data, 4096 characters]", "tripleo::profile::base::pacemaker::pcsd_bind_addr": "%{hiera('internal_api')}", "tripleo::profile::base::pacemaker::resource_op_defaults": {"bundle": {"name": "timeout", "value": "120s"}}}, {"apache::default_vhost": false, "placement::keystone::authtoken::project_name": "service", "placement::keystone::authtoken::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "placement::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "placement::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "placement::keystone::authtoken::region_name": "regionOne", "placement::keystone::authtoken::interface": "internal", "placement::keystone::authtoken::memcache_use_advanced_pool": true, "placement::logging::debug": true, "placement::policy::enforce_new_defaults": false, "placement::policy::enforce_scope": false, "placement::policy::policies": {}, "placement::wsgi::apache::api_port": "8778", "placement::wsgi::apache::ssl": false, "placement::wsgi::apache::bind_host": "%{hiera('internal_api')}", "placement::wsgi::apache::servername": "%{hiera('fqdn_internal_api')}", "placement::db::database_connection": "mysql+pymysql://placement:P6qVEsGa91uUJyG6cL2sqYpFU@172.17.0.2/placement?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "placement::wsgi::apache::workers": 1}, {}, {}, {"tripleo::profile::base::snmp::snmpd_user": "ro_snmp_user", "tripleo::profile::base::snmp::snmpd_password": "KY25uL2U94jHHfJNHHl8m07r", "snmp::agentaddress": ["udp:161", "udp6:[::1]:161"], "snmp::snmpd_options": "-LS0-5d"}, null, {"swift::swift_hash_path_suffix": "9DcVlAPcWbp0ySxp5PbD0EM2K", "swift::swift_hash_path_prefix": "", "swift::proxy::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "swift::proxy::authtoken::auth_url": "http://172.17.0.2:5000", "swift::proxy::authtoken::password": "8dzPI93pJLWng5a7IqjhGTq4v", "swift::proxy::authtoken::project_name": "service", "swift::proxy::authtoken::region_name": "regionOne", "swift::proxy::authtoken::interface": "internal", "swift::proxy::s3token::auth_uri": "http://172.17.0.2:5000/v3", "swift::proxy::node_timeout": 60, "swift::proxy::recoverable_node_timeout": 30, "swift::proxy::workers": "1", "swift::proxy::staticweb::url_base": "http://172.21.0.2:8080", "tripleo::profile::base::swift::proxy::ceilometer_enabled": false, "swift::proxy::keystone::operator_roles": ["admin", "swiftoperator", "ResellerAdmin"], "swift::proxy::versioned_writes::allow_versioned_writes": true, "swift::proxy::keystone::system_reader_roles": ["SwiftSystemReader"], "swift::proxy::pipeline": ["catch_errors", "gatekeeper", "healthcheck", "proxy-logging", "cache", "listing_formats", "ratelimit", "bulk", "tempurl", "formpost", "authtoken", "s3api", "s3token", "keystone", "staticweb", "copy", "container_quotas", "account_quotas", "slo", "dlo", "versioned_writes", "proxy-logging", "proxy-server"], "swift::proxy::account_autocreate": true, "tripleo::profile::base::swift::proxy::tls_proxy_bind_ip": "%{hiera('storage')}", "tripleo::profile::base::swift::proxy::tls_proxy_fqdn": "%{hiera('fqdn_storage')}", "tripleo::profile::base::swift::proxy::tls_proxy_port": "8080", "swift::proxy::port": "8080", "swift::proxy::proxy_local_net_ip": "%{hiera('storage')}", "swift::proxy::cache::tls_enabled": false}, {"tripleo::profile::base::swift::ringbuilder::skip_consistency_check": true, "tripleo::profile::base::swift::ringbuilder::swift_ring_get_tempurl": "", "tripleo::profile::base::swift::ringbuilder::swift_ring_put_tempurl": "", "tripleo::profile::base::swift::ringbuilder::build_ring": true, "tripleo::profile::base::swift::ringbuilder::replicas": 1, "tripleo::profile::base::swift::ringbuilder::part_power": 10, "tripleo::profile::base::swift::ringbuilder::min_part_hours": 1, "tripleo::profile::base::swift::ringbuilder::raw_disk_prefix": "r1z1-", "tripleo::profile::base::swift::ringbuilder::raw_disks": [":%PORT%/d1"]}, {"swift::swift_hash_path_suffix": "9DcVlAPcWbp0ySxp5PbD0EM2K", "swift::swift_hash_path_prefix": "", "swift::storage::all::mount_check": false, "tripleo::profile::base::swift::storage::use_local_dir": true, "swift::storage::all::incoming_chmod": "Du=rwx,g=rx,o=rx,Fu=rw,g=r,o=r", "swift::storage::all::outgoing_chmod": "Du=rwx,g=rx,o=rx,Fu=rw,g=r,o=r", "swift::storage::all::account_port": 6002, "swift::storage::all::container_port": 6001, "swift::storage::all::object_port": 6000, "swift::storage::all::object_pipeline": ["healthcheck", "recon", "object-server"], "swift::storage::all::container_pipeline": ["healthcheck", "recon", "container-server"], "swift::storage::all::account_pipeline": ["healthcheck", "recon", "account-server"], "swift::storage::disks::args": {}, "swift::storage::all::storage_local_net_ip": "%{hiera('storage_mgmt')}", "rsync::server::pid_file": "UNSET", "swift::objectexpirer::cache_tls_enabled": false}, null, null, null, {"tripleo::firewall::manage_firewall": false, "tripleo::firewall::purge_firewall_rules": false}, {"tripleo::packages::enable_install": true}, null], "tuple:('role_data', 'service_name')": ["boot_params_service", "barbican_api", "barbican_backend_simple_crypto", "ca_certs", "ceph_client", "ceph_mds", "ceph_mgr", "ceph_mon", "ceph_osd", "cinder_api", "cinder_backup", "cinder_scheduler", "cinder_volume", "clustercheck", "container_image_prepare", "logrotate_crond", "docker_registry", "glance_api", "glance_api_internal", "haproxy", "heat_api", "heat_api_cfn", "heat_engine", "horizon", "iscsid", "kernel", "keystone", "manila_api", "manila_backend_cephfs", "manila_scheduler", "manila_share", "memcached", "mysql", "mysql_client", "neutron_api", "neutron_plugin_ml2_ovn", "neutron_sriov_agent", "neutron_dhcp", "nova_api", "nova_conductor", "nova_metadata", "nova_scheduler", "nova_compute", "nova_libvirt", "nova_migration_target", "nova_vnc_proxy", "ovn_controller", "ovn_dbs", "ovn_metadata", "openstack_clients", "oslo_messaging_notify", "oslo_messaging_rpc", "pacemaker", "placement", "podman", "rsyslog_baremetal_ansible", "snmp", "sshd", "swift_proxy", "swift_ringbuilder", "swift_storage", "chrony", "timezone", "logrotate_tmpwatch", "tripleo_firewall", "tripleo_packages", "tuned"], "tuple:('role_data', 'container_config_scripts')": [null, null, null, null, null, null, null, null, null, null, {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}}, null, {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}}, null, null, null, null, null, null, {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}}, null, null, null, null, null, null, null, null, null, null, {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}}, null, {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}}, null, null, null, null, {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}}, {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "nova_wait_for_api_service.py": {"mode": "0755", "content": "#!/usr/bin/python3\n#\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\nimport argparse\nimport logging\nimport os\nimport six\nimport sys\nimport time\n\n\nfrom keystoneauth1 import loading\nfrom keystoneauth1 import session\n\nfrom novaclient import client\nfrom novaclient.exceptions import ClientException\n\n# In python3 SafeConfigParser was renamed to ConfigParser and the default\n# for duplicate options default to true. In case of nova it is valid to\n# have duplicate option lines, e.g. passthrough_whitelist which leads to\n# issues reading the nova.conf\n# https://bugs.launchpad.net/tripleo/+bug/1827775\nif six.PY3:\n    from six.moves.configparser import ConfigParser\n    config = ConfigParser(strict=False)\nelse:\n    from six.moves.configparser import SafeConfigParser\n    config = SafeConfigParser()\n\ndebug = os.getenv('__OS_DEBUG', 'false')\n\nif debug.lower() == 'true':\n    loglevel = logging.DEBUG\nelse:\n    loglevel = logging.INFO\n\nlogging.basicConfig(stream=sys.stdout, level=loglevel)\nLOG = logging.getLogger('nova_wait_for_api_service')\n\niterations = 60\ntimeout = 10\nnova_cfg = '/etc/nova/nova.conf'\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser(usage='%(prog)s [options]')\n    parser.add_argument('-k', '--insecure',\n                        action=\"store_false\",\n                        dest='insecure',\n                        default=True,\n                        help='Allow insecure connection when using SSL')\n\n    args = parser.parse_args()\n    LOG.debug('Running with parameter insecure = %s',\n              args.insecure)\n\n    if os.path.isfile(nova_cfg):\n        try:\n            config.read(nova_cfg)\n        except Exception:\n            LOG.exception('Error while reading nova.conf:')\n    else:\n        LOG.error('Nova configuration file %s does not exist', nova_cfg)\n        sys.exit(1)\n\n    loader = loading.get_plugin_loader('password')\n    auth = loader.load_from_options(\n        auth_url=config.get('neutron',\n                            'auth_url'),\n        username=config.get('neutron',\n                            'username'),\n        password=config.get('neutron',\n                            'password'),\n        project_name=config.get('neutron',\n                                'project_name'),\n        project_domain_name=config.get('neutron',\n                                       'project_domain_name'),\n        user_domain_name=config.get('neutron',\n                                    'user_domain_name'))\n    sess = session.Session(auth=auth, verify=args.insecure)\n\n    # Wait until this host is listed in the service list\n    for i in range(iterations):\n        try:\n            nova = client.Client('2.11', session=sess, endpoint_type='internal')\n            nova.versions.list()\n            LOG.info('Nova-api service active')\n            sys.exit(0)\n        except ClientException:\n            LOG.info('Waiting for nova-api service')\n        except Exception:\n            LOG.exception(\n                'Error while waiting for nova-api service')\n        time.sleep(timeout)\nsys.exit(1)\n\n# vim: set et ts=4 sw=4 :\n"}, "nova_api_ensure_default_cells.py": {"mode": "0755", "content": "#!/usr/bin/python3\n#\n# Copyright 2022 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\nfrom configparser import ConfigParser\nimport logging\nimport os\nimport subprocess\nimport sys\nfrom urllib import parse as urlparse\n\nconfig = ConfigParser(strict=False)\n\ndebug = os.getenv('__OS_DEBUG', 'false')\n\nif debug.lower() == 'true':\n    loglevel = logging.DEBUG\nelse:\n    loglevel = logging.INFO\n\nlogging.basicConfig(stream=sys.stdout, level=loglevel)\nLOG = logging.getLogger('nova_api_ensure_default_cells')\n\nNOVA_CFG = '/etc/nova/nova.conf'\nCELL0_ID = '00000000-0000-0000-0000-000000000000'\nDEFAULT_CELL_NAME = 'default'\n\n\ndef template_netloc_credentials(netloc, index=None):\n    if '@' in netloc:\n        userpass, hostport = netloc.split('@', 1)\n        has_pass = ':' in userpass\n        if index is None:\n            cred_template = '{username}'\n            if has_pass:\n                cred_template += ':{password}'\n        else:\n            cred_template = '{{username{index}}}'.format(index=index)\n            if has_pass:\n                cred_template += ':{{password{index}}}'.format(index=index)\n        return '@'.join((cred_template, hostport))\n    else:\n        return netloc\n\n\ndef template_url(url):\n    parsed = urlparse.urlparse(url)\n    if ',' in parsed.netloc:\n        orig_netlocs = parsed.netloc.split(',')\n        templ_netlocs = []\n        index = 0\n        for netloc in orig_netlocs:\n            index += 1\n            templ_netlocs.append(template_netloc_credentials(netloc, index))\n        new_netloc = ','.join(templ_netlocs)\n    else:\n        new_netloc = template_netloc_credentials(parsed.netloc)\n    return parsed._replace(netloc=new_netloc).geturl()\n\n\ndef parse_list_cells(list_cells_output):\n    list_cells_lines = list_cells_output.split('\\n')\n    if len(list_cells_lines) < 5:\n        raise ValueError('Invalid nova-manage cell_v2 list_cells output')\n\n    data_rows = list_cells_lines[3:-2]\n    by_name = {}\n    by_uuid = {}\n\n    for row in data_rows:\n        parts = row.split('|')\n        entry = {\n            'name': parts[1].strip(),\n            'uuid': parts[2].strip(),\n            'transport_url': parts[3].strip(),\n            'database_connection': parts[4].strip(),\n        }\n        by_name[entry['name']] = entry\n        by_uuid[entry['uuid']] = entry\n\n    return by_name, by_uuid\n\n\ndef create_or_update_default_cells(cell0_db, default_db, default_transport_url):\n    list_cells_cmd = ['/usr/bin/nova-manage', 'cell_v2', 'list_cells', '--verbose']\n    list_cells_output = subprocess.check_output(list_cells_cmd, encoding='utf-8')\n    cells_by_name, cells_by_uuid = parse_list_cells(list_cells_output)\n\n    if CELL0_ID in cells_by_uuid:\n        LOG.info('Setting cell0 database connection to \\'{}\\''.format(cell0_db))\n        cmd = [\n            '/usr/bin/nova-manage', 'cell_v2', 'update_cell',\n            '--cell_uuid', CELL0_ID,\n            '--database_connection', cell0_db,\n            '--transport-url', 'none:///'\n        ]\n    else:\n        LOG.info('Creating cell0 with database connection \\'{}\\''.format(cell0_db))\n        cmd = [\n            '/usr/bin/nova-manage', 'cell_v2', 'map_cell0',\n             '--database_connection', cell0_db\n        ]\n    subprocess.check_call(cmd)\n\n    if DEFAULT_CELL_NAME in cells_by_name:\n        LOG.info('Setting default cell database connection to \\'{}\\' and transport url to \\'{}\\''.format(\n            default_db, default_transport_url))\n        cmd = [\n            '/usr/bin/nova-manage', 'cell_v2', 'update_cell',\n            '--cell_uuid', cells_by_name[DEFAULT_CELL_NAME]['uuid'],\n            '--database_connection', default_db,\n            '--transport-url', default_transport_url\n        ]\n    else:\n        LOG.info('Creating default cell with database connection \\'{}\\' and transport url \\'{}\\''.format(\n            default_db, default_transport_url))\n        cmd = [\n            '/usr/bin/nova-manage', 'cell_v2', 'create_cell',\n            '--name', DEFAULT_CELL_NAME,\n             '--database_connection', default_db,\n            '--transport-url', default_transport_url\n        ]\n    subprocess.check_call(cmd)\n\n\ndef replace_db_name(db_url, db_name):\n    return urlparse.urlparse(db_url)._replace(path=db_name).geturl()\n\n\nif __name__ == '__main__':\n    if os.path.isfile(NOVA_CFG):\n        try:\n            config.read(NOVA_CFG)\n        except Exception:\n            LOG.exception('Error while reading nova.conf:')\n            sys.exit(1)\n    else:\n        LOG.error('Nova configuration file %s does not exist', NOVA_CFG)\n        sys.exit(1)\n\n    default_database_connection = config.get('database', 'connection')\n    cell0_database_connection = replace_db_name(default_database_connection, 'nova_cell0')\n    default_transport_url = config.get('DEFAULT', 'transport_url')\n\n    create_or_update_default_cells(\n        template_url(cell0_database_connection),\n        template_url(default_database_connection),\n        template_url(default_transport_url)\n    )\n"}}, null, null, null, {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "nova_statedir_ownership.py": {"mode": "0700", "content": "#!/usr/bin/python3\n#\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\nimport logging\nimport os\nimport pwd\nimport selinux\nimport stat\nimport sys\n\ndebug = os.getenv('__OS_DEBUG', 'false')\n\nif debug.lower() == 'true':\n    loglevel = logging.DEBUG\nelse:\n    loglevel = logging.INFO\n\nlogging.basicConfig(stream=sys.stdout, level=loglevel)\nLOG = logging.getLogger('nova_statedir')\n\n\nclass PathManager(object):\n    \"\"\"Helper class to manipulate ownership of a given path\"\"\"\n    def __init__(self, path):\n        self.path = path\n        self.uid = None\n        self.gid = None\n        self.is_dir = None\n        self.secontext = None\n        self._update()\n\n    def _update(self):\n        try:\n            statinfo = os.stat(self.path)\n            self.is_dir = stat.S_ISDIR(statinfo.st_mode)\n            self.uid = statinfo.st_uid\n            self.gid = statinfo.st_gid\n            self.secontext = selinux.lgetfilecon(self.path)[1]\n        except Exception:\n            LOG.exception('Could not update metadata for %s', self.path)\n            raise\n\n    def __str__(self):\n        return \"uid: {} gid: {} path: {}{}\".format(\n            self.uid,\n            self.gid,\n            self.path,\n            '/' if self.is_dir else ''\n        )\n\n    def has_owner(self, uid, gid):\n        return self.uid == uid and self.gid == gid\n\n    def has_either(self, uid, gid):\n        return self.uid == uid or self.gid == gid\n\n    def chown(self, uid, gid):\n        target_uid = -1\n        target_gid = -1\n        if self.uid != uid:\n            target_uid = uid\n        if self.gid != gid:\n            target_gid = gid\n        if (target_uid, target_gid) != (-1, -1):\n            LOG.info('Changing ownership of %s from %d:%d to %d:%d',\n                     self.path,\n                     self.uid,\n                     self.gid,\n                     self.uid if target_uid == -1 else target_uid,\n                     self.gid if target_gid == -1 else target_gid)\n            try:\n                os.chown(self.path, target_uid, target_gid)\n                self._update()\n            except Exception:\n                LOG.exception('Could not change ownership of %s: ',\n                              self.path)\n                raise\n        else:\n            LOG.info('Ownership of %s already %d:%d',\n                     self.path,\n                     uid,\n                     gid)\n\n    def chcon(self, context):\n        # If dir returns whether to recusively set context\n        try:\n            try:\n                selinux.lsetfilecon(self.path, context)\n                LOG.info('Setting selinux context of %s to %s',\n                     self.path, context)\n                return True\n            except OSError as e:\n                if self.is_dir and e.errno == 95:\n                    # Operation not supported, assume NFS mount and skip\n                    LOG.info('Setting selinux context not supported for %s',\n                             self.path)\n                    return False\n                else:\n                    raise\n        except Exception:\n            LOG.exception('Could not set selinux context of %s to %s:',\n                          self.path, context)\n            raise\n\n\nclass NovaStatedirOwnershipManager(object):\n    \"\"\"Class to manipulate the ownership of the nova statedir (/var/lib/nova).\n\n       The nova uid/gid differ on the host and container images. An upgrade\n       that switches from host systemd services to docker requires a change in\n       ownership. Previously this was a naive recursive chown, however this\n       causes issues if nova instance are shared via an NFS mount: any open\n       filehandles in qemu/libvirt fail with an I/O error (LP1778465).\n\n       Instead the upgrade/FFU ansible tasks now lay down a marker file when\n       stopping and disabling the host systemd services. We use this file to\n       determine the host nova uid/gid. We then walk the tree and update any\n       files that have the host uid/gid to the docker nova uid/gid. As files\n       owned by root/qemu etc... are ignored this avoids the issues with open\n       filehandles. The marker is removed once the tree has been walked.\n\n       For subsequent runs, or for a new deployment, we simply ensure that the\n       docker nova user/group owns all directories. This is required as the\n       directories are created with root ownership in host_prep_tasks (the\n       docker nova uid/gid is not known in this context).\n    \"\"\"\n    def __init__(self, statedir, upgrade_marker='upgrade_marker',\n                 nova_user='nova', secontext_marker='../_nova_secontext',\n                 exclude_paths=None):\n        self.statedir = statedir\n        self.nova_user = nova_user\n\n        self.upgrade_marker_path = os.path.join(statedir, upgrade_marker)\n        self.secontext_marker_path = os.path.normpath(os.path.join(statedir, secontext_marker))\n        self.upgrade = os.path.exists(self.upgrade_marker_path)\n\n        self.exclude_paths = [self.upgrade_marker_path]\n        if exclude_paths is not None:\n            for p in exclude_paths:\n                if not p.startswith(os.path.sep):\n                    p = os.path.join(self.statedir, p)\n                self.exclude_paths.append(p)\n\n        self.target_uid, self.target_gid = self._get_nova_ids()\n        self.previous_uid, self.previous_gid = self._get_previous_nova_ids()\n        self.id_change = (self.target_uid, self.target_gid) != \\\n            (self.previous_uid, self.previous_gid)\n        self.target_secontext = self._get_secontext()\n\n    def _get_nova_ids(self):\n        nova_uid, nova_gid = pwd.getpwnam(self.nova_user)[2:4]\n        return nova_uid, nova_gid\n\n    def _get_previous_nova_ids(self):\n        if self.upgrade:\n            statinfo = os.stat(self.upgrade_marker_path)\n            return statinfo.st_uid, statinfo.st_gid\n        else:\n            return self._get_nova_ids()\n\n    def _get_secontext(self):\n        if os.path.exists(self.secontext_marker_path):\n            return selinux.lgetfilecon(self.secontext_marker_path)[1]\n        else:\n            return None\n\n    def _walk(self, top, chcon=True):\n        for f in os.listdir(top):\n            pathname = os.path.join(top, f)\n\n            if pathname in self.exclude_paths:\n                continue\n\n            try:\n                pathinfo = PathManager(pathname)\n                LOG.info(\"Checking %s\", pathinfo)\n                if pathinfo.is_dir:\n                    # Always chown the directories\n                    pathinfo.chown(self.target_uid, self.target_gid)\n                    chcon_r = chcon\n                    if chcon:\n                        chcon_r = pathinfo.chcon(self.target_secontext)\n                    self._walk(pathname, chcon_r)\n                elif self.id_change:\n                    # Only chown files if it's an upgrade and the file is owned by\n                    # the host nova uid/gid\n                    pathinfo.chown(\n                        self.target_uid if pathinfo.uid == self.previous_uid\n                        else pathinfo.uid,\n                        self.target_gid if pathinfo.gid == self.previous_gid\n                        else pathinfo.gid\n                    )\n                    if chcon:\n                        pathinfo.chcon(self.target_secontext)\n            except Exception:\n                # Likely to have been caused by external systems\n                # interacting with this directory tree,\n                # especially on NFS e.g snapshot dirs.\n                # Just ignore it and continue on to the next entry\n                continue\n\n    def run(self):\n        LOG.info('Applying nova statedir ownership')\n        LOG.info('Target ownership for %s: %d:%d',\n                 self.statedir,\n                 self.target_uid,\n                 self.target_gid)\n\n        pathinfo = PathManager(self.statedir)\n        LOG.info(\"Checking %s\", pathinfo)\n        pathinfo.chown(self.target_uid, self.target_gid)\n        chcon = self.target_secontext is not None\n\n        if chcon:\n            pathinfo.chcon(self.target_secontext)\n\n        self._walk(self.statedir, chcon)\n\n        if self.upgrade:\n            LOG.info('Removing upgrade_marker %s',\n                     self.upgrade_marker_path)\n            os.unlink(self.upgrade_marker_path)\n\n        LOG.info('Nova statedir ownership complete')\n\n\ndef get_exclude_paths():\n    exclude_paths = os.environ.get('NOVA_STATEDIR_OWNERSHIP_SKIP')\n    if exclude_paths is not None:\n        exclude_paths = exclude_paths.split(os.pathsep)\n    return exclude_paths\n\n\nif __name__ == '__main__':\n    NovaStatedirOwnershipManager('/var/lib/nova', exclude_paths=get_exclude_paths()).run()\n"}, "nova_wait_for_compute_service.py": {"mode": "0755", "content": "#!/usr/bin/python3\n#\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\nimport argparse\nimport logging\nimport os\nimport six\nimport socket\nimport sys\nimport time\n\n\nfrom keystoneauth1 import loading\nfrom keystoneauth1 import session\n\nfrom novaclient import client\n\n# In python3 SafeConfigParser was renamed to ConfigParser and the default\n# for duplicate options default to true. In case of nova it is valid to\n# have duplicate option lines, e.g. passthrough_whitelist which leads to\n# issues reading the nova.conf\n# https://bugs.launchpad.net/tripleo/+bug/1827775\nif six.PY3:\n    from six.moves.configparser import ConfigParser\n    config = ConfigParser(strict=False)\nelse:\n    from six.moves.configparser import SafeConfigParser\n    config = SafeConfigParser()\n\ndebug = os.getenv('__OS_DEBUG', 'false')\n\nif debug.lower() == 'true':\n    loglevel = logging.DEBUG\nelse:\n    loglevel = logging.INFO\n\nlogging.basicConfig(stream=sys.stdout, level=loglevel)\nLOG = logging.getLogger('nova_wait_for_compute_service')\n\niterations = 60\ntimeout = 10\nnova_cfg = '/etc/nova/nova.conf'\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser(usage='%(prog)s [options]')\n    parser.add_argument('-k', '--insecure',\n                        action=\"store_false\",\n                        dest='insecure',\n                        default=True,\n                        help='Allow insecure connection when using SSL')\n\n    args = parser.parse_args()\n    LOG.debug('Running with parameter insecure = %s',\n              args.insecure)\n\n    if os.path.isfile(nova_cfg):\n        try:\n            config.read(nova_cfg)\n        except Exception:\n            LOG.exception('Error while reading nova.conf:')\n    else:\n        LOG.error('Nova configuration file %s does not exist', nova_cfg)\n        sys.exit(1)\n\n    my_host = config.get('DEFAULT', 'host')\n    if not my_host:\n        # If host isn't set nova defaults to this\n        my_host = socket.gethostname()\n\n    loader = loading.get_plugin_loader('password')\n    auth = loader.load_from_options(\n        auth_url=config.get('neutron',\n                            'auth_url'),\n        username=config.get('neutron',\n                            'username'),\n        password=config.get('neutron',\n                            'password'),\n        project_name=config.get('neutron',\n                                'project_name'),\n        project_domain_name=config.get('neutron',\n                                       'project_domain_name'),\n        user_domain_name=config.get('neutron',\n                                    'user_domain_name'))\n    sess = session.Session(auth=auth, verify=args.insecure)\n    nova = client.Client('2.11', session=sess, endpoint_type='internal',\n                         region_name=config.get('neutron', 'region_name'))\n\n    # Wait until this host is listed in the service list\n    for i in range(iterations):\n        try:\n            service_list = nova.services.list(binary='nova-compute')\n            for entry in service_list:\n                host = getattr(entry, 'host', '')\n                zone = getattr(entry, 'zone', '')\n                if host == my_host and zone != 'internal':\n                    LOG.info('Nova-compute service registered')\n                    sys.exit(0)\n            LOG.info('Waiting for nova-compute service to register')\n        except Exception:\n            LOG.exception(\n                'Error while waiting for nova-compute service to register')\n        time.sleep(timeout)\nsys.exit(1)\n\n# vim: set et ts=4 sw=4 :\n"}}, {"nova_libvirt_init_secret.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -e\n\nCEPH_INFO=($*)\n\nif [ -z \"$CEPH_INFO\" ]; then\n    echo \"error: At least one CLUSTER:CLIENT tuple must be specified\"\n    exit 1\nfi\n\necho \"------------------------------------------------\"\necho \"Initializing virsh secrets for: ${CEPH_INFO[@]}\"\n\nfor INFO in ${CEPH_INFO[@]}; do\n    IFS=: read CLUSTER CLIENT <<< $INFO\n    if [ ! -f /etc/ceph/${CLUSTER}.conf ]; then\n        echo \"Error: /etc/ceph/${CLUSTER}.conf was not found\"\n        echo \"Path to nova_libvirt_init_secret was ${CEPH_INFO}\"\n        exit 1\n    fi\n    FSID=$(awk '$1 == \"fsid\" {print $3}' /etc/ceph/${CLUSTER}.conf)\n    if [ -z \"${FSID}\" ]; then\n        echo \"Error: /etc/ceph/${CLUSTER}.conf contained an empty fsid definition\"\n        echo \"Check your ceph configuration\"\n        exit 1\n    fi\n\n    echo \"--------\"\n    echo \"Initializing the virsh secret for '$CLUSTER' cluster ($FSID) '$CLIENT' client\"\n\n    # Ensure the secret XML file exists. Puppet should have created a secret.xml\n    # file for the first cluster's secret, so detect when to use that file.\n    if grep -q $FSID /etc/nova/secret.xml; then\n        SECRET_FILE=\"/etc/nova/secret.xml\"\n        SECRET_NAME=\"client.${CLIENT} secret\"\n    else\n        SECRET_FILE=\"/etc/nova/${CLUSTER}-secret.xml\"\n        SECRET_NAME=\"${CLUSTER}.client.${CLIENT} secret\"\n    fi\n\n    if [ ! -f $SECRET_FILE ]; then\n        echo \"Creating $SECRET_FILE\"\n        cat <<EOF > $SECRET_FILE\n<secret ephemeral='no' private='no'>\n  <usage type='ceph'>\n    <name>${SECRET_NAME}</name>\n  </usage>\n  <uuid>${FSID}</uuid>\n</secret>\nEOF\n    else\n        echo \"The $SECRET_FILE file already exists\"\n    fi\n\n    # Ensure the libvirt secret is defined\n    if /usr/bin/virsh secret-list | grep -q $FSID; then\n        echo \"The virsh secret for $FSID has already been defined\"\n    else\n        /usr/bin/virsh secret-define --file $SECRET_FILE\n    fi\n\n    # Fetch the key from the keyring and ensure the secret is set\n    KEY=$(awk '$1 == \"key\" {print $3}' /etc/ceph/${CLUSTER}.client.${CLIENT}.keyring)\n    if /usr/bin/virsh secret-get-value $FSID 2>/dev/null | grep -q $KEY; then\n        echo \"The virsh secret for $FSID has already been set\"\n    else\n        /usr/bin/virsh secret-set-value --secret $FSID --base64 $KEY\n    fi\ndone\n"}}, null, null, null, null, {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}}, null, null, {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}}, null, {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "placement_wait_for_service.py": {"mode": "0755", "content": "#!/usr/bin/python3\n#\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\n# shell script to check if placement API is up after X attempts.\n# Default max is 60 iterations with 10s (default) timeout in between.\n\nimport logging\nimport os\nimport re\nimport six\nimport sys\nimport time\n\nfrom keystoneauth1.identity import v3\nfrom keystoneauth1 import session\nfrom keystoneclient.v3 import client\nimport requests\n\n# In python3 SafeConfigParser was renamed to ConfigParser and the default\n# for duplicate options default to true. In case of nova it is valid to\n# have duplicate option lines, e.g. passthrough_whitelist which leads to\n# issues reading the nova.conf\n# https://bugs.launchpad.net/tripleo/+bug/1827775\nif six.PY3:\n    from six.moves.configparser import ConfigParser\n    config = ConfigParser(strict=False)\nelse:\n    from six.moves.configparser import SafeConfigParser\n    config = SafeConfigParser()\n\n\ndebug = os.getenv('__OS_DEBUG', 'false')\n\nif debug.lower() == 'true':\n    loglevel = logging.DEBUG\nelse:\n    loglevel = logging.INFO\n\nlogging.basicConfig(stream=sys.stdout, level=loglevel)\nLOG = logging.getLogger('placement_wait_for_service')\n\niterations = 60\ntimeout = 10\nplacement_cfg = '/etc/placement/placement.conf'\n\nif __name__ == '__main__':\n    if os.path.isfile(placement_cfg):\n        try:\n            config.read(placement_cfg)\n        except Exception:\n            LOG.exception('Error while reading placement.conf:')\n    else:\n        LOG.error('Placement configuration file %s does not exist',\n                  placement_cfg)\n        sys.exit(1)\n\n    # get keystone client with details from [keystone_authtoken] section\n    auth = v3.Password(\n        user_domain_name=config.get('keystone_authtoken', 'user_domain_name'),\n        username=config.get('keystone_authtoken', 'username'),\n        password=config.get('keystone_authtoken', 'password'),\n        project_name=config.get('keystone_authtoken', 'project_name'),\n        project_domain_name=config.get('keystone_authtoken',\n                                       'project_domain_name'),\n        auth_url=config.get('keystone_authtoken', 'auth_url') + '/v3')\n    sess = session.Session(auth=auth, verify=False)\n    keystone = client.Client(session=sess, interface='internal')\n\n    iterations_endpoint = iterations\n    placement_endpoint_url = None\n    while iterations_endpoint > 1:\n        iterations_endpoint -= 1\n        try:\n            # get placement service id\n            placement_service_id = keystone.services.list(\n                name='placement')[0].id\n\n            # get placement endpoint\n            # Note: puppet-placement does not support setting the interface\n            #       until we have https://review.opendev.org/688862.\n            #       Lets hard code 'internal' for now.\n            placement_endpoint_url = keystone.endpoints.list(\n                service=placement_service_id,\n                region=config.get('keystone_authtoken', 'region_name'),\n                interface='internal')[0].url\n            if not placement_endpoint_url:\n                LOG.error('Failed to get placement service endpoint!')\n            else:\n                break\n        except Exception:\n            LOG.exception('Retry - Failed to get placement service endpoint:')\n        time.sleep(timeout)\n\n    if not placement_endpoint_url:\n        LOG.error('Failed to get placement service endpoint!')\n        sys.exit(1)\n\n    # we should have CURRENT in the request response from placement:\n    # {\"versions\": [{\"status\": \"CURRENT\", \"min_version\": \"1.0\", \"max_version\":\n    # \"1.29\", \"id\": \"v1.0\", \"links\": [{\"href\": \"\", \"rel\": \"self\"}]}]}\n    response_reg = re.compile('.*CURRENT,*')\n\n    while iterations > 1:\n        iterations -= 1\n        try:\n            r = requests.get(placement_endpoint_url + '/', verify=False)\n            if r.status_code == 200 and response_reg.match(r.text):\n                LOG.info('Placement service up! - %s', r.text)\n                sys.exit(0)\n                break\n            else:\n                LOG.info('response - %r', r)\n                LOG.info('Placement service not up - %s, %s',\n                         r.status_code,\n                         r.text)\n        except Exception:\n            LOG.exception('Error query the placement endpoint:')\n        time.sleep(timeout)\n\n    sys.exit(1)\n\n# vim: set et ts=4 sw=4 :\n"}}, null, null, null, null, {"create_swift_secret.sh": {"mode": "0700", "content": "#!/bin/bash\nexport OS_PROJECT_DOMAIN_ID=$(crudini --get /etc/swift/keymaster.conf kms_keymaster project_domain_id)\nexport OS_USER_DOMAIN_ID=$(crudini --get /etc/swift/keymaster.conf kms_keymaster user_domain_id)\nexport OS_PROJECT_NAME=$(crudini --get /etc/swift/keymaster.conf kms_keymaster project_name)\nexport OS_USERNAME=$(crudini --get /etc/swift/keymaster.conf kms_keymaster username)\nexport OS_PASSWORD=$(crudini --get /etc/swift/keymaster.conf kms_keymaster password)\nexport OS_AUTH_URL=$(crudini --get /etc/swift/keymaster.conf kms_keymaster auth_endpoint)\nexport OS_AUTH_TYPE=password\nexport OS_IDENTITY_API_VERSION=3\n\necho \"Check if secret already exists\"\nsecret_href=$(openstack secret list --name swift_root_secret_uuid)\nrc=$?\nif [[ $rc != 0 ]]; then\n  echo \"Failed to check secrets, check if Barbican in enabled and responding properly\"\n  exit $rc;\nfi\nif [ -z \"$secret_href\" ]; then\n  echo \"Create new secret\"\n  order_href=$(openstack secret order create --name swift_root_secret_uuid --payload-content-type=\"application/octet-stream\" --algorithm aes --bit-length 256 --mode ctr key -f value -c \"Order href\")\nfi\n"}, "set_swift_keymaster_key_id.sh": {"mode": "0700", "content": "#!/bin/bash\nexport OS_PROJECT_DOMAIN_ID=$(crudini --get /etc/swift/keymaster.conf kms_keymaster project_domain_id)\nexport OS_USER_DOMAIN_ID=$(crudini --get /etc/swift/keymaster.conf kms_keymaster user_domain_id)\nexport OS_PROJECT_NAME=$(crudini --get /etc/swift/keymaster.conf kms_keymaster project_name)\nexport OS_USERNAME=$(crudini --get /etc/swift/keymaster.conf kms_keymaster username)\nexport OS_PASSWORD=$(crudini --get /etc/swift/keymaster.conf kms_keymaster password)\nexport OS_AUTH_URL=$(crudini --get /etc/swift/keymaster.conf kms_keymaster auth_endpoint)\nexport OS_AUTH_TYPE=password\nexport OS_IDENTITY_API_VERSION=3\necho \"retrieve key_id\"\nloop_wait=2\nfor i in {0..5}; do\n  #TODO update uuid from mistral here too\n  secret_href=$(openstack secret list --name swift_root_secret_uuid)\n  if [ \"$secret_href\" ]; then\n    echo \"set key_id in keymaster.conf\"\n    secret_href=$(openstack secret list --name swift_root_secret_uuid -f value -c \"Secret href\")\n    crudini --set /etc/swift/keymaster.conf kms_keymaster key_id ${secret_href##*/}\n    exit 0\n  else\n    echo \"no key, wait for $loop_wait and check again\"\n    sleep $loop_wait\n    ((loop_wait++))\n  fi\ndone\necho \"Failed to set secret in keymaster.conf, check if Barbican is enabled and responding properly\"\nexit 1\n"}}, null, null, null, null, null, null, null, null], "role_data": [{"service_name": "boot_params_service", "deploy_steps_tasks": [{"name": "Configure tuned before reboot", "when": "step|int == 0", "include_role": {"name": "tuned"}, "vars": {"tuned_profile": "throughput-performance", "tuned_isolated_cores": ""}}, {"name": "Configure kernel args and reboot", "when": "step|int == 0", "include_role": {"name": "tripleo_kernel", "tasks_from": "kernelargs.yml"}, "vars": {"tripleo_kernel_args": "", "tripleo_kernel_hugepages": {}, "tripleo_kernel_hugepages_remove": false, "tripleo_kernel_defer_reboot": false, "tripleo_kernel_reboot_timeout": 900}}], "upgrade_tasks": [{"name": "upgrade prepare for leapp to align kernel arg shortcommings in leapp", "tags": ["never", "system_upgrade", "system_upgrade_prepare"], "when": ["step|int == 3", "upgrade_leapp_enabled"], "block": [{"name": "fix grub entries to have name start with GRUB_", "replace": {"path": "/etc/default/grub", "regexp": "^(TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS)(.*)", "replace": "GRUB_\\1\\2"}}, {"name": "fix grub entries in append statement", "replace": {"path": "/etc/default/grub", "regexp": "(.*){(TRIPLEO_HEAT_TEMPLATE_KERNEL_ARGS)}(.*)", "replace": "\\1{GRUB_\\2}\\3"}}]}]}, {"service_name": "barbican_api", "firewall_rules": {"117 barbican": {"dport": [9311]}}, "firewall_frontend_rules": {"100 barbican_haproxy_frontend": {"dport": [9311]}}, "firewall_ssl_frontend_rules": {"100 barbican_haproxy_frontend_ssl": {"dport": [13311]}}, "keystone_resources": {"barbican": {"endpoints": {"public": "http://172.21.0.2:9311", "internal": "http://172.17.0.2:9311", "admin": "http://172.17.0.2:9311"}, "users": {"barbican": {"password": "G5XhguvbNw19nfLNKira8K2xr"}}, "region": "regionOne", "service": "key-manager", "roles": ["key-manager:service-admin", "creator", "observer", "audit"]}}, "config_settings": {"apache::ip": "%{hiera('internal_api')}", "apache::default_vhost": false, "apache::trace_enable": "Off", "apache::server_signature": "Off", "apache::server_tokens": "Prod", "apache::timeout": 90, "apache::default_mods": false, "apache::mod::prefork::maxrequestworkers": 100, "apache::mod::prefork::serverlimit": 100, "apache::mod::remoteip::proxy_ips": ["172.17.0.0/24"], "apache::mod::alias::icons_options": "None", "barbican::keystone::authtoken::password": "G5XhguvbNw19nfLNKira8K2xr", "barbican::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "barbican::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "barbican::keystone::authtoken::project_name": "service", "barbican::keystone::authtoken::region_name": "regionOne", "barbican::keystone::authtoken::interface": "internal", "barbican::keystone::authtoken::memcache_use_advanced_pool": true, "barbican::keystone::notification::enable_keystone_notification": true, "barbican::keystone::notification::keystone_notification_topic": "barbican_notifications", "barbican::policy::policies": {}, "barbican::api::host_href": "http://172.21.0.2:9311", "barbican::api::db_auto_create": false, "barbican::api::enabled_certificate_plugins": ["simple_certificate"], "barbican::api::enable_queue": true, "barbican::api::logging::debug": true, "barbican::api::notification_driver": ["noop"], "barbican::api::rabbit_heartbeat_in_pthread": false, "barbican::api::service_name": "httpd", "barbican::wsgi::apache::bind_host": "%{hiera('internal_api')}", "barbican::wsgi::apache::ssl": false, "barbican::wsgi::apache::workers": "%{::processorcount}", "barbican::wsgi::apache::servername": "%{hiera('fqdn_internal_api')}", "barbican::db::database_connection": "mysql+pymysql://barbican:G5XhguvbNw19nfLNKira8K2xr@172.17.0.2/barbican?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo"}, "service_config_settings": {"nova_compute": {"nova::compute::keymgr_backend": "barbican", "nova::compute::barbican_endpoint": "http://172.17.0.2:9311", "nova::compute::barbican_auth_endpoint": "http://172.17.0.2:5000"}, "cinder_api": {"cinder::keymgr_backend": "barbican", "cinder::keymgr_encryption_api_url": "http://172.17.0.2:9311", "cinder::keymgr_encryption_auth_url": "http://172.17.0.2:5000"}, "cinder_volume": {"cinder::keymgr_backend": "barbican", "cinder::keymgr_encryption_api_url": "http://172.17.0.2:9311", "cinder::keymgr_encryption_auth_url": "http://172.17.0.2:5000"}, "cinder_backup": {"cinder::keymgr_backend": "barbican", "cinder::keymgr_encryption_api_url": "http://172.17.0.2:9311", "cinder::keymgr_encryption_auth_url": "http://172.17.0.2:5000"}, "glance_api": {"glance::api::keymgr_backend": "barbican", "glance::api::keymgr_encryption_api_url": "http://172.17.0.2:9311", "glance::api::keymgr_encryption_auth_url": "http://172.17.0.2:5000"}, "glance_api_edge": {"glance::api::keymgr_backend": "barbican", "glance::api::keymgr_encryption_api_url": "http://172.17.0.2:9311", "glance::api::keymgr_encryption_auth_url": "http://172.17.0.2:5000"}, "mysql": {"barbican::db::mysql::password": "G5XhguvbNw19nfLNKira8K2xr", "barbican::db::mysql::user": "barbican", "barbican::db::mysql::host": "%", "barbican::db::mysql::dbname": "barbican"}, "keystone": {"tripleo::profile::base::keystone::barbican_notification_topics": ["barbican_notifications"]}}, "puppet_config": {"config_volume": "barbican", "puppet_tags": "barbican_api_paste_ini,barbican_config", "step_config": "include tripleo::profile::base::barbican::api\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-barbican-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/barbican_api.json": {"command": "/usr/sbin/httpd -DFOREGROUND", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/barbican_keystone_listener.json": {"command": "/usr/bin/barbican-keystone-listener", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/barbican_worker.json": {"command": "/usr/bin/barbican-worker", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/barbican_api_db_sync.json": {"command": "/usr/bin/bootstrap_host_exec barbican_api su barbican -s /bin/bash -c '/usr/bin/barbican-manage  db upgrade '", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/barbican_api_create_mkek.json": {"command": "/usr/bin/bootstrap_host_exec barbican_api su barbican -s /bin/bash -c '/usr/bin/barbican-manage  hsm check_mkek --label  || /usr/bin/barbican-manage  hsm gen_mkek --label  '", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/barbican_api_create_hmac.json": {"command": "/usr/bin/bootstrap_host_exec barbican_api su barbican -s /bin/bash -c '/usr/bin/barbican-manage  hsm check_hmac --label  || /usr/bin/barbican-manage hsm gen_hmac --label  '", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/barbican_api_update_rfs_server.json": {"command": "/usr/bin/bootstrap_host_exec barbican_api /opt/nfast/bin/rfs-sync --commit", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/barbican_api_get_from_rfs.json": {"command": "/opt/nfast/bin/rfs-sync --update", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/barbican_api_secret_store_sync.json": {"command": "/usr/bin/bootstrap_host_exec barbican_api su barbican -s /bin/bash -c '/usr/bin/barbican-manage  db sync_secret_stores --verbose '", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/barbican_api_rewrap_pkeks.json": {"command": "/usr/bin/bootstrap_host_exec barbican_api su barbican -s /bin/bash -c '/usr/bin/barbican-manage  hsm rewrap_pkek '", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}}, "deploy_steps_tasks": [{"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"name": "Create dirs for certificates and keys", "file": {"path": "{{ item }}", "state": "directory", "serole": "object_r", "setype": "cert_t", "seuser": "system_u"}, "with_items": ["/etc/pki/tls/certs/httpd", "/etc/pki/tls/private/httpd"]}, {"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "httpd-ctlplane", "dns": "{{ fqdn_ctlplane }}", "principal": "HTTP/{{ fqdn_ctlplane }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-ctlplane.crt /etc/pki/tls/certs/httpd/httpd-ctlplane.crt\ncp /etc/pki/tls/private/httpd-ctlplane.key /etc/pki/tls/private/httpd/httpd-ctlplane.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage", "dns": "{{ fqdn_storage }}", "principal": "HTTP/{{ fqdn_storage }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage.crt /etc/pki/tls/certs/httpd/httpd-storage.crt\ncp /etc/pki/tls/private/httpd-storage.key /etc/pki/tls/private/httpd/httpd-storage.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage_mgmt", "dns": "{{ fqdn_storage_mgmt }}", "principal": "HTTP/{{ fqdn_storage_mgmt }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage_mgmt.crt /etc/pki/tls/certs/httpd/httpd-storage_mgmt.crt\ncp /etc/pki/tls/private/httpd-storage_mgmt.key /etc/pki/tls/private/httpd/httpd-storage_mgmt.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-internal_api", "dns": "{{ fqdn_internal_api }}", "principal": "HTTP/{{ fqdn_internal_api }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-internal_api.crt /etc/pki/tls/certs/httpd/httpd-internal_api.crt\ncp /etc/pki/tls/private/httpd-internal_api.key /etc/pki/tls/private/httpd/httpd-internal_api.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-external", "dns": "{{ fqdn_external }}", "principal": "HTTP/{{ fqdn_external }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-external.crt /etc/pki/tls/certs/httpd/httpd-external.crt\ncp /etc/pki/tls/private/httpd-external.key /etc/pki/tls/private/httpd/httpd-external.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}]}}]}], "docker_config": {"step_2": {"barbican_init_log": {"image": "registry.redhat.io/rhosp-rhel9/openstack-barbican-api:17.1", "net": "none", "user": "root", "volumes": ["/var/log/containers/barbican:/var/log/barbican:z", "/var/log/containers/httpd/barbican-api:/var/log/httpd:z"], "command": ["/bin/bash", "-c", "chown -R barbican:barbican /var/log/barbican"]}}, "step_3": {"barbican_api_db_sync": {"start_order": 3, "image": "registry.redhat.io/rhosp-rhel9/openstack-barbican-api:17.1", "cap_add": ["AUDIT_WRITE"], "net": "host", "detach": false, "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/barbican:/var/log/barbican:z", "/var/log/containers/httpd/barbican-api:/var/log/httpd:z", "/var/lib/config-data/puppet-generated/barbican:/var/lib/kolla/config_files/src:ro", "/var/lib/kolla/config_files/barbican_api_db_sync.json:/var/lib/kolla/config_files/config.json:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "barbican_api_secret_store_sync": {"start_order": 4, "image": "registry.redhat.io/rhosp-rhel9/openstack-barbican-api:17.1", "net": "host", "detach": false, "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/barbican:/var/log/barbican:z", "/var/log/containers/httpd/barbican-api:/var/log/httpd:z", "/var/lib/config-data/puppet-generated/barbican:/var/lib/kolla/config_files/src:ro", "/var/lib/kolla/config_files/barbican_api_secret_store_sync.json:/var/lib/kolla/config_files/config.json:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "barbican_api": {"start_order": 5, "image": "registry.redhat.io/rhosp-rhel9/openstack-barbican-api:17.1", "net": "host", "privileged": false, "restart": "always", "user": "root", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/barbican:/var/log/barbican:z", "/var/log/containers/httpd/barbican-api:/var/log/httpd:z", "/var/lib/kolla/config_files/barbican_api.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/barbican:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "barbican_keystone_listener": {"start_order": 6, "image": "registry.redhat.io/rhosp-rhel9/openstack-barbican-keystone-listener:17.1", "net": "host", "privileged": false, "restart": "always", "user": "barbican", "healthcheck": {"test": "/openstack/healthcheck 5672"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/barbican:/var/log/barbican:z", "/var/log/containers/httpd/barbican-api:/var/log/httpd:z", "/var/lib/kolla/config_files/barbican_keystone_listener.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/barbican:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "barbican_worker": {"start_order": 7, "image": "registry.redhat.io/rhosp-rhel9/openstack-barbican-worker:17.1", "net": "host", "privileged": false, "restart": "always", "user": "barbican", "healthcheck": {"test": "/openstack/healthcheck 5672"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/barbican:/var/log/barbican:z", "/var/log/containers/httpd/barbican-api:/var/log/httpd:z", "/var/lib/kolla/config_files/barbican_worker.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/barbican:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/barbican", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/log/containers/httpd/barbican-api", "setype": "container_file_t", "mode": "0750"}]}, {"name": "enable virt_sandbox_use_netlink for healthcheck", "seboolean": {"name": "virt_sandbox_use_netlink", "persistent": true, "state": true}, "when": ["ansible_facts.selinux is defined", "ansible_facts.selinux.status == \"enabled\""]}], "metadata_settings": null, "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop barbican api container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["barbican_api"], "tripleo_delegate_to": "{{ groups['barbican_api'] | difference(groups['excluded_overcloud']) }}"}}]}]}, {"service_name": "barbican_backend_simple_crypto", "config_settings": {"barbican::plugins::simple_crypto::simple_crypto_plugin_kek": "OYlzhn5Z789Pm8uX1xw6rb9SXqoG3q3o3GIkxbUm8OU=", "barbican::plugins::simple_crypto::global_default": true}}, {"service_name": "ca_certs", "config_settings": {"tripleo::trusted_cas::ca_map": {}}, "step_config": "include tripleo::trusted_cas"}, {"service_name": "ceph_client", "upgrade_tasks": [], "puppet_config": {}, "docker_config": {}, "external_deploy_tasks": [{"name": "ceph_base_external_deploy_task", "when": "step|int == 2", "block": [{"name": "create cephadm working directory and related files", "include_role": {"name": "tripleo_run_cephadm", "tasks_from": "prepare.yml"}, "vars": {"ceph_pools": {"gnocchi_pool": {"name": "metrics", "enabled": false}, "nova_pool": {"name": "vms", "enabled": true}, "glance_pool": {"name": "images", "enabled": {"or": [false, {"equals": ["rbd", "rbd"]}]}}, "cinder_pool": {"name": "volumes", "enabled": true, "cinder_extra_pools": []}, "cinder_backup_pool": {"name": "backups", "enabled": true}, "extra_pools": []}, "manila_pools": {"data": "manila_data", "metadata": "manila_metadata"}, "ceph_keys": {"openstack_client": {"name": "openstack", "key": "AQBP7ytpAAAAABAAXQLQTQkMxLJ7uGigDaf94Q=="}, "manila": {"name": "manila", "key": "AQBP7ytpAAAAABAAtMMHM73CN0teAlvUPYPESw=="}, "radosgw": {"name": "radosgw", "key": "AQBP7ytpAAAAABAA6+E+za7v5zDAbqU5G6Itow=="}, "extra_keys": []}, "ceph_config_overrides": {}, "tripleo_run_cephadm_spec_path": "{{ playbook_dir }}/cephadm/ceph_spec.yaml", "tripleo_cephadm_dynamic_spec": false, "ceph_spec_fqdn": true, "ceph_osd_spec": {"data_devices": {"all": true}}, "ceph_default_overrides": {"global": {}}, "cephadm_extra_vars": {"tripleo_cephadm_fsid": "55dae5e5-7ec8-55ff-b761-b30fedcc71d6", "tripleo_cephadm_cluster": "ceph", "tripleo_cephadm_container_cli": "podman", "tripleo_ceph_client_vars": "/root/overcloud-deploy/overcloud/ceph_client.yml", "tripleo_cephadm_dashboard_enabled": false, "cephfs": "cephfs", "tripleo_cephadm_container_ns": "registry.redhat.io", "tripleo_cephadm_container_image": "rhceph/rhceph-7-rhel9", "tripleo_cephadm_container_tag": "latest", "tripleo_cephadm_crush_rules": [], "tripleo_cephadm_debug": false, "tripleo_cephadm_verbose": false, "tripleo_cephadm_rbd_trash": "15", "tripleo_cephadm_enable_trash_scheduler": false, "tripleo_cephadm_apply_ceph_conf_overrides_on_update": true, "tripleo_cephadm_default_container": false, "ceph_container_registry_auth": false, "ceph_container_registry_username": "", "ceph_container_registry_password": "", "public_network": "172.18.0.0/24", "cluster_network": "172.20.0.0/24"}, "tripleo_cephadm_deployed_ceph": true}}, {"name": "Prepare cephadm user and keys", "include_role": {"name": "tripleo_run_cephadm", "tasks_from": "enable_ceph_admin_user.yml"}, "when": ["not external_ceph", "(not deployed_ceph and (groups['ceph_mon'] | default([]) | length > 0 or groups['ceph_nfs'] | default([]) | length > 0)) or (deployed_ceph and ((groups['ceph_rgw'] | default([]) != groups['ceph_mon'] | default([]) and groups['ceph_rgw'] | default([]) | length > 0) or (groups['ceph_mds'] | default([]) != groups['ceph_mon'] | default([]) and groups['ceph_mds'] | default([]) | length > 0) or (groups['ceph_nfs'] | default([]) != groups['ceph_mon'] | default([]) and groups['ceph_nfs'] | default([]) | length > 0) or (groups['ceph_rbdmirror'] | default([]) != groups['ceph_mon'] | default([]) and groups['ceph_rbdmirror'] | default([]) | length > 0)))"], "vars": {"deployed_ceph": true, "external_ceph": false}}, {"name": "Deploy or configure the cephadm Ceph cluster", "include_role": {"name": "tripleo_run_cephadm"}, "when": ["not external_ceph", "groups['ceph_mon'] | default([]) | length > 0 or groups['ceph_nfs'] | default([]) | length > 0"], "vars": {"external_ceph": false}}]}, {"name": "ceph_base_external_deploy_task", "when": ["(step | int) == 3", false, "not external_ceph"], "block": [{"name": "Pause cephadm", "include_role": {"name": "tripleo_run_cephadm", "tasks_from": "disable_cephadm.yml"}}], "vars": {"external_ceph": false}}, {"name": "ceph_base_external_deploy_init", "when": "step|int == 1", "block": [{"name": "Set some tripleo-ansible facts", "set_fact": {"ceph_external_multi_config": []}}]}, {"name": "Configure Ceph Clients", "when": "step|int == 2", "block": [{"name": "Check if the input variable file exists", "stat": {"path": "{{ tripleo_ceph_client_vars }}"}, "register": "ceph_input_vars", "vars": {"tripleo_ceph_client_vars": "/root/overcloud-deploy/overcloud/ceph_client.yml"}}, {"name": "Populate CephClientConfigVars from THT", "copy": {"dest": "{{ tripleo_ceph_client_vars }}", "content": "tripleo_ceph_client_fsid: \"{{ tripleo_ceph_client_fsid }}\"\nexternal_cluster_mon_ips: \"{{ external_cluster_mon_ips }}\"\n"}, "when": ["ceph_input_vars.stat.exists == False", "external_cluster_mon_ips | length > 0"], "vars": {"tripleo_ceph_client_vars": "/root/overcloud-deploy/overcloud/ceph_client.yml", "tripleo_ceph_client_fsid": "55dae5e5-7ec8-55ff-b761-b30fedcc71d6", "external_cluster_mon_ips": ""}}, {"name": "configure ceph clients", "include_role": {"name": "tripleo_ceph_client"}, "vars": {"tripleo_ceph_client_config_home": "/var/lib/tripleo-config/ceph", "tripleo_ceph_client_vars": "/root/overcloud-deploy/overcloud/ceph_client.yml"}}, {"include_role": {"name": "tripleo_ceph_client"}, "name": "tripleo client role", "vars": {"tripleo_ceph_client_config_home": "/var/lib/tripleo-config/ceph", "multiple": "{{ item }}"}, "loop": "{{ ceph_external_multi_config }}", "when": ["ceph_external_multi_config is defined"]}, {"name": "Load variables for standalone ganesha", "include_vars": "{{ playbook_dir }}/cephadm/cephadm-extra-vars-heat.yml", "when": ["groups['ceph_mon'] | default([]) | length == 0", "groups['ceph_nfs'] | default([]) | length > 0"]}, {"name": "Config Standalone Ganesha", "include_role": {"name": "tripleo_cephadm", "tasks_from": "nfs"}, "when": ["groups['ceph_mon'] | default([]) | length == 0", "groups['ceph_nfs'] | default([]) | length > 0"], "vars": {"tripleo_cephadm_config_home": "/var/lib/tripleo-config/ceph"}}]}], "external_update_tasks": [], "external_upgrade_tasks": [], "config_settings": {}}, {"service_name": "ceph_mds", "firewall_rules": {"112 ceph_mds": {"dport": ["6800-7300"]}}, "puppet_config": {}, "docker_config": {}, "external_deploy_tasks": [], "external_update_tasks": [], "external_upgrade_tasks": []}, {"service_name": "ceph_mgr", "firewall_rules": {"113 ceph_mgr": {"dport": ["6800-7300"]}}, "upgrade_tasks": [], "puppet_config": {}, "docker_config": {}, "external_deploy_tasks": [{"name": "ceph_base_external_deploy_task", "when": "step|int == 2", "block": [{"name": "create cephadm working directory and related files", "include_role": {"name": "tripleo_run_cephadm", "tasks_from": "prepare.yml"}, "vars": {"ceph_pools": {"gnocchi_pool": {"name": "metrics", "enabled": false}, "nova_pool": {"name": "vms", "enabled": true}, "glance_pool": {"name": "images", "enabled": {"or": [false, {"equals": ["rbd", "rbd"]}]}}, "cinder_pool": {"name": "volumes", "enabled": true, "cinder_extra_pools": []}, "cinder_backup_pool": {"name": "backups", "enabled": true}, "extra_pools": []}, "manila_pools": {"data": "manila_data", "metadata": "manila_metadata"}, "ceph_keys": {"openstack_client": {"name": "openstack", "key": "AQBP7ytpAAAAABAAXQLQTQkMxLJ7uGigDaf94Q=="}, "manila": {"name": "manila", "key": "AQBP7ytpAAAAABAAtMMHM73CN0teAlvUPYPESw=="}, "radosgw": {"name": "radosgw", "key": "AQBP7ytpAAAAABAA6+E+za7v5zDAbqU5G6Itow=="}, "extra_keys": []}, "ceph_config_overrides": {}, "tripleo_run_cephadm_spec_path": "{{ playbook_dir }}/cephadm/ceph_spec.yaml", "tripleo_cephadm_dynamic_spec": false, "ceph_spec_fqdn": true, "ceph_osd_spec": {"data_devices": {"all": true}}, "ceph_default_overrides": {"global": {}}, "cephadm_extra_vars": {"tripleo_cephadm_fsid": "55dae5e5-7ec8-55ff-b761-b30fedcc71d6", "tripleo_cephadm_cluster": "ceph", "tripleo_cephadm_container_cli": "podman", "tripleo_ceph_client_vars": "/root/overcloud-deploy/overcloud/ceph_client.yml", "tripleo_cephadm_dashboard_enabled": false, "cephfs": "cephfs", "tripleo_cephadm_container_ns": "registry.redhat.io", "tripleo_cephadm_container_image": "rhceph/rhceph-7-rhel9", "tripleo_cephadm_container_tag": "latest", "tripleo_cephadm_crush_rules": [], "tripleo_cephadm_debug": false, "tripleo_cephadm_verbose": false, "tripleo_cephadm_rbd_trash": "15", "tripleo_cephadm_enable_trash_scheduler": false, "tripleo_cephadm_apply_ceph_conf_overrides_on_update": true, "tripleo_cephadm_default_container": false, "ceph_container_registry_auth": false, "ceph_container_registry_username": "", "ceph_container_registry_password": "", "public_network": "172.18.0.0/24", "cluster_network": "172.20.0.0/24"}, "tripleo_cephadm_deployed_ceph": true}}, {"name": "Prepare cephadm user and keys", "include_role": {"name": "tripleo_run_cephadm", "tasks_from": "enable_ceph_admin_user.yml"}, "when": ["not external_ceph", "(not deployed_ceph and (groups['ceph_mon'] | default([]) | length > 0 or groups['ceph_nfs'] | default([]) | length > 0)) or (deployed_ceph and ((groups['ceph_rgw'] | default([]) != groups['ceph_mon'] | default([]) and groups['ceph_rgw'] | default([]) | length > 0) or (groups['ceph_mds'] | default([]) != groups['ceph_mon'] | default([]) and groups['ceph_mds'] | default([]) | length > 0) or (groups['ceph_nfs'] | default([]) != groups['ceph_mon'] | default([]) and groups['ceph_nfs'] | default([]) | length > 0) or (groups['ceph_rbdmirror'] | default([]) != groups['ceph_mon'] | default([]) and groups['ceph_rbdmirror'] | default([]) | length > 0)))"], "vars": {"deployed_ceph": true, "external_ceph": false}}, {"name": "Deploy or configure the cephadm Ceph cluster", "include_role": {"name": "tripleo_run_cephadm"}, "when": ["not external_ceph", "groups['ceph_mon'] | default([]) | length > 0 or groups['ceph_nfs'] | default([]) | length > 0"], "vars": {"external_ceph": false}}]}, {"name": "ceph_base_external_deploy_task", "when": ["(step | int) == 3", false, "not external_ceph"], "block": [{"name": "Pause cephadm", "include_role": {"name": "tripleo_run_cephadm", "tasks_from": "disable_cephadm.yml"}}], "vars": {"external_ceph": false}}]}, {"service_name": "ceph_mon", "firewall_rules": {"110 ceph_mon": {"dport": [6789, 3300]}}, "service_config_settings": {"collectd": {"tripleo.collectd.plugins.ceph_mon": ["ceph"], "collectd::plugin::ceph::daemons": []}}, "upgrade_tasks": [], "puppet_config": {}, "docker_config": {}, "external_deploy_tasks": []}, {"service_name": "ceph_osd", "firewall_rules": {"111 ceph_osd": {"dport": ["6800-7300"]}}, "service_config_settings": {"collectd": {"tripleo.collectd.plugins.ceph_osd": ["ceph"], "collectd::plugin::ceph::daemons": []}}, "upgrade_tasks": [{"name": "Set noout flag", "shell": "cephadm shell -c /etc/ceph/{{ tripleo_cephadm_cluster }}.conf -k /etc/ceph/{{ tripleo_cephadm_cluster }}.client.admin.keyring -- ceph osd set {{ item }}", "become": true, "with_items": ["noout", "norecover", "nobackfill", "norebalance", "nodeep-scrub"], "delegate_to": "{{ groups['ceph_mon'][0] }}", "tags": ["never", "system_upgrade", "system_upgrade_prepare"], "vars": {"tripleo_cephadm_cluster": "ceph"}, "when": ["step|int == 1", "upgrade_leapp_enabled"]}, {"name": "Unset noout flag", "shell": "cephadm shell -c /etc/ceph/{{ tripleo_cephadm_cluster }}.conf -k /etc/ceph/{{ tripleo_cephadm_cluster }}.client.admin.keyring -- ceph osd unset {{ item }}", "with_items": ["noout", "norecover", "nobackfill", "norebalance", "nodeep-scrub"], "delegate_to": "{{ groups['ceph_mon'][0] }}", "tags": ["never", "system_upgrade", "system_upgrade_prepare"], "vars": {"tripleo_cephadm_cluster": "ceph"}, "when": ["step|int == 5"]}], "puppet_config": {}, "docker_config": {}, "external_deploy_tasks": [{"name": "ceph_base_external_deploy_task", "when": "step|int == 2", "block": [{"name": "create cephadm working directory and related files", "include_role": {"name": "tripleo_run_cephadm", "tasks_from": "prepare.yml"}, "vars": {"ceph_pools": {"gnocchi_pool": {"name": "metrics", "enabled": false}, "nova_pool": {"name": "vms", "enabled": true}, "glance_pool": {"name": "images", "enabled": {"or": [false, {"equals": ["rbd", "rbd"]}]}}, "cinder_pool": {"name": "volumes", "enabled": true, "cinder_extra_pools": []}, "cinder_backup_pool": {"name": "backups", "enabled": true}, "extra_pools": []}, "manila_pools": {"data": "manila_data", "metadata": "manila_metadata"}, "ceph_keys": {"openstack_client": {"name": "openstack", "key": "AQBP7ytpAAAAABAAXQLQTQkMxLJ7uGigDaf94Q=="}, "manila": {"name": "manila", "key": "AQBP7ytpAAAAABAAtMMHM73CN0teAlvUPYPESw=="}, "radosgw": {"name": "radosgw", "key": "AQBP7ytpAAAAABAA6+E+za7v5zDAbqU5G6Itow=="}, "extra_keys": []}, "ceph_config_overrides": {}, "tripleo_run_cephadm_spec_path": "{{ playbook_dir }}/cephadm/ceph_spec.yaml", "tripleo_cephadm_dynamic_spec": false, "ceph_spec_fqdn": true, "ceph_osd_spec": {"data_devices": {"all": true}}, "ceph_default_overrides": {"global": {}}, "cephadm_extra_vars": {"tripleo_cephadm_fsid": "55dae5e5-7ec8-55ff-b761-b30fedcc71d6", "tripleo_cephadm_cluster": "ceph", "tripleo_cephadm_container_cli": "podman", "tripleo_ceph_client_vars": "/root/overcloud-deploy/overcloud/ceph_client.yml", "tripleo_cephadm_dashboard_enabled": false, "cephfs": "cephfs", "tripleo_cephadm_container_ns": "registry.redhat.io", "tripleo_cephadm_container_image": "rhceph/rhceph-7-rhel9", "tripleo_cephadm_container_tag": "latest", "tripleo_cephadm_crush_rules": [], "tripleo_cephadm_debug": false, "tripleo_cephadm_verbose": false, "tripleo_cephadm_rbd_trash": "15", "tripleo_cephadm_enable_trash_scheduler": false, "tripleo_cephadm_apply_ceph_conf_overrides_on_update": true, "tripleo_cephadm_default_container": false, "ceph_container_registry_auth": false, "ceph_container_registry_username": "", "ceph_container_registry_password": "", "public_network": "172.18.0.0/24", "cluster_network": "172.20.0.0/24"}, "tripleo_cephadm_deployed_ceph": true}}, {"name": "Prepare cephadm user and keys", "include_role": {"name": "tripleo_run_cephadm", "tasks_from": "enable_ceph_admin_user.yml"}, "when": ["not external_ceph", "(not deployed_ceph and (groups['ceph_mon'] | default([]) | length > 0 or groups['ceph_nfs'] | default([]) | length > 0)) or (deployed_ceph and ((groups['ceph_rgw'] | default([]) != groups['ceph_mon'] | default([]) and groups['ceph_rgw'] | default([]) | length > 0) or (groups['ceph_mds'] | default([]) != groups['ceph_mon'] | default([]) and groups['ceph_mds'] | default([]) | length > 0) or (groups['ceph_nfs'] | default([]) != groups['ceph_mon'] | default([]) and groups['ceph_nfs'] | default([]) | length > 0) or (groups['ceph_rbdmirror'] | default([]) != groups['ceph_mon'] | default([]) and groups['ceph_rbdmirror'] | default([]) | length > 0)))"], "vars": {"deployed_ceph": true, "external_ceph": false}}, {"name": "Deploy or configure the cephadm Ceph cluster", "include_role": {"name": "tripleo_run_cephadm"}, "when": ["not external_ceph", "groups['ceph_mon'] | default([]) | length > 0 or groups['ceph_nfs'] | default([]) | length > 0"], "vars": {"external_ceph": false}}]}, {"name": "ceph_base_external_deploy_task", "when": ["(step | int) == 3", false, "not external_ceph"], "block": [{"name": "Pause cephadm", "include_role": {"name": "tripleo_run_cephadm", "tasks_from": "disable_cephadm.yml"}}], "vars": {"external_ceph": false}}, {"name": "ceph_osd_external_deploy_init", "when": "step|int == 1", "block": [{"name": "Build disk list for cephadm", "set_fact": {"cephadm_disk_list": {"devices": [], "osd_scenario": "lvm", "osd_objectstore": "bluestore"}}}]}]}, {"service_name": "cinder_api", "firewall_rules": {"119 cinder": {"dport": [8776]}}, "firewall_frontend_rules": {"100 cinder_haproxy_frontend": {"dport": [8776]}}, "firewall_ssl_frontend_rules": {"100 cinder_haproxy_frontend_ssl": {"dport": [13776]}}, "keystone_resources": {"cinder": {"users": {"cinder": {"password": "Olf2iTUMR6eJ33irnMaVcSJQX", "roles": ["admin", "service"]}}}, "cinderv3": {"endpoints": {"public": "http://172.21.0.2:8776/v3/%(tenant_id)s", "internal": "http://172.17.0.2:8776/v3/%(tenant_id)s", "admin": "http://172.17.0.2:8776/v3/%(tenant_id)s"}, "users": {"cinderv3": {"password": "Olf2iTUMR6eJ33irnMaVcSJQX", "roles": ["admin", "service"]}}, "region": "regionOne", "service": "volumev3"}}, "monitoring_subscription": "overcloud-cinder-api", "config_settings": {"cinder::database_connection": "mysql+pymysql://cinder:Olf2iTUMR6eJ33irnMaVcSJQX@172.17.0.2/cinder?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "cinder::logging::debug": true, "cinder::storage_availability_zone": "nova", "cinder::rabbit_heartbeat_timeout_threshold": 60, "cinder::rpc_response_timeout": 60, "cinder::rabbit_heartbeat_in_pthread": false, "cinder::db::database_db_max_retries": -1, "cinder::db::database_max_retries": -1, "cinder::cron::db_purge::minute": "1", "cinder::cron::db_purge::hour": "0", "cinder::cron::db_purge::monthday": "*", "cinder::cron::db_purge::month": "*", "cinder::cron::db_purge::weekday": "*", "cinder::cron::db_purge::user": "cinder", "cinder::cron::db_purge::age": "30", "cinder::cron::db_purge::destination": "/var/log/cinder/cinder-rowsflush.log", "cinder::cron::db_purge::maxdelay": "3600", "cinder::glance::glance_api_servers": "http://172.17.0.2:9293", "cinder::keystone::service_user::send_service_user_token": true, "cinder::keystone::service_user::project_name": "service", "cinder::keystone::service_user::auth_type": "password", "cinder::keystone::service_user::auth_url": "http://172.17.0.2:5000/", "cinder::keystone::service_user::password": "Olf2iTUMR6eJ33irnMaVcSJQX", "cinder::keystone::service_user::region_name": "regionOne", "cinder::nova::interface": "admin", "cinder::nova::project_name": "service", "cinder::nova::auth_type": "password", "cinder::nova::auth_url": "http://172.17.0.2:5000/", "cinder::nova::password": "nad1A8ZvKDsT60yawIjN1cH4K", "cinder::nova::region_name": "regionOne", "apache::ip": "%{hiera('internal_api')}", "apache::default_vhost": false, "apache::trace_enable": "Off", "apache::server_signature": "Off", "apache::server_tokens": "Prod", "apache::timeout": 90, "apache::default_mods": false, "apache::mod::prefork::maxrequestworkers": 100, "apache::mod::prefork::serverlimit": 100, "apache::mod::remoteip::proxy_ips": ["172.17.0.0/24"], "apache::mod::alias::icons_options": "None", "keystone_resources_managed": false, "cinder::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "cinder::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "cinder::keystone::authtoken::password": "Olf2iTUMR6eJ33irnMaVcSJQX", "cinder::keystone::authtoken::project_name": "service", "cinder::keystone::authtoken::user_domain_name": "Default", "cinder::keystone::authtoken::project_domain_name": "Default", "cinder::keystone::authtoken::region_name": "regionOne", "cinder::keystone::authtoken::interface": "internal", "cinder::keystone::authtoken::memcache_use_advanced_pool": true, "cinder::policy::policies": {}, "cinder::notification_driver": ["noop"], "cinder::api::default_volume_type": "tripleo", "cinder::api::enable_proxy_headers_parsing": true, "tripleo::profile::base::cinder::cinder_enable_db_purge": true, "cinder::api::bind_host": "%{hiera('fqdn_internal_api')}", "cinder::wsgi::apache::ssl": false, "cinder::api::service_name": "httpd", "cinder::wsgi::apache::bind_host": "%{hiera('internal_api')}", "cinder::wsgi::apache::servername": "%{hiera('fqdn_internal_api')}", "cinder::wsgi::apache::vhost_custom_fragment": "Timeout 60", "cinder::wsgi::apache::workers": "1"}, "service_config_settings": {"rsyslog": {"tripleo_logging_sources_cinder_api": [{"tag": "openstack.cinder.api", "file": "/var/log/containers/cinder/cinder-api.log"}]}, "mysql": {"cinder::db::mysql::password": "Olf2iTUMR6eJ33irnMaVcSJQX", "cinder::db::mysql::user": "cinder", "cinder::db::mysql::host": "%", "cinder::db::mysql::dbname": "cinder"}}, "puppet_config": {"config_volume": "cinder", "puppet_tags": "cinder_config,file,concat,file_line,cinder_api_paste_ini", "step_config": "include tripleo::profile::base::cinder::api\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/cinder_api.json": {"command": "/usr/sbin/httpd -DFOREGROUND", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-tls/*", "dest": "/", "merge": true, "preserve_properties": true, "optional": true}], "permissions": [{"path": "/var/log/cinder", "owner": "cinder:cinder", "recurse": true}, {"path": "/etc/pki/tls/certs/etcd.crt", "owner": "cinder:cinder"}, {"path": "/etc/pki/tls/private/etcd.key", "owner": "cinder:cinder"}]}, "/var/lib/kolla/config_files/cinder_api_db_sync.json": {"command": "/usr/bin/bootstrap_host_exec cinder_api su cinder -s /bin/bash -c 'cinder-manage db sync --bump-versions'", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-tls/*", "dest": "/", "merge": true, "preserve_properties": true, "optional": true}], "permissions": [{"path": "/var/log/cinder", "owner": "cinder:cinder", "recurse": true}, {"path": "/etc/pki/tls/certs/etcd.crt", "owner": "cinder:cinder"}, {"path": "/etc/pki/tls/private/etcd.key", "owner": "cinder:cinder"}]}, "/var/lib/kolla/config_files/cinder_api_cron.json": {"command": "/usr/sbin/crond -n", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/cinder", "owner": "cinder:cinder", "recurse": true}]}}, "docker_config": {"step_2": {"cinder_api_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-api:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/cinder:/var/log/cinder:z", "/var/log/containers/httpd/cinder-api:/var/log/httpd:z"], "command": ["/bin/bash", "-c", "chown -R cinder:cinder /var/log/cinder"]}}, "step_3": {"cinder_api_db_sync": {"image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-api:17.1", "cap_add": ["AUDIT_WRITE"], "net": "host", "privileged": false, "detach": false, "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/cinder_api_db_sync.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro", "/var/log/containers/cinder:/var/log/cinder:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_4": {"cinder_api": {"image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-api:17.1", "net": "host", "privileged": false, "restart": "always", "user": "root", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/cinder_api.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro", "/var/log/containers/cinder:/var/log/cinder:z", "/var/log/containers/httpd/cinder-api:/var/log/httpd:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "cinder_api_cron": {"image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-api:17.1", "net": "host", "user": "root", "privileged": false, "restart": "always", "healthcheck": {"test": "/usr/share/openstack-tripleo-common/healthcheck/cron cinder"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/cinder_api_cron.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro", "/var/log/containers/cinder:/var/log/cinder:z", "/var/log/containers/httpd/cinder-api:/var/log/httpd:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "metadata_settings": null, "deploy_steps_tasks": [{"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"name": "Create dirs for certificates and keys", "file": {"path": "{{ item }}", "state": "directory", "serole": "object_r", "setype": "cert_t", "seuser": "system_u"}, "with_items": ["/etc/pki/tls/certs/httpd", "/etc/pki/tls/private/httpd"]}, {"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "httpd-ctlplane", "dns": "{{ fqdn_ctlplane }}", "principal": "HTTP/{{ fqdn_ctlplane }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-ctlplane.crt /etc/pki/tls/certs/httpd/httpd-ctlplane.crt\ncp /etc/pki/tls/private/httpd-ctlplane.key /etc/pki/tls/private/httpd/httpd-ctlplane.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage", "dns": "{{ fqdn_storage }}", "principal": "HTTP/{{ fqdn_storage }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage.crt /etc/pki/tls/certs/httpd/httpd-storage.crt\ncp /etc/pki/tls/private/httpd-storage.key /etc/pki/tls/private/httpd/httpd-storage.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage_mgmt", "dns": "{{ fqdn_storage_mgmt }}", "principal": "HTTP/{{ fqdn_storage_mgmt }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage_mgmt.crt /etc/pki/tls/certs/httpd/httpd-storage_mgmt.crt\ncp /etc/pki/tls/private/httpd-storage_mgmt.key /etc/pki/tls/private/httpd/httpd-storage_mgmt.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-internal_api", "dns": "{{ fqdn_internal_api }}", "principal": "HTTP/{{ fqdn_internal_api }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-internal_api.crt /etc/pki/tls/certs/httpd/httpd-internal_api.crt\ncp /etc/pki/tls/private/httpd-internal_api.key /etc/pki/tls/private/httpd/httpd-internal_api.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-external", "dns": "{{ fqdn_external }}", "principal": "HTTP/{{ fqdn_external }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-external.crt /etc/pki/tls/certs/httpd/httpd-external.crt\ncp /etc/pki/tls/private/httpd-external.key /etc/pki/tls/private/httpd/httpd-external.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}]}}]}], "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode|default(omit) }}"}, "with_items": [{"path": "/var/log/containers/cinder", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/log/containers/httpd/cinder-api", "setype": "container_file_t", "mode": "0750"}]}], "external_upgrade_tasks": [{"when": "step|int == 1", "block": [{"name": "Online data migration for Cinder", "command": "{{ container_cli }} exec cinder_api cinder-manage db online_data_migrations", "delegate_to": "{{ (groups['cinder_api'] | difference(groups['excluded_overcloud']))[0] }}", "become": true, "tags": ["online_upgrade", "online_upgrade_cinder"]}]}, {"when": "step|int == 1", "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop cinder api container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["cinder_api", "cinder_api_cron"], "tripleo_delegate_to": "{{ groups['cinder_api'] | difference(groups['excluded_overcloud']) }}"}}]}], "external_update_tasks": [{"when": "step|int == 1", "block": [{"name": "Online data migration for Cinder", "command": "{{ container_cli }} exec cinder_api cinder-manage db online_data_migrations", "delegate_to": "{{ (groups['cinder_api'] | difference(groups['excluded_overcloud']))[0] }}", "become": true, "tags": ["online_upgrade", "online_upgrade_cinder"]}]}], "external_deploy_tasks": [{"name": "Clean up legacy Cinder keystone catalog entries", "become": true, "openstack.cloud.catalog_service": {"cloud": "standalone", "name": "{{ item.service_name }}", "service_type": "{{ item.service_type }}", "state": "absent"}, "loop": [{"service_name": "cinderv2", "service_type": "volumev2"}, {"service_name": "cinderv3", "service_type": "volume"}], "loop_control": {"index_var": "cinder_api_service"}, "when": ["step|int == 4", "not ansible_check_mode|bool"]}, {"name": "Manage Cinder's default volume type", "become": true, "vars": {"default_volume_type": "tripleo"}, "environment": {"OS_CLOUD": "standalone"}, "when": ["step|int == 5", "not ansible_check_mode|bool"], "shell": "if ! openstack volume type show \"{{ default_volume_type }}\"; then\n    openstack volume type create --public \"{{ default_volume_type }}\"\nfi\neval $(openstack volume type show __DEFAULT__ -f shell -c id -c description)\nif [ -n \"$id\" ]; then\n    if ! openstack volume type delete $id ; then\n        tripleo_descr=\"For internal use, '{{ default_volume_type }}' is the default volume type\"\n        if [ \"$description\" != \"$tripleo_descr\" ]; then\n            openstack volume type set $id --description \"$tripleo_descr\"\n        fi\n    fi\nfi\n", "args": {"executable": "/bin/bash"}, "changed_when": false}]}, {"service_name": "cinder_backup", "monitoring_subscription": "overcloud-cinder-backup", "config_settings": {"cinder::database_connection": "mysql+pymysql://cinder:Olf2iTUMR6eJ33irnMaVcSJQX@172.17.0.2/cinder?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "cinder::logging::debug": true, "cinder::storage_availability_zone": "nova", "cinder::rabbit_heartbeat_timeout_threshold": 60, "cinder::rpc_response_timeout": 60, "cinder::rabbit_heartbeat_in_pthread": false, "cinder::db::database_db_max_retries": -1, "cinder::db::database_max_retries": -1, "cinder::cron::db_purge::minute": "1", "cinder::cron::db_purge::hour": "0", "cinder::cron::db_purge::monthday": "*", "cinder::cron::db_purge::month": "*", "cinder::cron::db_purge::weekday": "*", "cinder::cron::db_purge::user": "cinder", "cinder::cron::db_purge::age": "30", "cinder::cron::db_purge::destination": "/var/log/cinder/cinder-rowsflush.log", "cinder::cron::db_purge::maxdelay": "3600", "cinder::glance::glance_api_servers": "http://172.17.0.2:9293", "cinder::keystone::service_user::send_service_user_token": true, "cinder::keystone::service_user::project_name": "service", "cinder::keystone::service_user::auth_type": "password", "cinder::keystone::service_user::auth_url": "http://172.17.0.2:5000/", "cinder::keystone::service_user::password": "Olf2iTUMR6eJ33irnMaVcSJQX", "cinder::keystone::service_user::region_name": "regionOne", "cinder::nova::interface": "admin", "cinder::nova::project_name": "service", "cinder::nova::auth_type": "password", "cinder::nova::auth_url": "http://172.17.0.2:5000/", "cinder::nova::password": "nad1A8ZvKDsT60yawIjN1cH4K", "cinder::nova::region_name": "regionOne", "cinder::backup::backup_workers": 1, "cinder::backup::backup_max_operations": 15, "cinder::backup::ceph::backup_ceph_user": "openstack", "cinder::backup::ceph::backup_ceph_pool": "backups", "cinder::backup::ceph::backup_ceph_conf": "/etc/ceph/ceph.conf", "tripleo::profile::pacemaker::cinder::backup_bundle::cinder_backup_docker_image": "cluster.common.tag/cinder-backup:pcmklatest", "tripleo::profile::pacemaker::cinder::backup_bundle::docker_volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro", "/var/log/containers/cinder:/var/log/cinder:z", "/var/lib/kolla/config_files/cinder_backup.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/iscsid/etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/dev:/dev", "/run:/run", "/sys:/sys", "/lib/modules:/lib/modules:ro", "/var/lib/cinder:/var/lib/cinder:z", "/var/lib/iscsi:/var/lib/iscsi:z"], "tripleo::profile::pacemaker::cinder::backup_bundle::docker_environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}, "tripleo::profile::pacemaker::cinder::backup_bundle::container_backend": "podman", "tripleo::profile::pacemaker::cinder::backup_bundle::ceph_conf_path": "/var/lib/tripleo-config/ceph", "cinder::backup::manage_service": false, "cinder::backup::enabled": false}, "service_config_settings": {"rsyslog": {"tripleo_logging_sources_cinder_backup": [{"tag": "openstack.cinder.backup", "file": "/var/log/containers/cinder/cinder-backup.log"}]}}, "puppet_config": {"config_volume": "cinder", "puppet_tags": "cinder_config,file,concat,file_line", "step_config": "\ninclude tripleo::profile::base::cinder::backup::ceph\ninclude tripleo::profile::base::database::mysql::client\ninclude tripleo::profile::pacemaker::cinder::backup_bundle", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/cinder_backup.json": {"command": "/usr/bin/cinder-backup --config-file /usr/share/cinder/cinder-dist.conf --config-file /etc/cinder/cinder.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-ceph/", "dest": "/etc/ceph/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-iscsid/*", "dest": "/etc/iscsi/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-tls/*", "dest": "/", "merge": true, "preserve_properties": true, "optional": true}], "permissions": [{"path": "/var/log/cinder", "owner": "cinder:cinder", "recurse": true}, {"path": "/etc/ceph/ceph.client.openstack.keyring", "owner": "cinder:cinder", "perm": "0600"}, {"path": "/etc/pki/tls/certs/etcd.crt", "owner": "cinder:cinder"}, {"path": "/etc/pki/tls/private/etcd.key", "owner": "cinder:cinder"}]}}, "container_config_scripts": {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}}, "docker_config": {"step_3": {"cinder_backup_init_logs": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-backup:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/cinder:/var/log/cinder:z"], "command": ["/bin/bash", "-c", "chown -R cinder:cinder /var/log/cinder"]}}}, "host_prep_tasks": [{"name": "create fcontext entry for cinder data", "community.general.sefcontext": {"target": "/var/lib/cinder(/.*)?", "setype": "container_file_t", "state": "present"}}, {"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode|default(omit) }}"}, "with_items": [{"path": "/var/log/containers/cinder", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/lib/cinder", "setype": "container_file_t"}]}, {"name": "ensure ceph configurations exist", "file": {"path": "/var/lib/tripleo-config/ceph", "state": "directory"}}], "deploy_steps_tasks": [{"name": "Cinder Backup tag container image for pacemaker", "when": "step|int == 1", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-backup:17.1", "container_image_latest": "cluster.common.tag/cinder-backup:pcmklatest"}}, {"name": "Cinder Backup HA Wrappers Step", "when": "step|int == 5", "block": [{"name": "Cinder backup puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "cinder_backup", "tripleo_ha_wrapper_resource_name": "openstack-cinder-backup", "tripleo_ha_wrapper_bundle_name": "openstack-cinder-backup", "tripleo_ha_wrapper_resource_state": "_ Started", "tripleo_ha_wrapper_puppet_config_volume": "cinder", "tripleo_ha_wrapper_puppet_execute": "include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::cinder::backup_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ip,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".cinder_backup_previous_run"}}]}], "external_update_tasks": [{"when": "step|int == 1", "tags": "ha_image_update", "block": [{"block": [{"name": "Get cinder_backup image from pacemaker", "become": true, "register": "xmllint_pcmk_cinder_backup_image", "shell": "xmllint --xpath \"string(//bundle[@id='openstack-cinder-backup']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container cinder_backup image", "set_fact": {"cinder_backup_image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-backup:17.1", "cinder_backup_image_latest": "cluster.common.tag/cinder-backup:pcmklatest", "pcmk_cinder_backup_image": "{{xmllint_pcmk_cinder_backup_image.stdout}}"}}], "delegate_to": "{{ (groups[\"cinder_backup\"] | difference(groups[\"excluded_overcloud\"]))[0] }}"}, {"name": "cinder-backup temporary pacemaker container tag in case of image switch", "become": true, "import_role": {"name": "tripleo_ha_image_update"}, "vars": {"tripleo_ha_image_update_node_names": "{{ groups[\"cinder_backup\"] | difference(groups[\"excluded_overcloud\"]) }}", "tripleo_ha_image_update_bundle": "openstack-cinder-backup", "tripleo_ha_image_update_new_image": "{{cinder_backup_image_latest}}", "tripleo_ha_image_update_old_image": "{{pcmk_cinder_backup_image}}"}, "when": ["(pcmk_cinder_backup_image != \"\")", "(pcmk_cinder_backup_image != cinder_backup_image_latest)"]}]}], "update_tasks": [{"block": [{"name": "Get cinder_backup image from pacemaker", "become": true, "register": "xmllint_pcmk_cinder_backup_image", "shell": "xmllint --xpath \"string(//bundle[@id='openstack-cinder-backup']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container cinder_backup image", "set_fact": {"cinder_backup_image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-backup:17.1", "cinder_backup_image_latest": "cluster.common.tag/cinder-backup:pcmklatest", "pcmk_cinder_backup_image": "{{xmllint_pcmk_cinder_backup_image.stdout}}"}}], "when": "(step|int == 0 or step|int == 2)"}, {"name": "Check for update of cinder-backup container image name", "when": "step|int == 0", "assert": {"that": "pcmk_cinder_backup_image == cinder_backup_image_latest", "fail_msg": "cinder-backup image change detected, run overcloud external-update --tags ha_image_update first"}}, {"name": "Tear-down non-HA cinder-backup container", "when": ["step|int == 1"], "block": [{"name": "Remove non-HA cinder-backup container", "include_role": {"name": "tripleo_container_rm"}, "vars": {"tripleo_container_cli": "{{ container_cli }}", "tripleo_containers_to_rm": ["cinder_backup"]}}]}, {"name": "cinder_backup fetch and retag container image for pacemaker", "when": "step|int == 2", "block": [{"name": "Retag pcmklatest to latest cinder_backup image", "include_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{cinder_backup_image}}", "container_image_latest": "{{cinder_backup_image_latest}}"}}]}], "post_update_tasks": [{"name": "Cinder backup bundle post update", "when": "step|int == 1", "block": [{"name": "Cinder backup puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "cinder_backup", "tripleo_ha_wrapper_resource_name": "openstack-cinder-backup", "tripleo_ha_wrapper_bundle_name": "openstack-cinder-backup", "tripleo_ha_wrapper_resource_state": "_ Started", "tripleo_ha_wrapper_puppet_config_volume": "cinder", "tripleo_ha_wrapper_puppet_execute": "include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::cinder::backup_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ip,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".cinder_backup_previous_run"}}], "vars": {"tripleo_ha_wrapper_minor_update": true}}], "upgrade_tasks": [{"name": "Tear-down non-HA cinder_backup container", "when": ["step|int == 0"], "block": [{"name": "Remove non-HA cinder-backup container", "include_role": {"name": "tripleo_container_rm"}, "vars": {"tripleo_container_cli": "{{ container_cli }}", "tripleo_containers_to_rm": ["cinder_backup"]}}]}, {"name": "Prepare switch of cinder_backup image name", "when": ["step|int == 0"], "block": [{"name": "Get cinder_backup image id currently used by pacemaker", "shell": "pcs resource config openstack-cinder-backup | grep -Eo 'image=[^ ]+' | awk -F= '{print $2;}'", "register": "cinder_backup_image_current_res", "failed_when": false}, {"name": "cinder_backup image facts", "set_fact": {"cinder_backup_image_latest": "cluster.common.tag/cinder-backup:pcmklatest", "cinder_backup_image_current": "{{cinder_backup_image_current_res.stdout}}"}}, {"name": "Temporarily tag the current cinder_backup image id with the upgraded image name", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{cinder_backup_image_current}}", "container_image_latest": "{{cinder_backup_image_latest}}", "pull_image": false}, "when": ["cinder_backup_image_current != ''", "cinder_backup_image_current != cinder_backup_image_latest"]}, {"name": "Create cinder_backup retag statefile", "file": {"path": "/var/lib/tripleo/cinder_backup_needs_retag", "state": "touch"}, "when": ["cinder_backup_image_current != ''", "cinder_backup_image_current != cinder_backup_image_latest"]}]}, {"name": "Update cinder_backup pcs resource bundle for new container image - check", "when": ["step|int == 1"], "block": [{"name": "set is_cinder_backup_bootstrap_node fact", "tags": "common", "set_fact": "is_cinder_backup_bootstrap_node={{cinder_backup_short_bootstrap_node_name|lower == ansible_facts['hostname']|lower}}"}, {"name": "Check for cinder_backup retag statefile", "stat": {"path": "/var/lib/tripleo/cinder_backup_needs_retag"}, "register": "cinder_backup_retag_state_file"}]}, {"name": "Update cinder_backup pcs resource bundle for new container image", "when": ["step|int == 1", "is_cinder_backup_bootstrap_node", "cinder_backup_retag_state_file.stat.exists|bool"], "block": [{"name": "Disable the cinder_backup cluster resource before container upgrade step1", "pacemaker_resource": {"resource": "openstack-cinder-backup", "state": "disable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}, {"name": "Update the cinder_backup bundle to use the new container image name", "command": "pcs resource bundle update openstack-cinder-backup container image={{cinder_backup_image_latest}}"}, {"name": "Enable the cinder_backup cluster resource", "pacemaker_resource": {"resource": "openstack-cinder-backup", "state": "enable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}]}, {"name": "Check for cinder_backup retag statefile", "when": ["step|int == 3"], "stat": {"path": "/var/lib/tripleo/cinder_backup_needs_retag"}, "register": "cinder_backup_retag_state_file"}, {"name": "Retag the pacemaker image for cinder_backup", "when": ["step|int == 3", "cinder_backup_retag_state_file.stat.exists|bool"], "block": [{"name": "Disable the cinder_backup cluster resource before container upgrade step3", "pacemaker_resource": {"resource": "openstack-cinder-backup", "state": "disable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}, {"name": "Retag the pacemaker image if containerized", "block": [{"block": [{"name": "Get cinder_backup image from pacemaker", "become": true, "register": "xmllint_pcmk_cinder_backup_image", "shell": "xmllint --xpath \"string(//bundle[@id='openstack-cinder-backup']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container cinder_backup image", "set_fact": {"cinder_backup_image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-backup:17.1", "cinder_backup_image_latest": "cluster.common.tag/cinder-backup:pcmklatest", "pcmk_cinder_backup_image": "{{xmllint_pcmk_cinder_backup_image.stdout}}"}}]}, {"block": [{"name": "Retag pcmklatest to latest cinder_backup image", "include_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{cinder_backup_image}}", "container_image_latest": "{{cinder_backup_image_latest}}"}}]}]}, {"name": "Enable the cinder_backup cluster resource", "pacemaker_resource": {"resource": "openstack-cinder-backup", "state": "enable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}, {"name": "Remove cinder_backup retag statefile", "file": {"path": "/var/lib/tripleo/cinder_backup_needs_retag", "state": "absent"}}]}]}, {"service_name": "cinder_scheduler", "monitoring_subscription": "overcloud-cinder-scheduler", "config_settings": {"cinder::database_connection": "mysql+pymysql://cinder:Olf2iTUMR6eJ33irnMaVcSJQX@172.17.0.2/cinder?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "cinder::logging::debug": true, "cinder::storage_availability_zone": "nova", "cinder::rabbit_heartbeat_timeout_threshold": 60, "cinder::rpc_response_timeout": 60, "cinder::rabbit_heartbeat_in_pthread": false, "cinder::db::database_db_max_retries": -1, "cinder::db::database_max_retries": -1, "cinder::cron::db_purge::minute": "1", "cinder::cron::db_purge::hour": "0", "cinder::cron::db_purge::monthday": "*", "cinder::cron::db_purge::month": "*", "cinder::cron::db_purge::weekday": "*", "cinder::cron::db_purge::user": "cinder", "cinder::cron::db_purge::age": "30", "cinder::cron::db_purge::destination": "/var/log/cinder/cinder-rowsflush.log", "cinder::cron::db_purge::maxdelay": "3600", "cinder::glance::glance_api_servers": "http://172.17.0.2:9293", "cinder::keystone::service_user::send_service_user_token": true, "cinder::keystone::service_user::project_name": "service", "cinder::keystone::service_user::auth_type": "password", "cinder::keystone::service_user::auth_url": "http://172.17.0.2:5000/", "cinder::keystone::service_user::password": "Olf2iTUMR6eJ33irnMaVcSJQX", "cinder::keystone::service_user::region_name": "regionOne", "cinder::nova::interface": "admin", "cinder::nova::project_name": "service", "cinder::nova::auth_type": "password", "cinder::nova::auth_url": "http://172.17.0.2:5000/", "cinder::nova::password": "nad1A8ZvKDsT60yawIjN1cH4K", "cinder::nova::region_name": "regionOne", "cinder::scheduler::scheduler_driver": "cinder.scheduler.filter_scheduler.FilterScheduler"}, "service_config_settings": {"rsyslog": {"tripleo_logging_sources_cinder_scheduler": [{"tag": "openstack.cinder.scheduler", "file": "/var/log/containers/cinder/cinder-scheduler.log"}]}}, "puppet_config": {"config_volume": "cinder", "puppet_tags": "cinder_config,file,concat,file_line", "step_config": "include tripleo::profile::base::cinder::scheduler\n\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/cinder_scheduler.json": {"command": "/usr/bin/cinder-scheduler --config-file /usr/share/cinder/cinder-dist.conf --config-file /etc/cinder/cinder.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-tls/*", "dest": "/", "merge": true, "preserve_properties": true, "optional": true}], "permissions": [{"path": "/var/log/cinder", "owner": "cinder:cinder", "recurse": true}, {"path": "/etc/pki/tls/certs/etcd.crt", "owner": "cinder:cinder"}, {"path": "/etc/pki/tls/private/etcd.key", "owner": "cinder:cinder"}]}}, "docker_config": {"step_2": {"cinder_scheduler_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-scheduler:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/cinder:/var/log/cinder:z"], "command": ["/bin/bash", "-c", "chown -R cinder:cinder /var/log/cinder"]}}, "step_4": {"cinder_scheduler": {"image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-scheduler:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck 5672"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro", "/var/log/containers/cinder:/var/log/cinder:z", "/var/lib/kolla/config_files/cinder_scheduler.json:/var/lib/kolla/config_files/config.json:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/cinder", "setype": "container_file_t", "mode": "0750"}]}, {"name": "enable virt_sandbox_use_netlink for healthcheck", "seboolean": {"name": "virt_sandbox_use_netlink", "persistent": true, "state": true}, "when": ["ansible_facts.selinux is defined", "ansible_facts.selinux.status == \"enabled\""]}], "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop cinder scheduler container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["cinder_scheduler"], "tripleo_delegate_to": "{{ groups['cinder_scheduler'] | difference(groups['excluded_overcloud']) }}"}}]}]}, {"service_name": "cinder_volume", "firewall_rules": {"120 iscsi initiator": {"dport": 3260}}, "monitoring_subscription": "overcloud-cinder-volume", "config_settings": {"cinder::database_connection": "mysql+pymysql://cinder:Olf2iTUMR6eJ33irnMaVcSJQX@172.17.0.2/cinder?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "cinder::logging::debug": true, "cinder::storage_availability_zone": "nova", "cinder::rabbit_heartbeat_timeout_threshold": 60, "cinder::rpc_response_timeout": 60, "cinder::rabbit_heartbeat_in_pthread": false, "cinder::db::database_db_max_retries": -1, "cinder::db::database_max_retries": -1, "cinder::cron::db_purge::minute": "1", "cinder::cron::db_purge::hour": "0", "cinder::cron::db_purge::monthday": "*", "cinder::cron::db_purge::month": "*", "cinder::cron::db_purge::weekday": "*", "cinder::cron::db_purge::user": "cinder", "cinder::cron::db_purge::age": "30", "cinder::cron::db_purge::destination": "/var/log/cinder/cinder-rowsflush.log", "cinder::cron::db_purge::maxdelay": "3600", "cinder::glance::glance_api_servers": "http://172.17.0.2:9293", "cinder::keystone::service_user::send_service_user_token": true, "cinder::keystone::service_user::project_name": "service", "cinder::keystone::service_user::auth_type": "password", "cinder::keystone::service_user::auth_url": "http://172.17.0.2:5000/", "cinder::keystone::service_user::password": "Olf2iTUMR6eJ33irnMaVcSJQX", "cinder::keystone::service_user::region_name": "regionOne", "cinder::nova::interface": "admin", "cinder::nova::project_name": "service", "cinder::nova::auth_type": "password", "cinder::nova::auth_url": "http://172.17.0.2:5000/", "cinder::nova::password": "nad1A8ZvKDsT60yawIjN1cH4K", "cinder::nova::region_name": "regionOne", "tripleo::profile::base::lvm::enable_udev": false, "tripleo::profile::base::cinder::volume::cinder_enable_iscsi_backend": false, "tripleo::profile::base::cinder::volume::cinder_enable_nfs_backend": false, "tripleo::profile::base::cinder::volume::cinder_enable_rbd_backend": true, "tripleo::profile::base::cinder::volume::cinder_volume_cluster": "", "tripleo::profile::base::cinder::volume::nfs::cinder_nfs_mount_options": "context=system_u:object_r:container_file_t:s0", "tripleo::profile::base::cinder::volume::nfs::cinder_nfs_servers": [], "tripleo::profile::base::cinder::volume::nfs::cinder_nfs_snapshot_support": false, "tripleo::profile::base::cinder::volume::nfs::cinder_nas_secure_file_operations": "False", "tripleo::profile::base::cinder::volume::nfs::cinder_nas_secure_file_permissions": "False", "tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_helper": "lioadm", "tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_protocol": "iscsi", "tripleo::profile::base::cinder::volume::rbd::backend_name": ["tripleo_ceph"], "tripleo::profile::base::cinder::volume::rbd::cinder_rbd_ceph_conf": "/etc/ceph/ceph.conf", "tripleo::profile::base::cinder::volume::rbd::cinder_rbd_pool_name": "volumes", "tripleo::profile::base::cinder::volume::rbd::cinder_rbd_extra_pools": [], "tripleo::profile::base::cinder::volume::rbd::cinder_rbd_secret_uuid": "55dae5e5-7ec8-55ff-b761-b30fedcc71d6", "tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name": "openstack", "tripleo::profile::base::cinder::volume::rbd::cinder_rbd_flatten_volume_from_snapshot": false, "tripleo::profile::base::cinder::volume::rbd::multi_config": {}, "tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_address": "%{hiera('storage')}", "tripleo::profile::pacemaker::cinder::volume_bundle::cinder_volume_docker_image": "cluster.common.tag/cinder-volume:pcmklatest", "tripleo::profile::pacemaker::cinder::volume_bundle::docker_volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro", "/var/log/containers/cinder:/var/log/cinder:z", "/var/lib/kolla/config_files/cinder_volume.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/iscsid/etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/lib/modules:/lib/modules:ro", "/dev:/dev", "/run:/run", "/sys:/sys", "/var/lib/cinder:/var/lib/cinder:z", "/var/lib/iscsi:/var/lib/iscsi:z"], "tripleo::profile::pacemaker::cinder::volume_bundle::docker_environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}, "tripleo::profile::pacemaker::cinder::volume_bundle::container_backend": "podman", "tripleo::profile::pacemaker::cinder::volume_bundle::ceph_conf_path": "/var/lib/tripleo-config/ceph", "cinder::volume::manage_service": false, "cinder::volume::enabled": false, "cinder::backend_host": "hostgroup"}, "service_config_settings": {"rsyslog": {"tripleo_logging_sources_cinder_volume": [{"tag": "openstack.cinder.volume", "file": "/var/log/containers/cinder/cinder-volume.log"}]}}, "puppet_config": {"config_volume": "cinder", "puppet_tags": "cinder_config,file,concat,file_line", "step_config": "include tripleo::profile::base::lvm\ninclude tripleo::profile::pacemaker::cinder::volume_bundle\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/cinder_volume.json": {"command": "/usr/bin/cinder-volume --config-file /usr/share/cinder/cinder-dist.conf --config-file /etc/cinder/cinder.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-ceph/", "dest": "/etc/ceph/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-iscsid/*", "dest": "/etc/iscsi/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-tls/*", "dest": "/", "merge": true, "preserve_properties": true, "optional": true}], "permissions": [{"path": "/var/log/cinder", "owner": "cinder:cinder", "recurse": true}, {"path": "/etc/ceph/ceph.client.openstack.keyring", "owner": "cinder:cinder", "perm": "0600"}, {"path": "/etc/pki/tls/certs/etcd.crt", "owner": "cinder:cinder"}, {"path": "/etc/pki/tls/private/etcd.key", "owner": "cinder:cinder"}]}}, "container_config_scripts": {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}}, "docker_config": {"step_3": {"cinder_volume_init_logs": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-volume:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/cinder:/var/log/cinder"], "command": ["/bin/bash", "-c", "chown -R cinder:cinder /var/log/cinder"]}}}, "host_prep_tasks": [{"name": "create fcontext entry for cinder data", "community.general.sefcontext": {"target": "/var/lib/cinder(/.*)?", "setype": "container_file_t", "state": "present"}}, {"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode|default(omit) }}"}, "with_items": [{"path": "/var/log/containers/cinder", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/lib/cinder", "setype": "container_file_t"}]}, {"name": "ensure ceph configurations exist", "file": {"path": "/var/lib/tripleo-config/ceph", "state": "directory"}}, {"name": "Support using an NFS share for cinder image conversion", "vars": {"image_conversion_nfs_enabled": false}, "when": ["image_conversion_nfs_enabled|bool"], "block": [{"name": "Create cinder image conversion directory", "file": {"path": "/var/lib/cinder_image_conversion", "state": "directory"}}, {"name": "Mount cinder's image conversion NFS share", "vars": {"image_conversion_nfs_share": "", "image_conversion_nfs_options": "_netdev,bg,intr,context=system_u:object_r:container_file_t:s0"}, "mount": {"path": "/var/lib/cinder_image_conversion", "state": "mounted", "fstype": "nfs4", "src": "{{ image_conversion_nfs_share }}", "opts": "{{ image_conversion_nfs_options }}"}}]}, {"name": "cinder_configure_lvm fact", "set_fact": {"cinder_configure_lvm": false}}, {"when": "cinder_configure_lvm|bool", "block": [{"name": "ensure LVM rpm dependencies are installed", "package": {"name": "lvm2", "state": "latest"}}, {"name": "cinder create LVM volume group dd", "command": "dd if=/dev/zero of=/var/lib/cinder/cinder-volumes bs=1 count=0 seek=10280M", "args": {"creates": "/var/lib/cinder/cinder-volumes"}}, {"name": "Get or create LVM loopback device", "shell": "exit_code=0\nexisting_device=$(losetup -j /var/lib/cinder/cinder-volumes -l -n -O NAME)\nif [[ -z \"${existing_device}\" ]]; then\n    losetup -f /var/lib/cinder/cinder-volumes --show\n    exit_code=2\nelse\n    echo ${existing_device%$'\\n'*}\nfi\nexit ${exit_code}", "args": {"executable": "/bin/bash"}, "register": "_loopback_device", "changed_when": "_loopback_device.rc == 2", "failed_when": "_loopback_device.rc not in [0,2]"}, {"name": "Create LVM volume group", "community.general.lvg": {"vg": "cinder-volumes", "pvs": "{{ _loopback_device.stdout }}", "state": "present"}, "when": ["not (ansible_check_mode | bool)"]}, {"name": "cinder create service to run losetup for LVM on startup", "copy": {"dest": "/etc/systemd/system/cinder-lvm-losetup.service", "content": "[Unit]\nDescription=Cinder LVM losetup\nDefaultDependencies=no\nConflicts=umount.target\nRequires=lvm2-monitor.service systemd-udev-settle.service\nBefore=local-fs.target umount.target\nAfter=var.mount lvm2-monitor.service systemd-udev-settle.service\n\n[Service]\nType=oneshot\nExecStart=/sbin/losetup {{ _loopback_device.stdout }} /var/lib/cinder/cinder-volumes\nExecStop=/sbin/losetup -d {{ _loopback_device.stdout }}\nRemainAfterExit=yes\n\n[Install]\nWantedBy=local-fs-pre.target\n"}, "when": ["not (ansible_check_mode | bool)"]}, {"name": "cinder enable the LVM losetup service", "systemd": {"name": "cinder-lvm-losetup", "enabled": true, "daemon_reload": true}}]}], "deploy_steps_tasks": [{"name": "Cinder Volume tag container image for pacemaker", "when": "step|int == 1", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-volume:17.1", "container_image_latest": "cluster.common.tag/cinder-volume:pcmklatest"}}, {"name": "Cinder Volume HA Wrappers Step", "when": "step|int == 5", "block": [{"name": "Cinder volume puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "cinder_volume", "tripleo_ha_wrapper_resource_name": "openstack-cinder-volume", "tripleo_ha_wrapper_bundle_name": "openstack-cinder-volume", "tripleo_ha_wrapper_resource_state": "_ Started", "tripleo_ha_wrapper_puppet_config_volume": "cinder", "tripleo_ha_wrapper_puppet_execute": "include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::cinder::volume_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ip,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".cinder_volume_previous_run"}}]}], "external_update_tasks": [{"when": "step|int == 1", "tags": "ha_image_update", "block": [{"block": [{"name": "Get cinder_volume image from pacemaker", "become": true, "register": "xmllint_pcmk_cinder_volume_image", "shell": "xmllint --xpath \"string(//bundle[@id='openstack-cinder-volume']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container cinder_volume image", "set_fact": {"cinder_volume_image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-volume:17.1", "cinder_volume_image_latest": "cluster.common.tag/cinder-volume:pcmklatest", "pcmk_cinder_volume_image": "{{xmllint_pcmk_cinder_volume_image.stdout}}"}}], "delegate_to": "{{ (groups[\"cinder_volume\"] | difference(groups[\"excluded_overcloud\"]))[0] }}"}, {"name": "cinder-volume temporary pacemaker container tag in case of image switch", "become": true, "import_role": {"name": "tripleo_ha_image_update"}, "vars": {"tripleo_ha_image_update_node_names": "{{ groups[\"cinder_volume\"] | difference(groups[\"excluded_overcloud\"]) }}", "tripleo_ha_image_update_bundle": "openstack-cinder-volume", "tripleo_ha_image_update_new_image": "{{cinder_volume_image_latest}}", "tripleo_ha_image_update_old_image": "{{pcmk_cinder_volume_image}}"}, "when": ["(pcmk_cinder_volume_image != \"\")", "(pcmk_cinder_volume_image != cinder_volume_image_latest)"]}]}], "update_tasks": [{"block": [{"name": "Get cinder_volume image from pacemaker", "become": true, "register": "xmllint_pcmk_cinder_volume_image", "shell": "xmllint --xpath \"string(//bundle[@id='openstack-cinder-volume']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container cinder_volume image", "set_fact": {"cinder_volume_image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-volume:17.1", "cinder_volume_image_latest": "cluster.common.tag/cinder-volume:pcmklatest", "pcmk_cinder_volume_image": "{{xmllint_pcmk_cinder_volume_image.stdout}}"}}], "when": "(step|int == 0 or step|int == 2)"}, {"name": "Check for update of cinder-volume container image name", "when": "step|int == 0", "assert": {"that": "pcmk_cinder_volume_image == cinder_volume_image_latest", "fail_msg": "cinder-volume image change detected, run overcloud external-update --tags ha_image_update first"}}, {"name": "Tear-down non-HA cinder_volume container", "when": ["step|int == 1"], "block": [{"name": "Remove non-HA cinder_volume container", "include_role": {"name": "tripleo_container_rm"}, "vars": {"tripleo_container_cli": "{{ container_cli }}", "tripleo_containers_to_rm": ["cinder_volume"]}}]}, {"name": "cinder_volume fetch and retag container image for pacemaker", "when": "step|int == 2", "block": [{"name": "Retag pcmklatest to latest cinder_volume image", "include_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{cinder_volume_image}}", "container_image_latest": "{{cinder_volume_image_latest}}"}}]}], "post_update_tasks": [{"name": "Cinder volume bundle post update", "when": "step|int == 1", "block": [{"name": "Cinder volume puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "cinder_volume", "tripleo_ha_wrapper_resource_name": "openstack-cinder-volume", "tripleo_ha_wrapper_bundle_name": "openstack-cinder-volume", "tripleo_ha_wrapper_resource_state": "_ Started", "tripleo_ha_wrapper_puppet_config_volume": "cinder", "tripleo_ha_wrapper_puppet_execute": "include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::cinder::volume_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ip,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".cinder_volume_previous_run"}}], "vars": {"tripleo_ha_wrapper_minor_update": true}}], "upgrade_tasks": [{"name": "Tear-down non-HA cinder_volume container", "when": ["step|int == 0"], "block": [{"name": "Remove non-HA cinder_volume container", "include_role": {"name": "tripleo_container_rm"}, "vars": {"tripleo_container_cli": "{{ container_cli }}", "tripleo_containers_to_rm": ["cinder_volume"]}}]}, {"name": "Prepare switch of cinder_volume image name", "when": ["step|int == 0"], "block": [{"name": "Get cinder_volume image id currently used by pacemaker", "shell": "pcs resource config openstack-cinder-volume | grep -Eo 'image=[^ ]+' | awk -F= '{print $2;}'", "register": "cinder_volume_image_current_res", "failed_when": false}, {"name": "cinder_volume image facts", "set_fact": {"cinder_volume_image_latest": "cluster.common.tag/cinder-volume:pcmklatest", "cinder_volume_image_current": "{{cinder_volume_image_current_res.stdout}}"}}, {"name": "Temporarily tag the current cinder_volume image id with the upgraded image name", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{cinder_volume_image_current}}", "container_image_latest": "{{cinder_volume_image_latest}}", "pull_image": false}, "when": ["cinder_volume_image_current != ''", "cinder_volume_image_current != cinder_volume_image_latest"]}, {"name": "Create cinder_volume retag statefile", "file": {"path": "/var/lib/tripleo/cinder_volume_needs_retag", "state": "touch"}, "when": ["cinder_volume_image_current != ''", "cinder_volume_image_current != cinder_volume_image_latest"]}]}, {"name": "Update cinder_volume pcs resource bundle for new container image", "when": ["step|int == 1"], "block": [{"name": "set is_cinder_volume_bootstrap_node fact", "tags": "common", "set_fact": "is_cinder_volume_bootstrap_node={{cinder_volume_short_bootstrap_node_name|lower == ansible_facts['hostname']|lower}}"}, {"name": "Check for cinder_volume retag statefile", "stat": {"path": "/var/lib/tripleo/cinder_volume_needs_retag"}, "register": "cinder_volume_retag_state_file"}]}, {"name": "Update cinder_volume pcs resource bundle for new container image", "when": ["step|int == 1", "is_cinder_volume_bootstrap_node", "cinder_volume_retag_state_file.stat.exists|bool"], "block": [{"name": "Disable the cinder_volume cluster resource before container upgrade", "pacemaker_resource": {"resource": "openstack-cinder-volume", "state": "disable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}, {"name": "pcs resource bundle update cinder_volume for new container image name", "command": "pcs resource bundle update openstack-cinder-volume container image={{cinder_volume_image_latest}}"}, {"name": "Enable the cinder_volume cluster resource", "pacemaker_resource": {"resource": "openstack-cinder-volume", "state": "enable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}]}, {"name": "Check for cinder_volume retag statefile", "when": ["step|int == 3"], "stat": {"path": "/var/lib/tripleo/cinder_volume_needs_retag"}, "register": "cinder_volume_retag_state_file"}, {"name": "Retag pacemaker cinder_volume", "when": ["step|int == 3", "cinder_volume_retag_state_file.stat.exists|bool"], "block": [{"name": "Disable the cinder_volume cluster resource before container upgrade", "pacemaker_resource": {"resource": "openstack-cinder-volume", "state": "disable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}, {"name": "Retag the pacemaker image if containerized", "block": [{"block": [{"name": "Get cinder_volume image from pacemaker", "become": true, "register": "xmllint_pcmk_cinder_volume_image", "shell": "xmllint --xpath \"string(//bundle[@id='openstack-cinder-volume']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container cinder_volume image", "set_fact": {"cinder_volume_image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-volume:17.1", "cinder_volume_image_latest": "cluster.common.tag/cinder-volume:pcmklatest", "pcmk_cinder_volume_image": "{{xmllint_pcmk_cinder_volume_image.stdout}}"}}]}, {"block": [{"name": "Retag pcmklatest to latest cinder_volume image", "include_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{cinder_volume_image}}", "container_image_latest": "{{cinder_volume_image_latest}}"}}]}]}, {"name": "Enable the cinder_volume cluster resource", "pacemaker_resource": {"resource": "openstack-cinder-volume", "state": "enable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}, {"name": "Remove cinder_volume retag statefile", "file": {"path": "/var/lib/tripleo/cinder_volume_needs_retag", "state": "absent"}}]}, {"name": "cinder-volume pre system_upgrade tasks", "tags": ["never", "system_upgrade", "system_upgrade_run", "system_upgrade_nfsmounts"], "when": ["step|int == 3", "upgrade_leapp_enabled"], "block": [{"name": "Unmount cinder's image conversion NFS share", "vars": {"image_conversion_nfs_share": "", "image_conversion_nfs_options": "_netdev,bg,intr,context=system_u:object_r:container_file_t:s0", "image_conversion_nfs_enabled": false}, "when": "image_conversion_nfs_enabled|bool", "mount": {"path": "/var/lib/cinder_image_conversion", "state": "absent", "fstype": "nfs4"}}]}, {"name": "cinder-volume post system_upgrade tasks", "tags": ["never", "system_upgrade", "system_upgrade_run", "system_upgrade_nfsmounts"], "when": ["step|int == 5", "upgrade_leapp_enabled"], "block": [{"name": "Support using an NFS share for cinder image conversion", "vars": {"image_conversion_nfs_enabled": false}, "when": ["image_conversion_nfs_enabled|bool"], "block": [{"name": "Create cinder image conversion directory", "file": {"path": "/var/lib/cinder_image_conversion", "state": "directory"}}, {"name": "Mount cinder's image conversion NFS share", "vars": {"image_conversion_nfs_share": "", "image_conversion_nfs_options": "_netdev,bg,intr,context=system_u:object_r:container_file_t:s0"}, "mount": {"path": "/var/lib/cinder_image_conversion", "state": "mounted", "fstype": "nfs4", "src": "{{ image_conversion_nfs_share }}", "opts": "{{ image_conversion_nfs_options }}"}}]}]}], "post_upgrade_tasks": [{"name": "Start cinder_volume service (pacemaker)", "become": true, "when": "step|int == 1", "pacemaker_resource": {"resource": "openstack-cinder-volume", "state": "enable"}}]}, {"service_name": "clustercheck", "config_settings": {"mysql::server::package_name": "mariadb-galera-server", "mysql::server::manage_config_file": true, "mysql_ipv6": false, "mysql_max_connections": 4096, "mysql::server::root_password": "NhgQDgRVoo", "mysql_clustercheck_password": "sRxOzvWhwndqoiP4taJzvBzao", "enable_galera": true, "mysql_auth_ed25519": false, "mysql_bind_host": "%{hiera('internal_api')}", "tripleo::profile::base::database::mysql::bind_address": "%{hiera('fqdn_internal_api')}", "tripleo::profile::base::database::mysql::client_bind_address": "%{hiera('internal_api')}", "tripleo::profile::base::database::mysql::generate_dropin_file_limit": true, "innodb_buffer_pool_size": "128M", "tripleo::profile::pacemaker::database::mysql::bind_address": "%{hiera('fqdn_internal_api')}", "tripleo::profile::pacemaker::database::mysql::gmcast_listen_addr": "%{hiera('internal_api')}", "tripleo::profile::pacemaker::database::mysql::ca_file": "/etc/ipa/ca.crt", "tripleo::profile::pacemaker::database::mysql_bundle::mysql_docker_image": "cluster.common.tag/mariadb:pcmklatest", "tripleo::profile::pacemaker::database::mysql_bundle::control_port": 3123, "tripleo::profile::pacemaker::database::mysql_bundle::container_backend": "podman", "tripleo::profile::pacemaker::database::mysql_bundle::bind_address": "%{hiera('fqdn_internal_api')}", "tripleo::profile::pacemaker::database::mysql_bundle::sst_method": "rsync", "tripleo::profile::pacemaker::database::mysql_bundle::mariabackup_user": "mariabackup", "tripleo::profile::pacemaker::database::mysql_bundle::mariabackup_password": "3Qi03Zmdc69WCw00okVFEQzzM", "tripleo::profile::pacemaker::database::mysql_bundle::force_ocf": true, "tripleo::profile::pacemaker::database::mysql_bundle::gcomm_cipher": "AES128-SHA256", "tripleo::profile::pacemaker::clustercheck::bind_address": "%{hiera('internal_api')}"}, "puppet_config": {"config_volume": "clustercheck", "puppet_tags": "file", "step_config": "include tripleo::profile::pacemaker::clustercheck", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-mariadb:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/clustercheck.json": {"command": "bash -c $* -- eval source /etc/sysconfig/clustercheck; exec socat -T\"${TRIPLEO_HEALTHCHECK_TIMEOUT:-2}\" \"$TRIPLEO_SOCAT_BIND\" system:\"grep -qPe \\\"^\\\\r\\$\\\" && /usr/bin/clustercheck\"", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}}, "docker_config": {"step_2": {"clustercheck": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-mariadb:17.1", "restart": "always", "net": "host", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/clustercheck.json:/var/lib/kolla/config_files/config.json", "/var/lib/config-data/puppet-generated/clustercheck:/var/lib/kolla/config_files/src:ro", "/var/lib/mysql:/var/lib/mysql"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "host_prep_tasks": null, "upgrade_tasks": null, "update_tasks": null, "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop clustercheck container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["clustercheck"], "tripleo_delegate_to": "{{ groups['clustercheck'] | difference(groups['excluded_overcloud']) }}"}}]}]}, {"service_name": "container_image_prepare", "external_deploy_tasks": [{"name": "Run tripleo-container-image-prepare role", "import_role": {"name": "tripleo_container_image_prepare"}, "vars": {"tripleo_container_image_prepare_log_file": "/var/log/tripleo-container-image-prepare.log", "tripleo_container_image_prepare_debug": true, "tripleo_container_image_prepare_roles": [{"name": "Standalone", "description": "A standalone role that a minimal set of services. This can be used for\ntesting in a single node configuration with the\n'openstack tripleo deploy --standalone' command or via an Undercloud using\n'openstack overcloud deploy'.\n", "CountDefault": 1, "tags": ["primary", "controller", "standalone"], "default_route_networks": [], "networks": {"External": {"subnet": "external_subnet"}, "InternalApi": {"subnet": "internal_api_subnet"}, "Storage": {"subnet": "storage_subnet"}, "StorageMgmt": {"subnet": "storage_mgmt_subnet"}, "StorageNFS": {"subnet": "storage_nfs_subnet"}, "Tenant": {"subnet": "tenant_subnet"}}, "RoleParametersDefault": {"OVNCMSOptions": "enable-chassis-as-gw"}, "ServicesDefault": ["OS::TripleO::Services::Aide", "OS::TripleO::Services::AodhApi", "OS::TripleO::Services::AodhEvaluator", "OS::TripleO::Services::AodhListener", "OS::TripleO::Services::AodhNotifier", "OS::TripleO::Services::AuditD", "OS::TripleO::Services::BootParams", "OS::TripleO::Services::BarbicanApi", "OS::TripleO::Services::BarbicanBackendDogtag", "OS::TripleO::Services::BarbicanBackendKmip", "OS::TripleO::Services::BarbicanBackendPkcs11Crypto", "OS::TripleO::Services::BarbicanBackendSimpleCrypto", "OS::TripleO::Services::CACerts", "OS::TripleO::Services::CeilometerAgentCentral", "OS::TripleO::Services::CeilometerAgentNotification", "OS::TripleO::Services::CephClient", "OS::TripleO::Services::CephExternal", "OS::TripleO::Services::CephGrafana", "OS::TripleO::Services::CephMds", "OS::TripleO::Services::CephMgr", "OS::TripleO::Services::CephMon", "OS::TripleO::Services::CephNfs", "OS::TripleO::Services::CephRbdMirror", "OS::TripleO::Services::CephRgw", "OS::TripleO::Services::CephOSD", "OS::TripleO::Services::CinderApi", "OS::TripleO::Services::CinderBackendDellEMCPowerFlex", "OS::TripleO::Services::CinderBackendDellEMCPowermax", "OS::TripleO::Services::CinderBackendDellEMCPowerStore", "OS::TripleO::Services::CinderBackendDellEMCSc", "OS::TripleO::Services::CinderBackendDellEMCUnity", "OS::TripleO::Services::CinderBackendDellEMCVMAXISCSI", "OS::TripleO::Services::CinderBackendDellEMCVNX", "OS::TripleO::Services::CinderBackendDellEMCVxFlexOS", "OS::TripleO::Services::CinderBackendDellEMCXtremio", "OS::TripleO::Services::CinderBackendDellSc", "OS::TripleO::Services::CinderBackendNVMeOF", "OS::TripleO::Services::CinderBackendPure", "OS::TripleO::Services::CinderBackendNetApp", "OS::TripleO::Services::CinderBackendScaleIO", "OS::TripleO::Services::CinderBackup", "OS::TripleO::Services::CinderHPELeftHandISCSI", "OS::TripleO::Services::CinderScheduler", "OS::TripleO::Services::CinderVolume", "OS::TripleO::Services::Clustercheck", "OS::TripleO::Services::Collectd", "OS::TripleO::Services::ComputeCeilometerAgent", "OS::TripleO::Services::CeilometerAgentIpmi", "OS::TripleO::Services::ContainerImagePrepare", "OS::TripleO::Services::ContainersLogrotateCrond", "OS::TripleO::Services::DesignateApi", "OS::TripleO::Services::DesignateCentral", "OS::TripleO::Services::DesignateMDNS", "OS::TripleO::Services::DesignateProducer", "OS::TripleO::Services::DesignateSink", "OS::TripleO::Services::DesignateBind", "OS::TripleO::Services::DesignateWorker", "OS::TripleO::Services::DockerRegistry", "OS::TripleO::Services::Etcd", "OS::TripleO::Services::ExternalSwiftProxy", "OS::TripleO::Services::Frr", "OS::TripleO::Services::GlanceApi", "OS::TripleO::Services::GlanceApiInternal", "OS::TripleO::Services::GnocchiApi", "OS::TripleO::Services::GnocchiMetricd", "OS::TripleO::Services::GnocchiStatsd", "OS::TripleO::Services::HAproxy", "OS::TripleO::Services::HeatApi", "OS::TripleO::Services::HeatApiCfn", "OS::TripleO::Services::HeatApiCloudwatch", "OS::TripleO::Services::HeatEngine", "OS::TripleO::Services::Horizon", "OS::TripleO::Services::IpaClient", "OS::TripleO::Services::Ipsec", "OS::TripleO::Services::IronicApi", "OS::TripleO::Services::IronicConductor", "OS::TripleO::Services::IronicInspector", "OS::TripleO::Services::IronicNeutronAgent", "OS::TripleO::Services::IronicPxe", "OS::TripleO::Services::Iscsid", "OS::TripleO::Services::Kernel", "OS::TripleO::Services::Keystone", "OS::TripleO::Services::LoginDefs", "OS::TripleO::Services::ManilaApi", "OS::TripleO::Services::ManilaBackendCephFs", "OS::TripleO::Services::ManilaBackendIsilon", "OS::TripleO::Services::ManilaBackendNetapp", "OS::TripleO::Services::ManilaBackendPowerMax", "OS::TripleO::Services::ManilaBackendUnity", "OS::TripleO::Services::ManilaBackendVMAX", "OS::TripleO::Services::ManilaBackendVNX", "OS::TripleO::Services::ManilaScheduler", "OS::TripleO::Services::ManilaShare", "OS::TripleO::Services::MasqueradeNetworks", "OS::TripleO::Services::Memcached", "OS::TripleO::Services::MetricsQdr", "OS::TripleO::Services::Multipathd", "OS::TripleO::Services::MySQL", "OS::TripleO::Services::MySQLClient", "OS::TripleO::Services::NeutronApi", "OS::TripleO::Services::NeutronBgpVpnApi", "OS::TripleO::Services::NeutronBgpVpnBagpipe", "OS::TripleO::Services::NeutronCorePlugin", "OS::TripleO::Services::NeutronL2gwAgent", "OS::TripleO::Services::NeutronL2gwApi", "OS::TripleO::Services::NeutronL3Agent", "OS::TripleO::Services::NeutronLinuxbridgeAgent", "OS::TripleO::Services::NeutronMetadataAgent", "OS::TripleO::Services::NeutronOvsAgent", "OS::TripleO::Services::NeutronSfcApi", "OS::TripleO::Services::NeutronSriovAgent", "OS::TripleO::Services::NeutronDhcpAgent", "OS::TripleO::Services::NeutronVppAgent", "OS::TripleO::Services::NovaApi", "OS::TripleO::Services::NovaConductor", "OS::TripleO::Services::NovaIronic", "OS::TripleO::Services::NovaMetadata", "OS::TripleO::Services::NovaScheduler", "OS::TripleO::Services::NovaCompute", "OS::TripleO::Services::NovaLibvirt", "OS::TripleO::Services::NovaMigrationTarget", "OS::TripleO::Services::NovaVncProxy", "OS::TripleO::Services::OVNController", "OS::TripleO::Services::OVNDBs", "OS::TripleO::Services::OVNMetadataAgent", "OS::TripleO::Services::OctaviaApi", "OS::TripleO::Services::OctaviaDeploymentConfig", "OS::TripleO::Services::OctaviaHealthManager", "OS::TripleO::Services::OctaviaHousekeeping", "OS::TripleO::Services::OctaviaWorker", "OS::TripleO::Services::OpenStackClients", "OS::TripleO::Services::OsloMessagingNotify", "OS::TripleO::Services::OsloMessagingRpc", "OS::TripleO::Services::Pacemaker", "OS::TripleO::Services::PlacementApi", "OS::TripleO::Services::Podman", "OS::TripleO::Services::Redis", "OS::TripleO::Services::Rhsm", "OS::TripleO::Services::Rsyslog", "OS::TripleO::Services::RsyslogSidecar", "OS::TripleO::Services::Securetty", "OS::TripleO::Services::Snmp", "OS::TripleO::Services::Sshd", "OS::TripleO::Services::SwiftDispersion", "OS::TripleO::Services::SwiftProxy", "OS::TripleO::Services::SwiftRingBuilder", "OS::TripleO::Services::SwiftStorage", "OS::TripleO::Services::Timesync", "OS::TripleO::Services::Timezone", "OS::TripleO::Services::Tmpwatch", "OS::TripleO::Services::TripleoFirewall", "OS::TripleO::Services::TripleoPackages", "OS::TripleO::Services::Unbound", "OS::TripleO::Services::Tuned", "OS::TripleO::Services::Vpp"]}], "tripleo_container_image_prepare_content": {"parameter_defaults": {"ContainerImagePrepare": [{"set": {"ceph_alertmanager_image": "ose-prometheus-alertmanager", "ceph_alertmanager_namespace": "registry.redhat.io/openshift4", "ceph_alertmanager_tag": "v4.12", "ceph_grafana_image": "rhceph-6-dashboard-rhel9", "ceph_grafana_namespace": "registry.redhat.io/rhceph", "ceph_grafana_tag": "latest", "ceph_image": "rhceph-7-rhel9", "ceph_namespace": "registry.redhat.io/rhceph", "ceph_node_exporter_image": "ose-prometheus-node-exporter", "ceph_node_exporter_namespace": "registry.redhat.io/openshift4", "ceph_node_exporter_tag": "v4.12", "ceph_prometheus_image": "ose-prometheus", "ceph_prometheus_namespace": "registry.redhat.io/openshift4", "ceph_prometheus_tag": "v4.12", "ceph_tag": "latest", "name_prefix": "openstack-", "name_suffix": "", "namespace": "registry.redhat.io/rhosp-rhel9", "neutron_driver": "ovn", "rhel_containers": false, "tag": "17.1"}, "tag_from_label": "{version}-{release}"}], "DockerInsecureRegistryAddress": ["192.168.122.100:8787"], "NeutronMechanismDrivers": ["sriovnicswitch", "ovn"], "DockerRegistryMirror": "", "ContainerImageRegistryCredentials": {}, "AdditionalArchitectures": [], "StandaloneContainerImagePrepare": {}, "StandaloneServices": ["OS::TripleO::Services::BootParams", "OS::TripleO::Services::BarbicanApi", "OS::TripleO::Services::BarbicanBackendSimpleCrypto", "OS::TripleO::Services::CACerts", "OS::TripleO::Services::CephClient", "OS::TripleO::Services::CephMds", "OS::TripleO::Services::CephMgr", "OS::TripleO::Services::CephMon", "OS::TripleO::Services::CephOSD", "OS::TripleO::Services::CinderApi", "OS::TripleO::Services::CinderBackup", "OS::TripleO::Services::CinderScheduler", "OS::TripleO::Services::CinderVolume", "OS::TripleO::Services::Clustercheck", "OS::TripleO::Services::ContainerImagePrepare", "OS::TripleO::Services::ContainersLogrotateCrond", "OS::TripleO::Services::DockerRegistry", "OS::TripleO::Services::GlanceApi", "OS::TripleO::Services::GlanceApiInternal", "OS::TripleO::Services::HAproxy", "OS::TripleO::Services::HeatApi", "OS::TripleO::Services::HeatApiCfn", "OS::TripleO::Services::HeatEngine", "OS::TripleO::Services::Horizon", "OS::TripleO::Services::Iscsid", "OS::TripleO::Services::Kernel", "OS::TripleO::Services::Keystone", "OS::TripleO::Services::ManilaApi", "OS::TripleO::Services::ManilaBackendCephFs", "OS::TripleO::Services::ManilaScheduler", "OS::TripleO::Services::ManilaShare", "OS::TripleO::Services::Memcached", "OS::TripleO::Services::MySQL", "OS::TripleO::Services::MySQLClient", "OS::TripleO::Services::NeutronApi", "OS::TripleO::Services::NeutronCorePlugin", "OS::TripleO::Services::NeutronSriovAgent", "OS::TripleO::Services::NeutronDhcpAgent", "OS::TripleO::Services::NovaApi", "OS::TripleO::Services::NovaConductor", "OS::TripleO::Services::NovaMetadata", "OS::TripleO::Services::NovaScheduler", "OS::TripleO::Services::NovaCompute", "OS::TripleO::Services::NovaLibvirt", "OS::TripleO::Services::NovaMigrationTarget", "OS::TripleO::Services::NovaVncProxy", "OS::TripleO::Services::OVNController", "OS::TripleO::Services::OVNDBs", "OS::TripleO::Services::OVNMetadataAgent", "OS::TripleO::Services::OpenStackClients", "OS::TripleO::Services::OsloMessagingNotify", "OS::TripleO::Services::OsloMessagingRpc", "OS::TripleO::Services::Pacemaker", "OS::TripleO::Services::PlacementApi", "OS::TripleO::Services::Podman", "OS::TripleO::Services::Rsyslog", "OS::TripleO::Services::Snmp", "OS::TripleO::Services::Sshd", "OS::TripleO::Services::SwiftProxy", "OS::TripleO::Services::SwiftRingBuilder", "OS::TripleO::Services::SwiftStorage", "OS::TripleO::Services::Timesync", "OS::TripleO::Services::Timezone", "OS::TripleO::Services::Tmpwatch", "OS::TripleO::Services::TripleoFirewall", "OS::TripleO::Services::TripleoPackages", "OS::TripleO::Services::Tuned"], "StandaloneCount": 1}}}, "when": ["(step | int) == 1"]}]}, {"service_name": "logrotate_crond", "config_settings": {"tripleo::profile::base::logging::logrotate::maxsize": "10M", "tripleo::profile::base::logging::logrotate::rotation": "daily", "tripleo::profile::base::logging::logrotate::rotate": "14", "tripleo::profile::base::logging::logrotate::purge_after_days": "14", "tripleo::profile::base::logging::logrotate::dateext": false}, "host_prep_tasks": [{"name": "allow logrotate to read inside containers", "seboolean": {"name": "logrotate_read_inside_containers", "persistent": true, "state": true}, "when": ["ansible_facts.selinux is defined", "ansible_facts.selinux.status == \"enabled\""]}], "deploy_steps_tasks": [{"name": "configure tmpwatch on the host", "when": "step|int == 2", "block": [{"name": "Push script", "copy": {"dest": "/usr/local/sbin/containers-tmpwatch", "owner": "root", "group": "root", "mode": 493, "content": "#!/bin/sh\ntmpwatch --nodirs \\\n  -X \"/var/log/containers/*/*log\" \\\n  -X \"/var/log/containers/*/*/*log\" \\\n  -X \"/var/log/containers/*/*err\" \\\n  {{ LogrotatePurgeAfterDays|int +1 }}d \\\n  /var/log/containers/ 2>&1 | logger -t container-tmpwatch\n"}, "vars": {"LogrotatePurgeAfterDays": "14"}}, {"name": "Insert cronjob in root crontab", "cron": {"name": "Remove old logs", "special_time": "daily", "user": "root", "job": "/usr/local/sbin/containers-tmpwatch"}}]}], "update_tasks": [{"name": "Ensure old cron.daily is absent", "when": "step|int == 1", "file": {"path": "/etc/cron.daily/containers-tmpwatch", "state": "absent"}}], "upgrade_tasks": [{"name": "Ensure old cron.daily is absent", "when": "step|int == 1", "file": {"path": "/etc/cron.daily/containers-tmpwatch", "state": "absent"}}], "puppet_config": {"config_volume": "crond", "step_config": "include tripleo::profile::base::logging::logrotate", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-cron:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/logrotate-crond.json": {"command": "/usr/sbin/crond -s -n", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}}, "docker_config": {"step_4": {"logrotate_crond": {"image": "registry.redhat.io/rhosp-rhel9/openstack-cron:17.1", "net": "none", "pid": "host", "privileged": true, "user": "root", "restart": "always", "healthcheck": {"test": "/usr/share/openstack-tripleo-common/healthcheck/cron"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/logrotate-crond.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/crond:/var/lib/kolla/config_files/src:ro", "/var/log/containers:/var/log/containers:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop logrotate container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["logrotate_crond"], "tripleo_delegate_to": "{{ groups['logrotate_crond'] | difference(groups['excluded_overcloud']) }}"}}]}]}, {"service_name": "docker_registry", "firewall_rules": {"155 docker-registry": {"dport": [8787]}}, "firewall_frontend_rules": {"100 docker_registry_haproxy_frontend": {"dport": [8787]}}, "firewall_ssl_frontend_rules": {"100 docker_registry_haproxy_frontend_ssl": {"dport": [13787]}}, "host_prep_tasks": [{"name": "authorize httpd to listen on registry ports", "community.general.seport": {"ports": ["8787", "13787"], "proto": "tcp", "setype": "http_port_t", "state": "present"}}, {"name": "Install, Configure and Run Apache to serve container images", "block": [{"set_fact": {"container_registry_host": "192.168.122.99", "container_registry_port": "8787"}}, {"include_role": {"name": "tripleo_image_serve"}}]}], "upgrade_tasks": [{"name": "Uninstall docker-distribution", "when": "step|int == 3", "block": [{"name": "check if docker-distribution is installed", "command": "/usr/bin/rpm -q docker-distribution", "failed_when": false, "register": "docker_distribution_installed", "check_mode": false}, {"name": "Stop, disable docker-distribution", "systemd": {"enabled": false, "state": "stopped", "name": "docker-distribution"}, "when": "docker_distribution_installed.rc == 0"}, {"name": "Uninstall docker-distribution", "package": "name=docker-distribution state=absent", "when": "docker_distribution_installed.rc == 0"}]}]}, {"service_name": "glance_api", "firewall_rules": {"112 glance_api": {"dport": [9292]}}, "firewall_frontend_rules": {"100 glance_api_haproxy_frontend": {"dport": [9292]}}, "firewall_ssl_frontend_rules": {"100 glance_api_haproxy_frontend_ssl": {"dport": [13292]}}, "keystone_resources": {"glance": {"endpoints": {"public": "http://172.21.0.2:9292", "internal": "http://172.17.0.2:9293", "admin": "http://172.17.0.2:9293"}, "users": {"glance": {"password": "PmWCSVDW3B2uUjMQtJIVdd5mn"}}, "region": "regionOne", "service": "image"}}, "monitoring_subscription": "overcloud-glance-api", "config_settings": {"glance::api::db::database_connection": "mysql+pymysql://glance:PmWCSVDW3B2uUjMQtJIVdd5mn@172.17.0.2/glance?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "glance::api::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "glance::api::authtoken::auth_url": "http://172.17.0.2:5000", "glance::api::enable_v1_api": false, "glance::api::enable_v2_api": true, "glance::api::authtoken::password": "PmWCSVDW3B2uUjMQtJIVdd5mn", "glance::api::enable_proxy_headers_parsing": true, "glance::api::logging::debug": true, "glance::policy::policies": {}, "glance::api::authtoken::project_name": "service", "glance::api::authtoken::region_name": "regionOne", "glance::api::authtoken::user_domain_name": "Default", "glance::api::authtoken::project_domain_name": "Default", "glance::api::authtoken::interface": "internal", "glance::api::authtoken::memcache_use_advanced_pool": true, "glance::api::pipeline": "keystone", "glance::api::os_region_name": "regionOne", "glance::api::image_member_quota": 128, "glance::api::enabled_import_methods": ["web-download"], "glance::api::node_staging_uri": "file:///var/lib/glance/staging", "glance::api::image_import_plugins": ["no_op", "image_conversion"], "glance::api::image_conversion_output_format": "raw", "glance::api::inject_metadata_properties": [], "glance::api::ignore_user_roles": ["admin"], "tripleo::profile::base::glance::api::tls_proxy_bind_ip": "%{hiera('internal_api')}", "tripleo::profile::base::glance::api::tls_proxy_fqdn": "%{hiera('fqdn_internal_api')}", "tripleo::profile::base::glance::api::tls_proxy_port": 9292, "glance::api::bind_host": "%{hiera('internal_api')}", "glance_notifier_strategy": "noop", "glance_log_file": "", "glance::backend::rbd::rbd_thin_provisioning": false, "glance::backend::file::filesystem_thin_provisioning": false, "glance::backend::swift::swift_store_auth_address": "http://172.17.0.2:5000/v3", "glance::backend::swift::swift_store_user": "service:glance", "glance::backend::swift::swift_store_key": "PmWCSVDW3B2uUjMQtJIVdd5mn", "glance::backend::swift::swift_store_create_container_on_put": true, "glance::backend::swift::swift_store_auth_version": 3, "glance::backend::rbd::rbd_store_ceph_conf": "/etc/ceph/ceph.conf", "glance::backend::rbd::rbd_store_pool": "images", "glance::backend::rbd::rbd_store_user": "openstack", "glance_backend": "rbd", "tripleo::profile::base::glance::api::glance_backend_id": "default_backend", "tripleo::profile::base::glance::api::glance_store_description": "Default glance store backend.", "tripleo::profile::base::glance::api::multistore_config": {}, "tripleo::profile::base::glance::backend::rbd::glance_rbd_ceph_conf_path": "/var/lib/tripleo-config/ceph", "glance::notify::rabbitmq::notification_driver": ["noop"], "glance::notify::rabbitmq::rabbit_heartbeat_in_pthread": false, "glance::cron::db_purge::minute": "1", "glance::cron::db_purge::hour": "0", "glance::cron::db_purge::monthday": "*", "glance::cron::db_purge::month": "*", "glance::cron::db_purge::weekday": "*", "glance::cron::db_purge::user": "glance", "glance::cron::db_purge::age": "30", "glance::cron::db_purge::max_rows": "100", "glance::cron::db_purge::destination": "/var/log/glance/glance-rowsflush.log", "glance::cron::db_purge::maxdelay": "3600", "glance::api::workers": "1"}, "service_config_settings": {"mysql": {"glance::db::mysql::password": "PmWCSVDW3B2uUjMQtJIVdd5mn", "glance::db::mysql::user": "glance", "glance::db::mysql::host": "%", "glance::db::mysql::dbname": "glance"}, "rsyslog": {"tripleo_logging_sources_glance_api": [{"tag": "openstack.glance.api", "file": "/var/log/containers/glance/api.log"}]}}, "puppet_config": {"config_volume": "glance_api", "puppet_tags": "glance_api_config,glance_api_paste_ini,glance_swift_config,glance_cache_config,glance_image_import_config", "step_config": "include tripleo::profile::base::glance::api\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-glance-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/glance_api.json": {"command": "/usr/bin/glance-api --config-file /usr/share/glance/glance-api-dist.conf --config-file /etc/glance/glance-api.conf --config-file /etc/glance/glance-image-import.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-ceph/", "dest": "/etc/ceph/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-iscsid/*", "dest": "/etc/iscsi/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/lib/glance", "owner": "glance:glance", "recurse": true}, {"path": "/etc/ceph/ceph.client.openstack.keyring", "owner": "glance:glance", "perm": "0600"}]}, "/var/lib/kolla/config_files/glance_api_tls_proxy.json": {"command": "/usr/sbin/httpd -DFOREGROUND", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/glance_api_cron.json": {"command": "/usr/sbin/crond -n", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/glance", "owner": "glance:glance", "recurse": true}]}}, "docker_config": {"step_2": {"glance_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-glance-api:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/glance:/var/log/glance:z", "/var/log/containers/httpd/glance:/var/log/httpd:z"], "command": ["/bin/bash", "-c", "chown -R glance:glance /var/log/glance"]}}, "step_3": {"glance_api_db_sync": {"image": "registry.redhat.io/rhosp-rhel9/openstack-glance-api:17.1", "cap_add": ["AUDIT_WRITE"], "net": "host", "privileged": false, "detach": false, "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/glance:/var/log/glance:z", "/var/log/containers/httpd/glance:/var/log/httpd:z", "/var/lib/kolla/config_files/glance_api.json:/var/lib/kolla/config_files/config.json", "/var/lib/config-data/puppet-generated/glance_api:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/config-data/puppet-generated/iscsid/etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro", "/var/lib/glance:/var/lib/glance:shared"], "environment": {"KOLLA_BOOTSTRAP": true, "KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}, "command": "/usr/bin/bootstrap_host_exec glance_api su glance -s /bin/bash -c '/usr/local/bin/kolla_start'"}}, "step_4": {"glance_api": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-glance-api:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/glance:/var/log/glance:z", "/var/log/containers/httpd/glance:/var/log/httpd:z", "/var/lib/kolla/config_files/glance_api.json:/var/lib/kolla/config_files/config.json", "/var/lib/config-data/puppet-generated/glance_api:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/config-data/puppet-generated/iscsid/etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro", "/var/lib/glance:/var/lib/glance:shared"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "glance_api_cron": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-glance-api:17.1", "net": "host", "user": "root", "privileged": false, "restart": "always", "healthcheck": {"test": "/usr/share/openstack-tripleo-common/healthcheck/cron glance"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/glance:/var/log/glance:z", "/var/log/containers/httpd/glance:/var/log/httpd:z", "/var/lib/kolla/config_files/glance_api_cron.json:/var/lib/kolla/config_files/config.json", "/var/lib/config-data/puppet-generated/glance_api:/var/lib/kolla/config_files/src:ro", "/var/lib/glance:/var/lib/glance:shared"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "host_prep_tasks": [{"name": "create persistent logs directory", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/glance", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/log/containers/httpd/glance", "setype": "container_file_t", "mode": "0750"}]}, {"name": "Mount NFS on host", "vars": {"nfs_backend_enabled": false, "glance_netapp_nfs_enabled": false, "glance_nfs_share": "", "netapp_share_location": "", "nfs_share": "{{ glance_nfs_share if (glance_nfs_share) else netapp_share_location }}", "nfs_options": "_netdev,bg,intr,context=system_u:object_r:container_file_t:s0"}, "mount": {"name": "/var/lib/glance/images", "state": "mounted", "fstype": "nfs", "src": "{{nfs_share}}", "opts": "{{nfs_options}}"}, "when": "nfs_backend_enabled or glance_netapp_nfs_enabled"}, {"name": "Mount Node Staging Location", "vars": {"glance_node_staging_uri": "file:///var/lib/glance/staging", "glance_staging_nfs_share": "", "glance_nfs_options": "_netdev,bg,intr,context=system_u:object_r:container_file_t:s0"}, "mount": {"name": "{{glance_node_staging_uri[7:]}}", "state": "mounted", "fstype": "nfs", "src": "{{glance_staging_nfs_share}}", "opts": "{{glance_nfs_options}}"}, "when": "glance_staging_nfs_share != ''"}, {"name": "ensure ceph configurations exist", "file": {"path": "/var/lib/tripleo-config/ceph", "state": "directory"}}, {"name": "ensure /var/lib/glance exists", "file": {"path": "/var/lib/glance", "state": "directory", "setype": "container_file_t"}}], "upgrade_tasks": [{"name": "glance-api pre system_upgrade tasks", "tags": ["never", "system_upgrade", "system_upgrade_run", "system_upgrade_nfsmounts"], "when": ["step|int == 3", "upgrade_leapp_enabled"], "block": [{"name": "Unmount and remove NFS glance entry", "vars": {"nfs_backend_enabled": false, "glance_netapp_nfs_enabled": false}, "mount": {"name": "/var/lib/glance/images", "fstype": "nfs", "state": "absent"}, "when": "nfs_backend_enabled or glance_netapp_nfs_enabled"}, {"name": "Unmount and remove NFS glance-staging entry", "vars": {"glance_node_staging_uri": "file:///var/lib/glance/staging", "glance_staging_nfs_share": ""}, "mount": {"name": "{{glance_node_staging_uri[7:]}}", "fstype": "nfs", "state": "absent"}, "when": "glance_staging_nfs_share != ''"}]}, {"name": "glance-api post system_upgrade tasks", "tags": ["never", "system_upgrade", "system_upgrade_run", "system_upgrade_nfsmounts"], "when": ["step|int == 5", "upgrade_leapp_enabled"], "block": [{"name": "Mount NFS on host", "vars": {"nfs_backend_enabled": false, "glance_netapp_nfs_enabled": false, "glance_nfs_share": "", "netapp_share_location": "", "nfs_share": "{{ glance_nfs_share if (glance_nfs_share) else netapp_share_location }}", "nfs_options": "_netdev,bg,intr,context=system_u:object_r:container_file_t:s0"}, "mount": {"name": "/var/lib/glance/images", "state": "mounted", "fstype": "nfs", "src": "{{nfs_share}}", "opts": "{{nfs_options}}"}, "when": "nfs_backend_enabled or glance_netapp_nfs_enabled"}, {"name": "Mount Node Staging Location", "vars": {"glance_node_staging_uri": "file:///var/lib/glance/staging", "glance_staging_nfs_share": "", "glance_nfs_options": "_netdev,bg,intr,context=system_u:object_r:container_file_t:s0"}, "mount": {"name": "{{glance_node_staging_uri[7:]}}", "state": "mounted", "fstype": "nfs", "src": "{{glance_staging_nfs_share}}", "opts": "{{glance_nfs_options}}"}, "when": "glance_staging_nfs_share != ''"}]}], "metadata_settings": null, "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop glance api container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["glance_api", "glance_api_cron"], "tripleo_delegate_to": "{{ groups['glance_api'] | difference(groups['excluded_overcloud']) }}"}}]}], "deploy_steps_tasks": null}, {"service_name": "glance_api_internal", "firewall_rules": {"112 glance_api_internal": {"dport": ["9293"]}}, "firewall_frontend_rules": {"100 glance_api_internal_haproxy_frontend": {"dport": ["9293"]}}, "firewall_ssl_frontend_rules": {"100 glance_api_haproxy_frontend_ssl": {"dport": [13292]}}, "keystone_resources": {}, "monitoring_subscription": "overcloud-glance-api", "config_settings": {"glance::api::db::database_connection": "mysql+pymysql://glance:PmWCSVDW3B2uUjMQtJIVdd5mn@172.17.0.2/glance?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "glance::api::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "glance::api::authtoken::auth_url": "http://172.17.0.2:5000", "glance::api::enable_v1_api": false, "glance::api::enable_v2_api": true, "glance::api::authtoken::password": "PmWCSVDW3B2uUjMQtJIVdd5mn", "glance::api::enable_proxy_headers_parsing": true, "glance::api::logging::debug": true, "glance::policy::policies": {}, "glance::api::authtoken::project_name": "service", "glance::api::authtoken::region_name": "regionOne", "glance::api::authtoken::user_domain_name": "Default", "glance::api::authtoken::project_domain_name": "Default", "glance::api::authtoken::interface": "internal", "glance::api::authtoken::memcache_use_advanced_pool": true, "glance::api::pipeline": "keystone", "glance::api::os_region_name": "regionOne", "glance::api::image_member_quota": 128, "glance::api::enabled_import_methods": ["web-download"], "glance::api::node_staging_uri": "file:///var/lib/glance/staging", "glance::api::image_import_plugins": ["no_op", "image_conversion"], "glance::api::image_conversion_output_format": "raw", "glance::api::inject_metadata_properties": [], "glance::api::ignore_user_roles": ["admin"], "tripleo::profile::base::glance::api::tls_proxy_bind_ip": "%{hiera('internal_api')}", "tripleo::profile::base::glance::api::tls_proxy_fqdn": "%{hiera('fqdn_internal_api')}", "tripleo::profile::base::glance::api::tls_proxy_port": 9292, "glance::api::bind_host": "%{hiera('internal_api')}", "glance_notifier_strategy": "noop", "glance_log_file": "", "glance::backend::rbd::rbd_thin_provisioning": false, "glance::backend::file::filesystem_thin_provisioning": false, "glance::backend::swift::swift_store_auth_address": "http://172.17.0.2:5000/v3", "glance::backend::swift::swift_store_user": "service:glance", "glance::backend::swift::swift_store_key": "PmWCSVDW3B2uUjMQtJIVdd5mn", "glance::backend::swift::swift_store_create_container_on_put": true, "glance::backend::swift::swift_store_auth_version": 3, "glance::backend::rbd::rbd_store_ceph_conf": "/etc/ceph/ceph.conf", "glance::backend::rbd::rbd_store_pool": "images", "glance::backend::rbd::rbd_store_user": "openstack", "glance_backend": "rbd", "tripleo::profile::base::glance::api::glance_backend_id": "default_backend", "tripleo::profile::base::glance::api::glance_store_description": "Default glance store backend.", "tripleo::profile::base::glance::api::multistore_config": {}, "tripleo::profile::base::glance::backend::rbd::glance_rbd_ceph_conf_path": "/var/lib/tripleo-config/ceph", "glance::notify::rabbitmq::notification_driver": ["noop"], "glance::notify::rabbitmq::rabbit_heartbeat_in_pthread": false, "glance::cron::db_purge::minute": "1", "glance::cron::db_purge::hour": "0", "glance::cron::db_purge::monthday": "*", "glance::cron::db_purge::month": "*", "glance::cron::db_purge::weekday": "*", "glance::cron::db_purge::user": "glance", "glance::cron::db_purge::age": "30", "glance::cron::db_purge::max_rows": "100", "glance::cron::db_purge::destination": "/var/log/glance/glance-rowsflush.log", "glance::cron::db_purge::maxdelay": "3600", "glance::api::workers": "1"}, "service_config_settings": {"mysql": {"glance::db::mysql::password": "PmWCSVDW3B2uUjMQtJIVdd5mn", "glance::db::mysql::user": "glance", "glance::db::mysql::host": "%", "glance::db::mysql::dbname": "glance"}, "rsyslog": {"tripleo_logging_sources_glance_api_internal": [{"tag": "openstack.glance.api", "file": "/var/log/containers/glance/api_internal.log"}]}}, "puppet_config": {"config_volume": "glance_api_internal", "puppet_tags": "glance_api_config,glance_api_paste_ini,glance_swift_config,glance_cache_config,glance_image_import_config", "step_config": "class { 'tripleo::profile::base::glance::api':\n  bind_port => 9293,\n  tls_proxy_port => 9293,\n  log_file => '/var/log/glance/api_internal.log',\n  show_image_direct_url => true,\n  show_multiple_locations => true,\n}\n\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-glance-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/glance_api.json": {"command": "/usr/bin/glance-api --config-file /usr/share/glance/glance-api-dist.conf --config-file /etc/glance/glance-api.conf --config-file /etc/glance/glance-image-import.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-ceph/", "dest": "/etc/ceph/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-iscsid/*", "dest": "/etc/iscsi/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/lib/glance", "owner": "glance:glance", "recurse": true}, {"path": "/etc/ceph/ceph.client.openstack.keyring", "owner": "glance:glance", "perm": "0600"}]}, "/var/lib/kolla/config_files/glance_api_tls_proxy.json": {"command": "/usr/sbin/httpd -DFOREGROUND", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/glance_api_cron.json": {"command": "/usr/sbin/crond -n", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/glance", "owner": "glance:glance", "recurse": true}]}}, "docker_config": {"step_2": {"glance_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-glance-api:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/glance:/var/log/glance:z", "/var/log/containers/httpd/glance:/var/log/httpd:z"], "command": ["/bin/bash", "-c", "chown -R glance:glance /var/log/glance"]}}, "step_4": {"glance_api_internal": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-glance-api:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/glance:/var/log/glance:z", "/var/log/containers/httpd/glance:/var/log/httpd:z", "/var/lib/kolla/config_files/glance_api.json:/var/lib/kolla/config_files/config.json", "/var/lib/config-data/puppet-generated/glance_api_internal:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/config-data/puppet-generated/iscsid/etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro", "/var/lib/glance:/var/lib/glance:shared"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "host_prep_tasks": [{"name": "create persistent logs directory", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/glance", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/log/containers/httpd/glance", "setype": "container_file_t", "mode": "0750"}]}, {"name": "Mount NFS on host", "vars": {"nfs_backend_enabled": false, "glance_netapp_nfs_enabled": false, "glance_nfs_share": "", "netapp_share_location": "", "nfs_share": "{{ glance_nfs_share if (glance_nfs_share) else netapp_share_location }}", "nfs_options": "_netdev,bg,intr,context=system_u:object_r:container_file_t:s0"}, "mount": {"name": "/var/lib/glance/images", "state": "mounted", "fstype": "nfs", "src": "{{nfs_share}}", "opts": "{{nfs_options}}"}, "when": "nfs_backend_enabled or glance_netapp_nfs_enabled"}, {"name": "Mount Node Staging Location", "vars": {"glance_node_staging_uri": "file:///var/lib/glance/staging", "glance_staging_nfs_share": "", "glance_nfs_options": "_netdev,bg,intr,context=system_u:object_r:container_file_t:s0"}, "mount": {"name": "{{glance_node_staging_uri[7:]}}", "state": "mounted", "fstype": "nfs", "src": "{{glance_staging_nfs_share}}", "opts": "{{glance_nfs_options}}"}, "when": "glance_staging_nfs_share != ''"}, {"name": "ensure ceph configurations exist", "file": {"path": "/var/lib/tripleo-config/ceph", "state": "directory"}}, {"name": "ensure /var/lib/glance exists", "file": {"path": "/var/lib/glance", "state": "directory", "setype": "container_file_t"}}], "upgrade_tasks": [{"name": "glance-api pre system_upgrade tasks", "tags": ["never", "system_upgrade", "system_upgrade_run", "system_upgrade_nfsmounts"], "when": ["step|int == 3", "upgrade_leapp_enabled"], "block": [{"name": "Unmount and remove NFS glance entry", "vars": {"nfs_backend_enabled": false, "glance_netapp_nfs_enabled": false}, "mount": {"name": "/var/lib/glance/images", "fstype": "nfs", "state": "absent"}, "when": "nfs_backend_enabled or glance_netapp_nfs_enabled"}, {"name": "Unmount and remove NFS glance-staging entry", "vars": {"glance_node_staging_uri": "file:///var/lib/glance/staging", "glance_staging_nfs_share": ""}, "mount": {"name": "{{glance_node_staging_uri[7:]}}", "fstype": "nfs", "state": "absent"}, "when": "glance_staging_nfs_share != ''"}]}, {"name": "glance-api post system_upgrade tasks", "tags": ["never", "system_upgrade", "system_upgrade_run", "system_upgrade_nfsmounts"], "when": ["step|int == 5", "upgrade_leapp_enabled"], "block": [{"name": "Mount NFS on host", "vars": {"nfs_backend_enabled": false, "glance_netapp_nfs_enabled": false, "glance_nfs_share": "", "netapp_share_location": "", "nfs_share": "{{ glance_nfs_share if (glance_nfs_share) else netapp_share_location }}", "nfs_options": "_netdev,bg,intr,context=system_u:object_r:container_file_t:s0"}, "mount": {"name": "/var/lib/glance/images", "state": "mounted", "fstype": "nfs", "src": "{{nfs_share}}", "opts": "{{nfs_options}}"}, "when": "nfs_backend_enabled or glance_netapp_nfs_enabled"}, {"name": "Mount Node Staging Location", "vars": {"glance_node_staging_uri": "file:///var/lib/glance/staging", "glance_staging_nfs_share": "", "glance_nfs_options": "_netdev,bg,intr,context=system_u:object_r:container_file_t:s0"}, "mount": {"name": "{{glance_node_staging_uri[7:]}}", "state": "mounted", "fstype": "nfs", "src": "{{glance_staging_nfs_share}}", "opts": "{{glance_nfs_options}}"}, "when": "glance_staging_nfs_share != ''"}]}], "metadata_settings": null, "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop glance api internal container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["glance_api_internal"], "tripleo_delegate_to": "{{ groups['glance_api_internal'] | default([]) }}"}}]}], "deploy_steps_tasks": null}, {"service_name": "haproxy", "firewall_rules": {"107 haproxy stats": {"dport": 1993}}, "monitoring_subscription": "overcloud-haproxy", "ansible_group_vars": {"tripleo_firewall_frontend_enabled": true}, "config_settings": {"tripleo::haproxy::haproxy_service_manage": false, "tripleo::haproxy::haproxy_log_address": "/dev/log", "tripleo::haproxy::haproxy_log_facility": "local0", "tripleo::haproxy::haproxy_stats_user": "admin", "tripleo::haproxy::haproxy_stats_password": "1BkjVM1bzatAC1q8QQRKR2F3K", "tripleo::haproxy::haproxy_stats_bind_address": "%{hiera('ctlplane')}", "tripleo::haproxy::redis_password": "SxsXCWFxPxaSD4h7eclNJxxIm", "tripleo::haproxy::crl_file": null, "tripleo::haproxy::haproxy_stats": true, "enable_load_balancer": true, "tripleo::profile::base::haproxy::certificates_specs": {}, "tripleo::haproxy::mysql_clustercheck": true, "haproxy_docker": true, "tripleo::profile::pacemaker::haproxy_bundle::container_backend": "podman", "tripleo::profile::pacemaker::haproxy_bundle::tls_mapping": [], "tripleo::profile::pacemaker::haproxy_bundle::tls_mapping_init_bundle": [], "tripleo::profile::pacemaker::haproxy_bundle::internal_certs_directory": "/etc/pki/tls/certs/haproxy", "tripleo::profile::pacemaker::haproxy_bundle::internal_keys_directory": "/etc/pki/tls/private/haproxy", "tripleo::profile::pacemaker::haproxy_bundle::haproxy_docker_image": "cluster.common.tag/haproxy:pcmklatest"}, "service_config_settings": {"rsyslog": {"tripleo_logging_sources_haproxy": [{"tag": "openstack.haproxy", "file": "/var/log/containers/haproxy/haproxy.log", "startmsg.regex": "^[a-zA-Z]{3} [ 123][0-9] [:0-9]{8}"}]}}, "puppet_config": {"config_volume": "haproxy", "puppet_tags": "haproxy_config", "step_config": "exec {'wait-for-settle': command => '/bin/true' }\nclass tripleo::firewall(){}; define tripleo::firewall::rule( $port = undef, $dport = undef, $sport = undef, $proto = undef, $action = undef, $state = undef, $source = undef, $iniface = undef, $chain = undef, $destination = undef, $extras = undef){}\n['pcmk_bundle', 'pcmk_resource', 'pcmk_property', 'pcmk_constraint', 'pcmk_resource_default'].each |String $val| { noop_resource($val) }\ninclude tripleo::profile::pacemaker::haproxy_bundle", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-haproxy:17.1", "volumes": []}, "kolla_config": {"/var/lib/kolla/config_files/haproxy.json": {"command": "bash -c $* -- eval if [ -f /usr/sbin/haproxy-systemd-wrapper ]; then exec /usr/sbin/haproxy-systemd-wrapper -f /etc/haproxy/haproxy.cfg; else exec /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -Ws; fi", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true, "optional": true}, {"source": "/var/lib/kolla/config_files/src-tls/*", "dest": "/", "merge": true, "optional": true, "preserve_properties": true}], "permissions": [{"path": "/var/lib/haproxy", "owner": "haproxy:haproxy", "recurse": true}, {"path": "/etc/pki/tls/certs/haproxy/*", "owner": "haproxy:haproxy", "perm": "0600", "optional": true}, {"path": "/etc/pki/tls/private/haproxy/*", "owner": "haproxy:haproxy", "perm": "0600", "optional": true}]}}, "container_config_scripts": {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}}, "host_prep_tasks": [{"name": "get parameters", "set_fact": {"cert_path": "/etc/pki/tls/private/overcloud_endpoint.pem", "cert_content": "", "chain_content": "", "key_content": ""}, "no_log": "{{ hide_sensitive_logs | bool }}"}, {"name": "manage certificate", "when": ["cert_content is defined", "cert_content != ''"], "block": [{"name": "get DeployedSSLCertificatePath attributes", "register": "attr_cert_path", "stat": {"path": "{{cert_path}}"}}, {"name": "set is_haproxy_bootstrap_node fact", "set_fact": "is_haproxy_bootstrap_node={{haproxy_short_bootstrap_node_name | lower == ansible_facts['hostname'] | lower}}", "when": ["haproxy_short_bootstrap_node_name|default(false)"]}, {"name": "get haproxy status", "register": "haproxy_state", "systemd": {"name": "haproxy"}}, {"name": "get pacemaker status", "register": "pacemaker_state", "systemd": {"name": "pacemaker"}}, {"name": "get docker status", "register": "docker_state", "systemd": {"name": "docker"}}, {"name": "get container_id", "when": ["docker_state.status.ActiveState == 'active' or container_cli == 'podman'", "attr_cert_path.stat.exists", "attr_cert_path.stat.isdir == False"], "command": "{{ container_cli }} ps -q -f name='^haproxy(-bundle)?'", "register": "container_id"}, {"name": "get pcs resource name for haproxy container", "when": ["bootstrap_node is defined", "is_haproxy_bootstrap_node", "pacemaker_state.status.ActiveState == 'active'", "attr_cert_path.stat.exists", "attr_cert_path.stat.isdir"], "shell": "pcs status resources | sed -n 's/^.*container.*: \\(haproxy.*\\) .*/\\1/p'\n", "register": "pacemaker_resource"}, {"name": "remove DeployedSSLCertificatePath if is dir", "when": "attr_cert_path.stat.isdir is defined and attr_cert_path.stat.isdir", "file": {"path": "{{cert_path}}", "state": "absent"}}, {"name": "push certificate content", "no_log": "{{ hide_sensitive_logs | bool }}", "copy": {"dest": "{{cert_path}}", "mode": 288, "owner": "root", "content": "{{cert_content}}\n{{chain_content}}\n{{key_content}}\n"}}, {"name": "BM haproxy non-pacemaker context", "when": "haproxy_state.status.ActiveState == 'active'", "block": [{"name": "set certificate ownership", "file": {"path": "{{cert_path}}", "group": "haproxy"}}, {"name": "reload haproxy if enabled", "service": {"name": "haproxy", "state": "reloaded"}}]}, {"name": "restart pacemaker resource for haproxy", "when": ["pacemaker_resource is defined", "pacemaker_resource.stdout is defined", "pacemaker_resource.stdout != ''"], "command": "pcs resource restart \"{{pacemaker_resource.stdout}}\""}, {"name": "dedicated part for containers", "when": ["container_id is defined", "container_id.stdout is defined", "container_id.stdout != ''"], "block": [{"name": "copy certificate, chgrp, restart haproxy", "shell": "set -e\nif {{ container_cli }} ps -f \"id={{ item }}\" --format \"{{ '{{' }}.Names{{ '}}' }}\" | grep -q \"^haproxy-bundle\"; then\n  tar -c {{ cert_path }} | {{container_cli}} exec -i {{ item }} tar -C / -xv\nelse\n  {{ container_cli }} cp {{ cert_path }} {{ item }}:{{ cert_path }}\nfi\n{{ container_cli }} exec --user root {{ item }} chgrp haproxy {{ cert_path }}\n{{ container_cli }} kill --signal=HUP {{ item }}\n", "register": "container_kill_result", "failed_when": ["container_kill_result.rc != 0", "(\"no such container\" not in container_kill_result.stderr)", "(\"container state improper\" not in container_kill_result.stderr)"], "with_items": "{{ container_id.stdout.split('\n') }}"}]}]}, {"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode|default(omit) }}"}, "with_items": [{"path": "/var/log/containers/haproxy", "setype": "var_log_t", "mode": "0750"}, {"path": "/var/lib/haproxy", "setype": "container_file_t"}]}, {"name": "Run puppet on the host to apply IPtables rules", "no_log": true, "shell": "puppet apply {{ (puppet_debug|bool) | ternary('--debug --verbose', '') }} --detailed-exitcodes --summarize --color=false \\\n    --modulepath '{{ puppet_modulepath }}' --tags '{{ puppet_tags }}' -e '{{ puppet_execute }}'\n", "register": "puppet_host_outputs", "changed_when": "puppet_host_outputs.rc == 2", "failed_when": false, "vars": {"puppet_execute": "if hiera('enable_load_balancer', true) { class {'::tripleo::haproxy': use_internal_certificates => false, manage_firewall => hiera('tripleo::firewall::manage_firewall', true), }}", "puppet_tags": "tripleo::firewall::rule", "puppet_modulepath": "/etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules", "puppet_debug": false}}, {"name": "Debug output for task: Run puppet on the host to apply IPtables rules", "debug": {"var": "puppet_host_outputs.stdout_lines | default([]) | union(puppet_host_outputs.stderr_lines | default([]))"}, "when": ["not (ansible_check_mode | bool)", "puppet_host_outputs.rc is defined"], "failed_when": "puppet_host_outputs.rc not in [0, 2]"}], "metadata_settings": [], "deploy_steps_tasks": [{"name": "Configure rsyslog for HAproxy container managed by Pacemaker", "when": "step|int == 1", "block": [{"name": "Check if rsyslog exists", "shell": "systemctl is-active rsyslog", "register": "rsyslog_config"}, {"when": ["rsyslog_config is changed", "rsyslog_config.rc == 0"], "block": [{"name": "Forward logging to haproxy.log file", "blockinfile": {"content": "if $syslogfacility-text == '{{facility}}' and $programname == 'haproxy' then -/var/log/containers/haproxy/haproxy.log\n& stop\n", "create": true, "path": "/etc/rsyslog.d/openstack-haproxy.conf"}, "vars": {"facility": "local0"}, "register": "logconfig"}, {"name": "restart rsyslog service after logging conf change", "service": {"name": "rsyslog", "state": "restarted"}, "when": "logconfig is changed"}]}]}, {"name": "HAproxy tag container image for pacemaker", "when": "step|int == 1", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "registry.redhat.io/rhosp-rhel9/openstack-haproxy:17.1", "container_image_latest": "cluster.common.tag/haproxy:pcmklatest"}}, {"name": "HAproxy HA Wrappers Step", "when": "step|int == 2", "block": [{"name": "HAproxy puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "haproxy", "tripleo_ha_wrapper_resource_name": "haproxy-bundle", "tripleo_ha_wrapper_bundle_name": "haproxy-bundle", "tripleo_ha_wrapper_resource_state": "Started", "tripleo_ha_wrapper_puppet_config_volume": "haproxy", "tripleo_ha_wrapper_puppet_execute": "include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::haproxy_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ip,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".previous_run"}}]}], "external_update_tasks": [{"when": "step|int == 1", "tags": "ha_image_update", "block": [{"block": [{"name": "Get haproxy image from pacemaker", "become": true, "register": "xmllint_pcmk_haproxy_image", "shell": "xmllint --xpath \"string(//bundle[@id='haproxy-bundle']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container haproxy image", "set_fact": {"haproxy_image": "registry.redhat.io/rhosp-rhel9/openstack-haproxy:17.1", "haproxy_image_latest": "cluster.common.tag/haproxy:pcmklatest", "pcmk_haproxy_image": "{{xmllint_pcmk_haproxy_image.stdout}}"}}], "delegate_to": "{{ (groups[\"haproxy\"] | difference(groups[\"excluded_overcloud\"]))[0] }}"}, {"name": "haproxy temporary pacemaker container tag in case of image switch", "become": true, "import_role": {"name": "tripleo_ha_image_update"}, "vars": {"tripleo_ha_image_update_node_names": "{{ groups[\"haproxy\"] | difference(groups[\"excluded_overcloud\"]) }}", "tripleo_ha_image_update_bundle": "haproxy-bundle", "tripleo_ha_image_update_new_image": "{{haproxy_image_latest}}", "tripleo_ha_image_update_old_image": "{{pcmk_haproxy_image}}"}, "when": ["(pcmk_haproxy_image != \"\")", "(pcmk_haproxy_image != haproxy_image_latest)"]}]}], "update_tasks": [{"block": [{"name": "Get haproxy image from pacemaker", "become": true, "register": "xmllint_pcmk_haproxy_image", "shell": "xmllint --xpath \"string(//bundle[@id='haproxy-bundle']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container haproxy image", "set_fact": {"haproxy_image": "registry.redhat.io/rhosp-rhel9/openstack-haproxy:17.1", "haproxy_image_latest": "cluster.common.tag/haproxy:pcmklatest", "pcmk_haproxy_image": "{{xmllint_pcmk_haproxy_image.stdout}}"}}], "when": "(step|int == 0 or step|int == 2)"}, {"name": "Check for update of haproxy container image name", "when": "step|int == 0", "assert": {"that": "pcmk_haproxy_image == haproxy_image_latest", "fail_msg": "haproxy image change detected, run overcloud external-update --tags ha_image_update first"}}, {"name": "Tear-down non-HA haproxy container", "when": ["step|int == 1"], "block": [{"name": "Remove non-HA haproxy container", "include_role": {"name": "tripleo_container_rm"}, "vars": {"tripleo_container_cli": "{{ container_cli }}", "tripleo_containers_to_rm": ["haproxy"]}}]}, {"name": "Set HAProxy upgrade facts", "when": ["step|int == 1"], "block": [{"name": "set is_haproxy_bootstrap_node fact", "tags": "common", "set_fact": "is_haproxy_bootstrap_node={{haproxy_short_bootstrap_node_name|lower == ansible_facts['hostname']|lower}}", "when": ["haproxy_short_bootstrap_node_name|default(false)"]}]}, {"name": "Mount TLS cert if needed", "when": ["step|int == 1", "is_haproxy_bootstrap_node"], "block": [{"name": "Check haproxy public certificate configuration in pacemaker", "command": "cibadmin --query --xpath \"//storage-mapping[@id='haproxy-cert']\"", "failed_when": false, "register": "haproxy_cert_mounted"}, {"name": "Disable the haproxy cluster resource", "pacemaker_resource": {"resource": "haproxy-bundle", "state": "disable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0", "when": "haproxy_cert_mounted.rc == 6"}, {"name": "Set HAProxy public cert volume mount fact", "set_fact": {"haproxy_public_cert_path": "/etc/pki/tls/private/overcloud_endpoint.pem", "haproxy_public_tls_enabled": false}}, {"name": "Add a bind mount for public certificate in the haproxy bundle", "command": "pcs resource bundle update haproxy-bundle storage-map add id=haproxy-cert source-dir={{ haproxy_public_cert_path }} target-dir=/var/lib/kolla/config_files/src-tls/{{ haproxy_public_cert_path }} options=ro", "when": "haproxy_cert_mounted.rc == 6 and haproxy_public_tls_enabled|bool"}, {"name": "Enable the haproxy cluster resource", "pacemaker_resource": {"resource": "haproxy-bundle", "state": "enable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0", "when": "haproxy_cert_mounted.rc == 6"}]}, {"name": "Haproxy fetch and retag container image for pacemaker", "tags": "haproxy_syn_block", "when": ["step|int == 2"], "block": [{"name": "Retag pcmklatest to latest haproxy image", "include_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{haproxy_image}}", "container_image_latest": "{{haproxy_image_latest}}"}}]}, {"name": "Move virtual IPs to another node before stopping pacemaker", "when": ["step|int == 1", "hostvars[inventory_hostname][\"haproxy_node_names\"]|default([])|length > 1"], "shell": "CLUSTER_NODE=$(crm_node -n)\necho \"Retrieving all the VIPs which are hosted on this node\"\nVIPS_TO_MOVE=$(crm_mon --as-xml | xmllint --xpath '//resource[@resource_agent=\"ocf:heartbeat:IPaddr2\" and @role = \"Started\" and @managed = \"true\" and ./node[@name = \"'${CLUSTER_NODE}'\"]]/@id' - | sed -e 's/id=//g' -e 's/\"//g')\nfor v in ${VIPS_TO_MOVE}; do\n    echo \"Moving VIP $v on another node\"\n    pcs resource ban $v ${CLUSTER_NODE} --wait=300\ndone\necho \"Removing the location constraints that were created to move the VIPs\"\nfor v in ${VIPS_TO_MOVE}; do\n    echo \"Removing location ban for VIP $v\"\n    ban_id=$(cibadmin --query | xmllint --xpath 'string(//rsc_location[@rsc=\"'${v}'\" and @node=\"'${CLUSTER_NODE}'\" and @score=\"-INFINITY\"]/@id)' -)\n    if [ -n \"$ban_id\" ]; then\n        pcs constraint remove ${ban_id}\n    else\n        echo \"Could not retrieve and clear location constraint for VIP $v\" 2>&1\n    fi\ndone\n"}, {"name": "Wait for 10s to settle connections on new VIPs", "wait_for": {"timeout": 10}, "when": ["step|int == 1", "hostvars[inventory_hostname][\"haproxy_node_names\"]|default([])|length > 1"]}, {"name": "Block local INPUT SYN packets on the backends except mysql", "tags": "haproxy_syn_block", "when": ["step|int == 1"], "shell": "# server controller-0.storage.redhat.local 172.17.3.93:8080 check fall 5 inter 2000 rise 2\n# server controller-0.internalapi.redhat.local fd00:fd00:fd00:2000::176:9292 check fall 5 inter 2000 rise 2\nset -o pipefail\nsource /etc/os-release; test  \"${VERSION_ID%*}\" = \"9.0\" && exit 0\ngrep {{ ansible_facts[\"hostname\"]|lower }} /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg  | grep -v \":3306 \" | \\\nawk '{print $3}' | \\\nwhile read BACKEND; do\n    IP=${BACKEND%:*}\n    PORT=${BACKEND#\"$IP:\"}\n    if [[ $IP =~ .*:.* ]]\n      then PROTOCOL=\"ip6\"\n      else PROTOCOL=\"ip\"\n    fi\necho \"insert rule $PROTOCOL filter INPUT $PROTOCOL daddr $IP tcp dport $PORT tcp flags syn / fin,syn,rst,ack meta time\"\ndone | xargs -i nft {} $(date +%s)-$(date -d'+20 minutes' +%s) counter drop comment \"{{ ansible_facts[\"hostname\"]|lower }}_haproxy_drop\"\n"}, {"name": "Generate block for other nodes OUTPUT SYN packets on the backends except mysql", "tags": "haproxy_syn_block", "when": ["step|int == 1"], "shell": "set -o pipefail\nsource /etc/os-release; test  \"${VERSION_ID%*}\" = \"9.0\" && exit 0\ngrep {{ ansible_facts[\"hostname\"]|lower }} /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg  | grep -v \":3306 \" | \\\nawk '{print $3}' | \\\nwhile read BACKEND; do\n    IP=${BACKEND%:*}\n    PORT=${BACKEND#\"$IP:\"}\n    if [[ $IP =~ .*:.* ]]\n      then PROTOCOL=\"ip6\"\n      else PROTOCOL=\"ip\"\n    fi\n echo \"nft insert rule $PROTOCOL \\$TABLE OUTPUT $PROTOCOL daddr $IP tcp dport $PORT tcp flags syn / fin,syn,rst,ack meta time $(date +%s)-$(date -d'+20 minutes' +%s) counter drop comment \\\"{{ ansible_facts[\"hostname\"]|lower }}_haproxy_drop\\\" \"\ndone\n", "register": "haproxy_iptables_block"}, {"name": "Block OUTPUT SYN packets to this node on other haproxy nodes", "tags": "haproxy_syn_block", "delegate_to": "{{ item }}", "when": ["step|int == 1"], "shell": "set -o pipefail\ngrep {{ ansible_facts[\"hostname\"]|lower }} /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg | head -n 1 \\\nawk '{print $3}' | while read BACKEND; do\n    IP=${BACKEND%:*}\n    if [[ $IP =~ .*:.* ]]\n      then PROTOCOL=\"ip6\"\n      else PROTOCOL=\"ip\"\n    fi\ndone\nTABLE=$(nft list tables | grep -q \"$PROTOCOL raw\" && echo raw || echo filter )\necho \"{{ haproxy_iptables_block.stdout}}\" | while read i; do bash -c \"$i\"; done\n", "loop": "{{ groups[\"haproxy\"] | difference(groups[\"excluded_overcloud\"]) | difference(ansible_facts[\"hostname\"]|lower) }}"}], "post_update_tasks": [{"name": "HAProxy bundle post update", "when": "step|int == 1", "block": [{"name": "HAproxy puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "haproxy", "tripleo_ha_wrapper_resource_name": "haproxy-bundle", "tripleo_ha_wrapper_bundle_name": "haproxy-bundle", "tripleo_ha_wrapper_resource_state": "Started", "tripleo_ha_wrapper_puppet_config_volume": "haproxy", "tripleo_ha_wrapper_puppet_execute": "include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::haproxy_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ip,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".previous_run"}}], "vars": {"tripleo_ha_wrapper_minor_update": true}}, {"name": "Cleanup local SYN DROP rules", "tags": "haproxy_syn_block", "when": "step|int == 1", "shell": "nft -a list chain filter INPUT | grep {{ ansible_facts[\"hostname\"]|lower }} | grep haproxy_drop | \\\nawk -F# '{print \"nft delete rule filter INPUT \"$2}' | \\\nwhile read i; do bash -c \"$i\"; done\n"}, {"name": "Cleanup SYN DROP rules from other haproxy nodes", "tags": "haproxy_syn_block", "when": "step|int == 1", "delegate_to": "{{ item }}", "loop": "{{ groups[\"haproxy\"] | difference(groups[\"excluded_overcloud\"]) | difference(ansible_facts[\"hostname\"]|lower) }}", "shell": "nft -a list chain raw OUTPUT | grep {{ ansible_facts[\"hostname\"]|lower }} | grep haproxy_drop | \\\nawk -F# '{print \"nft delete rule raw OUTPUT \"$2}' | \\\nwhile read i; do bash -c \"$i\"; done\n"}], "upgrade_tasks": [{"name": "Tear-down non-HA haproxy container", "when": ["step|int == 0"], "block": [{"name": "Remove non-HA haproxy container", "include_role": {"name": "tripleo_container_rm"}, "vars": {"tripleo_container_cli": "{{ container_cli }}", "tripleo_containers_to_rm": ["haproxy"]}}]}, {"name": "Prepare switch of haproxy image name", "when": ["step|int == 0"], "block": [{"name": "Get haproxy image id currently used by pacemaker", "shell": "pcs resource config haproxy-bundle | grep -Eo 'image=[^ ]+' | awk -F= '{print $2;}'", "register": "haproxy_image_current_res", "failed_when": false}, {"name": "Image facts for haproxy", "set_fact": {"haproxy_image_latest": "cluster.common.tag/haproxy:pcmklatest", "haproxy_image_current": "{{haproxy_image_current_res.stdout}}"}}, {"name": "Temporarily tag the current haproxy image id with the upgraded image name", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{haproxy_image_current}}", "container_image_latest": "{{haproxy_image_latest}}", "pull_image": false}, "when": ["haproxy_image_current != ''", "haproxy_image_current != haproxy_image_latest"]}, {"name": "Create haproxy retag statefile", "file": {"path": "/var/lib/tripleo/haproxy_needs_retag", "state": "touch"}, "when": ["haproxy_image_current != ''", "haproxy_image_current != haproxy_image_latest"]}]}, {"name": "Update haproxy pcs resource bundle for new container image", "when": ["step|int == 1"], "block": [{"name": "Set upgrade haproxy facts", "set_fact": {"is_haproxy_bootstrap_node": "{{haproxy_short_bootstrap_node_name|lower == ansible_facts['hostname']|lower}}"}}, {"name": "Check for haproxy retag statefile", "stat": {"path": "/var/lib/tripleo/haproxy_needs_retag"}, "register": "haproxy_retag_state_file"}]}, {"name": "Update haproxy pcs resource bundle for new container image", "when": ["step|int == 1", "is_haproxy_bootstrap_node|bool", "haproxy_retag_state_file.stat.exists|bool"], "block": [{"name": "Disable the haproxy cluster resource before container upgrade", "pacemaker_resource": {"resource": "haproxy-bundle", "state": "disable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}, {"name": "Expose HAProxy stats socket on the host and mount TLS cert if needed", "block": [{"name": "Check haproxy stats socket configuration in pacemaker", "command": "cibadmin --query --xpath \"//storage-mapping[@id='haproxy-var-lib']\"", "failed_when": false, "register": "haproxy_stats_exposed"}, {"name": "Check haproxy public certificate configuration in pacemaker", "command": "cibadmin --query --xpath \"//storage-mapping[@id='haproxy-cert']\"", "failed_when": false, "register": "haproxy_cert_mounted"}, {"name": "Add a bind mount for stats socket in the haproxy bundle", "command": "pcs resource bundle update haproxy-bundle storage-map add id=haproxy-var-lib source-dir=/var/lib/haproxy target-dir=/var/lib/haproxy options=rw", "when": "haproxy_stats_exposed.rc == 6"}, {"name": "Set HAProxy public cert volume mount fact", "set_fact": {"haproxy_public_cert_path": "/etc/pki/tls/private/overcloud_endpoint.pem", "haproxy_public_tls_enabled": false}}, {"name": "Add a bind mount for public certificate in the haproxy bundle", "command": "pcs resource bundle update haproxy-bundle storage-map add id=haproxy-cert source-dir={{ haproxy_public_cert_path }} target-dir=/var/lib/kolla/config_files/src-tls/{{ haproxy_public_cert_path }} options=ro", "when": ["haproxy_cert_mounted.rc == 6", "haproxy_public_tls_enabled|bool"]}]}, {"name": "Update the haproxy bundle to use the new container image name", "command": "pcs resource bundle update haproxy-bundle container image={{haproxy_image_latest}}"}, {"name": "Enable the haproxy cluster resource", "pacemaker_resource": {"resource": "haproxy-bundle", "state": "enable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}, {"name": "Remove haproxy retag statefile", "file": {"path": "/var/lib/tripleo/haproxy_needs_retag", "state": "absent"}}]}, {"name": "Retag the pacemaker image if containerized", "when": ["step|int == 3"], "block": [{"block": [{"name": "Get haproxy image from pacemaker", "become": true, "register": "xmllint_pcmk_haproxy_image", "shell": "xmllint --xpath \"string(//bundle[@id='haproxy-bundle']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container haproxy image", "set_fact": {"haproxy_image": "registry.redhat.io/rhosp-rhel9/openstack-haproxy:17.1", "haproxy_image_latest": "cluster.common.tag/haproxy:pcmklatest", "pcmk_haproxy_image": "{{xmllint_pcmk_haproxy_image.stdout}}"}}]}, {"block": [{"name": "Retag pcmklatest to latest haproxy image", "include_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{haproxy_image}}", "container_image_latest": "{{haproxy_image_latest}}"}}]}, {"name": "Ensure config works for the new config", "shell": "set -o pipefail\nawk -i inplace -v INPLACE_SUFFIX=.bak '/  rsprep/ {print \"  http-response replace-header Location \" $3\" \"$5; next;}; {print} ' /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg"}]}]}, {"service_name": "heat_api", "firewall_rules": {"125 heat_api": {"dport": [8004]}}, "firewall_frontend_rules": {"100 heat_api_haproxy_frontend": {"dport": [8004]}}, "firewall_ssl_frontend_rules": {"100 heat_api_haproxy_frontend_ssl": {"dport": [13004]}}, "keystone_resources": {"heat": {"endpoints": {"public": "http://172.21.0.2:8004/v1/%(tenant_id)s", "internal": "http://172.17.0.2:8004/v1/%(tenant_id)s", "admin": "http://172.17.0.2:8004/v1/%(tenant_id)s"}, "users": {"heat": {"password": "zoQ75Cm0pXTer2ifVDLy2roE1"}, "heat_stack_domain_admin": {"password": "n4UsoSw5TVdj2qe8wfVPQKKbC", "roles": ["admin"], "domain": "heat_stack"}}, "region": "regionOne", "service": "orchestration", "roles": ["heat_stack_user"], "domains": ["heat_stack"]}}, "monitoring_subscription": "overcloud-heat-api", "config_settings": {"heat::notification_driver": ["noop"], "heat::logging::debug": true, "heat::policy::policies": {}, "heat::enable_proxy_headers_parsing": true, "heat::rpc_response_timeout": 600, "heat::rabbit_heartbeat_timeout_threshold": 60, "heat::rabbit_heartbeat_in_pthread": false, "heat::region_name": "regionOne", "heat::keystone::authtoken::project_name": "service", "heat::keystone::authtoken::user_domain_name": "Default", "heat::keystone::authtoken::project_domain_name": "Default", "heat::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "heat::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "heat::keystone::authtoken::password": "zoQ75Cm0pXTer2ifVDLy2roE1", "heat::keystone::authtoken::region_name": "regionOne", "heat::keystone::authtoken::interface": "internal", "heat::keystone::authtoken::memcache_use_advanced_pool": true, "heat::keystone::domain::domain_name": "heat_stack", "heat::keystone::domain::domain_admin": "heat_stack_domain_admin", "heat::keystone::domain::domain_admin_email": "heat_stack_domain_admin@localhost", "heat::db::database_db_max_retries": -1, "heat::db::database_max_retries": -1, "heat::yaql_memory_quota": 100000, "heat::yaql_limit_iterators": 1000, "heat::cors::max_age": 3600, "heat::cors::allow_headers": "Content-Type,Cache-Control,Content-Language,Expires,Last-Modified,Pragma,X-Auth-Token", "heat::cors::expose_headers": "Content-Type,Cache-Control,Content-Language,Expires,Last-Modified,Pragma", "heat::cron::purge_deleted::ensure": "present", "heat::cron::purge_deleted::minute": "1", "heat::cron::purge_deleted::hour": "0", "heat::cron::purge_deleted::monthday": "*", "heat::cron::purge_deleted::month": "*", "heat::cron::purge_deleted::weekday": "*", "heat::cron::purge_deleted::maxdelay": "3600", "heat::cron::purge_deleted::user": "heat", "heat::cron::purge_deleted::age": "30", "heat::cron::purge_deleted::age_type": "days", "heat::cron::purge_deleted::destination": "/dev/null", "heat::max_json_body_size": 4194304, "heat::cache::enabled": true, "heat::cache::tls_enabled": false, "heat::cache::resource_finder_caching": false, "heat::cache::backend": "dogpile.cache.memcached", "apache::ip": "%{hiera('internal_api')}", "apache::default_vhost": false, "apache::trace_enable": "Off", "apache::server_signature": "Off", "apache::server_tokens": "Prod", "apache::timeout": 90, "apache::default_mods": false, "apache::mod::prefork::maxrequestworkers": 100, "apache::mod::prefork::serverlimit": 100, "apache::mod::remoteip::proxy_ips": ["172.17.0.0/24"], "apache::mod::alias::icons_options": "None", "heat::api::bind_host": "%{hiera('internal_api')}", "heat::wsgi::apache_api::ssl": false, "heat::wsgi::apache_api::vhost_custom_fragment": "Timeout 600", "heat::api::service_name": "httpd", "heat::wsgi::apache_api::bind_host": "%{hiera('internal_api')}", "heat::wsgi::apache_api::servername": "%{hiera('fqdn_internal_api')}", "heat::wsgi::apache_api::workers": 1}, "service_config_settings": {"rsyslog": {"tripleo_logging_sources_heat_api": [{"tag": "openstack.heat.api", "file": "/var/log/containers/heat/heat_api.log"}]}}, "puppet_config": {"config_volume": "heat_api", "puppet_tags": "heat_config,file,concat,file_line,heat_api_paste_ini", "step_config": "include tripleo::profile::base::heat::api\n", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-heat-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/heat_api.json": {"command": "/usr/sbin/httpd -DFOREGROUND", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/heat", "owner": "heat:heat", "recurse": true}]}, "/var/lib/kolla/config_files/heat_api_cron.json": {"command": "/usr/sbin/crond -n", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/heat", "owner": "heat:heat", "recurse": true}]}}, "docker_config": {"step_2": {}, "step_4": {"heat_api": {"image": "registry.redhat.io/rhosp-rhel9/openstack-heat-api:17.1", "net": "host", "privileged": false, "restart": "always", "user": "root", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/heat:/var/log/heat:z", "/var/log/containers/httpd/heat-api:/var/log/httpd:z", "/var/lib/kolla/config_files/heat_api.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/heat_api:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "heat_api_cron": {"image": "registry.redhat.io/rhosp-rhel9/openstack-heat-api:17.1", "net": "host", "user": "root", "privileged": false, "restart": "always", "healthcheck": {"test": "/usr/share/openstack-tripleo-common/healthcheck/cron heat"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/heat:/var/log/heat:z", "/var/log/containers/httpd/heat-api:/var/log/httpd:z", "/var/lib/kolla/config_files/heat_api_cron.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/heat_api:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/heat", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/log/containers/httpd/heat-api", "setype": "container_file_t", "mode": "0750"}]}], "upgrade_tasks": [], "metadata_settings": null, "deploy_steps_tasks": [{"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"name": "Create dirs for certificates and keys", "file": {"path": "{{ item }}", "state": "directory", "serole": "object_r", "setype": "cert_t", "seuser": "system_u"}, "with_items": ["/etc/pki/tls/certs/httpd", "/etc/pki/tls/private/httpd"]}, {"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "httpd-ctlplane", "dns": "{{ fqdn_ctlplane }}", "principal": "HTTP/{{ fqdn_ctlplane }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-ctlplane.crt /etc/pki/tls/certs/httpd/httpd-ctlplane.crt\ncp /etc/pki/tls/private/httpd-ctlplane.key /etc/pki/tls/private/httpd/httpd-ctlplane.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage", "dns": "{{ fqdn_storage }}", "principal": "HTTP/{{ fqdn_storage }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage.crt /etc/pki/tls/certs/httpd/httpd-storage.crt\ncp /etc/pki/tls/private/httpd-storage.key /etc/pki/tls/private/httpd/httpd-storage.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage_mgmt", "dns": "{{ fqdn_storage_mgmt }}", "principal": "HTTP/{{ fqdn_storage_mgmt }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage_mgmt.crt /etc/pki/tls/certs/httpd/httpd-storage_mgmt.crt\ncp /etc/pki/tls/private/httpd-storage_mgmt.key /etc/pki/tls/private/httpd/httpd-storage_mgmt.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-internal_api", "dns": "{{ fqdn_internal_api }}", "principal": "HTTP/{{ fqdn_internal_api }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-internal_api.crt /etc/pki/tls/certs/httpd/httpd-internal_api.crt\ncp /etc/pki/tls/private/httpd-internal_api.key /etc/pki/tls/private/httpd/httpd-internal_api.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-external", "dns": "{{ fqdn_external }}", "principal": "HTTP/{{ fqdn_external }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-external.crt /etc/pki/tls/certs/httpd/httpd-external.crt\ncp /etc/pki/tls/private/httpd-external.key /etc/pki/tls/private/httpd/httpd-external.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}]}}]}], "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop heat api container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["heat_api", "heat_api_cron"], "tripleo_delegate_to": "{{ groups['heat_api'] | difference(groups['excluded_overcloud']) }}"}}]}]}, {"service_name": "heat_api_cfn", "firewall_rules": {"125 heat_cfn": {"dport": [8000]}}, "firewall_frontend_rules": {"100 heat_api_cfn_haproxy_frontend": {"dport": [8000]}}, "firewall_ssl_frontend_rules": {"100 heat_api_cfn_haproxy_frontend_ssl": {"dport": [13005]}}, "keystone_resources": {"heat-cfn": {"endpoints": {"public": "http://172.21.0.2:8000/v1", "internal": "http://172.17.0.2:8000/v1", "admin": "http://172.17.0.2:8000/v1"}, "users": {"heat-cfn": {"password": "zoQ75Cm0pXTer2ifVDLy2roE1"}}, "region": "regionOne", "service": "cloudformation"}}, "monitoring_subscription": "overcloud-heat-api-cfn", "config_settings": {"heat::notification_driver": ["noop"], "heat::logging::debug": true, "heat::policy::policies": {}, "heat::enable_proxy_headers_parsing": true, "heat::rpc_response_timeout": 600, "heat::rabbit_heartbeat_timeout_threshold": 60, "heat::rabbit_heartbeat_in_pthread": false, "heat::region_name": "regionOne", "heat::keystone::authtoken::project_name": "service", "heat::keystone::authtoken::user_domain_name": "Default", "heat::keystone::authtoken::project_domain_name": "Default", "heat::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "heat::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "heat::keystone::authtoken::password": "zoQ75Cm0pXTer2ifVDLy2roE1", "heat::keystone::authtoken::region_name": "regionOne", "heat::keystone::authtoken::interface": "internal", "heat::keystone::authtoken::memcache_use_advanced_pool": true, "heat::keystone::domain::domain_name": "heat_stack", "heat::keystone::domain::domain_admin": "heat_stack_domain_admin", "heat::keystone::domain::domain_admin_email": "heat_stack_domain_admin@localhost", "heat::db::database_db_max_retries": -1, "heat::db::database_max_retries": -1, "heat::yaql_memory_quota": 100000, "heat::yaql_limit_iterators": 1000, "heat::cors::max_age": 3600, "heat::cors::allow_headers": "Content-Type,Cache-Control,Content-Language,Expires,Last-Modified,Pragma,X-Auth-Token", "heat::cors::expose_headers": "Content-Type,Cache-Control,Content-Language,Expires,Last-Modified,Pragma", "heat::cron::purge_deleted::ensure": "present", "heat::cron::purge_deleted::minute": "1", "heat::cron::purge_deleted::hour": "0", "heat::cron::purge_deleted::monthday": "*", "heat::cron::purge_deleted::month": "*", "heat::cron::purge_deleted::weekday": "*", "heat::cron::purge_deleted::maxdelay": "3600", "heat::cron::purge_deleted::user": "heat", "heat::cron::purge_deleted::age": "30", "heat::cron::purge_deleted::age_type": "days", "heat::cron::purge_deleted::destination": "/dev/null", "heat::max_json_body_size": 4194304, "heat::cache::enabled": true, "heat::cache::tls_enabled": false, "heat::cache::resource_finder_caching": false, "heat::cache::backend": "dogpile.cache.memcached", "apache::default_vhost": false, "heat::api_cfn::bind_host": "%{hiera('internal_api')}", "heat::wsgi::apache_api_cfn::ssl": false, "heat::api_cfn::service_name": "httpd", "heat::wsgi::apache_api_cfn::bind_host": "%{hiera('internal_api')}", "heat::wsgi::apache_api_cfn::servername": "%{hiera('fqdn_internal_api')}", "heat::wsgi::apache_api_cfn::workers": 1}, "service_config_settings": {"rsyslog": {"tripleo_logging_sources_heat_api_cfn": [{"tag": "openstack.heat.api.cfn", "file": "/var/log/containers/heat/heat_api_cfn.log"}]}}, "puppet_config": {"config_volume": "heat_api_cfn", "puppet_tags": "heat_config,file,concat,file_line", "step_config": "include tripleo::profile::base::heat::api_cfn\n", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-heat-api-cfn:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/heat_api_cfn.json": {"command": "/usr/sbin/httpd -DFOREGROUND", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/heat", "owner": "heat:heat", "recurse": true}]}}, "docker_config": {"step_2": {}, "step_4": {"heat_api_cfn": {"image": "registry.redhat.io/rhosp-rhel9/openstack-heat-api-cfn:17.1", "net": "host", "privileged": false, "restart": "always", "user": "root", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/heat:/var/log/heat:z", "/var/log/containers/httpd/heat-api-cfn:/var/log/httpd:z", "/var/lib/kolla/config_files/heat_api_cfn.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/heat_api_cfn:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/heat", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/log/containers/httpd/heat-api-cfn", "setype": "container_file_t", "mode": "0750"}]}], "upgrade_tasks": [], "metadata_settings": null, "deploy_steps_tasks": [{"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"name": "Create dirs for certificates and keys", "file": {"path": "{{ item }}", "state": "directory", "serole": "object_r", "setype": "cert_t", "seuser": "system_u"}, "with_items": ["/etc/pki/tls/certs/httpd", "/etc/pki/tls/private/httpd"]}, {"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "httpd-ctlplane", "dns": "{{ fqdn_ctlplane }}", "principal": "HTTP/{{ fqdn_ctlplane }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-ctlplane.crt /etc/pki/tls/certs/httpd/httpd-ctlplane.crt\ncp /etc/pki/tls/private/httpd-ctlplane.key /etc/pki/tls/private/httpd/httpd-ctlplane.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage", "dns": "{{ fqdn_storage }}", "principal": "HTTP/{{ fqdn_storage }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage.crt /etc/pki/tls/certs/httpd/httpd-storage.crt\ncp /etc/pki/tls/private/httpd-storage.key /etc/pki/tls/private/httpd/httpd-storage.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage_mgmt", "dns": "{{ fqdn_storage_mgmt }}", "principal": "HTTP/{{ fqdn_storage_mgmt }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage_mgmt.crt /etc/pki/tls/certs/httpd/httpd-storage_mgmt.crt\ncp /etc/pki/tls/private/httpd-storage_mgmt.key /etc/pki/tls/private/httpd/httpd-storage_mgmt.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-internal_api", "dns": "{{ fqdn_internal_api }}", "principal": "HTTP/{{ fqdn_internal_api }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-internal_api.crt /etc/pki/tls/certs/httpd/httpd-internal_api.crt\ncp /etc/pki/tls/private/httpd-internal_api.key /etc/pki/tls/private/httpd/httpd-internal_api.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-external", "dns": "{{ fqdn_external }}", "principal": "HTTP/{{ fqdn_external }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-external.crt /etc/pki/tls/certs/httpd/httpd-external.crt\ncp /etc/pki/tls/private/httpd-external.key /etc/pki/tls/private/httpd/httpd-external.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}]}}]}], "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop heat cfn container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["heat_api_cfn"], "tripleo_delegate_to": "{{ groups['heat_api_cfn'] | difference(groups['excluded_overcloud']) }}"}}]}]}, {"service_name": "heat_engine", "monitoring_subscription": "overcloud-heat-engine", "config_settings": {"heat::notification_driver": ["noop"], "heat::logging::debug": true, "heat::policy::policies": {}, "heat::enable_proxy_headers_parsing": true, "heat::rpc_response_timeout": 600, "heat::rabbit_heartbeat_timeout_threshold": 60, "heat::rabbit_heartbeat_in_pthread": false, "heat::region_name": "regionOne", "heat::keystone::authtoken::project_name": "service", "heat::keystone::authtoken::user_domain_name": "Default", "heat::keystone::authtoken::project_domain_name": "Default", "heat::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "heat::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "heat::keystone::authtoken::password": "zoQ75Cm0pXTer2ifVDLy2roE1", "heat::keystone::authtoken::region_name": "regionOne", "heat::keystone::authtoken::interface": "internal", "heat::keystone::authtoken::memcache_use_advanced_pool": true, "heat::keystone::domain::domain_name": "heat_stack", "heat::keystone::domain::domain_admin": "heat_stack_domain_admin", "heat::keystone::domain::domain_admin_email": "heat_stack_domain_admin@localhost", "heat::db::database_db_max_retries": -1, "heat::db::database_max_retries": -1, "heat::yaql_memory_quota": 100000, "heat::yaql_limit_iterators": 1000, "heat::cors::max_age": 3600, "heat::cors::allow_headers": "Content-Type,Cache-Control,Content-Language,Expires,Last-Modified,Pragma,X-Auth-Token", "heat::cors::expose_headers": "Content-Type,Cache-Control,Content-Language,Expires,Last-Modified,Pragma", "heat::cron::purge_deleted::ensure": "present", "heat::cron::purge_deleted::minute": "1", "heat::cron::purge_deleted::hour": "0", "heat::cron::purge_deleted::monthday": "*", "heat::cron::purge_deleted::month": "*", "heat::cron::purge_deleted::weekday": "*", "heat::cron::purge_deleted::maxdelay": "3600", "heat::cron::purge_deleted::user": "heat", "heat::cron::purge_deleted::age": "30", "heat::cron::purge_deleted::age_type": "days", "heat::cron::purge_deleted::destination": "/dev/null", "heat::max_json_body_size": 4194304, "heat::cache::enabled": true, "heat::cache::tls_enabled": false, "heat::cache::resource_finder_caching": false, "heat::cache::backend": "dogpile.cache.memcached", "apache::default_vhost": false, "heat::engine::configure_delegated_roles": false, "heat::engine::trusts_delegated_roles": [], "heat::engine::max_nested_stack_depth": 6, "heat::engine::max_resources_per_stack": 1000, "heat::engine::reauthentication_auth_method": "", "heat::engine::heat_metadata_server_url": "http://172.21.0.2:8000", "heat::engine::heat_waitcondition_server_url": "http://172.21.0.2:8000/v1/waitcondition", "heat::engine::convergence_engine": true, "heat::engine::client_retry_limit": 2, "tripleo::profile::base::heat::manage_db_purge": true, "heat::database_connection": "mysql+pymysql://heat:zoQ75Cm0pXTer2ifVDLy2roE1@172.17.0.2/heat?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "heat::keystone_ec2_uri": "http://172.17.0.2:5000/v3/ec2tokens", "heat::keystone::domain::domain_password": "n4UsoSw5TVdj2qe8wfVPQKKbC", "heat::engine::auth_encryption_key": "7OJ84fd3Qs6InvwvKVASVKB0UZMMnbNC", "heat::engine::plugin_dirs": [], "heat::engine::num_engine_workers": 1}, "service_config_settings": {"rsyslog": {"tripleo_logging_sources_heat_engine": [{"tag": "openstack.heat.engine", "file": "/var/log/containers/heat/heat-engine.log"}]}, "mysql": {"heat::db::mysql::password": "zoQ75Cm0pXTer2ifVDLy2roE1", "heat::db::mysql::user": "heat", "heat::db::mysql::host": "%", "heat::db::mysql::dbname": "heat"}}, "puppet_config": {"config_volume": "heat", "puppet_tags": "heat_config,file,concat,file_line", "step_config": "include tripleo::profile::base::heat::engine\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-heat-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/heat_engine.json": {"command": "/usr/bin/heat-engine --config-file /usr/share/heat/heat-dist.conf --config-file /etc/heat/heat.conf ", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/heat", "owner": "heat:heat", "recurse": true}]}, "/var/lib/kolla/config_files/heat_engine_db_sync.json": {"command": "/usr/bin/bootstrap_host_exec heat_engine su heat -s /bin/bash -c 'heat-manage db_sync'", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/heat", "owner": "heat:heat", "recurse": true}]}}, "docker_config": {"step_2": {"heat_init_log": {"image": "registry.redhat.io/rhosp-rhel9/openstack-heat-engine:17.1", "net": "none", "user": "root", "volumes": ["/var/log/containers/heat:/var/log/heat:z"], "command": ["/bin/bash", "-c", "chown -R heat:heat /var/log/heat"]}}, "step_3": {"heat_engine_db_sync": {"image": "registry.redhat.io/rhosp-rhel9/openstack-heat-engine:17.1", "cap_add": ["AUDIT_WRITE"], "net": "host", "privileged": false, "detach": false, "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/heat:/var/log/heat:z", "/var/lib/kolla/config_files/heat_engine_db_sync.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/heat:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_4": {"heat_engine": {"image": "registry.redhat.io/rhosp-rhel9/openstack-heat-engine:17.1", "net": "host", "privileged": false, "stop_grace_period": 60, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck 5672"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/heat:/var/log/heat:z", "/var/lib/kolla/config_files/heat_engine.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/heat:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/heat", "setype": "container_file_t", "mode": "0750"}]}, {"name": "enable virt_sandbox_use_netlink for healthcheck", "seboolean": {"name": "virt_sandbox_use_netlink", "persistent": true, "state": true}, "when": ["ansible_facts.selinux is defined", "ansible_facts.selinux.status == \"enabled\""]}], "upgrade_tasks": [], "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop heat engine container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["heat_engine"], "tripleo_delegate_to": "{{ groups['heat_engine'] | difference(groups['excluded_overcloud']) }}"}}]}]}, {"service_name": "horizon", "firewall_rules": {"126 horizon": {"dport": [80]}}, "firewall_frontend_rules": {"100 horizon_haproxy_frontend": {"dport": [80]}}, "firewall_ssl_frontend_rules": {"100 horizon_haproxy_frontend_ssl": {"dport": [443]}}, "monitoring_subscription": "overcloud-horizon", "config_settings": {"horizon::allowed_hosts": ["*"], "horizon::enable_secure_proxy_ssl_header": true, "horizon::secure_proxy_addr_header": "HTTP_X_FORWARDED_FOR", "horizon::disable_password_reveal": true, "horizon::enforce_password_check": true, "horizon::disallow_iframe_embed": true, "horizon::cache_backend": "django.core.cache.backends.memcached.MemcachedCache", "horizon::django_session_engine": "django.contrib.sessions.backends.cache", "horizon::vhost_extra_params": {"add_listen": true, "priority": 10, "access_log_format": "%a %l %u %t \\\"%r\\\" %>s %b \\\"%%{}{Referer}i\\\" \\\"%%{}{User-Agent}i\\\"", "options": ["FollowSymLinks", "MultiViews"]}, "horizon::bind_address": "%{hiera('internal_api')}", "horizon::keystone_url": "http://172.21.0.2:5000/v3", "horizon::password_validator": "", "horizon::password_validator_help": "", "horizon::secret_key": "NiewE8UjEw", "horizon::secure_cookies": false, "horizon::session_timeout": 1800, "memcached_ipv6": false, "horizon::servername": "%{hiera('fqdn_internal_api')}", "horizon::listen_ssl": false, "horizon::customization_module": "", "horizon::timezone": "UTC", "horizon::file_upload_temp_dir": "/var/tmp", "horizon::help_url": "http://docs.openstack.org", "horizon::enable_user_pass": "UNSET", "horizon::django_debug": true, "horizon::log_level": "DEBUG"}, "ansible_group_vars": {"keystone_enable_member": true}, "service_config_settings": {"rsyslog": {"tripleo_logging_sources_horizon": [{"tag": "openstack.horizon", "file": "/var/log/containers/horizon/horizon.log"}]}}, "puppet_config": {"config_volume": "horizon", "puppet_tags": "horizon_config", "step_config": "include tripleo::profile::base::horizon\n", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-horizon:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/horizon.json": {"command": "/usr/sbin/httpd -DFOREGROUND", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/horizon/", "owner": "apache:apache", "recurse": true}, {"path": "/etc/openstack-dashboard/", "owner": "apache:apache", "recurse": true}, {"path": "/usr/share/openstack-dashboard/openstack_dashboard/local/", "owner": "apache:apache", "recurse": false}, {"path": "/usr/share/openstack-dashboard/openstack_dashboard/local/local_settings.d/", "owner": "apache:apache", "recurse": false}]}}, "docker_config": {"step_2": {"horizon_fix_perms": {"image": "registry.redhat.io/rhosp-rhel9/openstack-horizon:17.1", "net": "none", "user": "root", "command": ["/bin/bash", "-c", "touch /var/log/horizon/horizon.log ; chown -R apache:apache /var/log/horizon && chmod -R a+rx /etc/openstack-dashboard"], "volumes": ["/var/log/containers/horizon:/var/log/horizon:z", "/var/log/containers/httpd/horizon:/var/log/httpd:z", "/var/lib/config-data/puppet-generated/horizon/etc/openstack-dashboard:/etc/openstack-dashboard"]}}, "step_3": {"horizon": {"image": "registry.redhat.io/rhosp-rhel9/openstack-horizon:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/horizon.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/horizon:/var/lib/kolla/config_files/src:ro", "/var/log/containers/horizon:/var/log/horizon:z", "/var/log/containers/httpd/horizon:/var/log/httpd:z", "/var/tmp/horizon:/var/tmp:z", "/var/www:/var/www:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "ENABLE_DESIGNATE": "yes", "ENABLE_HEAT": "yes", "ENABLE_IRONIC": "yes", "ENABLE_MANILA": "yes", "ENABLE_OCTAVIA": "yes"}}}}, "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode|default(omit) }}"}, "with_items": [{"path": "/var/log/containers/horizon", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/log/containers/httpd/horizon", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/www", "setype": "container_file_t"}, {"path": "/var/tmp/horizon", "setype": "container_file_t", "mode": "01777"}]}, {"name": "ensure /var/tmp/horizon exists on boot", "copy": {"dest": "/etc/tmpfiles.d/var-tmp-horizon.conf", "content": "d /var/tmp/horizon 01777 root root - -\n"}}], "upgrade_tasks": [{"name": "Anchor for upgrade and update tasks", "when": "step|int == 0", "block": [{"name": "Reset selinux label on /var/tmp", "file": {"path": "/var/tmp", "state": "directory", "setype": "tmp_t", "mode": 1023}}]}], "update_tasks": [{"name": "Anchor for upgrade and update tasks", "when": "step|int == 0", "block": [{"name": "Reset selinux label on /var/tmp", "file": {"path": "/var/tmp", "state": "directory", "setype": "tmp_t", "mode": 1023}}]}], "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop horizon container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["horizon"], "tripleo_delegate_to": "{{ groups['horizon'] | difference(groups['excluded_overcloud']) }}"}}]}]}, {"service_name": "iscsid", "config_settings": {"tripleo::profile::base::iscsid::chap_algs": "SHA3-256,SHA256,SHA1,MD5"}, "service_config_settings": {}, "deploy_steps_tasks": [{"name": "Run lvmfilter role", "include_role": {"name": "tripleo_lvmfilter"}, "when": ["step|int == 1"]}], "ansible_group_vars": {"tripleo_tripleo_lvmfilter_enabled": false, "tripleo_tripleo_lvmfilter_devices_allowlist": [], "tripleo_tripleo_lvmfilter_devices_denylist": [".*"], "ContainerIscsidImage": "registry.redhat.io/rhosp-rhel9/openstack-iscsid:17.1", "ContainerIscsidConfigImage": "registry.redhat.io/rhosp-rhel9/openstack-iscsid:17.1"}, "puppet_config": {"config_volume": "iscsid", "puppet_tags": "iscsid_config", "step_config": "include tripleo::profile::base::iscsid\n", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-iscsid:17.1", "volumes": ["/etc/iscsi:/tmp/iscsi.host:z"]}, "kolla_config": {"/var/lib/kolla/config_files/iscsid.json": {"command": "/usr/sbin/iscsid -f", "config_files": [{"source": "/var/lib/kolla/config_files/src-iscsid/", "dest": "/etc/iscsi/", "merge": true, "preserve_properties": true}]}}, "docker_config": {"step_3": {"iscsid": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-iscsid:17.1", "net": "host", "privileged": true, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/iscsid.json:/var/lib/kolla/config_files/config.json:ro", "/dev:/dev", "/run:/run", "/sys:/sys", "/lib/modules:/lib/modules:ro", "/var/lib/config-data/puppet-generated/iscsid/etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro", "/etc/target:/etc/target:z", "/var/lib/iscsi:/var/lib/iscsi:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "host_prep_tasks": [{"name": "create fcontext entry for iscsi", "community.general.sefcontext": {"target": "{{ item.path }}(/.*)?", "setype": "{{ item.setype }}", "state": "present"}, "with_items": [{"path": "/etc/iscsi", "setype": "container_file_t"}, {"path": "/etc/target", "setype": "container_file_t"}, {"path": "/var/lib/iscsi", "setype": "container_file_t"}]}, {"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}"}, "with_items": [{"path": "/etc/iscsi", "setype": "container_file_t"}, {"path": "/etc/target", "setype": "container_file_t"}, {"path": "/var/lib/iscsi", "setype": "container_file_t"}]}, {"name": "stat /lib/systemd/system/iscsid.socket", "stat": "path=/lib/systemd/system/iscsid.socket", "register": "stat_iscsid_socket"}, {"name": "Stop and disable iscsid.socket service", "service": "name=iscsid.socket state=stopped enabled=no", "when": "stat_iscsid_socket.stat.exists"}, {"name": "Check if iscsi.service is enabled", "command": "systemctl is-enabled --quiet iscsi.service", "failed_when": false, "register": "iscsi_service_enabled_result"}, {"name": "Stop iscsi.service", "service": "name=iscsi.service state=stopped enabled=no", "when": ["iscsi_service_enabled_result is changed", "iscsi_service_enabled_result.rc == 0"]}], "upgrade_tasks": []}, {"service_name": "kernel", "host_prep_tasks": [{"include_role": {"name": "tripleo_kernel"}}], "ansible_group_vars": {"hieradata_localhost_address": "localhost", "tripleo_kernel_extra_modules": {}, "tripleo_kernel_extra_packages": {}, "tripleo_kernel_sysctl_extra_settings": {"net.ipv6.conf.default.disable_ipv6": {"value": 0}, "net.ipv4.ip_local_reserved_ports": {"value": "35357,49000-49001"}, "net.ipv6.conf.all.disable_ipv6": {"value": 0}, "net.ipv6.conf.lo.disable_ipv6": {"value": 0}, "net.ipv4.ip_forward": {"value": 1}, "net.ipv4.conf.all.rp_filter": {"value": 1}, "net.ipv6.conf.all.forwarding": {"value": 0}, "net.ipv4.ip_nonlocal_bind": {"value": 1}, "net.ipv6.ip_nonlocal_bind": {"value": 1}, "kernel.pid_max": {"value": 1048576}, "net.ipv4.neigh.default.gc_thresh1": {"value": 1024}, "net.ipv4.neigh.default.gc_thresh2": {"value": 2048}, "net.ipv4.neigh.default.gc_thresh3": {"value": 4096}, "net.bridge.bridge-nf-call-arptables": {"value": 1}, "net.bridge.bridge-nf-call-iptables": {"value": 1}, "net.bridge.bridge-nf-call-ip6tables": {"value": 1}, "fs.inotify.max_user_instances": {"value": 1024}}}}, {"service_name": "keystone", "firewall_rules": {"111 keystone": {"dport": [5000, "35357"]}}, "firewall_frontend_rules": {"100 keystone_public_haproxy_frontend": {"dport": [5000]}, "100 keystone_admin_haproxy_frontend": {"dport": ["35357"]}}, "firewall_ssl_frontend_rules": {"100 keystone_public_haproxy_frontend_ssl": {"dport": [13000]}}, "monitoring_subscription": "overcloud-keystone", "config_settings": {"apache::ip": "%{hiera('internal_api')}", "apache::default_vhost": false, "apache::trace_enable": "Off", "apache::server_signature": "Off", "apache::server_tokens": "Prod", "apache::timeout": 90, "apache::default_mods": false, "apache::mod::prefork::maxrequestworkers": 100, "apache::mod::prefork::serverlimit": 100, "apache::mod::remoteip::proxy_ips": ["172.17.0.0/24"], "apache::mod::alias::icons_options": "None", "keystone::database_connection": "mysql+pymysql://keystone:1atINBzulchyizB7OeFxWH1ed@172.17.0.2/keystone?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "keystone::token_expiration": 3600, "keystone::policy::policies": {}, "keystone_ssl_certificate": "", "keystone_ssl_certificate_key": "", "keystone::token_provider": "fernet", "keystone::enable_fernet_setup": true, "keystone::fernet_max_active_keys": 5, "keystone::enable_proxy_headers_parsing": true, "keystone::enable_credential_setup": true, "keystone::credential_keys": {"/etc/keystone/credential-keys/0": {"content": "vn8Ahvk3PcwUIjPlU2cQ4ssV0LZrx4qHN7sXq0cJe3w="}, "/etc/keystone/credential-keys/1": {"content": "6Q7wVkDLZrhAceec2rpUf5jUKuaRp2ETFnn1S1CcQWQ="}}, "keystone::fernet_keys": {"/etc/keystone/fernet-keys/0": {"content": "puYLH9X_q7AKedRexvfvqE0Kg7bkZG-Eci3u88ckVag="}, "/etc/keystone/fernet-keys/1": {"content": "VmF8KXPGfie7IK3_G6iiqoS_459PwX0nBsoQkN893bM="}}, "keystone::fernet_replace_keys": true, "keystone::logging::debug": true, "keystone::notification_driver": ["messagingv2"], "keystone::notification_format": "basic", "tripleo::profile::base::keystone::extra_notification_topics": [], "keystone_enable_db_purge": true, "keystone::cron::trust_flush::ensure": "present", "keystone::cron::trust_flush::minute": "1", "keystone::cron::trust_flush::hour": "*", "keystone::cron::trust_flush::monthday": "*", "keystone::cron::trust_flush::month": "*", "keystone::cron::trust_flush::weekday": "*", "keystone::cron::trust_flush::maxdelay": 0, "keystone::cron::trust_flush::destination": "/var/log/keystone/keystone-trustflush.log", "keystone::cron::trust_flush::user": "keystone", "keystone::rabbit_heartbeat_timeout_threshold": 60, "keystone::rabbit_heartbeat_in_pthread": false, "keystone::service_name": "httpd", "keystone::enable_ssl": false, "keystone::wsgi::apache::api_port": [5000, "35357"], "keystone::wsgi::apache::ssl": false, "keystone::wsgi::apache::servername": "%{hiera('fqdn_internal_api')}", "keystone::wsgi::apache::workers": "1", "keystone::wsgi::apache::threads": 1, "keystone::db::database_db_max_retries": -1, "keystone::db::database_max_retries": -1, "keystone::wsgi::apache::bind_host": ["%{hiera('internal_api')}", "%{hiera('ctlplane')}"], "keystone::cache::enabled": true, "keystone::cache::backend_argument": ["memcached_expire_time: 3600"], "keystone::cache::tls_enabled": false, "keystone::cache::backend": "dogpile.cache.memcached"}, "service_config_settings": {"rsyslog": {"tripleo_logging_sources_keystone": {"tag": "openstack.keystone", "file": "/var/log/containers/keystone/keystone.log"}}, "mysql": {"keystone::db::mysql::password": "1atINBzulchyizB7OeFxWH1ed", "keystone::db::mysql::user": "keystone", "keystone::db::mysql::host": "%", "keystone::db::mysql::dbname": "keystone"}, "pacemaker": {"keystone::endpoint::public_url": "http://172.21.0.2:5000", "keystone::endpoint::internal_url": "http://172.17.0.2:5000", "keystone::endpoint::admin_url": "http://192.168.122.99:35357", "keystone::endpoint::region": "regionOne", "keystone::admin_password": "79Ijk5YjIkKxsXqqoJYsL9qZ5"}}, "puppet_config": {"config_volume": "keystone", "puppet_tags": "keystone_config,keystone_domain_config", "step_config": "['Keystone_user', 'Keystone_endpoint', 'Keystone_domain', 'Keystone_tenant', 'Keystone_user_role', 'Keystone_role', 'Keystone_service'].each |String $val| { noop_resource($val) }\ninclude tripleo::profile::base::keystone\n\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-keystone:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/keystone.json": {"command": "/usr/sbin/httpd", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/keystone/fernet-keys", "dest": "/etc/keystone/fernet-keys", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/keystone_cron.json": {"command": "/usr/sbin/crond -n", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}}, "docker_config": {"step_2": {"keystone_init_log": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-keystone:17.1", "net": "none", "user": "root", "command": ["/bin/bash", "-c", "chown -R keystone:keystone /var/log/keystone"], "volumes": ["/var/log/containers/keystone:/var/log/keystone:z", "/var/log/containers/httpd/keystone:/var/log/httpd:z"]}}, "step_3": {"keystone_db_sync": {"image": "registry.redhat.io/rhosp-rhel9/openstack-keystone:17.1", "net": "host", "user": "root", "privileged": false, "detach": false, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/keystone:/var/log/keystone:z", "/var/log/containers/httpd/keystone:/var/log/httpd:z", "/etc/openldap:/etc/openldap:ro", "/var/lib/kolla/config_files/keystone.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/keystone:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_BOOTSTRAP": true, "KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}, "command": ["/usr/bin/bootstrap_host_exec", "keystone", "/usr/local/bin/kolla_start"]}, "keystone": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-keystone:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/keystone:/var/log/keystone:z", "/var/log/containers/httpd/keystone:/var/log/httpd:z", "/etc/openldap:/etc/openldap:ro", "/var/lib/kolla/config_files/keystone.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/keystone:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "keystone_bootstrap": {"start_order": 3, "action": "exec", "user": "root", "command": ["keystone", "/usr/bin/bootstrap_host_exec", "keystone", "keystone-manage", "bootstrap"], "environment": {"KOLLA_BOOTSTRAP": true, "OS_BOOTSTRAP_PASSWORD": "79Ijk5YjIkKxsXqqoJYsL9qZ5", "OS_BOOTSTRAP_USERNAME": "admin", "OS_BOOTSTRAP_PROJECT_NAME": "admin", "OS_BOOTSTRAP_ROLE_NAME": "admin", "OS_BOOTSTRAP_SERVICE_NAME": "keystone", "OS_BOOTSTRAP_ADMIN_URL": "http://192.168.122.99:35357", "OS_BOOTSTRAP_PUBLIC_URL": "http://172.21.0.2:5000", "OS_BOOTSTRAP_INTERNAL_URL": "http://172.17.0.2:5000", "OS_BOOTSTRAP_REGION_ID": "regionOne"}}, "keystone_cron": {"start_order": 4, "image": "registry.redhat.io/rhosp-rhel9/openstack-keystone:17.1", "user": "root", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/usr/share/openstack-tripleo-common/healthcheck/cron keystone"}, "command": ["/bin/bash", "-c", "/usr/local/bin/kolla_set_configs && /usr/sbin/crond -n"], "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/keystone:/var/log/keystone:z", "/var/log/containers/httpd/keystone:/var/log/httpd:z", "/var/lib/kolla/config_files/keystone_cron.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/keystone:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}, "step_4": {"keystone_refresh": {"start_order": 1, "action": "exec", "user": "root", "command": ["keystone", "pkill", "--signal", "USR1", "httpd"]}}}, "external_deploy_tasks": [{"name": "Manage clouds.yaml files", "when": ["step|int == 1", "not ansible_check_mode|bool"], "block": [{"name": "Create /etc/openstack directory if it does not exist", "become": true, "file": {"mode": "0755", "owner": "root", "path": "/etc/openstack", "state": "directory"}}, {"name": "Configure /etc/openstack/clouds.yaml", "include_role": {"name": "tripleo_keystone_resources", "tasks_from": "clouds"}, "vars": {"tripleo_keystone_resources_cloud_name": "standalone", "tripleo_keystone_resources_cloud_config": {"auth": {"auth_url": "http://172.21.0.2:5000", "password": "79Ijk5YjIkKxsXqqoJYsL9qZ5", "project_domain_name": "Default", "project_name": "admin", "user_domain_name": "Default", "username": "admin"}, "cacert": "", "identity_api_version": "3", "volume_api_version": "3", "region_name": "regionOne"}}}]}, {"name": "Manage Keystone resources", "become": true, "when": ["step|int == 4", "not ansible_check_mode|bool"], "block": [{"name": "Manage Keystone resources for OpenStack services", "include_role": {"name": "tripleo_keystone_resources"}, "vars": {"tripleo_keystone_resources_catalog_config": "{{ keystone_resources }}", "tripleo_keystone_resources_service_project": "service", "tripleo_keystone_resources_cloud_name": "standalone", "tripleo_keystone_resources_region": "regionOne", "tripleo_keystone_resources_admin_endpoint": "http://192.168.122.99:35357", "tripleo_keystone_resources_public_endpoint": "http://172.21.0.2:5000", "tripleo_keystone_resources_internal_endpoint": "http://172.17.0.2:5000", "tripleo_keystone_resources_admin_password": "79Ijk5YjIkKxsXqqoJYsL9qZ5", "tripleo_keystone_resources_member_role_enabled": false}}, {"name": "is Keystone LDAP enabled", "set_fact": {"keystone_ldap_domain_enabled": false}}, {"name": "Set fact for tripleo_keystone_ldap_domains", "set_fact": {"tripleo_keystone_ldap_domains": {}}, "when": "keystone_ldap_domain_enabled|bool"}, {"name": "Manage Keystone domains from LDAP config", "when": "keystone_ldap_domain_enabled|bool", "include_role": {"name": "tripleo_keystone_resources", "tasks_from": "domains"}, "vars": {"tripleo_keystone_resources_catalog_config": "{{ keystone_resources }}", "tripleo_keystone_resources_cloud_name": "standalone", "batched_tripleo_keystone_resources_domains": "{{ tripleo_keystone_ldap_domains | list }}"}}]}], "deploy_steps_tasks": [{"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"name": "Create dirs for certificates and keys", "file": {"path": "{{ item }}", "state": "directory", "serole": "object_r", "setype": "cert_t", "seuser": "system_u"}, "with_items": ["/etc/pki/tls/certs/httpd", "/etc/pki/tls/private/httpd"]}, {"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "httpd-ctlplane", "dns": "{{ fqdn_ctlplane }}", "principal": "HTTP/{{ fqdn_ctlplane }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-ctlplane.crt /etc/pki/tls/certs/httpd/httpd-ctlplane.crt\ncp /etc/pki/tls/private/httpd-ctlplane.key /etc/pki/tls/private/httpd/httpd-ctlplane.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage", "dns": "{{ fqdn_storage }}", "principal": "HTTP/{{ fqdn_storage }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage.crt /etc/pki/tls/certs/httpd/httpd-storage.crt\ncp /etc/pki/tls/private/httpd-storage.key /etc/pki/tls/private/httpd/httpd-storage.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage_mgmt", "dns": "{{ fqdn_storage_mgmt }}", "principal": "HTTP/{{ fqdn_storage_mgmt }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage_mgmt.crt /etc/pki/tls/certs/httpd/httpd-storage_mgmt.crt\ncp /etc/pki/tls/private/httpd-storage_mgmt.key /etc/pki/tls/private/httpd/httpd-storage_mgmt.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-internal_api", "dns": "{{ fqdn_internal_api }}", "principal": "HTTP/{{ fqdn_internal_api }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-internal_api.crt /etc/pki/tls/certs/httpd/httpd-internal_api.crt\ncp /etc/pki/tls/private/httpd-internal_api.key /etc/pki/tls/private/httpd/httpd-internal_api.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-external", "dns": "{{ fqdn_external }}", "principal": "HTTP/{{ fqdn_external }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-external.crt /etc/pki/tls/certs/httpd/httpd-external.crt\ncp /etc/pki/tls/private/httpd-external.key /etc/pki/tls/private/httpd/httpd-external.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}]}}]}, {"name": "validate keystone container state", "containers.podman.podman_container_info": {"name": "keystone"}, "register": "keystone_infos", "failed_when": ["keystone_infos.containers.0.Healthcheck.Status is defined", "'healthy' not in keystone_infos.containers.0.Healthcheck.Status"], "retries": 10, "delay": 30, "tags": ["opendev-validation", "opendev-validation-keystone"], "when": ["container_cli == 'podman'", "not container_healthcheck_disabled", "step|int == 4"]}], "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/keystone", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/log/containers/httpd/keystone", "setype": "container_file_t", "mode": "0750"}]}, {"name": "Check if file certs_valid exist", "stat": {"path": "/etc/openldap/certs/certs_valid"}, "register": "certs_valid_stat"}, {"name": "Check if file cert9.db exist", "stat": {"path": "/etc/openldap/certs/cert9.db"}, "register": "cert9_stat", "when": "not certs_valid_stat.stat.exists"}, {"name": "Check if file key4.db exist", "stat": {"path": "/etc/openldap/certs/key4.db"}, "register": "key4_stat", "when": "not certs_valid_stat.stat.exists"}, {"fail": {"msg": "Keys and/or certificates were found in /etc/openldap/certs but these conflicts with keystone python LDAP calls.  If you know those certificates are valid and not causing any conflicts, you can touch /etc/openldap/certs/certs_valid in order to skip this failure and retry or you can delete the files located in /etc/openldap/certs and retry.\n"}, "when": "( cert9_stat.stat.exists or key4_stat.stat.exists ) and not certs_valid_stat.stat.exists"}], "metadata_settings": null, "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop keystone container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["keystone", "keystone_cron"], "tripleo_delegate_to": "{{ groups['keystone'] | difference(groups['excluded_overcloud']) }}"}}]}]}, {"service_name": "manila_api", "firewall_rules": {"150 manila": {"dport": [8786]}}, "firewall_frontend_rules": {"100 manila_haproxy_frontend": {"dport": [8786]}}, "firewall_ssl_frontend_rules": {"100 manila_haproxy_frontend_ssl": {"dport": [13786]}}, "keystone_resources": {"manila": {"endpoints": {"public": "http://172.21.0.2:8786/v1/%(tenant_id)s", "internal": "http://172.17.0.2:8786/v1/%(tenant_id)s", "admin": "http://172.17.0.2:8786/v1/%(tenant_id)s"}, "users": {"manila": {"password": "2R4FJ91KHJyj5ppSY0XKBqLJg"}}, "region": "regionOne", "service": "share"}, "manilav2": {"endpoints": {"public": "http://172.21.0.2:8786/v2", "internal": "http://172.17.0.2:8786/v2", "admin": "http://172.17.0.2:8786/v2"}, "users": {"manilav2": {"password": "2R4FJ91KHJyj5ppSY0XKBqLJg"}}, "region": "regionOne", "service": "sharev2"}}, "monitoring_subscription": "overcloud-manila-api", "config_settings": {"manila::storage_availability_zone": "nova", "manila::notification_driver": ["noop"], "manila::logging::debug": true, "manila::db::database_db_max_retries": -1, "manila::db::database_max_retries": -1, "manila::sql_connection": "mysql+pymysql://manila:2R4FJ91KHJyj5ppSY0XKBqLJg@172.17.0.2/manila?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "manila::network::neutron::network_plugin_ipv6_enabled": false, "manila::network::neutron::network_plugin_ipv4_enabled": true, "manila::rabbit_heartbeat_in_pthread": false, "apache::ip": "%{hiera('internal_api')}", "apache::default_vhost": false, "apache::trace_enable": "Off", "apache::server_signature": "Off", "apache::server_tokens": "Prod", "apache::timeout": 90, "apache::default_mods": false, "apache::mod::prefork::maxrequestworkers": 100, "apache::mod::prefork::serverlimit": 100, "apache::mod::remoteip::proxy_ips": ["172.17.0.0/24"], "apache::mod::alias::icons_options": "None", "manila::keystone::authtoken::password": "2R4FJ91KHJyj5ppSY0XKBqLJg", "manila::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "manila::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "manila::keystone::authtoken::project_name": "service", "manila::keystone::authtoken::user_domain_name": "Default", "manila::keystone::authtoken::project_domain_name": "Default", "manila::keystone::authtoken::region_name": "regionOne", "manila::keystone::authtoken::interface": "internal", "manila::keystone::authtoken::memcache_use_advanced_pool": true, "manila::api::bind_host": "%{hiera('internal_api')}", "manila::wsgi::apache::bind_host": "%{hiera('internal_api')}", "manila::wsgi::apache::ssl": false, "manila::api::service_name": "httpd", "manila::api::enable_proxy_headers_parsing": true, "manila::api::default_share_type": "default", "manila_enabled_share_protocols": [], "manila::policy::policies": {}, "manila::cron::db_purge::minute": "1", "manila::cron::db_purge::hour": "0", "manila::cron::db_purge::monthday": "*", "manila::cron::db_purge::month": "*", "manila::cron::db_purge::weekday": "*", "manila::cron::db_purge::user": "manila", "manila::cron::db_purge::age": "30", "manila::cron::db_purge::destination": "/var/log/manila/manila-rowsflush.log", "manila::cron::db_purge::maxdelay": "3600", "manila::wsgi::apache::servername": "%{hiera('fqdn_internal_api')}", "manila::wsgi::apache::workers": "%{::os_workers}"}, "service_config_settings": {"mysql": {"manila::db::mysql::password": "2R4FJ91KHJyj5ppSY0XKBqLJg", "manila::db::mysql::user": "manila", "manila::db::mysql::host": "%", "manila::db::mysql::dbname": "manila"}}, "puppet_config": {"config_volume": "manila", "puppet_tags": "manila_config,manila_api_paste_ini", "step_config": "include tripleo::profile::base::manila::api\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-manila-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/manila_api.json": {"command": "/usr/sbin/httpd -DFOREGROUND", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/manila", "owner": "manila:manila", "recurse": true}]}, "/var/lib/kolla/config_files/manila_api_db_sync.json": {"command": "/usr/bin/bootstrap_host_exec manila_api su manila -s /bin/bash -c '/usr/bin/manila-manage db sync'", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/manila", "owner": "manila:manila", "recurse": true}]}, "/var/lib/kolla/config_files/manila_api_cron.json": {"command": "/usr/sbin/crond -n", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/manila", "owner": "manila:manila", "recurse": true}]}}, "docker_config": {"step_2": {"manila_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-manila-api:17.1", "net": "none", "user": "root", "volumes": ["/var/log/containers/manila:/var/log/manila:z", "/var/log/containers/httpd/manila-api:/var/log/httpd:z"], "command": ["/bin/bash", "-c", "chown -R manila:manila /var/log/manila"]}}, "step_3": {"manila_api_db_sync": {"user": "root", "image": "registry.redhat.io/rhosp-rhel9/openstack-manila-api:17.1", "cap_add": ["AUDIT_WRITE"], "net": "host", "detach": false, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/manila_api_db_sync.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/manila:/var/lib/kolla/config_files/src:ro", "/var/log/containers/manila:/var/log/manila:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}, "step_4": {"manila_api": {"image": "registry.redhat.io/rhosp-rhel9/openstack-manila-api:17.1", "net": "host", "restart": "always", "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/manila_api.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/manila:/var/lib/kolla/config_files/src:ro", "/var/log/containers/manila:/var/log/manila:z", "/var/log/containers/httpd/manila-api:/var/log/httpd:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "manila_api_cron": {"image": "registry.redhat.io/rhosp-rhel9/openstack-manila-api:17.1", "net": "host", "user": "root", "privileged": false, "restart": "always", "healthcheck": {"test": "/usr/share/openstack-tripleo-common/healthcheck/cron manila"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/manila_api_cron.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/manila:/var/lib/kolla/config_files/src:ro", "/var/log/containers/manila:/var/log/manila:z", "/var/log/containers/httpd/manila-api:/var/log/httpd:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "metadata_settings": null, "deploy_steps_tasks": [{"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"name": "Create dirs for certificates and keys", "file": {"path": "{{ item }}", "state": "directory", "serole": "object_r", "setype": "cert_t", "seuser": "system_u"}, "with_items": ["/etc/pki/tls/certs/httpd", "/etc/pki/tls/private/httpd"]}, {"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "httpd-ctlplane", "dns": "{{ fqdn_ctlplane }}", "principal": "HTTP/{{ fqdn_ctlplane }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-ctlplane.crt /etc/pki/tls/certs/httpd/httpd-ctlplane.crt\ncp /etc/pki/tls/private/httpd-ctlplane.key /etc/pki/tls/private/httpd/httpd-ctlplane.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage", "dns": "{{ fqdn_storage }}", "principal": "HTTP/{{ fqdn_storage }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage.crt /etc/pki/tls/certs/httpd/httpd-storage.crt\ncp /etc/pki/tls/private/httpd-storage.key /etc/pki/tls/private/httpd/httpd-storage.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage_mgmt", "dns": "{{ fqdn_storage_mgmt }}", "principal": "HTTP/{{ fqdn_storage_mgmt }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage_mgmt.crt /etc/pki/tls/certs/httpd/httpd-storage_mgmt.crt\ncp /etc/pki/tls/private/httpd-storage_mgmt.key /etc/pki/tls/private/httpd/httpd-storage_mgmt.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-internal_api", "dns": "{{ fqdn_internal_api }}", "principal": "HTTP/{{ fqdn_internal_api }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-internal_api.crt /etc/pki/tls/certs/httpd/httpd-internal_api.crt\ncp /etc/pki/tls/private/httpd-internal_api.key /etc/pki/tls/private/httpd/httpd-internal_api.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-external", "dns": "{{ fqdn_external }}", "principal": "HTTP/{{ fqdn_external }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-external.crt /etc/pki/tls/certs/httpd/httpd-external.crt\ncp /etc/pki/tls/private/httpd-external.key /etc/pki/tls/private/httpd/httpd-external.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}]}}]}], "host_prep_tasks": [{"name": "Create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode|default(omit) }}"}, "with_items": [{"path": "/var/log/containers/manila", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/log/containers/httpd/manila-api", "setype": "container_file_t", "mode": "0750"}]}], "upgrade_tasks": [], "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop manila api container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["manila_api"], "tripleo_delegate_to": "{{ groups['manila_api'] | difference(groups['excluded_overcloud']) }}"}}]}]}, {"service_name": "manila_backend_cephfs", "config_settings": {"manila::backend::cephfs::title": "cephfs", "manila::backend::cephfs::driver_handles_share_servers": false, "manila::backend::cephfs::pool_name": "manila_data", "manila::backend::cephfs::share_backend_name": "cephfs", "manila::backend::cephfs::cephfs_filesystem_name": "cephfs", "manila_cephfs_ceph_conf_path": "/var/lib/tripleo-config/ceph", "manila::backend::cephfs::cephfs_conf_path": "/etc/ceph/ceph.conf", "manila::backend::cephfs::cephfs_auth_id": "manila", "manila::backend::cephfs::cephfs_cluster_name": "ceph", "manila::backend::cephfs::cephfs_volume_mode": "0755", "manila::backend::cephfs::cephfs_protocol_helper_type": "CEPHFS"}}, {"service_name": "manila_scheduler", "monitoring_subscription": "overcloud-manila-scheduler", "config_settings": {"manila::storage_availability_zone": "nova", "manila::notification_driver": ["noop"], "manila::logging::debug": true, "manila::db::database_db_max_retries": -1, "manila::db::database_max_retries": -1, "manila::sql_connection": "mysql+pymysql://manila:2R4FJ91KHJyj5ppSY0XKBqLJg@172.17.0.2/manila?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "manila::network::neutron::network_plugin_ipv6_enabled": false, "manila::network::neutron::network_plugin_ipv4_enabled": true, "manila::rabbit_heartbeat_in_pthread": false}, "service_config_settings": {"mysql": {"manila::db::mysql::password": "2R4FJ91KHJyj5ppSY0XKBqLJg", "manila::db::mysql::user": "manila", "manila::db::mysql::host": "%", "manila::db::mysql::dbname": "manila"}}, "puppet_config": {"config_volume": "manila", "puppet_tags": "manila_config,manila_scheduler_paste_ini", "step_config": "include tripleo::profile::base::manila::scheduler\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-manila-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/manila_scheduler.json": {"command": "/usr/bin/manila-scheduler --config-file /usr/share/manila/manila-dist.conf --config-file /etc/manila/manila.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/manila", "owner": "manila:manila", "recurse": true}]}}, "docker_config": {"step_4": {"manila_scheduler": {"image": "registry.redhat.io/rhosp-rhel9/openstack-manila-scheduler:17.1", "net": "host", "restart": "always", "healthcheck": {"test": "/openstack/healthcheck 5672"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/manila_scheduler.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/manila:/var/lib/kolla/config_files/src:ro", "/var/log/containers/manila:/var/log/manila:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/manila", "setype": "container_file_t", "mode": "0750"}]}, {"name": "enable virt_sandbox_use_netlink for healthcheck", "seboolean": {"name": "virt_sandbox_use_netlink", "persistent": true, "state": true}, "when": ["ansible_facts.selinux is defined", "ansible_facts.selinux.status == \"enabled\""]}], "upgrade_tasks": [], "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop manila scheduler container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["manila_scheduler"], "tripleo_delegate_to": "{{ groups['manila_scheduler'] | difference(groups['excluded_overcloud']) }}"}}]}]}, {"service_name": "manila_share", "config_settings": {"manila::storage_availability_zone": "nova", "manila::notification_driver": ["noop"], "manila::logging::debug": true, "manila::db::database_db_max_retries": -1, "manila::db::database_max_retries": -1, "manila::sql_connection": "mysql+pymysql://manila:2R4FJ91KHJyj5ppSY0XKBqLJg@172.17.0.2/manila?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "manila::network::neutron::network_plugin_ipv6_enabled": false, "manila::network::neutron::network_plugin_ipv4_enabled": true, "manila::rabbit_heartbeat_in_pthread": false, "manila::keystone::authtoken::password": "2R4FJ91KHJyj5ppSY0XKBqLJg", "manila::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000/", "manila::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "manila::keystone::authtoken::project_name": "service", "manila::keystone::authtoken::user_domain_name": "Default", "manila::keystone::authtoken::project_domain_name": "Default", "manila::keystone::authtoken::region_name": "regionOne", "manila::keystone::authtoken::interface": "internal", "manila::compute::nova::username": "manila", "manila::compute::nova::password": "2R4FJ91KHJyj5ppSY0XKBqLJg", "manila::compute::nova::auth_type": "password", "manila::compute::nova::project_name": "service", "manila::compute::nova::user_domain_name": "Default", "manila::compute::nova::project_domain_name": "Default", "manila::compute::nova::auth_url": "http://172.17.0.2:5000", "manila::compute::nova::region_name": "regionOne", "manila::network::neutron::username": "manila", "manila::network::neutron::password": "2R4FJ91KHJyj5ppSY0XKBqLJg", "manila::network::neutron::auth_type": "password", "manila::network::neutron::project_name": "service", "manila::network::neutron::user_domain_name": "Default", "manila::network::neutron::project_domain_name": "Default", "manila::network::neutron::auth_url": "http://172.17.0.2:5000", "manila::network::neutron::region_name": "regionOne", "manila::volume::cinder::username": "manila", "manila::volume::cinder::password": "2R4FJ91KHJyj5ppSY0XKBqLJg", "manila::volume::cinder::auth_type": "password", "manila::volume::cinder::project_name": "service", "manila::volume::cinder::user_domain_name": "Default", "manila::volume::cinder::project_domain_name": "Default", "manila::volume::cinder::auth_url": "http://172.17.0.2:5000", "manila::volume::cinder::region_name": "regionOne", "tripleo::profile::pacemaker::manila::share_bundle::manila_share_docker_image": "cluster.common.tag/manila-share:pcmklatest", "tripleo::profile::pacemaker::manila::share_bundle::docker_volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/manila_share.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/manila:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/dev:/dev", "/run:/run", "/sys:/sys", "/lib/modules:/lib/modules:ro", "/var/lib/manila:/var/lib/manila:z", "/var/log/containers/manila:/var/log/manila:z"], "tripleo::profile::pacemaker::manila::share_bundle::docker_environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}, "tripleo::profile::pacemaker::manila::share_bundle::container_backend": "podman", "tripleo::profile::pacemaker::manila::share_bundle::ceph_conf_path": "/var/lib/tripleo-config/ceph", "manila::share::manage_service": false, "manila::share::enabled": false, "manila::host": "hostgroup"}, "service_config_settings": {"mysql": {"manila::db::mysql::password": "2R4FJ91KHJyj5ppSY0XKBqLJg", "manila::db::mysql::user": "manila", "manila::db::mysql::host": "%", "manila::db::mysql::dbname": "manila"}}, "puppet_config": {"config_volume": "manila", "puppet_tags": "manila_config,file,concat,file_line", "step_config": "include tripleo::profile::pacemaker::manila::share_bundle\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-manila-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/manila_share.json": {"command": "/usr/bin/manila-share --config-file /usr/share/manila/manila-dist.conf --config-file /etc/manila/manila.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-ceph/", "dest": "/etc/ceph/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/manila", "owner": "manila:manila", "recurse": true}]}}, "container_config_scripts": {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}}, "docker_config": {"step_3": {"manila_share_init_logs": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-manila-share:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/manila:/var/log/manila:z"], "command": ["/bin/bash", "-c", "chown -R manila:manila /var/log/manila"]}}}, "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode|default(omit) }}"}, "with_items": [{"path": "/var/log/containers/manila", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/lib/manila", "setype": "container_file_t"}]}, {"name": "ensure ceph configurations exist", "file": {"path": "/var/lib/tripleo-config/ceph", "state": "directory"}}], "deploy_steps_tasks": [{"name": "Manila Share tag container image for pacemaker", "when": "step|int == 1", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "registry.redhat.io/rhosp-rhel9/openstack-manila-share:17.1", "container_image_latest": "cluster.common.tag/manila-share:pcmklatest"}}, {"name": "Manila-Share HA Wrappers Step", "when": "step|int == 5", "block": [{"name": "Manila-Share puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "manila_share", "tripleo_ha_wrapper_resource_name": "openstack-manila-share", "tripleo_ha_wrapper_bundle_name": "openstack-manila-share", "tripleo_ha_wrapper_resource_state": "_ Started", "tripleo_ha_wrapper_puppet_config_volume": "manila", "tripleo_ha_wrapper_puppet_execute": "include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::manila::share_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ip,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".previous_run"}}]}], "external_update_tasks": [{"when": "step|int == 1", "tags": "ha_image_update", "block": [{"block": [{"name": "Get manila_share image from pacemaker", "become": true, "register": "xmllint_pcmk_manila_share_image", "shell": "xmllint --xpath \"string(//bundle[@id='openstack-manila-share']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container manila_share image", "set_fact": {"manila_share_image": "registry.redhat.io/rhosp-rhel9/openstack-manila-share:17.1", "manila_share_image_latest": "cluster.common.tag/manila-share:pcmklatest", "pcmk_manila_share_image": "{{xmllint_pcmk_manila_share_image.stdout}}"}}], "delegate_to": "{{ (groups[\"manila_share\"] | difference(groups[\"excluded_overcloud\"]))[0] }}"}, {"name": "manila-share temporary pacemaker container tag in case of image switch", "become": true, "import_role": {"name": "tripleo_ha_image_update"}, "vars": {"tripleo_ha_image_update_node_names": "{{ groups[\"manila_share\"] | difference(groups[\"excluded_overcloud\"]) }}", "tripleo_ha_image_update_bundle": "openstack-manila-share", "tripleo_ha_image_update_new_image": "{{manila_share_image_latest}}", "tripleo_ha_image_update_old_image": "{{pcmk_manila_share_image}}"}, "when": ["(pcmk_manila_share_image != \"\")", "(pcmk_manila_share_image != manila_share_image_latest)"]}]}], "update_tasks": [{"block": [{"name": "Get manila_share image from pacemaker", "become": true, "register": "xmllint_pcmk_manila_share_image", "shell": "xmllint --xpath \"string(//bundle[@id='openstack-manila-share']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container manila_share image", "set_fact": {"manila_share_image": "registry.redhat.io/rhosp-rhel9/openstack-manila-share:17.1", "manila_share_image_latest": "cluster.common.tag/manila-share:pcmklatest", "pcmk_manila_share_image": "{{xmllint_pcmk_manila_share_image.stdout}}"}}], "when": "(step|int == 0 or step|int == 2)"}, {"name": "Check for update of manila-share container image name", "when": "step|int == 0", "assert": {"that": "pcmk_manila_share_image == manila_share_image_latest", "fail_msg": "manila-share image change detected, run overcloud external-update --tags ha_image_update first"}}, {"name": "manila_share fetch and retag container image for pacemaker", "when": "step|int == 2", "block": [{"name": "Retag pcmklatest to latest manila_share image", "include_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{manila_share_image}}", "container_image_latest": "{{manila_share_image_latest}}"}}]}], "post_update_tasks": [{"name": "Manila-share bundle post update", "when": "step|int == 1", "block": [{"name": "Manila-Share puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "manila_share", "tripleo_ha_wrapper_resource_name": "openstack-manila-share", "tripleo_ha_wrapper_bundle_name": "openstack-manila-share", "tripleo_ha_wrapper_resource_state": "_ Started", "tripleo_ha_wrapper_puppet_config_volume": "manila", "tripleo_ha_wrapper_puppet_execute": "include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::manila::share_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ip,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".previous_run"}}], "vars": {"tripleo_ha_wrapper_minor_update": true}}], "upgrade_tasks": [{"name": "Prepare switch of manila_share image name", "when": ["step|int == 0"], "block": [{"name": "Get manila_share image id currently used by pacemaker", "shell": "pcs resource config openstack-manila-share | grep -Eo 'image=[^ ]+' | awk -F= '{print $2;}'", "register": "manila_share_image_current_res", "failed_when": false}, {"name": "manila_share image facts", "set_fact": {"manila_share_image_latest": "cluster.common.tag/manila-share:pcmklatest", "manila_share_image_current": "{{manila_share_image_current_res.stdout}}"}}, {"name": "Temporarily tag the current manila_share image id with the upgraded image name", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{manila_share_image_current}}", "container_image_latest": "{{manila_share_image_latest}}", "pull_image": false}, "when": ["manila_share_image_current != ''", "manila_share_image_current != manila_share_image_latest"]}, {"name": "Create manila_share retag statefile", "file": {"path": "/var/lib/tripleo/manila_share_needs_retag", "state": "touch"}, "when": ["manila_share_image_current != ''", "manila_share_image_current != manila_share_image_latest"]}]}, {"name": "Update openstack-manila-share pcs resource bundle for new container image", "when": ["step|int == 1"], "block": [{"name": "set is_manila_share_bootstrap_node fact", "tags": "common", "set_fact": "is_manila_share_bootstrap_node={{manila_share_short_bootstrap_node_name|lower == ansible_facts['hostname']|lower}}"}, {"name": "Check for manila_share retag statefile", "stat": {"path": "/var/lib/tripleo/manila_share_needs_retag"}, "register": "manila_share_retag_state_file"}]}, {"name": "Update openstack-manila-share pcs resource bundle for new container image", "when": ["step|int == 1", "is_manila_share_bootstrap_node", "manila_share_retag_state_file.stat.exists|bool"], "block": [{"name": "Disable the manila_share cluster resource before container upgrade", "pacemaker_resource": {"resource": "openstack-manila-share", "state": "disable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}, {"name": "pcs resource bundle update manila_share for new container image name", "command": "pcs resource bundle update openstack-manila-share container image={{manila_share_image_latest}}"}, {"name": "Enable the manila_share cluster resource", "pacemaker_resource": {"resource": "openstack-manila-share", "state": "enable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}]}, {"name": "Check for manila_share retag statefile", "when": ["step|int == 3"], "stat": {"path": "/var/lib/tripleo/manila_share_needs_retag"}, "register": "manila_share_retag_state_file"}, {"name": "Retag openstack-manila-share container image", "when": ["step|int == 3"], "block": [{"name": "Disable the manila_share cluster resource before container upgrade", "pacemaker_resource": {"resource": "openstack-manila-share", "state": "disable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}, {"name": "Retag the pacemaker image if containerized", "block": [{"block": [{"name": "Get manila_share image from pacemaker", "become": true, "register": "xmllint_pcmk_manila_share_image", "shell": "xmllint --xpath \"string(//bundle[@id='openstack-manila-share']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container manila_share image", "set_fact": {"manila_share_image": "registry.redhat.io/rhosp-rhel9/openstack-manila-share:17.1", "manila_share_image_latest": "cluster.common.tag/manila-share:pcmklatest", "pcmk_manila_share_image": "{{xmllint_pcmk_manila_share_image.stdout}}"}}]}, {"block": [{"name": "Retag pcmklatest to latest manila_share image", "include_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{manila_share_image}}", "container_image_latest": "{{manila_share_image_latest}}"}}]}]}, {"name": "Enable the manila_share cluster resource", "pacemaker_resource": {"resource": "openstack-manila-share", "state": "enable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}, {"name": "Remove manila_share retag statefile", "file": {"path": "/var/lib/tripleo/manila_share_needs_retag", "state": "absent"}}]}]}, {"service_name": "memcached", "firewall_rules": {"121 memcached 172.17.0.0/24": {"dport": [11211], "proto": "tcp", "source": "172.17.0.0/24"}}, "monitoring_subscription": "overcloud-memcached", "config_settings": {"memcached::listen": ["127.0.0.1", "%{hiera('internal_api')}"], "memcached::listen_ip": ["127.0.0.1", "%{hiera('internal_api')}"], "memcached::listen_ip_uri": ["127.0.0.1", "%{hiera('internal_api_uri')}"], "memcached::tcp_port": 11211, "memcached::max_connections": 8192, "memcached::max_memory": "50%", "memcached::udp_port": 0, "memcached::verbosity": "vv", "memcached::disable_cachedump": true, "memcached::logstdout": true, "tripleo::profile::base::memcached::enable_internal_memcached_tls": false}, "service_config_settings": {"collectd": {"tripleo.collectd.plugins.memcached": ["memcached"], "collectd::plugin::memcached::instances": {"local": {"host": "%{hiera('memcached::listen_ip_uri')}", "port": [11211]}}}}, "global_config_settings": {"memcached_port": 11211, "memcached_authtoken_port": 11211}, "puppet_config": {"config_volume": "memcached", "puppet_tags": "file", "step_config": "include tripleo::profile::base::memcached\n", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-memcached:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/memcached.json": {"command": "bash -c $* -- eval source /etc/sysconfig/memcached; exec /usr/bin/memcached -p ${PORT} -u ${USER} -m ${CACHESIZE} -c ${MAXCONN} $OPTIONS >> /var/log/memcached/memcached.log 2>&1", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-tls/*", "dest": "/", "merge": true, "preserve_properties": true, "optional": true}], "permissions": [{"path": "/var/log/memcached", "owner": "memcached:memcached", "recurse": true}, {"path": "/etc/pki/tls/certs/memcached.crt", "owner": "memcached:memcached", "optional": true}, {"path": "/etc/pki/tls/private/memcached.key", "owner": "memcached:memcached", "optional": true}]}}, "docker_config": {"step_1": {"memcached": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-memcached:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/memcached.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/memcached:/var/lib/kolla/config_files/src:rw,z", "/var/log/containers/memcached:/var/log/memcached:rw"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/memcached", "setype": "container_file_t", "mode": "0750"}]}], "upgrade_tasks": [], "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop memcached container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["memcached"], "tripleo_delegate_to": "{{ groups['memcached'] | difference(groups['excluded_overcloud']) }}"}}]}]}, {"service_name": "mysql", "firewall_rules": {"104 mysql galera-bundle": {"dport": [873, 3123, 3306, 4444, 4567, 4568, 9200]}}, "firewall_frontend_rules": {"100 mysql_haproxy": {"dport": [3306]}}, "config_settings": {"mysql::server::package_name": "mariadb-galera-server", "mysql::server::manage_config_file": true, "mysql_ipv6": false, "mysql_max_connections": 4096, "mysql::server::root_password": "NhgQDgRVoo", "mysql_clustercheck_password": "sRxOzvWhwndqoiP4taJzvBzao", "enable_galera": true, "mysql_auth_ed25519": false, "mysql_bind_host": "%{hiera('internal_api')}", "tripleo::profile::base::database::mysql::bind_address": "%{hiera('fqdn_internal_api')}", "tripleo::profile::base::database::mysql::client_bind_address": "%{hiera('internal_api')}", "tripleo::profile::base::database::mysql::generate_dropin_file_limit": true, "innodb_buffer_pool_size": "128M", "tripleo::profile::pacemaker::database::mysql::bind_address": "%{hiera('fqdn_internal_api')}", "tripleo::profile::pacemaker::database::mysql::gmcast_listen_addr": "%{hiera('internal_api')}", "tripleo::profile::pacemaker::database::mysql::ca_file": "/etc/ipa/ca.crt", "tripleo::profile::pacemaker::database::mysql_bundle::mysql_docker_image": "cluster.common.tag/mariadb:pcmklatest", "tripleo::profile::pacemaker::database::mysql_bundle::control_port": 3123, "tripleo::profile::pacemaker::database::mysql_bundle::container_backend": "podman", "tripleo::profile::pacemaker::database::mysql_bundle::bind_address": "%{hiera('fqdn_internal_api')}", "tripleo::profile::pacemaker::database::mysql_bundle::sst_method": "rsync", "tripleo::profile::pacemaker::database::mysql_bundle::mariabackup_user": "mariabackup", "tripleo::profile::pacemaker::database::mysql_bundle::mariabackup_password": "3Qi03Zmdc69WCw00okVFEQzzM", "tripleo::profile::pacemaker::database::mysql_bundle::force_ocf": true, "tripleo::profile::pacemaker::database::mysql_bundle::gcomm_cipher": "AES128-SHA256"}, "puppet_config": {"config_volume": "mysql", "puppet_tags": "file", "step_config": "['Mysql_datadir', 'Mysql_user', 'Mysql_database', 'Mysql_grant', 'Mysql_plugin'].each |String $val| { noop_resource($val) }\nexec {'wait-for-settle': command => '/bin/true' }\ninclude tripleo::profile::pacemaker::database::mysql_bundle", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-mariadb:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/mysql.json": {"command": "/usr/sbin/pacemaker_remoted", "config_files": [{"dest": "/etc/libqb/force-filesystem-sockets", "source": "/dev/null", "owner": "root", "perm": "0644"}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-tls/*", "dest": "/", "merge": true, "optional": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/mysql", "owner": "mysql:mysql", "recurse": true}, {"path": "/etc/pki/tls/certs/mysql.crt", "owner": "mysql:mysql", "perm": "0600", "optional": true}, {"path": "/etc/pki/tls/private/mysql.key", "owner": "mysql:mysql", "perm": "0600", "optional": true}]}}, "container_config_scripts": {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}}, "docker_config": {"step_1": {"mysql_data_ownership": {"start_order": 0, "detach": false, "image": "registry.redhat.io/rhosp-rhel9/openstack-mariadb:17.1", "net": "host", "user": "root", "command": ["chown", "-R", "mysql:", "/var/lib/mysql"], "volumes": ["/var/lib/mysql:/var/lib/mysql:z"]}, "mysql_bootstrap": {"start_order": 1, "detach": false, "image": "registry.redhat.io/rhosp-rhel9/openstack-mariadb:17.1", "net": "host", "user": "root", "command": ["bash", "-ec", "if [ -e /var/lib/mysql/mysql ]; then exit 0; fi\necho -e \"\\n[mysqld]\\nwsrep_provider=none\" >> /etc/my.cnf\nexport DB_ROOT_PASSWORD=$(hiera 'mysql::server::root_password')\nkolla_set_configs\nsudo -u mysql -E kolla_extend_start\ntimeout ${DB_MAX_TIMEOUT} /bin/bash -c 'while pgrep -af /usr/bin/mysqld_safe | grep -q -v grep; do sleep 1; done'\nmysqld_safe --skip-networking --wsrep-on=OFF &\ntimeout ${DB_MAX_TIMEOUT} /bin/bash -c 'until mysqladmin -uroot -p\"$(hiera 'mysql::server::root_password')\" ping 2>/dev/null; do sleep 1; done'\nmysql -uroot -p\"$(hiera 'mysql::server::root_password')\" -e \"CREATE USER 'clustercheck'@'localhost' IDENTIFIED BY '$(hiera mysql_clustercheck_password)';\"\nmysql -uroot -p\"$(hiera 'mysql::server::root_password')\" -e \"GRANT PROCESS ON *.* TO 'clustercheck'@'localhost' WITH GRANT OPTION;\"\nmysql -uroot -p\"$(hiera 'mysql::server::root_password')\" -e \"DELETE FROM mysql.user WHERE user = 'root' AND host NOT IN ('%','localhost');\"\ntimeout ${DB_MAX_TIMEOUT} mysqladmin -uroot -p\"$(hiera 'mysql::server::root_password')\" shutdown"], "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:rw,z", "/var/lib/config-data/puppet-generated/mysql:/var/lib/kolla/config_files/src:ro,z", "/var/lib/mysql:/var/lib/mysql:rw,z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "KOLLA_BOOTSTRAP": true, "DB_MAX_TIMEOUT": 60, "DB_MARIABACKUP_USER": "mariabackup", "DB_MARIABACKUP_PASSWORD": "3Qi03Zmdc69WCw00okVFEQzzM"}}}, "step_2": {"mysql_wait_bundle": {"start_order": 0, "detach": false, "net": "host", "ipc": "host", "user": "root", "command": ["/container_puppet_apply.sh", "2", "file,file_line,concat,augeas,galera_ready,mysql_database,mysql_grant,mysql_user", "include tripleo::profile::pacemaker::database::mysql_bundle"], "image": "registry.redhat.io/rhosp-rhel9/openstack-mariadb:17.1", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/var/lib/container-config-scripts/container_puppet_apply.sh:/container_puppet_apply.sh:ro", "/etc/puppet:/tmp/puppet-etc:ro", "/usr/share/openstack-puppet/modules:/usr/share/openstack-puppet/modules:ro", "/var/lib/mysql:/var/lib/mysql:rw,z", "/var/lib/config-data/puppet-generated/mysql/root:/root:rw"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}}, "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode|default(omit) }}"}, "with_items": [{"path": "/var/log/containers/mysql", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/lib/mysql", "setype": "container_file_t"}, {"path": "/var/log/mariadb", "setype": "container_file_t", "mode": "0750"}]}], "metadata_settings": null, "deploy_steps_tasks": [{"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "mysql", "dns": ["{{fqdn_internal_api}}", "{{cloud_names.cloud_name_internal_api}}"], "principal": "mysql/{{fqdn_internal_api}}@{{idm_realm}}", "key_size": "2048", "ca": "ipa"}]}}]}, {"name": "MySQL tag container image for pacemaker", "when": "step|int == 1", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "registry.redhat.io/rhosp-rhel9/openstack-mariadb:17.1", "container_image_latest": "cluster.common.tag/mariadb:pcmklatest"}}, {"name": "MySQL HA Wrappers Step", "when": "step|int == 2", "block": [{"name": "Mysql puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "mysql", "tripleo_ha_wrapper_resource_name": "galera", "tripleo_ha_wrapper_bundle_name": "galera-bundle", "tripleo_ha_wrapper_resource_state": "Master", "tripleo_ha_wrapper_puppet_config_volume": "mysql", "tripleo_ha_wrapper_puppet_execute": "[\"Mysql_datadir\", \"Mysql_user\", \"Mysql_database\", \"Mysql_grant\", \"Mysql_plugin\"].each |String $val| { noop_resource($val) }; include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::database::mysql_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".previous_run"}}]}], "external_update_tasks": [{"when": "step|int == 1", "tags": "ha_image_update", "block": [{"block": [{"name": "Get galera image from pacemaker", "become": true, "register": "xmllint_pcmk_galera_image", "shell": "xmllint --xpath \"string(//bundle[@id='galera-bundle']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container galera image", "set_fact": {"galera_image": "registry.redhat.io/rhosp-rhel9/openstack-mariadb:17.1", "galera_image_latest": "cluster.common.tag/mariadb:pcmklatest", "pcmk_galera_image": "{{xmllint_pcmk_galera_image.stdout}}"}}], "delegate_to": "{{ (groups[\"mysql\"] | difference(groups[\"excluded_overcloud\"]))[0] }}"}, {"name": "MySQL temporary pacemaker container tag in case of image switch", "become": true, "import_role": {"name": "tripleo_ha_image_update"}, "vars": {"tripleo_ha_image_update_node_names": "{{ groups[\"mysql\"] | difference(groups[\"excluded_overcloud\"]) }}", "tripleo_ha_image_update_bundle": "galera-bundle", "tripleo_ha_image_update_new_image": "{{galera_image_latest}}", "tripleo_ha_image_update_old_image": "{{pcmk_galera_image}}"}, "when": ["(pcmk_galera_image != \"\")", "(pcmk_galera_image != galera_image_latest)"]}]}], "update_tasks": [{"block": [{"name": "Get galera image from pacemaker", "become": true, "register": "xmllint_pcmk_galera_image", "shell": "xmllint --xpath \"string(//bundle[@id='galera-bundle']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container galera image", "set_fact": {"galera_image": "registry.redhat.io/rhosp-rhel9/openstack-mariadb:17.1", "galera_image_latest": "cluster.common.tag/mariadb:pcmklatest", "pcmk_galera_image": "{{xmllint_pcmk_galera_image.stdout}}"}}], "when": "(step|int == 0 or step|int == 2)"}, {"name": "Check for update of galera container image name", "when": "step|int == 0", "assert": {"that": "pcmk_galera_image == galera_image_latest", "fail_msg": "galera image change detected, run overcloud external-update --tags ha_image_update first"}}, {"name": "Tear-down non-HA mysql container", "when": ["step|int == 1"], "block": [{"name": "stat mysql container", "command": "podman container exists mysql", "failed_when": false, "changed_when": false, "register": "stat_mysql_container"}, {"name": "Create clustercheck user and permissions", "command": {"argv": "{{ mysql_exec_data | container_exec_cmd }}"}, "changed_when": true, "when": ["stat_mysql_container.rc == 0"], "vars": {"mysql_exec_data": {"environment": {"CLUSTERCHECK_PASSWORD": "sRxOzvWhwndqoiP4taJzvBzao"}, "command": ["mysql", "/bin/sh", "-c", "mysql -e \"CREATE USER IF NOT EXISTS 'clustercheck'@'localhost' IDENTIFIED BY '${CLUSTERCHECK_PASSWORD}'; GRANT PROCESS ON *.* TO 'clustercheck'@'localhost' WITH GRANT OPTION;\""]}}}, {"name": "Remove non-HA mysql container", "include_role": {"name": "tripleo_container_rm"}, "vars": {"tripleo_container_cli": "{{ container_cli }}", "tripleo_containers_to_rm": ["mysql"]}}]}, {"name": "Mariadb fetch and retag container image for pacemaker", "when": "step|int == 2", "block": [{"name": "Retag pcmklatest to latest galera image", "include_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{galera_image}}", "container_image_latest": "{{galera_image_latest}}"}}]}, {"name": "Ensure mariadb-server is not installed on the host", "when": "step|int == 2", "yum": {"name": "mariadb-server*", "state": "absent"}}], "post_update_tasks": [{"name": "Mysql bundle post update", "when": "step|int == 1", "block": [{"name": "Mysql puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "mysql", "tripleo_ha_wrapper_resource_name": "galera", "tripleo_ha_wrapper_bundle_name": "galera-bundle", "tripleo_ha_wrapper_resource_state": "Master", "tripleo_ha_wrapper_puppet_config_volume": "mysql", "tripleo_ha_wrapper_puppet_execute": "[\"Mysql_datadir\", \"Mysql_user\", \"Mysql_database\", \"Mysql_grant\", \"Mysql_plugin\"].each |String $val| { noop_resource($val) }; include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::database::mysql_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".previous_run"}}], "vars": {"tripleo_ha_wrapper_minor_update": true}}], "upgrade_tasks": [{"name": "Tear-down non-HA mysql container", "when": ["step|int == 0"], "block": [{"name": "stat mysql container", "command": "podman container exists mysql", "failed_when": false, "changed_when": false, "register": "stat_mysql_container"}, {"name": "Create clustercheck user and permissions", "command": {"argv": "{{ mysql_exec_data | container_exec_cmd }}"}, "changed_when": true, "when": ["stat_mysql_container.rc == 0"], "vars": {"mysql_exec_data": {"environment": {"CLUSTERCHECK_PASSWORD": "sRxOzvWhwndqoiP4taJzvBzao"}, "command": ["mysql", "/bin/sh", "-c", "mysql -e \"CREATE USER IF NOT EXISTS 'clustercheck'@'localhost' IDENTIFIED BY '${CLUSTERCHECK_PASSWORD}'; GRANT PROCESS ON *.* TO 'clustercheck'@'localhost' WITH GRANT OPTION;\""]}}}, {"name": "Remove non-HA mysql container", "include_role": {"name": "tripleo_container_rm"}, "vars": {"tripleo_container_cli": "{{ container_cli }}", "tripleo_containers_to_rm": ["mysql"]}}]}, {"name": "set mysql container name fact", "when": ["step|int == 1"], "set_fact": {"mysql_container_name": "galera-bundle"}}, {"name": "Drop unused OpenStack DB users", "when": ["step|int == 1", "mysql_short_bootstrap_node_name|lower == ansible_facts['hostname']|lower"], "block": [{"name": "mysql container id", "command": "podman ps -q --filter name={{ mysql_container_name }} --filter status=running", "register": "mysql_container_id"}, {"name": "set mysql container id fact", "set_fact": {"mysql_container": "{{ mysql_container_id.stdout }}"}}, {"name": "Mysql script to drop unused DB users", "when": "( mysql_container | length ) > 0", "block": [{"name": "Get the list of all OpenStack DB users", "shell": "jq -r 'to_entries[] | select(.key|endswith(\"::db::mysql::user\")) | .value' /etc/puppet/hieradata/service_configs.json", "register": "openstack_db_users"}, {"name": "List all DB users that match the DB users to be dropped", "shell": "for u in {{ openstack_db_users.stdout_lines | join(' ') }}; do podman exec -u root -it \"{{ mysql_container }}\" mysql -sNe \"select concat('\\`',user,'\\`@\\`',host,'\\`') from mysql.user where user = '$u' and host != '%';\"; done", "register": "mysql_db_users"}, {"name": "resulting DB users to be dropped", "debug": {"msg": "{{ mysql_db_users.stdout_lines }}"}}, {"name": "Drop all unneeded Openstack DB users", "shell": "podman exec -u root -it \"{{ mysql_container }}\" mysql -sNe 'drop user {{ item }};'", "loop": "{{ mysql_db_users.stdout_lines }}"}]}]}, {"vars": {"mysql_upgrade_persist": false}, "when": ["step|int == 3", "mysql_upgrade_persist"], "tags": ["never", "system_upgrade", "system_upgrade_prepare"], "block": [{"name": "Persist mysql data", "include_role": {"name": "tripleo_persist", "tasks_from": "persist.yml"}, "vars": {"tripleo_persist_dir": "/var/lib/mysql"}}]}, {"vars": {"mysql_upgrade_persist": false}, "when": ["step|int == 5", "mysql_upgrade_persist"], "tags": ["never", "system_upgrade", "system_upgrade_run"], "block": [{"name": "Restore mysql data", "include_role": {"name": "tripleo_persist", "tasks_from": "restore.yml"}, "vars": {"tripleo_persist_dir": "/var/lib/mysql"}}]}, {"name": "Prepare switch of galera image name", "when": ["step|int == 0"], "block": [{"name": "Get galera image id currently used by pacemaker", "shell": "pcs resource config galera-bundle | grep -Eo 'image=[^ ]+' | awk -F= '{print $2;}'", "register": "galera_image_current_res", "failed_when": false}, {"name": "Image facts for galera", "set_fact": {"galera_image_latest": "cluster.common.tag/mariadb:pcmklatest", "galera_image_current": "{{galera_image_current_res.stdout}}"}}, {"name": "Temporarily tag the current galera image id with the upgraded image name", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{galera_image_current}}", "container_image_latest": "{{galera_image_latest}}", "pull_image": false}, "when": ["galera_image_current != ''", "galera_image_current != galera_image_latest"]}, {"name": "Create galera retag statefile", "file": {"path": "/var/lib/tripleo/galera_needs_retag", "state": "touch"}, "when": ["galera_image_current != ''", "galera_image_current != galera_image_latest"]}]}, {"name": "Update galera pcs resource bundle for new container image - check", "when": ["step|int == 1"], "block": [{"name": "set is_mysql_bootstrap_node fact", "tags": "common", "set_fact": "is_mysql_bootstrap_node={{mysql_short_bootstrap_node_name|lower == ansible_facts['hostname']|lower}}"}, {"name": "Check for galera retag statefile", "stat": {"path": "/var/lib/tripleo/galera_needs_retag"}, "register": "galera_retag_state_file"}]}, {"name": "Update galera pcs resource bundle for new container image", "when": ["step|int == 1", "is_mysql_bootstrap_node|bool", "galera_retag_state_file.stat.exists|bool"], "block": [{"name": "Disable the galera cluster resource before container upgrade", "pacemaker_resource": {"resource": "galera-bundle", "state": "disable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}, {"name": "Move Mysql logging to /var/log/containers", "block": [{"name": "Check Mysql logging configuration in pacemaker", "command": "cibadmin --query --xpath \"//storage-mapping[@id='mysql-log']\"", "failed_when": false, "register": "mysql_logs_moved"}, {"name": "Change Mysql logging configuration in pacemaker", "when": "mysql_logs_moved.rc == 6", "block": [{"name": "Add a bind mount for logging in the galera bundle", "command": "pcs resource bundle update galera-bundle storage-map add id=mysql-log source-dir=/var/log/containers/mysql target-dir=/var/log/mysql options=rw"}, {"name": "Reconfigure Mysql log file in the galera resource agent", "command": "pcs resource update galera log=/var/log/mysql/mysqld.log"}]}]}, {"name": "Update the galera bundle to use the new container image name", "command": "pcs resource bundle update galera-bundle container image={{galera_image_latest}}"}, {"name": "Enable the galera cluster resource", "pacemaker_resource": {"resource": "galera-bundle", "state": "enable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}]}, {"name": "Upgrade galera in step3", "when": ["step|int == 3"], "block": [{"name": "Bind mounts for temporary container", "set_fact": {"mysql_upgrade_db_bind_mounts": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:rw,z", "/var/lib/config-data/puppet-generated/mysql:/var/lib/kolla/config_files/src:ro,z", "/var/lib/mysql:/var/lib/mysql:rw,z", "/var/log/containers/mysql:/var/log/mysql:rw,z"]}}, {"name": "Check if Galera needs upgrade", "shell": "{{ container_cli }} run --rm --log-driver=k8s-file --log-opt path=/var/log/containers/mysql/db-upgrade.log \\ -u root --net=host -e \"KOLLA_CONFIG_STRATEGY=COPY_ALWAYS\" -v {{ mysql_upgrade_db_bind_mounts | join(' -v ')}} \"cluster.common.tag/mariadb:pcmklatest\" /bin/bash -ecx \"$UPGRADE_SCRIPT\"", "environment": {"UPGRADE_SCRIPT": "kolla_set_configs\nupgraded_ver=$(cat /var/lib/mysql/mysql_upgrade_info 2>/dev/null || true)\nmysql_ver=$(mysql --version | awk -F'[ ,]*' '{print $5}')\nif [ \"${upgraded_ver}\" = \"${mysql_ver}\" ]; then\n    echo \"mysql already upgraded\"\nelse\n    echo \"${upgraded_ver} VS ${mysql_ver}\"\nfi\n"}, "register": "mysql_upgrade_needed"}, {"name": "Set fact upgrade_mysql", "set_fact": {"upgrade_mysql": "{{ (mysql_upgrade_needed.stdout != \"mysql already upgraded\") | bool }}"}}, {"debug": {"msg": "MYSQL check - {{ mysql_upgrade_needed.stdout }} - Upgrade needed: {{ upgrade_mysql }}"}}, {"name": "Disable the galera cluster resource before container upgrade", "when": ["upgrade_mysql|bool"], "pacemaker_resource": {"resource": "galera-bundle", "state": "disable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}, {"block": [{"name": "Get galera image from pacemaker", "become": true, "register": "xmllint_pcmk_galera_image", "shell": "xmllint --xpath \"string(//bundle[@id='galera-bundle']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container galera image", "set_fact": {"galera_image": "registry.redhat.io/rhosp-rhel9/openstack-mariadb:17.1", "galera_image_latest": "cluster.common.tag/mariadb:pcmklatest", "pcmk_galera_image": "{{xmllint_pcmk_galera_image.stdout}}"}}]}, {"name": "Check for galera retag statefile", "stat": {"path": "/var/lib/tripleo/galera_needs_retag"}, "register": "galera_retag_state_file"}, {"name": "Retag the pacemaker image if containerized", "when": ["galera_retag_state_file.stat.exists|bool"], "block": [{"name": "Retag pcmklatest to latest galera image", "include_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{galera_image}}", "container_image_latest": "{{galera_image_latest}}"}}]}, {"name": "Remove galera retag statefile", "file": {"path": "/var/lib/tripleo/galera_needs_retag", "state": "absent"}}, {"name": "Mysql upgrade script", "when": ["upgrade_mysql|bool"], "set_fact": {"mysql_upgrade_script": "kolla_set_configs\nif mysqladmin ping --silent; then exit 0; fi\nupgraded_ver=$(cat /var/lib/mysql/mysql_upgrade_info 2>/dev/null || true)\nmysql_ver=$(mysql --version | awk -F'[ ,]*' '{print $5}')\nif [ \"${upgraded_ver}\" = \"${mysql_ver}\" ]; then\n    echo \"mysql already upgraded\"\n    exit 0\nfi\nchown -R mysql:mysql /var/lib/mysql\nchown -R mysql:mysql /var/log/mysql\nmysqld_safe --user=mysql --wsrep-provider=none --skip-networking --wsrep-on=off --log-error=/var/log/mysql/mysqld-upgrade.log &\n\n#!/usr/bin/bash\n\nset -e\n\n# Wait until we know the mysql server is up and responding\ntimeout ${DB_MAX_TIMEOUT:-60} /bin/bash -c 'until mysqladmin -uroot ping 2>/dev/null; do sleep 1; done'\n\n# After an upgrade, make sure that the running mysql had a chance to\n# update its data table on disk.\nmysql_upgrade\n\n# Upgrade to 10.3: the default table row format changed from COMPACT\n# to DYNAMIC, so upgrade the existing tables.\ncompact_tables=$(mysql -se 'SELECT CONCAT(\"`\",TABLE_SCHEMA,\"`.`\",TABLE_NAME,\"`\") FROM information_schema.tables WHERE ENGINE = \"InnoDB\" and ROW_FORMAT = \"Compact\";');\nfor i in $compact_tables; do echo converting row format of table $i; mysql -e \"ALTER TABLE $i ROW_FORMAT=DYNAMIC;\"; done;\n\nmysqladmin shutdown"}}, {"name": "Upgrade Mysql database from a temporary container", "when": ["upgrade_mysql|bool"], "shell": "{{ container_cli }} run --rm --log-driver=k8s-file --log-opt path=/var/log/containers/mysql/db-upgrade.log \\ -u root --net=host -e \"KOLLA_CONFIG_STRATEGY=COPY_ALWAYS\" -v {{ mysql_upgrade_db_bind_mounts | join(' -v ')}} \"cluster.common.tag/mariadb:pcmklatest\" /bin/bash -ecx \"$UPGRADE_SCRIPT\"", "environment": {"UPGRADE_SCRIPT": "{{ mysql_upgrade_script }}"}}, {"name": "Enable the galera cluster resource", "when": ["upgrade_mysql|bool"], "pacemaker_resource": {"resource": "galera-bundle", "state": "enable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}]}], "external_upgrade_tasks": [{"vars": {"mysql_upgrade_transfer": true}, "when": ["step|int == 2", "mysql_upgrade_transfer"], "tags": ["never", "system_upgrade_transfer_data"], "block": [{"name": "Transfer mysql data", "include_role": {"name": "tripleo_transfer"}, "vars": {"tripleo_transfer_src_dir": "/var/lib/mysql", "tripleo_transfer_src_host": "{{hostvars[groups['overcloud'][0]]['mysql_short_node_names'][1]}}", "tripleo_transfer_dest_dir": "/var/lib/mysql", "tripleo_transfer_dest_host": "{{hostvars[groups['overcloud'][0]]['mysql_short_bootstrap_node_name']}}", "tripleo_transfer_flag_file": "/var/lib/tripleo/transfer-flags/var-lib-mysql", "tripleo_transfer_sync_options": "--whole-file --ignore-times --inplace --archive --delete", "tripleo_transfer_include": ["/ibdata*", "/ib_lru_dump", "/ib_logfile[0-9]*", "/undo*", "/*/"], "tripleo_transfer_exclude": "/*"}}]}]}, {"service_name": "mysql_client", "config_settings": {"tripleo::profile::base::database::mysql::client::mysql_client_bind_address": "%{hiera('internal_api')}", "tripleo::profile::base::database::mysql::client::enable_ssl": false, "tripleo::profile::base::database::mysql::client::ssl_ca": "/etc/ipa/ca.crt"}, "step_config": "include tripleo::profile::base::database::mysql::client"}, {"service_name": "neutron_api", "firewall_rules": {"114 neutron api": {"dport": [9696]}}, "firewall_frontend_rules": {"100 neutron_haproxy_frontend": {"dport": [9696]}}, "firewall_ssl_frontend_rules": {"100 neutron_haproxy_frontend_ssl": {"dport": [13696]}}, "keystone_resources": {"neutron": {"endpoints": {"public": "http://172.21.0.2:9696", "internal": "http://172.17.0.2:9696", "admin": "http://172.17.0.2:9696"}, "users": {"neutron": {"password": "Y7cJplS04DjBDFMUWgbx3kdiC"}}, "region": "regionOne", "service": "network"}}, "monitoring_subscription": "overcloud-neutron-server", "config_settings": {"neutron::notification_driver": ["noop"], "neutron::core_plugin": "ml2", "neutron::service_plugins": ["qos", "ovn-router", "trunk", "segments", "port_forwarding", "log"], "neutron::report_interval": 300, "neutron::logging::debug": true, "neutron::purge_config": false, "neutron::allow_overlapping_ips": true, "neutron::dhcp_agent_notification": true, "neutron::dns_domain": "localdomain", "neutron::rabbit_heartbeat_timeout_threshold": 60, "neutron::host": "%{hiera('fqdn_canonical')}", "neutron::db::database_db_max_retries": -1, "neutron::db::database_max_retries": -1, "neutron::db::sync::db_sync_timeout": 900, "neutron::global_physnet_mtu": 1500, "neutron::db::sync::extra_params": "", "neutron::vlan_transparent": false, "neutron::rabbit_heartbeat_in_pthread": false, "neutron::db::database_connection": "mysql+pymysql://neutron:Y7cJplS04DjBDFMUWgbx3kdiC@172.17.0.2/ovs_neutron?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "neutron::policy::policies": {}, "if": [false, {"neutron::policy::enforce_scope": true, "neutron::policy::enforce_new_defaults": true}], "neutron::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "neutron::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "neutron::server::agent_down_time": 600, "neutron::server::allow_automatic_l3agent_failover": "True", "neutron::server::enable_proxy_headers_parsing": true, "neutron::server::igmp_snooping_enable": false, "neutron::keystone::authtoken::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "neutron::server::notifications::nova::auth_url": "http://172.17.0.2:5000", "neutron::server::notifications::nova::project_name": "service", "neutron::server::notifications::nova::user_domain_name": "Default", "neutron::server::notifications::nova::project_domain_name": "Default", "neutron::server::notifications::nova::region_name": "regionOne", "neutron::server::notifications::nova::password": "nad1A8ZvKDsT60yawIjN1cH4K", "neutron::server::notifications::nova::endpoint_type": "internal", "neutron::keystone::authtoken::project_name": "service", "neutron::keystone::authtoken::user_domain_name": "Default", "neutron::keystone::authtoken::project_domain_name": "Default", "neutron::keystone::authtoken::region_name": "regionOne", "neutron::keystone::authtoken::interface": "internal", "neutron::keystone::authtoken::memcache_use_advanced_pool": true, "neutron::quota::quota_port": "500", "neutron::quota::quota_security_group": "10", "neutron::server::placement::password": "nad1A8ZvKDsT60yawIjN1cH4K", "neutron::server::placement::project_domain_name": "Default", "neutron::server::placement::project_name": "service", "neutron::server::placement::user_domain_name": "Default", "neutron::server::placement::username": "nova", "neutron::server::placement::auth_url": "http://172.17.0.2:5000", "neutron::server::placement::auth_type": "password", "neutron::server::placement::region_name": "regionOne", "tripleo::profile::base::neutron::server::tls_proxy_bind_ip": "%{hiera('internal_api')}", "tripleo::profile::base::neutron::server::tls_proxy_fqdn": "%{hiera('fqdn_internal_api')}", "tripleo::profile::base::neutron::server::tls_proxy_port": "9696", "neutron::bind_host": "%{hiera('internal_api')}", "tripleo::profile::base::neutron::server::l3_ha_override": "", "neutron::server::api_workers": "1", "neutron::server::rpc_workers": "1"}, "service_config_settings": {"rsyslog": {"tripleo_logging_sources_neutron_api": [{"tag": "openstack.neutron.api", "file": "/var/log/containers/neutron/server.log"}]}, "mysql": {"neutron::db::mysql::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "neutron::db::mysql::user": "neutron", "neutron::db::mysql::host": "%", "neutron::db::mysql::dbname": "ovs_neutron"}}, "puppet_config": {"config_volume": "neutron", "puppet_tags": "neutron_config,neutron_api_paste_ini", "step_config": "include tripleo::profile::base::neutron::server\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-server:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/neutron_api.json": {"command": "/usr/bin/neutron-server --config-file /usr/share/neutron/neutron-dist.conf --config-dir /usr/share/neutron/server --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugin.ini --config-dir /etc/neutron/conf.d/common --config-dir /etc/neutron/conf.d/neutron-server --log-file=/var/log/neutron/server.log", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-tls/*", "dest": "/", "merge": true, "optional": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/neutron", "owner": "neutron:neutron", "recurse": true}, {"path": "/etc/pki/tls/certs/neutron_ovn.crt", "owner": "neutron:neutron", "optional": true, "perm": "0644"}, {"path": "/etc/pki/tls/private/neutron_ovn.key", "owner": "neutron:neutron", "optional": true, "perm": "0644"}]}, "/var/lib/kolla/config_files/neutron_api_db_sync.json": {"command": "/usr/bin/bootstrap_host_exec neutron_api neutron-db-manage upgrade heads", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-tls/*", "dest": "/", "merge": true, "optional": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/neutron", "owner": "neutron:neutron", "recurse": true}, {"path": "/etc/pki/tls/certs/neutron_ovn.crt", "owner": "neutron:neutron", "optional": true, "perm": "0644"}, {"path": "/etc/pki/tls/private/neutron_ovn.key", "owner": "neutron:neutron", "optional": true, "perm": "0644"}]}, "/var/lib/kolla/config_files/neutron_server_tls_proxy.json": {"command": "/usr/sbin/httpd -DFOREGROUND", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}]}}, "docker_config": {"step_2": {"neutron_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-server:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/neutron:/var/log/neutron:z", "/var/log/containers/httpd/neutron-api:/var/log/httpd:z"], "command": ["/bin/bash", "-c", "chown -R neutron:neutron /var/log/neutron"]}}, "step_3": {"neutron_db_sync": {"image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-server:17.1", "cap_add": ["AUDIT_WRITE"], "net": "host", "privileged": false, "detach": false, "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/neutron:/var/log/neutron:z", "/var/log/containers/httpd/neutron-api:/var/log/httpd:z", "/var/lib/kolla/config_files/neutron_api_db_sync.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/neutron:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_4": {"neutron_api": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-server:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/neutron:/var/log/neutron:z", "/var/log/containers/httpd/neutron-api:/var/log/httpd:z", "/var/lib/kolla/config_files/neutron_api.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/neutron:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/neutron", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/log/containers/httpd/neutron-api", "setype": "container_file_t", "mode": "0750"}]}], "metadata_settings": [], "deploy_steps_tasks": null, "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop neutron api container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["neutron_api"], "tripleo_delegate_to": "{{ groups['neutron_api'] | difference(groups['excluded_overcloud']) }}"}}]}], "external_update_tasks": [{"name": "Check if python3-openvswitch3.1 workaround is needed", "delegate_to": "{{groups['neutron_api'][0]}}", "become": true, "tags": ["ovn", "ovn_image"], "when": "step|int == 1", "shell": "podman exec neutron_api rpm -q python3-openvswitch3.1\n", "register": "neutron_python_ovs_3_1", "failed_when": false}, {"name": "Update neutron_api first everywhere", "become": true, "tags": ["ovn", "ovn_image"], "when": ["step|int == 1", "neutron_python_ovs_3_1.rc == 0"], "block": [{"name": "Move virtual IPs to another node before stopping pacemaker", "delegate_to": "{{groups['neutron_api'][0]}}", "when": ["hostvars[groups['neutron_api'][0]][\"haproxy_node_names\"]|default([])|length > 1"], "shell": "CLUSTER_NODE=$(crm_node -n)\necho \"Retrieving all the VIPs which are hosted on this node\"\nVIPS_TO_MOVE=$(crm_mon --as-xml | xmllint --xpath '//resource[@resource_agent=\"ocf:heartbeat:IPaddr2\" and @role = \"Started\" and @managed = \"true\" and ./node[@name = \"'${CLUSTER_NODE}'\"]]/@id' - | sed -e 's/id=//g' -e 's/\"//g')\nfor v in ${VIPS_TO_MOVE}; do\n    echo \"Moving VIP $v on another node\"\n    pcs resource ban $v ${CLUSTER_NODE} --wait=300\ndone\necho \"Removing the location constraints that were created to move the VIPs\"\nfor v in ${VIPS_TO_MOVE}; do\n    echo \"Removing location ban for VIP $v\"\n    ban_id=$(cibadmin --query | xmllint --xpath 'string(//rsc_location[@rsc=\"'${v}'\" and @node=\"'${CLUSTER_NODE}'\" and @score=\"-INFINITY\"]/@id)' -)\n    if [ -n \"$ban_id\" ]; then\n        pcs constraint remove ${ban_id}\n    else\n        echo \"Could not retrieve and clear location constraint for VIP $v\" 2>&1\n    fi\ndone\n"}, {"name": "Create firewall nat prerouting and postrouting", "delegate_to": "{{groups['neutron_api'][0]}}", "shell": "set -o pipefail\nLOCAL_BACKEND=$(grep {{groups['neutron_api'][0]}} /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg | \\\n  grep \":9696 \" | awk '{print $3}')\nIP=${LOCAL_BACKEND%:*}\nif [[ $IP =~ .*:.* ]]\n  then\n       PROTOCOL=\"ip6\"\n       # we need to temporarly allow ipv6 forwarding, this will get reset by deployment run\n       sysctl -w net.ipv6.conf.all.forwarding=1\n  else PROTOCOL=\"ip\"\nfi\nnft add table $PROTOCOL nat\nnft -- add chain $PROTOCOL nat prerouting { type nat hook prerouting priority -100 \\; }\nnft add chain $PROTOCOL nat postrouting { type nat hook postrouting priority 100 \\; }\n"}, {"name": "Redirect neutron backend traffic for 60s to other host", "delegate_to": "{{groups['neutron_api'][0]}}", "shell": "set -o pipefail\nsource /etc/os-release; test  \"${VERSION_ID%*}\" = \"9.0\" && exit 0\nLOCAL_BACKEND=$(grep {{groups['neutron_api'][0]}} /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg | \\\n  grep \":9696 \" | awk '{print $3}')\nREMOTE_BACKEND=$(grep \":9696 \" /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg | \\\n  grep server | grep -v {{groups['neutron_api'][0]}} | head -n1 | awk '{print $3}')\nIP=${LOCAL_BACKEND%:*}\nPORT=9696\nif [[ $IP =~ .*:.* ]]\n  then PROTOCOL=\"ip6\"\n  else PROTOCOL=\"ip\"\nfi\nREMOTE_IP=${REMOTE_BACKEND%:*}\nnft add rule $PROTOCOL nat postrouting $PROTOCOL daddr $REMOTE_IP tcp dport 9696 counter meta time $(date +%s)-$(date -d'+60 seconds' +%s) masquerade comment neutron_api\nnft add rule $PROTOCOL nat prerouting $PROTOCOL daddr $IP tcp dport 9696 counter meta time $(date +%s)-$(date -d'+60 seconds' +%s) dnat to $REMOTE_IP comment neutron_api\n"}, {"name": "Insanity neutron update first", "delegate_to": "{{groups['neutron_api'][0]}}", "become": true, "tripleo_container_manage": {"config_dir": "/var/lib/tripleo-config/container-startup-config/step_4", "config_id": ["neutron_api"], "config_patterns": "*neutron_api.json", "config_overrides": {".*neutron_api": {"image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-server:17.1", "name": "neutron_api"}}}}, {"wait_for": {"timeout": 50}}, {"name": "Delete redirect rules", "delegate_to": "{{groups['neutron_api'][0]}}", "shell": "set -o pipefail\nLOCAL_BACKEND=$(grep {{groups['neutron_api'][0]}} /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg | \\\n  grep \":9696 \" | awk '{print $3}')\nIP=${LOCAL_BACKEND%:*}\nif [[ $IP =~ .*:.* ]]\n  then\n       PROTOCOL=\"ip6\"\n  else PROTOCOL=\"ip\"\nfi\nnft -a list chain $PROTOCOL nat postrouting | grep neutron_api | awk -F# '{print \"nat postrouting \"$2}' | \\\nwhile read i; do bash -c \"nft delete rule $PROTOCOL $i\"; done\nnft -a list chain $PROTOCOL nat prerouting | grep neutron_api | awk -F# '{print \"nat prerouting \"$2}' | \\\nwhile read i; do bash -c \"nft delete rule $PROTOCOL $i\"; done\n"}, {"name": "Move virtual IPs to another node before stopping pacemaker", "delegate_to": "{{groups['neutron_api'][1]}}", "when": ["hostvars[groups['neutron_api'][1]][\"haproxy_node_names\"]|default([])|length > 1", "groups['neutron_api'] | default([]) | length > 1"], "shell": "CLUSTER_NODE=$(crm_node -n)\necho \"Retrieving all the VIPs which are hosted on this node\"\nVIPS_TO_MOVE=$(crm_mon --as-xml | xmllint --xpath '//resource[@resource_agent=\"ocf:heartbeat:IPaddr2\" and @role = \"Started\" and @managed = \"true\" and ./node[@name = \"'${CLUSTER_NODE}'\"]]/@id' - | sed -e 's/id=//g' -e 's/\"//g')\nfor v in ${VIPS_TO_MOVE}; do\n    echo \"Moving VIP $v on another node\"\n    pcs resource ban $v ${CLUSTER_NODE} --wait=300\ndone\necho \"Removing the location constraints that were created to move the VIPs\"\nfor v in ${VIPS_TO_MOVE}; do\n    echo \"Removing location ban for VIP $v\"\n    ban_id=$(cibadmin --query | xmllint --xpath 'string(//rsc_location[@rsc=\"'${v}'\" and @node=\"'${CLUSTER_NODE}'\" and @score=\"-INFINITY\"]/@id)' -)\n    if [ -n \"$ban_id\" ]; then\n        pcs constraint remove ${ban_id}\n    else\n        echo \"Could not retrieve and clear location constraint for VIP $v\" 2>&1\n    fi\ndone\n"}, {"name": "Create firewall nat prerouting and postrouting", "delegate_to": "{{groups['neutron_api'][1]}}", "when": ["groups['neutron_api'] | default([]) | length > 1"], "shell": "set -o pipefail\nLOCAL_BACKEND=$(grep {{groups['neutron_api'][1]}} /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg | \\\n  grep \":9696 \" | awk '{print $3}')\nIP=${LOCAL_BACKEND%:*}\nif [[ $IP =~ .*:.* ]]\n  then\n       PROTOCOL=\"ip6\"\n       # we need to temporarly allow ipv6 forwarding, this will get reset by deployment run\n       sysctl -w net.ipv6.conf.all.forwarding=1\n  else PROTOCOL=\"ip\"\nfi\nnft add table $PROTOCOL nat\nnft -- add chain $PROTOCOL nat prerouting { type nat hook prerouting priority -100 \\; }\nnft add chain $PROTOCOL nat postrouting { type nat hook postrouting priority 100 \\; }\n"}, {"name": "Redirect neutron backend traffic for 60s to other host", "delegate_to": "{{groups['neutron_api'][1]}}", "when": ["groups['neutron_api'] | default([]) | length > 1"], "shell": "set -o pipefail\nsource /etc/os-release; test  \"${VERSION_ID%*}\" = \"9.0\" && exit 0\nLOCAL_BACKEND=$(grep {{groups['neutron_api'][1]}} /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg | \\\n  grep \":9696 \" | awk '{print $3}')\nREMOTE_BACKEND=$(grep \":9696 \" /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg | \\\n  grep server | grep -v {{groups['neutron_api'][1]}} | head -n1 | awk '{print $3}')\nIP=${LOCAL_BACKEND%:*}\nPORT=9696\nif [[ $IP =~ .*:.* ]]\n  then PROTOCOL=\"ip6\"\n  else PROTOCOL=\"ip\"\nfi\nREMOTE_IP=${REMOTE_BACKEND%:*}\nnft add rule $PROTOCOL nat postrouting $PROTOCOL daddr $REMOTE_IP tcp dport 9696 counter meta time $(date +%s)-$(date -d'+60 seconds' +%s) masquerade comment neutron_api\nnft add rule $PROTOCOL nat prerouting $PROTOCOL daddr $IP tcp dport 9696 counter meta time $(date +%s)-$(date -d'+60 seconds' +%s) dnat to $REMOTE_IP comment neutron_api\n"}, {"name": "Insanity neutron update first", "delegate_to": "{{groups['neutron_api'][1]}}", "become": true, "tripleo_container_manage": {"config_dir": "/var/lib/tripleo-config/container-startup-config/step_4", "config_id": ["neutron_api"], "config_patterns": "*neutron_api.json", "config_overrides": {".*neutron_api": {"image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-server:17.1", "name": "neutron_api"}}}, "when": ["groups['neutron_api'] | default([]) | length > 1"]}, {"wait_for": {"timeout": 50}, "when": ["groups['neutron_api'] | default([]) | length > 1"]}, {"name": "Delete redirect rules", "delegate_to": "{{groups['neutron_api'][1]}}", "when": ["groups['neutron_api'] | default([]) | length > 1"], "shell": "set -o pipefail\nLOCAL_BACKEND=$(grep {{groups['neutron_api'][1]}} /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg | \\\n  grep \":9696 \" | awk '{print $3}')\nIP=${LOCAL_BACKEND%:*}\nif [[ $IP =~ .*:.* ]]\n  then PROTOCOL=\"ip6\"\n  else PROTOCOL=\"ip\"\nfi\nnft -a list chain $PROTOCOL nat postrouting | grep neutron_api | awk -F# '{print \"nat postrouting \"$2}' | \\\nwhile read i; do bash -c \"nft delete rule $PROTOCOL $i\"; done\nnft -a list chain $PROTOCOL nat prerouting | grep neutron_api | awk -F# '{print \"nat prerouting \"$2}' | \\\nwhile read i; do bash -c \"nft delete rule $PROTOCOL $i\"; done\n"}, {"name": "Move virtual IPs to another node before stopping pacemaker", "delegate_to": "{{groups['neutron_api'][2]}}", "when": ["hostvars[groups['neutron_api'][2]][\"haproxy_node_names\"]|default([])|length > 1", "groups['neutron_api'] | default([]) | length > 2"], "shell": "CLUSTER_NODE=$(crm_node -n)\necho \"Retrieving all the VIPs which are hosted on this node\"\nVIPS_TO_MOVE=$(crm_mon --as-xml | xmllint --xpath '//resource[@resource_agent=\"ocf:heartbeat:IPaddr2\" and @role = \"Started\" and @managed = \"true\" and ./node[@name = \"'${CLUSTER_NODE}'\"]]/@id' - | sed -e 's/id=//g' -e 's/\"//g')\nfor v in ${VIPS_TO_MOVE}; do\n    echo \"Moving VIP $v on another node\"\n    pcs resource ban $v ${CLUSTER_NODE} --wait=300\ndone\necho \"Removing the location constraints that were created to move the VIPs\"\nfor v in ${VIPS_TO_MOVE}; do\n    echo \"Removing location ban for VIP $v\"\n    ban_id=$(cibadmin --query | xmllint --xpath 'string(//rsc_location[@rsc=\"'${v}'\" and @node=\"'${CLUSTER_NODE}'\" and @score=\"-INFINITY\"]/@id)' -)\n    if [ -n \"$ban_id\" ]; then\n        pcs constraint remove ${ban_id}\n    else\n        echo \"Could not retrieve and clear location constraint for VIP $v\" 2>&1\n    fi\ndone\n"}, {"name": "Create firewall nat prerouting and postrouting", "delegate_to": "{{groups['neutron_api'][2]}}", "when": ["groups['neutron_api'] | default([]) | length > 2"], "shell": "set -o pipefail\nLOCAL_BACKEND=$(grep {{groups['neutron_api'][2]}} /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg | \\\n  grep \":9696 \" | awk '{print $3}')\nIP=${LOCAL_BACKEND%:*}\nif [[ $IP =~ .*:.* ]]\n  then\n       PROTOCOL=\"ip6\"\n       # we need to temporarly allow ipv6 forwarding, this will get reset by deployment run\n       sysctl -w net.ipv6.conf.all.forwarding=1\n  else PROTOCOL=\"ip\"\nfi\nnft add table $PROTOCOL nat\nnft -- add chain $PROTOCOL nat prerouting { type nat hook prerouting priority -100 \\; }\nnft add chain $PROTOCOL nat postrouting { type nat hook postrouting priority 100 \\; }\n"}, {"name": "Redirect neutron backend traffic for 60s to other host", "delegate_to": "{{groups['neutron_api'][2]}}", "when": ["groups['neutron_api'] | default([]) | length > 2"], "shell": "set -o pipefail\nsource /etc/os-release; test  \"${VERSION_ID%*}\" = \"9.0\" && exit 0\nLOCAL_BACKEND=$(grep {{groups['neutron_api'][2]}} /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg | \\\n  grep \":9696 \" | awk '{print $3}')\nREMOTE_BACKEND=$(grep \":9696 \" /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg | \\\n  grep server | grep -v {{groups['neutron_api'][2]}} | head -n1 | awk '{print $3}')\nIP=${LOCAL_BACKEND%:*}\nPORT=9696\nif [[ $IP =~ .*:.* ]]\n  then PROTOCOL=\"ip6\"\n  else PROTOCOL=\"ip\"\nfi\nREMOTE_IP=${REMOTE_BACKEND%:*}\nnft add rule $PROTOCOL nat postrouting $PROTOCOL daddr $REMOTE_IP tcp dport 9696 counter meta time $(date +%s)-$(date -d'+60 seconds' +%s) masquerade comment neutron_api\nnft add rule $PROTOCOL nat prerouting $PROTOCOL daddr $IP tcp dport 9696 counter meta time $(date +%s)-$(date -d'+60 seconds' +%s) dnat to $REMOTE_IP comment neutron_api\n"}, {"name": "Insanity neutron update first", "delegate_to": "{{groups['neutron_api'][2]}}", "become": true, "tripleo_container_manage": {"config_dir": "/var/lib/tripleo-config/container-startup-config/step_4", "config_id": ["neutron_api"], "config_patterns": "*neutron_api.json", "config_overrides": {".*neutron_api": {"image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-server:17.1", "name": "neutron_api"}}}, "when": ["groups['neutron_api'] | default([]) | length > 2"]}, {"wait_for": {"timeout": 50}, "when": ["groups['neutron_api'] | default([]) | length > 2"]}, {"name": "Delete redirect rules", "delegate_to": "{{groups['neutron_api'][2]}}", "shell": "set -o pipefail\nLOCAL_BACKEND=$(grep {{groups['neutron_api'][2]}} /var/lib/config-data/puppet-generated/haproxy/etc/haproxy/haproxy.cfg | \\\n  grep \":9696 \" | awk '{print $3}')\nIP=${LOCAL_BACKEND%:*}\nif [[ $IP =~ .*:.* ]]\n  then PROTOCOL=\"ip6\"\n  else PROTOCOL=\"ip\"\nfi\nnft -a list chain $PROTOCOL nat postrouting | grep neutron_api | awk -F# '{print \"nat postrouting \"$2}' | \\\nwhile read i; do bash -c \"nft delete rule $PROTOCOL $i\"; done\nnft -a list chain $PROTOCOL nat prerouting | grep neutron_api | awk -F# '{print \"nat prerouting \"$2}' | \\\nwhile read i; do bash -c \"nft delete rule $PROTOCOL $i\"; done\n", "when": ["groups['neutron_api'] | default([]) | length > 2"]}]}]}, {"service_name": "neutron_plugin_ml2_ovn", "config_settings": {"neutron::notification_driver": ["noop"], "neutron::core_plugin": "ml2", "neutron::service_plugins": ["qos", "ovn-router", "trunk", "segments", "port_forwarding", "log"], "neutron::report_interval": 300, "neutron::logging::debug": true, "neutron::purge_config": false, "neutron::allow_overlapping_ips": true, "neutron::dhcp_agent_notification": true, "neutron::dns_domain": "localdomain", "neutron::rabbit_heartbeat_timeout_threshold": 60, "neutron::host": "%{hiera('fqdn_canonical')}", "neutron::db::database_db_max_retries": -1, "neutron::db::database_max_retries": -1, "neutron::db::sync::db_sync_timeout": 900, "neutron::global_physnet_mtu": 1500, "neutron::db::sync::extra_params": "", "neutron::vlan_transparent": false, "neutron::rabbit_heartbeat_in_pthread": false, "neutron::plugins::ml2::mechanism_drivers": ["sriovnicswitch", "ovn"], "neutron::plugins::ml2::type_drivers": ["geneve", "vxlan", "vlan", "flat"], "neutron::plugins::ml2::flat_networks": ["datacentre"], "neutron::plugins::ml2::extension_drivers": ["qos", "port_security", "dns_domain_ports"], "neutron::plugins::ml2::network_vlan_ranges": ["datacentre:1:1000"], "neutron::plugins::ml2::tunnel_id_ranges": ["1:4094"], "neutron::plugins::ml2::vni_ranges": ["1:65536"], "neutron::plugins::ml2::tenant_network_types": ["geneve"], "neutron::plugins::ml2::ovs_driver::vnic_type_blacklist": [], "neutron::plugins::ml2::sriov_driver::vnic_type_blacklist": [], "neutron::plugins::ml2::overlay_ip_version": 4, "ovn::southbound::port": 6642, "ovn::northbound::port": 6641, "neutron::plugins::ml2::ovn::ovsdb_connection_timeout": 180, "neutron::plugins::ml2::ovn::ovsdb_probe_interval": 60000, "neutron::plugins::ml2::ovn::neutron_sync_mode": "log", "neutron::plugins::ml2::ovn::ovn_metadata_enabled": true, "neutron::server::qos_notification_drivers": "ovn-qos", "neutron::server::igmp_snooping_enable": false, "neutron::plugins::ml2::max_header_size": 38, "neutron::plugins::ml2::ovn::dns_servers": [], "neutron::plugins::ml2::ovn::ovn_emit_need_to_frag": true, "neutron::plugins::ml2::ovn::dvr_enabled": true, "neutron::plugins::ml2::ovn::network_log_rate_limit": 100, "neutron::plugins::ml2::ovn::network_log_burst_limit": 25}, "service_config_settings": null, "metadata_settings": null, "puppet_config": {"config_volume": "neutron", "puppet_tags": "neutron_plugin_ml2", "step_config": "include tripleo::profile::base::neutron::plugins::ml2\n", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-server:17.1"}, "kolla_config": {}, "docker_config": {}, "host_prep_tasks": []}, {"service_name": "neutron_sriov_agent", "config_settings": {"neutron::notification_driver": ["noop"], "neutron::core_plugin": "ml2", "neutron::service_plugins": ["qos", "ovn-router", "trunk", "segments", "port_forwarding", "log"], "neutron::report_interval": 300, "neutron::logging::debug": true, "neutron::purge_config": false, "neutron::allow_overlapping_ips": true, "neutron::dhcp_agent_notification": true, "neutron::dns_domain": "localdomain", "neutron::rabbit_heartbeat_timeout_threshold": 60, "neutron::host": "%{hiera('fqdn_canonical')}", "neutron::db::database_db_max_retries": -1, "neutron::db::database_max_retries": -1, "neutron::db::sync::db_sync_timeout": 900, "neutron::global_physnet_mtu": 1500, "neutron::db::sync::extra_params": "", "neutron::vlan_transparent": false, "neutron::rabbit_heartbeat_in_pthread": false, "neutron::agents::ml2::sriov::physical_device_mappings": ["dummy_sriov_net:dummy-dev"], "neutron::agents::ml2::sriov::exclude_devices": [], "neutron::agents::ml2::sriov::extensions": [], "neutron::agents::ml2::sriov::resource_provider_bandwidths": [], "ContainerNeutronSriovImage": "registry.redhat.io/rhosp-rhel9/openstack-neutron-sriov-agent:17.1", "ContainerNeutronConfigImage": "registry.redhat.io/rhosp-rhel9/openstack-neutron-server:17.1", "neutron::agents::ml2::sriov::resource_provider_default_hypervisor": "%{hiera('fqdn_canonical')}"}, "puppet_config": {"config_volume": "neutron", "puppet_tags": "neutron_config,neutron_agent_sriov_numvfs,neutron_sriov_agent_config", "step_config": "include tripleo::profile::base::neutron::sriov\n", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-server:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/neutron_sriov_agent.json": {"command": "/usr/bin/neutron-sriov-nic-agent --config-file /usr/share/neutron/neutron-dist.conf --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugins/ml2/sriov_agent.ini --config-dir /etc/neutron/conf.d/common --log-file=/var/log/neutron/sriov-nic-agent.log", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/neutron", "owner": "neutron:neutron", "recurse": true}]}}, "docker_config": {"step_4": {"neutron_sriov_agent": {"start_order": 10, "image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-sriov-agent:17.1", "net": "host", "pid": "host", "privileged": true, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck 5672"}, "ulimit": ["nofile=16384"], "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/neutron:/var/log/neutron:z", "/var/lib/kolla/config_files/neutron_sriov_agent.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/neutron:/var/lib/kolla/config_files/src:ro", "/lib/modules:/lib/modules:ro", "/run:/run", "/sys/class/net:/sys/class/net:rw"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/neutron", "setype": "container_file_t", "mode": "0750"}]}, {"name": "enable virt_sandbox_use_netlink for healthcheck", "seboolean": {"name": "virt_sandbox_use_netlink", "persistent": true, "state": true}, "when": ["ansible_facts.selinux is defined", "ansible_facts.selinux.status == \"enabled\""]}, [{"name": "creating directory", "file": {"state": "directory", "path": "/var/lib/pci_passthrough_whitelist_scripts", "owner": "root", "group": "root", "mode": 488}}, {"name": "derive pci passthrough whitelist", "copy": {"src": "/usr/share/openstack-tripleo-heat-templates/deployment/neutron/derive_pci_passthrough_whitelist.py", "dest": "/var/lib/pci_passthrough_whitelist_scripts/derive_pci_passthrough_whitelist.py", "mode": 448}}, {"name": "run derive_pci_passthrough_whitelist.py", "command": "/var/lib/pci_passthrough_whitelist_scripts/derive_pci_passthrough_whitelist.py"}]], "metadata_settings": null, "upgrade_tasks": []}, {"service_name": "neutron_dhcp", "firewall_rules": {"115 neutron dhcp input": {"ipversion": "ipv4", "proto": "udp", "dport": 67}, "116 neutron dhcp output": {"ipversion": "ipv4", "proto": "udp", "chain": "OUTPUT", "dport": 68}, "115 neutron dhcpv6 input": {"ipversion": "ipv6", "proto": "udp", "dport": 547}, "116 neutron dhcpv6 output": {"ipversion": "ipv6", "proto": "udp", "chain": "OUTPUT", "dport": 546}, "116 neutron dhcpv6 relay output": {"ipversion": "ipv6", "proto": "udp", "chain": "OUTPUT", "dport": 547}}, "monitoring_subscription": "overcloud-neutron-dhcp", "config_settings": {"neutron::notification_driver": ["noop"], "neutron::core_plugin": "ml2", "neutron::service_plugins": ["qos", "ovn-router", "trunk", "segments", "port_forwarding", "log"], "neutron::report_interval": 300, "neutron::logging::debug": true, "neutron::purge_config": false, "neutron::allow_overlapping_ips": true, "neutron::dhcp_agent_notification": true, "neutron::dns_domain": "localdomain", "neutron::rabbit_heartbeat_timeout_threshold": 60, "neutron::host": "%{hiera('fqdn_canonical')}", "neutron::db::database_db_max_retries": -1, "neutron::db::database_max_retries": -1, "neutron::db::sync::db_sync_timeout": 900, "neutron::global_physnet_mtu": 1500, "neutron::db::sync::extra_params": "", "neutron::vlan_transparent": false, "neutron::rabbit_heartbeat_in_pthread": false, "tripleo::profile::base::neutron::dhcp_agent_wrappers::enable_dnsmasq_wrapper": true, "tripleo::profile::base::neutron::dhcp_agent_wrappers::dnsmasq_process_wrapper": "/var/lib/neutron/dnsmasq_wrapper", "tripleo::profile::base::neutron::dhcp_agent_wrappers::dnsmasq_image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-dhcp-agent:17.1", "tripleo::profile::base::neutron::dhcp_agent_wrappers::enable_haproxy_wrapper": true, "tripleo::profile::base::neutron::dhcp_agent_wrappers::haproxy_process_wrapper": "/var/lib/neutron/dhcp_haproxy_wrapper", "tripleo::profile::base::neutron::dhcp_agent_wrappers::haproxy_image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-dhcp-agent:17.1", "tripleo::profile::base::neutron::dhcp_agent_wrappers::debug": true, "tripleo::profile::base::neutron::container_cli": "podman", "neutron::agents::dhcp::enable_isolated_metadata": false, "neutron::agents::dhcp::enable_force_metadata": true, "neutron::agents::dhcp::enable_metadata_network": false, "neutron::agents::dhcp::dnsmasq_local_resolv": false, "neutron::agents::dhcp::dnsmasq_dns_servers": [], "neutron::agents::dhcp::interface_driver": "neutron.agent.linux.interface.OVSInterfaceDriver", "neutron::agents::dhcp::dhcp_broadcast_reply": false, "neutron::agents::dhcp::dnsmasq_enable_addr6_list": true, "neutron::agents::dhcp::debug": true}, "service_config_settings": {"rsyslog": {"tripleo_logging_sources_neutron_dhcp": [{"tag": "openstack.neutron.agent.dhcp", "file": "/var/log/containers/neutron/dhcp-agent.log"}]}}, "puppet_config": {"config_volume": "neutron", "puppet_tags": "neutron_config,neutron_dhcp_agent_config", "step_config": "include tripleo::profile::base::neutron::dhcp\n", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-server:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/neutron_dhcp.json": {"command": "/usr/bin/neutron-dhcp-agent --config-file /usr/share/neutron/neutron-dist.conf --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/dhcp_agent.ini --config-dir /etc/neutron/conf.d/common --config-dir /etc/neutron/conf.d/neutron-dhcp-agent --log-file=/var/log/neutron/dhcp-agent.log", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-tls/*", "dest": "/", "merge": true, "preserve_properties": true, "optional": true}], "permissions": [{"path": "/var/log/neutron", "owner": "neutron:neutron", "recurse": true}, {"path": "/var/lib/neutron", "owner": "neutron:neutron", "recurse": true}, {"path": "/etc/pki/tls/certs/neutron.crt", "owner": "neutron:neutron"}, {"path": "/etc/pki/tls/private/neutron.key", "owner": "neutron:neutron"}]}}, "container_config_scripts": {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}}, "docker_config": {"step_2": {"create_dnsmasq_wrapper": {"start_order": 1, "detach": false, "net": "host", "pid": "host", "user": "root", "command": ["/container_puppet_apply.sh", "4", "file", "include ::tripleo::profile::base::neutron::dhcp_agent_wrappers"], "image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-dhcp-agent:17.1", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/var/lib/container-config-scripts/container_puppet_apply.sh:/container_puppet_apply.sh:ro", "/etc/puppet:/tmp/puppet-etc:ro", "/usr/share/openstack-puppet/modules:/usr/share/openstack-puppet/modules:ro", "/run/openvswitch:/run/openvswitch:shared,z", "/var/lib/neutron:/var/lib/neutron:shared,z"]}}, "step_4": {"neutron_dhcp": {"start_order": 10, "image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-dhcp-agent:17.1", "net": "host", "pid": "host", "cgroupns": "host", "privileged": true, "restart": "always", "security_opt": ["label=disable"], "depends_on": ["openvswitch.service"], "healthcheck": {"test": "/openstack/healthcheck 5672"}, "ulimit": ["nofile=16384"], "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/neutron:/var/log/neutron:z", "/var/lib/kolla/config_files/neutron_dhcp.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/neutron:/var/lib/kolla/config_files/src:ro", "/lib/modules:/lib/modules:ro", "/run/openvswitch:/run/openvswitch:shared,z", "/var/lib/neutron:/var/lib/neutron:shared,z", "/run/netns:/run/netns:shared", "/var/lib/neutron/kill_scripts:/etc/neutron/kill_scripts:shared,z", "/var/lib/neutron/dnsmasq_wrapper:/usr/local/bin/dnsmasq:ro", "/var/lib/neutron/dhcp_haproxy_wrapper:/usr/local/bin/haproxy:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "metadata_settings": null, "deploy_steps_tasks": [{"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "neutron", "dns": "{{fqdn_internal_api}}", "principal": "neutron/{{fqdn_internal_api}}@{{idm_realm}}", "run_after": "container_name=$({{container_cli}} ps --format=\\{\\{.Names\\}\\} | grep neutron_dhcp)\n# The certificate is also installed on the computes, but neutron_dhcp is only\n# present on the controllers, so we exit if the container could not be found.\n[[ -z $container_name ]] && exit 0\n\nservice_crt=\"/etc/pki/tls/certs/neutron.crt\"\nservice_key=\"/etc/pki/tls/private/neutron.key\"\n# Copy the new cert from the mount-point to the real path\n{{container_cli}} exec -u root \"$container_name\" cp \"/var/lib/kolla/config_files/src-tls$service_crt\" \"$service_crt\"\n# Copy the new key from the mount-point to the real path\n{{container_cli}} exec -u root \"$container_name\" cp \"/var/lib/kolla/config_files/src-tls$service_key \"$service_key\"\n# No need to trigger a reload for neutron dhcpd since the cert is not cached\n", "key_size": "2048", "ca": "ipa"}]}}]}], "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/neutron", "setype": "container_file_t", "mode": "0750"}]}, {"name": "create /run/netns with temp namespace", "command": "ip netns add ns_temp", "register": "ipnetns_add_result", "failed_when": false}, {"name": "remove temp namespace", "command": "ip netns delete ns_temp", "failed_when": false, "when": ["ipnetns_add_result.rc is defined", "ipnetns_add_result.rc == 0"]}, {"name": "create /var/lib/neutron", "file": {"path": "/var/lib/neutron", "state": "directory", "setype": "container_file_t"}}, {"name": "enable virt_sandbox_use_netlink for healthcheck", "seboolean": {"name": "virt_sandbox_use_netlink", "persistent": true, "state": true}, "when": ["ansible_facts.selinux is defined", "ansible_facts.selinux.status == \"enabled\""]}, {"name": "set conditions", "set_fact": {"dnsmasq_wrapper_enabled": true, "haproxy_wrapper_enabled": true, "debug_enabled": true, "docker_additional_sockets": ["/var/lib/openstack/docker.sock"]}}, {"name": "create kill_scripts directory within /var/lib/neutron", "file": {"state": "directory", "path": "/var/lib/neutron/kill_scripts"}}, {"name": "create dnsmasq dhcp kill script", "when": "dnsmasq_wrapper_enabled|bool", "copy": {"dest": "/var/lib/neutron/kill_scripts/dnsmasq-kill", "mode": 493, "content": "#!/bin/bash\n{% if debug_enabled|bool -%}\nset -x\n{% endif -%}\nadd_date() {\n  echo \"$(date) $@\"\n}\n\n# Set up script logging for debugging purpose.\n# It will be taken care of by logrotate since there is the .log\n# suffix.\nexec 3>&1 4>&2\ntrap 'exec 2>&4 1>&3' 0 1 2 3\nexec 1>>/var/log/neutron/kill-script.log 2>&1\n\nSIG=$1\nPID=$2\nNETNS=$(ip netns identify ${PID})\n\nif [ \"x${NETNS}\" == \"x\" ]; then\n  CLI=\"nsenter --all --preserve-credentials -t 1 podman\"\n  SIG=9\nelse\n  CLI=\"nsenter --net=/run/netns/${NETNS} --preserve-credentials -m -t 1 podman\"\nfi\n\nkill_container() {\n  add_date \"Killing container $1 ($2)\"\n  signal_container $1 $2 9\n}\n\nstop_container() {\n  add_date \"Stopping container $1 ($2)\"\n  $CLI stop $2\n}\n\nsignal_container() {\n  SIGNAL=$3\n  if [ -z \"$SIGNAL\" ]; then\n      SIGNAL=\"HUP\"\n  fi\n  add_date \"Sending signal '$SIGNAL' to $1 ($2)\"\n  $CLI kill --signal $SIGNAL $2\n}\n\ndelete_container() {\n  add_date \"Deleting container $1 ($2)\"\n  $CLI rm $2 || echo \"Deleting container $1 ($2) failed\"\n}\n\n\n{% raw -%}\nif [ -f /proc/$PID/cgroup ]; then\n  # Get container ID based on process cgroups\n  CT_ID=$(awk 'BEGIN {FS=\".scope|-\"} /\\/libpod-/ {if ($(NF-1)) print $(NF-1);exit}' /proc/$PID/cgroup)\n  CT_NAME=$($CLI inspect -f '{{.Name}}' $CT_ID)\n\n  case $SIG in\n    HUP)\n      signal_container $CT_NAME $CT_ID\n      ;;\n    9)\n      kill_container $CT_NAME $CT_ID\n      delete_container $CT_NAME $CT_ID\n      ;;\n    15)\n      stop_container $CT_NAME $CT_ID\n      delete_container $CT_NAME $CT_ID\n      ;;\n    *)\n      add_date \"Unknown action ${SIG} for ${CT_NAME} ${CT_ID}\"\n      exit 1\n      ;;\n  esac\n\nelse\n  add_date \"No such PID: ${PID}\"\n  exit 1\nfi\n{% endraw %}\n"}}, {"name": "create haproxy kill script", "when": "haproxy_wrapper_enabled|bool", "copy": {"dest": "/var/lib/neutron/kill_scripts/haproxy-kill", "mode": 493, "content": "#!/bin/bash\n{% if debug_enabled|bool -%}\nset -x\n{% endif -%}\nadd_date() {\n  echo \"$(date) $@\"\n}\n\n# Set up script logging for debugging purpose.\n# It will be taken care of by logrotate since there is the .log\n# suffix.\nexec 3>&1 4>&2\ntrap 'exec 2>&4 1>&3' 0 1 2 3\nexec 1>>/var/log/neutron/kill-script.log 2>&1\n\nSIG=$1\nPID=$2\nNETNS=$(ip netns identify ${PID})\n\nif [ \"x${NETNS}\" == \"x\" ]; then\n  CLI=\"nsenter --all --preserve-credentials -t 1 podman\"\n  SIG=9\nelse\n  CLI=\"nsenter --net=/run/netns/${NETNS} --preserve-credentials -m -t 1 podman\"\nfi\n\nkill_container() {\n  add_date \"Killing container $1 ($2)\"\n  signal_container $1 $2 9\n}\n\nstop_container() {\n  add_date \"Stopping container $1 ($2)\"\n  $CLI stop $2\n}\n\nsignal_container() {\n  SIGNAL=$3\n  if [ -z \"$SIGNAL\" ]; then\n      SIGNAL=\"HUP\"\n  fi\n  add_date \"Sending signal '$SIGNAL' to $1 ($2)\"\n  $CLI kill --signal $SIGNAL $2\n}\n\ndelete_container() {\n  add_date \"Deleting container $1 ($2)\"\n  $CLI rm $2 || echo \"Deleting container $1 ($2) failed\"\n}\n\n\n{% raw -%}\nif [ -f /proc/$PID/cgroup ]; then\n  # Get container ID based on process cgroups\n  CT_ID=$(awk 'BEGIN {FS=\".scope|-\"} /\\/libpod-/ {if ($(NF-1)) print $(NF-1);exit}' /proc/$PID/cgroup)\n  CT_NAME=$($CLI inspect -f '{{.Name}}' $CT_ID)\n\n  case $SIG in\n    HUP)\n      signal_container $CT_NAME $CT_ID\n      ;;\n    9)\n      kill_container $CT_NAME $CT_ID\n      delete_container $CT_NAME $CT_ID\n      ;;\n    15)\n      stop_container $CT_NAME $CT_ID\n      delete_container $CT_NAME $CT_ID\n      ;;\n    *)\n      add_date \"Unknown action ${SIG} for ${CT_NAME} ${CT_ID}\"\n      exit 1\n      ;;\n  esac\n\nelse\n  add_date \"No such PID: ${PID}\"\n  exit 1\nfi\n{% endraw %}\n"}}], "upgrade_tasks": []}, {"service_name": "nova_api", "firewall_rules": {"113 nova_api": {"dport": [8774]}}, "firewall_frontend_rules": {"100 nova_osapi_haproxy_frontend": {"dport": [8774]}}, "firewall_ssl_frontend_rules": {"100 nova_osapi_haproxy_frontend_ssl": {"dport": [13774]}}, "keystone_resources": {"nova": {"endpoints": {"public": "http://172.21.0.2:8774/v2.1", "internal": "http://172.17.0.2:8774/v2.1", "admin": "http://172.17.0.2:8774/v2.1"}, "users": {"nova": {"roles": ["admin", "service"], "password": "nad1A8ZvKDsT60yawIjN1cH4K"}}, "region": "regionOne", "service": "compute"}}, "monitoring_subscription": "overcloud-nova-api", "config_settings": {"nova::my_ip": "%{hiera('internal_api')}", "nova::rabbit_use_ssl": "False", "nova::keystone::service_user::send_service_user_token": true, "nova::keystone::service_user::project_name": "service", "nova::keystone::service_user::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::service_user::auth_url": "http://172.17.0.2:5000", "nova::keystone::service_user::region_name": "regionOne", "nova::placement::project_name": "service", "nova::placement::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "nova::placement::auth_url": "http://172.17.0.2:5000", "nova::placement::region_name": "regionOne", "nova::placement::valid_interfaces": "internal", "nova::os_region_name": "regionOne", "nova::logging::debug": true, "nova::purge_config": false, "nova::network::neutron::project_name": "service", "nova::network::neutron::username": "neutron", "nova::network::neutron::region_name": "regionOne", "nova::dhcp_domain": "", "nova::network::neutron::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "nova::network::neutron::auth_url": "http://172.17.0.2:5000/v3", "nova::network::neutron::valid_interfaces": "internal", "nova::glance::valid_interfaces": "internal", "nova::rabbit_heartbeat_timeout_threshold": 60, "nova::cinder_catalog_info": "volumev3:cinderv3:internalURL", "nova::host": "%{hiera('fqdn_canonical')}", "nova::notify_on_state_change": "vm_and_task_state", "nova::notification_driver": ["noop"], "nova::notification_format": "unversioned", "nova::network::neutron::auth_type": "v3password", "nova::db::database_db_max_retries": -1, "nova::db::database_max_retries": -1, "nova::network::neutron::ovs_bridge": "br-int", "nova::compute::sync_power_state_interval": 600, "nova_is_additional_cell": false, "nova::cross_az_attach": true, "nova::cache::enabled": true, "nova::cache::tls_enabled": false, "nova::cache::backend": "dogpile.cache.memcached", "nova::rabbit_heartbeat_in_pthread": false, "nova::ovsdb_connection": "", "nova::api_database_connection": "mysql+pymysql://nova_api:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova_api?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "nova::database_connection": "mysql+pymysql://nova:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "apache::default_vhost": false, "nova::keystone::authtoken::project_name": "service", "nova::keystone::authtoken::user_domain_name": "Default", "nova::keystone::authtoken::project_domain_name": "Default", "nova::keystone::authtoken::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "nova::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "nova::keystone::authtoken::region_name": "regionOne", "nova::keystone::authtoken::interface": "internal", "nova::keystone::authtoken::memcache_use_advanced_pool": true, "nova::api::max_limit": 1000, "nova::api::enabled": true, "nova::network::neutron::default_floating_pool": "public", "nova::api::enable_proxy_headers_parsing": true, "nova::api::api_bind_address": "%{hiera('fqdn_internal_api')}", "nova::api::service_name": "httpd", "nova::wsgi::apache_api::ssl": false, "nova::wsgi::apache_api::bind_host": "%{hiera('internal_api')}", "nova::wsgi::apache_api::servername": "%{hiera('fqdn_internal_api')}", "nova::api::instance_name_template": "instance-%08x", "nova::policy::policies": {}, "nova::api::allow_resize_to_same_host": false, "nova_enable_db_purge": true, "nova::cron::purge_shadow_tables::minute": "0", "nova::cron::purge_shadow_tables::hour": "5", "nova::cron::purge_shadow_tables::monthday": "*", "nova::cron::purge_shadow_tables::month": "*", "nova::cron::purge_shadow_tables::weekday": "*", "nova::cron::purge_shadow_tables::user": "nova", "nova::cron::purge_shadow_tables::destination": "/var/log/nova/nova-rowspurge.log", "nova::cron::purge_shadow_tables::maxdelay": "3600", "nova::cron::purge_shadow_tables::age": 14, "nova::cron::purge_shadow_tables::verbose": false, "nova::cron::purge_shadow_tables::all_cells": true, "nova_enable_db_archive": true, "nova::cron::archive_deleted_rows::minute": "1", "nova::cron::archive_deleted_rows::hour": "0", "nova::cron::archive_deleted_rows::monthday": "*", "nova::cron::archive_deleted_rows::month": "*", "nova::cron::archive_deleted_rows::weekday": "*", "nova::cron::archive_deleted_rows::max_rows": "1000", "nova::cron::archive_deleted_rows::user": "nova", "nova::cron::archive_deleted_rows::destination": "/var/log/nova/nova-rowsflush.log", "nova::cron::archive_deleted_rows::maxdelay": "3600", "nova::cron::archive_deleted_rows::until_complete": true, "nova::cron::archive_deleted_rows::purge": false, "nova::cron::archive_deleted_rows::all_cells": true, "nova::cron::archive_deleted_rows::age": 90, "nova::policy::enforce_new_defaults": false, "nova::policy::enforce_scope": false, "nova::api::osapi_compute_workers": 1, "nova::wsgi::apache_api::workers": 1}, "service_config_settings": {"rabbitmq": {"get_attr": ["NovaBase", "role_data", "service_config_settings"], "rabbitmq": null}, "mysql": {"nova::db::mysql_api::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::db::mysql_api::user": "nova_api", "nova::db::mysql_api::host": "%", "nova::db::mysql_api::dbname": "nova_api", "nova::db::mysql::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::db::mysql::user": "nova", "nova::db::mysql::host": "%", "nova::db::mysql::dbname": "nova"}, "rsyslog": {"tripleo_logging_sources_nova_api": [{"tag": "openstack.nova.api", "file": "/var/log/containers/nova/nova-api.log"}]}}, "puppet_config": {"config_volume": "nova", "puppet_tags": "nova_config,nova_api_paste_ini", "step_config": "['Nova_cell_v2'].each |String $val| { noop_resource($val) }\ninclude tripleo::profile::base::nova::api\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/nova_api.json": {"command": "/usr/sbin/httpd -DFOREGROUND", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/nova", "owner": "nova:nova", "recurse": true}]}, "/var/lib/kolla/config_files/nova_api_db_sync.json": {"command": "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/usr/bin/nova-manage api_db sync'", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/nova", "owner": "nova:nova", "recurse": true}]}, "/var/lib/kolla/config_files/nova_api_ensure_default_cells.json": {"command": "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/container-config-scripts/pyshim.sh /container-config-scripts/nova_api_ensure_default_cells.py'", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/nova", "owner": "nova:nova", "recurse": true}]}, "/var/lib/kolla/config_files/nova_api_cron.json": {"command": "/usr/sbin/crond -n", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/nova", "owner": "nova:nova", "recurse": true}]}, "/var/lib/kolla/config_files/nova_wait_for_api_service.json": {"command": "/usr/bin/bootstrap_host_exec nova_api su nova -s /bin/bash -c '/container-config-scripts/pyshim.sh /container-config-scripts/nova_wait_for_api_service.py'", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/nova", "owner": "nova:nova", "recurse": true}]}}, "container_config_scripts": {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "nova_wait_for_api_service.py": {"mode": "0755", "content": "#!/usr/bin/python3\n#\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\nimport argparse\nimport logging\nimport os\nimport six\nimport sys\nimport time\n\n\nfrom keystoneauth1 import loading\nfrom keystoneauth1 import session\n\nfrom novaclient import client\nfrom novaclient.exceptions import ClientException\n\n# In python3 SafeConfigParser was renamed to ConfigParser and the default\n# for duplicate options default to true. In case of nova it is valid to\n# have duplicate option lines, e.g. passthrough_whitelist which leads to\n# issues reading the nova.conf\n# https://bugs.launchpad.net/tripleo/+bug/1827775\nif six.PY3:\n    from six.moves.configparser import ConfigParser\n    config = ConfigParser(strict=False)\nelse:\n    from six.moves.configparser import SafeConfigParser\n    config = SafeConfigParser()\n\ndebug = os.getenv('__OS_DEBUG', 'false')\n\nif debug.lower() == 'true':\n    loglevel = logging.DEBUG\nelse:\n    loglevel = logging.INFO\n\nlogging.basicConfig(stream=sys.stdout, level=loglevel)\nLOG = logging.getLogger('nova_wait_for_api_service')\n\niterations = 60\ntimeout = 10\nnova_cfg = '/etc/nova/nova.conf'\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser(usage='%(prog)s [options]')\n    parser.add_argument('-k', '--insecure',\n                        action=\"store_false\",\n                        dest='insecure',\n                        default=True,\n                        help='Allow insecure connection when using SSL')\n\n    args = parser.parse_args()\n    LOG.debug('Running with parameter insecure = %s',\n              args.insecure)\n\n    if os.path.isfile(nova_cfg):\n        try:\n            config.read(nova_cfg)\n        except Exception:\n            LOG.exception('Error while reading nova.conf:')\n    else:\n        LOG.error('Nova configuration file %s does not exist', nova_cfg)\n        sys.exit(1)\n\n    loader = loading.get_plugin_loader('password')\n    auth = loader.load_from_options(\n        auth_url=config.get('neutron',\n                            'auth_url'),\n        username=config.get('neutron',\n                            'username'),\n        password=config.get('neutron',\n                            'password'),\n        project_name=config.get('neutron',\n                                'project_name'),\n        project_domain_name=config.get('neutron',\n                                       'project_domain_name'),\n        user_domain_name=config.get('neutron',\n                                    'user_domain_name'))\n    sess = session.Session(auth=auth, verify=args.insecure)\n\n    # Wait until this host is listed in the service list\n    for i in range(iterations):\n        try:\n            nova = client.Client('2.11', session=sess, endpoint_type='internal')\n            nova.versions.list()\n            LOG.info('Nova-api service active')\n            sys.exit(0)\n        except ClientException:\n            LOG.info('Waiting for nova-api service')\n        except Exception:\n            LOG.exception(\n                'Error while waiting for nova-api service')\n        time.sleep(timeout)\nsys.exit(1)\n\n# vim: set et ts=4 sw=4 :\n"}, "nova_api_ensure_default_cells.py": {"mode": "0755", "content": "#!/usr/bin/python3\n#\n# Copyright 2022 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\nfrom configparser import ConfigParser\nimport logging\nimport os\nimport subprocess\nimport sys\nfrom urllib import parse as urlparse\n\nconfig = ConfigParser(strict=False)\n\ndebug = os.getenv('__OS_DEBUG', 'false')\n\nif debug.lower() == 'true':\n    loglevel = logging.DEBUG\nelse:\n    loglevel = logging.INFO\n\nlogging.basicConfig(stream=sys.stdout, level=loglevel)\nLOG = logging.getLogger('nova_api_ensure_default_cells')\n\nNOVA_CFG = '/etc/nova/nova.conf'\nCELL0_ID = '00000000-0000-0000-0000-000000000000'\nDEFAULT_CELL_NAME = 'default'\n\n\ndef template_netloc_credentials(netloc, index=None):\n    if '@' in netloc:\n        userpass, hostport = netloc.split('@', 1)\n        has_pass = ':' in userpass\n        if index is None:\n            cred_template = '{username}'\n            if has_pass:\n                cred_template += ':{password}'\n        else:\n            cred_template = '{{username{index}}}'.format(index=index)\n            if has_pass:\n                cred_template += ':{{password{index}}}'.format(index=index)\n        return '@'.join((cred_template, hostport))\n    else:\n        return netloc\n\n\ndef template_url(url):\n    parsed = urlparse.urlparse(url)\n    if ',' in parsed.netloc:\n        orig_netlocs = parsed.netloc.split(',')\n        templ_netlocs = []\n        index = 0\n        for netloc in orig_netlocs:\n            index += 1\n            templ_netlocs.append(template_netloc_credentials(netloc, index))\n        new_netloc = ','.join(templ_netlocs)\n    else:\n        new_netloc = template_netloc_credentials(parsed.netloc)\n    return parsed._replace(netloc=new_netloc).geturl()\n\n\ndef parse_list_cells(list_cells_output):\n    list_cells_lines = list_cells_output.split('\\n')\n    if len(list_cells_lines) < 5:\n        raise ValueError('Invalid nova-manage cell_v2 list_cells output')\n\n    data_rows = list_cells_lines[3:-2]\n    by_name = {}\n    by_uuid = {}\n\n    for row in data_rows:\n        parts = row.split('|')\n        entry = {\n            'name': parts[1].strip(),\n            'uuid': parts[2].strip(),\n            'transport_url': parts[3].strip(),\n            'database_connection': parts[4].strip(),\n        }\n        by_name[entry['name']] = entry\n        by_uuid[entry['uuid']] = entry\n\n    return by_name, by_uuid\n\n\ndef create_or_update_default_cells(cell0_db, default_db, default_transport_url):\n    list_cells_cmd = ['/usr/bin/nova-manage', 'cell_v2', 'list_cells', '--verbose']\n    list_cells_output = subprocess.check_output(list_cells_cmd, encoding='utf-8')\n    cells_by_name, cells_by_uuid = parse_list_cells(list_cells_output)\n\n    if CELL0_ID in cells_by_uuid:\n        LOG.info('Setting cell0 database connection to \\'{}\\''.format(cell0_db))\n        cmd = [\n            '/usr/bin/nova-manage', 'cell_v2', 'update_cell',\n            '--cell_uuid', CELL0_ID,\n            '--database_connection', cell0_db,\n            '--transport-url', 'none:///'\n        ]\n    else:\n        LOG.info('Creating cell0 with database connection \\'{}\\''.format(cell0_db))\n        cmd = [\n            '/usr/bin/nova-manage', 'cell_v2', 'map_cell0',\n             '--database_connection', cell0_db\n        ]\n    subprocess.check_call(cmd)\n\n    if DEFAULT_CELL_NAME in cells_by_name:\n        LOG.info('Setting default cell database connection to \\'{}\\' and transport url to \\'{}\\''.format(\n            default_db, default_transport_url))\n        cmd = [\n            '/usr/bin/nova-manage', 'cell_v2', 'update_cell',\n            '--cell_uuid', cells_by_name[DEFAULT_CELL_NAME]['uuid'],\n            '--database_connection', default_db,\n            '--transport-url', default_transport_url\n        ]\n    else:\n        LOG.info('Creating default cell with database connection \\'{}\\' and transport url \\'{}\\''.format(\n            default_db, default_transport_url))\n        cmd = [\n            '/usr/bin/nova-manage', 'cell_v2', 'create_cell',\n            '--name', DEFAULT_CELL_NAME,\n             '--database_connection', default_db,\n            '--transport-url', default_transport_url\n        ]\n    subprocess.check_call(cmd)\n\n\ndef replace_db_name(db_url, db_name):\n    return urlparse.urlparse(db_url)._replace(path=db_name).geturl()\n\n\nif __name__ == '__main__':\n    if os.path.isfile(NOVA_CFG):\n        try:\n            config.read(NOVA_CFG)\n        except Exception:\n            LOG.exception('Error while reading nova.conf:')\n            sys.exit(1)\n    else:\n        LOG.error('Nova configuration file %s does not exist', NOVA_CFG)\n        sys.exit(1)\n\n    default_database_connection = config.get('database', 'connection')\n    cell0_database_connection = replace_db_name(default_database_connection, 'nova_cell0')\n    default_transport_url = config.get('DEFAULT', 'transport_url')\n\n    create_or_update_default_cells(\n        template_url(cell0_database_connection),\n        template_url(default_database_connection),\n        template_url(default_transport_url)\n    )\n"}}, "docker_config": {"step_2": {"nova_api_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/nova:/var/log/nova:z", "/var/log/containers/httpd/nova-api:/var/log/httpd:z"], "command": ["/bin/bash", "-c", "chown -R nova:nova /var/log/nova"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_3": {"nova_api_db_sync": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1", "cap_add": ["AUDIT_WRITE"], "net": "host", "detach": false, "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova:z", "/var/log/containers/httpd/nova-api:/var/log/httpd:z", "/var/lib/kolla/config_files/nova_api_db_sync.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}, "nova_api_ensure_default_cells": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1", "user": "root", "net": "host", "privileged": false, "detach": false, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova:z", "/var/log/containers/httpd/nova-api:/var/log/httpd:z", "/var/lib/kolla/config_files/nova_api_ensure_default_cells.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro", "/var/lib/container-config-scripts:/container-config-scripts:z"], "environment": {"__OS_DEBUG": "true", "KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}, "step_4": {"nova_api": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1", "net": "host", "user": "root", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova:z", "/var/log/containers/httpd/nova-api:/var/log/httpd:z", "/var/lib/kolla/config_files/nova_api.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "nova_wait_for_api_service": {"start_order": 3, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1", "user": "root", "net": "host", "privileged": false, "detach": false, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova:z", "/var/log/containers/httpd/nova-api:/var/log/httpd:z", "/var/lib/kolla/config_files/nova_wait_for_api_service.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro", "/var/lib/container-config-scripts:/container-config-scripts:z"], "environment": {"__OS_DEBUG": "true", "KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "nova_api_cron": {"start_order": 4, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1", "net": "host", "user": "root", "privileged": false, "restart": "always", "healthcheck": {"test": "/usr/share/openstack-tripleo-common/healthcheck/cron nova"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova:z", "/var/log/containers/httpd/nova-api:/var/log/httpd:z", "/var/lib/kolla/config_files/nova_api_cron.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "metadata_settings": null, "deploy_steps_tasks": [{"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"name": "Create dirs for certificates and keys", "file": {"path": "{{ item }}", "state": "directory", "serole": "object_r", "setype": "cert_t", "seuser": "system_u"}, "with_items": ["/etc/pki/tls/certs/httpd", "/etc/pki/tls/private/httpd"]}, {"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "httpd-ctlplane", "dns": "{{ fqdn_ctlplane }}", "principal": "HTTP/{{ fqdn_ctlplane }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-ctlplane.crt /etc/pki/tls/certs/httpd/httpd-ctlplane.crt\ncp /etc/pki/tls/private/httpd-ctlplane.key /etc/pki/tls/private/httpd/httpd-ctlplane.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage", "dns": "{{ fqdn_storage }}", "principal": "HTTP/{{ fqdn_storage }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage.crt /etc/pki/tls/certs/httpd/httpd-storage.crt\ncp /etc/pki/tls/private/httpd-storage.key /etc/pki/tls/private/httpd/httpd-storage.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage_mgmt", "dns": "{{ fqdn_storage_mgmt }}", "principal": "HTTP/{{ fqdn_storage_mgmt }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage_mgmt.crt /etc/pki/tls/certs/httpd/httpd-storage_mgmt.crt\ncp /etc/pki/tls/private/httpd-storage_mgmt.key /etc/pki/tls/private/httpd/httpd-storage_mgmt.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-internal_api", "dns": "{{ fqdn_internal_api }}", "principal": "HTTP/{{ fqdn_internal_api }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-internal_api.crt /etc/pki/tls/certs/httpd/httpd-internal_api.crt\ncp /etc/pki/tls/private/httpd-internal_api.key /etc/pki/tls/private/httpd/httpd-internal_api.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-external", "dns": "{{ fqdn_external }}", "principal": "HTTP/{{ fqdn_external }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-external.crt /etc/pki/tls/certs/httpd/httpd-external.crt\ncp /etc/pki/tls/private/httpd-external.key /etc/pki/tls/private/httpd/httpd-external.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}]}}]}, {"name": "validate nova-api container state", "containers.podman.podman_container_info": {"name": "nova_api"}, "register": "nova_api_infos", "failed_when": ["nova_api_infos.containers.0.Healthcheck.Status is defined", "'healthy' not in nova_api_infos.containers.0.Healthcheck.Status"], "retries": 10, "delay": 30, "tags": ["opendev-validation", "opendev-validation-nova"], "when": ["container_cli == 'podman'", "not container_healthcheck_disabled", "step|int == 4"]}], "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "setype": "{{ item.setype }}", "state": "directory", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/nova", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/log/containers/httpd/nova-api", "setype": "container_file_t", "mode": "0750"}]}], "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop nova api container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["nova_api", "nova_api_cron"], "tripleo_delegate_to": "{{ groups['nova_api'] | difference(groups['excluded_overcloud']) }}"}}]}]}, {"service_name": "nova_conductor", "monitoring_subscription": "overcloud-nova-conductor", "config_settings": {"nova::my_ip": "%{hiera('internal_api')}", "nova::rabbit_use_ssl": "False", "nova::keystone::service_user::send_service_user_token": true, "nova::keystone::service_user::project_name": "service", "nova::keystone::service_user::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::service_user::auth_url": "http://172.17.0.2:5000", "nova::keystone::service_user::region_name": "regionOne", "nova::placement::project_name": "service", "nova::placement::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "nova::placement::auth_url": "http://172.17.0.2:5000", "nova::placement::region_name": "regionOne", "nova::placement::valid_interfaces": "internal", "nova::os_region_name": "regionOne", "nova::logging::debug": true, "nova::purge_config": false, "nova::network::neutron::project_name": "service", "nova::network::neutron::username": "neutron", "nova::network::neutron::region_name": "regionOne", "nova::dhcp_domain": "", "nova::network::neutron::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "nova::network::neutron::auth_url": "http://172.17.0.2:5000/v3", "nova::network::neutron::valid_interfaces": "internal", "nova::glance::valid_interfaces": "internal", "nova::rabbit_heartbeat_timeout_threshold": 60, "nova::cinder_catalog_info": "volumev3:cinderv3:internalURL", "nova::host": "%{hiera('fqdn_canonical')}", "nova::notify_on_state_change": "vm_and_task_state", "nova::notification_driver": ["noop"], "nova::notification_format": "unversioned", "nova::network::neutron::auth_type": "v3password", "nova::db::database_db_max_retries": -1, "nova::db::database_max_retries": -1, "nova::network::neutron::ovs_bridge": "br-int", "nova::compute::sync_power_state_interval": 600, "nova_is_additional_cell": false, "nova::cross_az_attach": true, "nova::cache::enabled": true, "nova::cache::tls_enabled": false, "nova::cache::backend": "dogpile.cache.memcached", "nova::rabbit_heartbeat_in_pthread": false, "nova::ovsdb_connection": "", "nova::api_database_connection": "mysql+pymysql://nova_api:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova_api?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "nova::database_connection": "mysql+pymysql://nova:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "nova::conductor::workers": 1}, "service_config_settings": {"rabbitmq": {"get_attr": ["NovaBase", "role_data", "service_config_settings"], "rabbitmq": null}, "mysql": {"nova::db::mysql_api::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::db::mysql_api::user": "nova_api", "nova::db::mysql_api::host": "%", "nova::db::mysql_api::dbname": "nova_api", "nova::db::mysql::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::db::mysql::user": "nova", "nova::db::mysql::host": "%", "nova::db::mysql::dbname": "nova"}, "rsyslog": {"tripleo_logging_sources_nova_conductor": [{"tag": "openstack.nova.conductor", "file": "/var/log/containers/nova/nova-conductor.log"}]}}, "puppet_config": {"config_volume": "nova", "puppet_tags": "nova_config", "step_config": "include tripleo::profile::base::nova::conductor\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/nova_conductor.json": {"command": "/usr/bin/nova-conductor ", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/nova", "owner": "nova:nova", "recurse": true}]}, "/var/lib/kolla/config_files/nova_conductor_db_sync.json": {"command": "/usr/bin/bootstrap_host_exec nova_conductor su nova -s /bin/bash -c '/usr/bin/nova-manage db sync '", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/nova", "owner": "nova:nova", "recurse": true}]}}, "docker_config": {"step_2": {"nova_conductor_init_log": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-conductor:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/nova:/var/log/nova:z"], "command": ["/bin/bash", "-c", "chown -R nova:nova /var/log/nova"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_3": {"nova_db_sync": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-conductor:17.1", "cap_add": ["AUDIT_WRITE"], "start_order": 3, "net": "host", "detach": false, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova", "/var/lib/kolla/config_files/nova_conductor_db_sync.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro"], "user": "root", "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_4": {"nova_conductor": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-conductor:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck 5672"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova", "/var/lib/kolla/config_files/nova_conductor.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "deploy_steps_tasks": [{"name": "validate nova-conductor container state", "containers.podman.podman_container_info": {"name": "nova_conductor"}, "register": "nova_conductor_infos", "failed_when": ["nova_conductor_infos.containers.0.Healthcheck.Status is defined", "'healthy' not in nova_conductor_infos.containers.0.Healthcheck.Status"], "retries": 10, "delay": 30, "tags": ["opendev-validation", "opendev-validation-nova"], "when": ["container_cli == 'podman'", "not container_healthcheck_disabled", "step|int == 5"]}], "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "setype": "{{ item.setype }}", "state": "directory", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/nova", "setype": "container_file_t", "mode": "0750"}]}, {"name": "enable virt_sandbox_use_netlink for healthcheck", "seboolean": {"name": "virt_sandbox_use_netlink", "persistent": true, "state": true}, "when": ["ansible_facts.selinux is defined", "ansible_facts.selinux.status == \"enabled\""]}], "external_upgrade_tasks": [{"when": "step|int == 1", "block": [{"name": "Set facts for running online DB migrations", "set_fact": {"is_additional_cell": false}, "tags": ["online_upgrade", "online_upgrade_nova"]}, {"name": "Online data migration for Nova", "when": "not is_additional_cell|bool", "command": "{{ container_cli }} exec nova_conductor nova-manage db online_data_migrations", "delegate_to": "{{ (groups['nova_conductor'] | difference(groups['excluded_overcloud']))[0] }}", "become": true, "tags": ["online_upgrade", "online_upgrade_nova"]}]}, {"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop nova conductor container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["nova_conductor"], "tripleo_delegate_to": "{{ groups['nova_conductor'] | difference(groups['excluded_overcloud']) }}"}}]}], "external_update_tasks": [{"when": "step|int == 1", "block": [{"name": "Set facts for running online DB migrations", "set_fact": {"is_additional_cell": false}, "tags": ["online_upgrade", "online_upgrade_nova"]}, {"name": "Online data migration for Nova", "when": "not is_additional_cell|bool", "command": "{{ container_cli }} exec nova_conductor nova-manage db online_data_migrations", "delegate_to": "{{ (groups['nova_conductor'] | difference(groups['excluded_overcloud']))[0] }}", "become": true, "tags": ["online_upgrade", "online_upgrade_nova"]}]}]}, {"service_name": "nova_metadata", "firewall_rules": {"139 nova_metadata": {"dport": [8775]}}, "firewall_frontend_rules": {"100 nova_metadatahaproxy_frontend": {"dport": [8775]}}, "firewall_ssl_frontend_rules": {"100 nova_metadatahaproxy_frontend_ssl": {"dport": [13775]}}, "monitoring_subscription": "overcloud-nova-metadata", "config_settings": {"nova::my_ip": "%{hiera('internal_api')}", "nova::rabbit_use_ssl": "False", "nova::keystone::service_user::send_service_user_token": true, "nova::keystone::service_user::project_name": "service", "nova::keystone::service_user::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::service_user::auth_url": "http://172.17.0.2:5000", "nova::keystone::service_user::region_name": "regionOne", "nova::placement::project_name": "service", "nova::placement::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "nova::placement::auth_url": "http://172.17.0.2:5000", "nova::placement::region_name": "regionOne", "nova::placement::valid_interfaces": "internal", "nova::os_region_name": "regionOne", "nova::logging::debug": true, "nova::purge_config": false, "nova::network::neutron::project_name": "service", "nova::network::neutron::username": "neutron", "nova::network::neutron::region_name": "regionOne", "nova::dhcp_domain": "", "nova::network::neutron::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "nova::network::neutron::auth_url": "http://172.17.0.2:5000/v3", "nova::network::neutron::valid_interfaces": "internal", "nova::glance::valid_interfaces": "internal", "nova::rabbit_heartbeat_timeout_threshold": 60, "nova::cinder_catalog_info": "volumev3:cinderv3:internalURL", "nova::host": "%{hiera('fqdn_canonical')}", "nova::notify_on_state_change": "vm_and_task_state", "nova::notification_driver": ["noop"], "nova::notification_format": "unversioned", "nova::network::neutron::auth_type": "v3password", "nova::db::database_db_max_retries": -1, "nova::db::database_max_retries": -1, "nova::network::neutron::ovs_bridge": "br-int", "nova::compute::sync_power_state_interval": 600, "nova_is_additional_cell": false, "nova::cross_az_attach": true, "nova::cache::enabled": true, "nova::cache::tls_enabled": false, "nova::cache::backend": "dogpile.cache.memcached", "nova::rabbit_heartbeat_in_pthread": false, "nova::ovsdb_connection": "", "nova::api_database_connection": "mysql+pymysql://nova_api:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova_api?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "nova::database_connection": "mysql+pymysql://nova:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "apache::ip": "%{hiera('internal_api')}", "apache::default_vhost": false, "apache::trace_enable": "Off", "apache::server_signature": "Off", "apache::server_tokens": "Prod", "apache::timeout": 90, "apache::default_mods": false, "apache::mod::prefork::maxrequestworkers": 100, "apache::mod::prefork::serverlimit": 100, "apache::mod::remoteip::proxy_ips": ["172.17.0.0/24"], "apache::mod::alias::icons_options": "None", "nova::keystone::authtoken::project_name": "service", "nova::keystone::authtoken::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "nova::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "nova::keystone::authtoken::region_name": "regionOne", "nova::keystone::authtoken::interface": "internal", "nova::keystone::authtoken::memcache_use_advanced_pool": true, "nova::wsgi::apache_metadata::api_port": "8775", "nova::wsgi::apache_metadata::ssl": false, "nova::metadata::local_metadata_per_cell": false, "nova::wsgi::apache_metadata::bind_host": "%{hiera('internal_api')}", "nova::wsgi::apache_metadata::servername": "%{hiera('fqdn_internal_api')}", "nova::wsgi::apache_metadata::workers": 1, "nova::metadata::neutron_metadata_proxy_shared_secret": "mfZEPZHNi0vmUaK2wU20kjgr2"}, "service_config_settings": {"rabbitmq": {"get_attr": ["NovaBase", "role_data", "service_config_settings"], "rabbitmq": null}, "mysql": {"nova::db::mysql_api::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::db::mysql_api::user": "nova_api", "nova::db::mysql_api::host": "%", "nova::db::mysql_api::dbname": "nova_api", "nova::db::mysql::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::db::mysql::user": "nova", "nova::db::mysql::host": "%", "nova::db::mysql::dbname": "nova"}, "rsyslog": {"tripleo_logging_sources_nova_metadata": [{"tag": "openstack.nova.api.metadata", "file": "/var/log/containers/nova/nova-metadata-api.log"}]}}, "puppet_config": {"config_volume": "nova_metadata", "puppet_tags": "nova_config,nova_api_paste_ini", "step_config": "include tripleo::profile::base::nova::metadata\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/nova_metadata.json": {"command": "/usr/sbin/httpd -DFOREGROUND", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/nova", "owner": "nova:nova", "recurse": true}]}}, "docker_config": {"step_2": {"nova_metadata_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/nova:/var/log/nova:z", "/var/log/containers/httpd/nova-metadata:/var/log/httpd:z"], "command": ["/bin/bash", "-c", "chown -R nova:nova /var/log/nova"]}}, "step_4": {"nova_metadata": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1", "net": "host", "user": "root", "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova:z", "/var/log/containers/httpd/nova-metadata:/var/log/httpd:z", "/var/lib/kolla/config_files/nova_metadata.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova_metadata:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "deploy_steps_tasks": [{"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"name": "Create dirs for certificates and keys", "file": {"path": "{{ item }}", "state": "directory", "serole": "object_r", "setype": "cert_t", "seuser": "system_u"}, "with_items": ["/etc/pki/tls/certs/httpd", "/etc/pki/tls/private/httpd"]}, {"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "httpd-ctlplane", "dns": "{{ fqdn_ctlplane }}", "principal": "HTTP/{{ fqdn_ctlplane }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-ctlplane.crt /etc/pki/tls/certs/httpd/httpd-ctlplane.crt\ncp /etc/pki/tls/private/httpd-ctlplane.key /etc/pki/tls/private/httpd/httpd-ctlplane.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage", "dns": "{{ fqdn_storage }}", "principal": "HTTP/{{ fqdn_storage }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage.crt /etc/pki/tls/certs/httpd/httpd-storage.crt\ncp /etc/pki/tls/private/httpd-storage.key /etc/pki/tls/private/httpd/httpd-storage.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage_mgmt", "dns": "{{ fqdn_storage_mgmt }}", "principal": "HTTP/{{ fqdn_storage_mgmt }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage_mgmt.crt /etc/pki/tls/certs/httpd/httpd-storage_mgmt.crt\ncp /etc/pki/tls/private/httpd-storage_mgmt.key /etc/pki/tls/private/httpd/httpd-storage_mgmt.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-internal_api", "dns": "{{ fqdn_internal_api }}", "principal": "HTTP/{{ fqdn_internal_api }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-internal_api.crt /etc/pki/tls/certs/httpd/httpd-internal_api.crt\ncp /etc/pki/tls/private/httpd-internal_api.key /etc/pki/tls/private/httpd/httpd-internal_api.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-external", "dns": "{{ fqdn_external }}", "principal": "HTTP/{{ fqdn_external }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-external.crt /etc/pki/tls/certs/httpd/httpd-external.crt\ncp /etc/pki/tls/private/httpd-external.key /etc/pki/tls/private/httpd/httpd-external.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}]}}]}, {"name": "validate nova-metadata container state", "containers.podman.podman_container_info": {"name": "nova_metadata"}, "register": "nova_metadata_infos", "failed_when": ["nova_metadata_infos.containers.0.Healthcheck.Status is defined", "'healthy' not in nova_metadata_infos.containers.0.Healthcheck.Status"], "retries": 10, "delay": 30, "tags": ["opendev-validation", "opendev-validation-nova"], "when": ["container_cli == 'podman'", "not container_healthcheck_disabled", "step|int == 5"]}], "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/nova", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/log/containers/httpd/nova-metadata", "setype": "container_file_t", "mode": "0750"}]}], "metadata_settings": null, "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop nova metadata container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["nova_metadata"], "tripleo_delegate_to": "{{ groups['nova_metadata'] | difference(groups['excluded_overcloud']) }}"}}]}]}, {"service_name": "nova_scheduler", "monitoring_subscription": "overcloud-nova-scheduler", "config_settings": {"nova::my_ip": "%{hiera('internal_api')}", "nova::rabbit_use_ssl": "False", "nova::keystone::service_user::send_service_user_token": true, "nova::keystone::service_user::project_name": "service", "nova::keystone::service_user::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::service_user::auth_url": "http://172.17.0.2:5000", "nova::keystone::service_user::region_name": "regionOne", "nova::placement::project_name": "service", "nova::placement::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "nova::placement::auth_url": "http://172.17.0.2:5000", "nova::placement::region_name": "regionOne", "nova::placement::valid_interfaces": "internal", "nova::os_region_name": "regionOne", "nova::logging::debug": true, "nova::purge_config": false, "nova::network::neutron::project_name": "service", "nova::network::neutron::username": "neutron", "nova::network::neutron::region_name": "regionOne", "nova::dhcp_domain": "", "nova::network::neutron::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "nova::network::neutron::auth_url": "http://172.17.0.2:5000/v3", "nova::network::neutron::valid_interfaces": "internal", "nova::glance::valid_interfaces": "internal", "nova::rabbit_heartbeat_timeout_threshold": 60, "nova::cinder_catalog_info": "volumev3:cinderv3:internalURL", "nova::host": "%{hiera('fqdn_canonical')}", "nova::notify_on_state_change": "vm_and_task_state", "nova::notification_driver": ["noop"], "nova::notification_format": "unversioned", "nova::network::neutron::auth_type": "v3password", "nova::db::database_db_max_retries": -1, "nova::db::database_max_retries": -1, "nova::network::neutron::ovs_bridge": "br-int", "nova::compute::sync_power_state_interval": 600, "nova_is_additional_cell": false, "nova::cross_az_attach": true, "nova::cache::enabled": true, "nova::cache::tls_enabled": false, "nova::cache::backend": "dogpile.cache.memcached", "nova::rabbit_heartbeat_in_pthread": false, "nova::ovsdb_connection": "", "nova::api_database_connection": "mysql+pymysql://nova_api:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova_api?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "nova::database_connection": "mysql+pymysql://nova:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "nova::scheduler::filter::scheduler_available_filters": ["nova.scheduler.filters.all_filters", "nova.scheduler.filters.pci_passthrough_filter.PciPassthroughFilter"], "nova::scheduler::filter::scheduler_enabled_filters": ["AvailabilityZoneFilter", "ComputeFilter", "ComputeCapabilitiesFilter", "ImagePropertiesFilter", "ServerGroupAntiAffinityFilter", "ServerGroupAffinityFilter", "PciPassthroughFilter"], "nova::scheduler::filter::scheduler_max_attempts": 3, "nova::scheduler::filter::scheduler_host_subset_size": 1, "nova::scheduler::filter::shuffle_best_same_weighed_hosts": false, "nova::scheduler::discover_hosts_in_cells_interval": -1, "nova::scheduler::query_placement_for_image_type_support": true, "nova::scheduler::limit_tenants_to_placement_aggregate": false, "nova::scheduler::placement_aggregate_required_for_tenants": false, "nova::scheduler::enable_isolated_aggregate_filtering": false, "nova::scheduler::query_placement_for_availability_zone": false, "nova::scheduler::query_placement_for_routed_network_aggregates": false, "nova::scheduler::workers": 1}, "service_config_settings": {"rabbitmq": {"get_attr": ["NovaBase", "role_data", "service_config_settings"], "rabbitmq": null}, "mysql": {"nova::db::mysql_api::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::db::mysql_api::user": "nova_api", "nova::db::mysql_api::host": "%", "nova::db::mysql_api::dbname": "nova_api", "nova::db::mysql::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::db::mysql::user": "nova", "nova::db::mysql::host": "%", "nova::db::mysql::dbname": "nova"}, "rsyslog": {"tripleo_logging_sources_nova_scheduler": [{"tag": "openstack.nova.scheduler", "file": "/var/log/containers/nova/nova-scheduler.log"}]}}, "puppet_config": {"config_volume": "nova", "puppet_tags": "nova_config", "step_config": "include tripleo::profile::base::nova::scheduler\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/nova_scheduler.json": {"command": "/usr/bin/nova-scheduler ", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/nova", "owner": "nova:nova", "recurse": true}]}}, "docker_config": {"step_4": {"nova_scheduler": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-scheduler:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck 5672"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova", "/var/lib/kolla/config_files/nova_scheduler.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "deploy_steps_tasks": [{"name": "validate nova-scheduler container state", "containers.podman.podman_container_info": {"name": "nova_scheduler"}, "register": "nova_scheduler_infos", "failed_when": ["nova_scheduler_infos.containers.0.Healthcheck.Status is defined", "'healthy' not in nova_scheduler_infos.containers.0.Healthcheck.Status"], "retries": 10, "delay": 30, "tags": ["opendev-validation", "opendev-validation-nova"], "when": ["container_cli == 'podman'", "not container_healthcheck_disabled", "step|int == 5"]}], "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "setype": "{{ item.setype }}", "state": "directory", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/nova", "setype": "container_file_t", "mode": "0750"}]}, {"name": "enable virt_sandbox_use_netlink for healthcheck", "seboolean": {"name": "virt_sandbox_use_netlink", "persistent": true, "state": true}, "when": ["ansible_facts.selinux is defined", "ansible_facts.selinux.status == \"enabled\""]}], "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop nova scheduler container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["nova_scheduler"], "tripleo_delegate_to": "{{ groups['nova_scheduler'] | difference(groups['excluded_overcloud']) }}"}}]}]}, {"service_name": "nova_compute", "monitoring_subscription": "overcloud-nova-compute", "config_settings": {"nova::my_ip": "%{hiera('internal_api')}", "nova::rabbit_use_ssl": "False", "nova::keystone::service_user::send_service_user_token": true, "nova::keystone::service_user::project_name": "service", "nova::keystone::service_user::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::service_user::auth_url": "http://172.17.0.2:5000", "nova::keystone::service_user::region_name": "regionOne", "nova::placement::project_name": "service", "nova::placement::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "nova::placement::auth_url": "http://172.17.0.2:5000", "nova::placement::region_name": "regionOne", "nova::placement::valid_interfaces": "internal", "nova::os_region_name": "regionOne", "nova::logging::debug": true, "nova::purge_config": false, "nova::network::neutron::project_name": "service", "nova::network::neutron::username": "neutron", "nova::network::neutron::region_name": "regionOne", "nova::dhcp_domain": "", "nova::network::neutron::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "nova::network::neutron::auth_url": "http://172.17.0.2:5000/v3", "nova::network::neutron::valid_interfaces": "internal", "nova::glance::valid_interfaces": "internal", "nova::rabbit_heartbeat_timeout_threshold": 60, "nova::cinder_catalog_info": "volumev3:cinderv3:internalURL", "nova::host": "%{hiera('fqdn_canonical')}", "nova::notify_on_state_change": "vm_and_task_state", "nova::notification_driver": ["noop"], "nova::notification_format": "unversioned", "nova::network::neutron::auth_type": "v3password", "nova::db::database_db_max_retries": -1, "nova::db::database_max_retries": -1, "nova::network::neutron::ovs_bridge": "br-int", "nova::compute::sync_power_state_interval": 600, "nova_is_additional_cell": false, "nova::cross_az_attach": true, "nova::cache::enabled": true, "nova::cache::tls_enabled": false, "nova::cache::backend": "dogpile.cache.memcached", "nova::rabbit_heartbeat_in_pthread": false, "nova::ovsdb_connection": "", "nova::compute::vcpu_pin_set": [], "nova::compute::cpu_shared_set": [], "nova::compute::cpu_dedicated_set": [], "nova::compute::reserved_host_memory": 512, "nova::compute::reserved_huge_pages": [], "nova::compute::neutron_physnets_numa_nodes_mapping": {}, "nova::compute::neutron_tunnel_numa_nodes": [], "nova::compute::resume_guests_state_on_host_boot": false, "nova::compute::libvirt::rx_queue_size": 512, "nova::compute::libvirt::tx_queue_size": 512, "nova::compute::libvirt::file_backed_memory": 0, "nova::compute::libvirt::volume_use_multipath": false, "nova::compute::libvirt::hw_machine_type": "x86_64=pc-q35-rhel9.0.0", "nova::compute::mkisofs_cmd": "mkisofs", "compute_enable_ksm": false, "nova::compute::rbd::libvirt_images_rbd_pool": "vms", "tripleo::profile::base::nova::compute::nova_nfs_enabled": false, "nfs_backend_enable": false, "nfs_share": "", "nfs_options": "context=system_u:object_r:nfs_t:s0", "nfs_vers": "4.2", "nova::compute::libvirt::num_pcie_ports": 16, "nova::compute::libvirt::mem_stats_period_seconds": 10, "nova::compute::rbd::ephemeral_storage": true, "resume_guests_state_on_host_boot": false, "nova::cpu_allocation_ratio": 0.0, "nova::ram_allocation_ratio": 1.0, "nova::disk_allocation_ratio": 0.0, "nova::compute::max_disk_devices_to_attach": -1, "nova::compute::libvirt::pmem_namespaces": "", "nova_pmem_namespaces": "", "nova::compute::image_cache::remove_unused_original_minimum_age_seconds": 86400, "nova::compute::libvirt::swtpm_enabled": false, "nova::compute::vgpu::vgpu_types_device_addresses_mapping": {}, "compute_disable_irqbalance": false, "nova::compute::libvirt::cpu_mode": "host-model", "nova::compute::libvirt::cpu_models": [], "nova::compute::libvirt::cpu_model_extra_flags": "", "nova_compute_opt_volumes": [], "nova_compute_startup_delay": 0, "nova_compute_opt_env_vars": {}, "nova::workarounds::never_download_image_if_on_rbd": false, "nova::compute::image_type_exclude_list": [], "nova_permit_post_copy": true, "nova_permit_auto_converge": true, "nova_libvirt_max_queues": 0, "nova_glance_enable_rbd_download": false, "nova_glance_rbd_download_multistore_id": "", "nova::compute::force_raw_images": true, "nova::compute::use_cow_images": true, "nova::compute::libvirt::preallocate_images": "none", "nova::compute::image_cache::manager_interval": 2400, "nova::compute::image_cache::remove_unused_base_images": true, "nova::compute::image_cache::remove_unused_resized_minimum_age_seconds": 3600, "nova::compute::image_cache::precache_concurrency": 1, "nova::compute::provider::custom_inventories": {}, "nova::compute::rbd::libvirt_images_rbd_glance_copy_poll_interval": 15, "nova::compute::rbd::libvirt_images_rbd_glance_copy_timeout": 600, "ContainerNovaComputeImage": "registry.redhat.io/rhosp-rhel9/openstack-nova-compute:17.1", "ContainerNovaLibvirtConfigImage": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "nova::compute::libvirt::manage_libvirt_services": false, "nova::compute::pci::passthrough": "[{\"devname\": \"dummy-dev\", \"physical_network\": \"dummy_sriov_net\"}]", "nova::compute::libvirt::migration_support": false, "tripleo::profile::base::nova::migration::client::nova_compute_enabled": true, "tripleo::profile::base::nova::migration::client::ssh_private_key": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEApEW+iSDuQm4pBk/4gct8N21CUktJkYPZh42iMMBRmjvtdesa\nzByskHhSHzsdcZVXB6QMglFiP2qlX3OCmioW5N0jqTmpTbU/69vDerntqnzxNWpY\nJdycEi7LLztxn6UOlIQykb9KiEY+ou9NO4k8SAU72a5PplhZVPN0PTVs303ASP1r\nSV/mwd6ybKKqzEZm7bqeAAY/v34jevkPmPP0b7E31jDLDHMyWlEBrQQc9dzpnfMA\nzm4XJ7FoY/fxKYhzT1BFCpXJidTGvskPlta9yshXNigWBtbcQaJvt1bIEoofl5WD\n/BXa20TrltlD/nQ6VTbBHADz9uCbRQxaHI0e6QIDAQABAoIBADmks6CA6rpq14B8\njXJWSidU8qHYA1/JiYqUe6Zpxfi+MKxhLI0VPsSPRBnrb4CM6/YlbbJJBpXEQXWG\nHf0k6dWzd7EJJ8y53k1QV374QVrC7w1UXMwkZqvGucvAKzXfAIfC/8++9VNuespH\n+Kz8kieflonjLdXsSKSTrWVRGvwcRohAQ6k5YofMeOfke+Tk3/QNDmNTOOofyEN0\npeGouReKeh9N7FV1Uy0QzEi1Bq8odpcK7dHUNXPJM4NlISMXl3hsCu19OOenxLT4\nMPAcQodjIYO795S9hN9HL7tWNTwEReq5n3X6U3ZuDH5nedy5ngEATt9cEIboTjSh\nmNdbXuUCgYEAtobrDEiywQXXOjGTejM8l1G/cteml6aJPS5hc9qUJZr6UzW+HkfH\nBK3eh0ACoFxsiy1pP1wXCG79V1nROJu2josvXWqkkmJ5MUgIsg1NO4qhBppMhlEX\n1qhEbrf0k05ykrsp2ZLkAYkRbQeCyVbSud0flZxPWO7fTJqk5ZxC7Y8CgYEA5mW5\nUSYt3j5T8BhEkdHKj7DacnQE6CWBck1iMZ61W/gbhJgqUlb14v8fvf2guVgSI+nk\nZJwBo7A/O+yWxMCHVkzSiEvn7VQUA04cK8vvyA0sXq5XpR6smlh6kXYd8z2/Ke6U\n8M1iUsipNwV4QBOa1jvAkLPWEy2TvCBN24ytYAcCgYEAoZG0Jl+Du9ELv3t67ETG\ncbvvGB1EXiEQgpO3t/HYCaGxzE5aSDuJM98kh9xldsXZXymHbIp76P82Skg7JzzO\n0EbsWmi33MxGMxVsfQ/BOd9wwoN7Ebpeu6ifStKTMIHUcYYzeWDJa4KFZlmrr8Wn\nKtdNygVfR33LM+f/5y/3HokCgYAp7Rz6DW6PYNw5dtETkA/CcjZuWXqDHEAgIsDZ\nsr3iDJZFXMNm1HhOZIrMMhlLYSF785YiUNljhjEboIQXLcnzBGvTbay43XH+3/n0\nZlouF2eZmzRL0fcq5yr2yX3Nw9DDuqzT3SwlUGqqno4xat/kWF9EddVhDAw0UdSZ\nPVJXhQKBgBJuA1NrpGyERbKfwmTriaQ/4jXXZwcIZf+U0KOKchDtaR9PmY05qQm8\nbwH5USjLpLcfIZCxGRh+KPBusLOnvnKX4PZwv397Zfq9j5y8UcsWKab71RDqVY8V\nZj25fDMEw29QQyaxQ+SRV78MG20YJtsb9TieU1XvJkpFURUpypNX\n-----END RSA PRIVATE KEY-----\n", "tripleo::profile::base::nova::migration::client::ssh_port": 2022, "nova::compute::rbd::libvirt_images_rbd_glance_store_name": "default_backend", "nova::compute::rbd::libvirt_images_rbd_ceph_conf": "/etc/ceph/ceph.conf", "nova::compute::rbd::libvirt_rbd_user": "openstack", "nova::compute::rbd::rbd_keyring": "client.openstack", "tripleo::profile::base::nova::compute::cinder_nfs_backend": false, "rbd_persistent_storage": true, "rbd_disk_cachemodes": ["network=writeback"], "nova::cinder::username": "cinder", "nova::cinder::auth_type": "v3password", "nova::cinder::project_name": "service", "nova::cinder::password": "Olf2iTUMR6eJ33irnMaVcSJQX", "nova::cinder::auth_url": "http://172.17.0.2:5000/v3", "nova::cinder::region_name": "regionOne", "nova::compute::rbd::libvirt_rbd_secret_key": "AQBP7ytpAAAAABAAXQLQTQkMxLJ7uGigDaf94Q==", "nova::compute::rbd::libvirt_rbd_secret_uuid": "55dae5e5-7ec8-55ff-b761-b30fedcc71d6", "nova::compute::instance_usage_audit": true, "nova::compute::instance_usage_audit_period": "hour", "nova::compute::consecutive_build_service_disable_threshold": "10", "nova::compute::live_migration_wait_for_vif_plug": true, "nova::migration::libvirt::live_migration_permit_post_copy": true, "nova::migration::libvirt::live_migration_permit_auto_converge": true, "nova::migration::libvirt::live_migration_tunnelled": false, "nova::compute::vncserver_proxyclient_address": "%{hiera('internal_api')}", "nova::compute::vncproxy_host": "172.21.0.2", "nova::vncproxy::common::vncproxy_protocol": "http", "nova::vncproxy::common::vncproxy_host": "172.21.0.2", "nova::vncproxy::common::vncproxy_port": "6080", "nova::compute::verify_glance_signatures": false, "nova::workarounds::disable_compute_service_check_for_ffu": false}, "service_config_settings": {"rsyslog": {"tripleo_logging_sources_nova_compute": [{"tag": "openstack.nova.compute", "file": "/var/log/containers/nova/nova-compute.log"}]}, "collectd": {"tripleo.collectd.plugins.nova_compute": ["virt"], "collectd::plugin::virt::connection": "qemu:///system"}}, "puppet_config": {"config_volume": "nova_libvirt", "puppet_tags": "nova_config", "step_config": "# TODO(emilien): figure how to deal with libvirt profile.\n# We'll probably treat it like we do with Neutron plugins.\n# Until then, just include it in the default nova-compute role.\ninclude tripleo::profile::base::nova::compute::libvirt\n", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/nova_compute.json": {"command": "/usr/bin/nova-compute ", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-iscsid/*", "dest": "/etc/iscsi/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-ceph/", "dest": "/etc/ceph/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/nova", "owner": "nova:nova", "recurse": true}, {"path": "/etc/ceph/ceph.client.openstack.keyring", "owner": "nova:nova", "perm": "0600"}]}, "/var/lib/kolla/config_files/nova_compute_wait_for_compute_service.json": {"command": "/container-config-scripts/pyshim.sh /container-config-scripts/nova_wait_for_compute_service.py", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/nova", "owner": "nova:nova", "recurse": true}]}}, "container_config_scripts": {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "nova_statedir_ownership.py": {"mode": "0700", "content": "#!/usr/bin/python3\n#\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\nimport logging\nimport os\nimport pwd\nimport selinux\nimport stat\nimport sys\n\ndebug = os.getenv('__OS_DEBUG', 'false')\n\nif debug.lower() == 'true':\n    loglevel = logging.DEBUG\nelse:\n    loglevel = logging.INFO\n\nlogging.basicConfig(stream=sys.stdout, level=loglevel)\nLOG = logging.getLogger('nova_statedir')\n\n\nclass PathManager(object):\n    \"\"\"Helper class to manipulate ownership of a given path\"\"\"\n    def __init__(self, path):\n        self.path = path\n        self.uid = None\n        self.gid = None\n        self.is_dir = None\n        self.secontext = None\n        self._update()\n\n    def _update(self):\n        try:\n            statinfo = os.stat(self.path)\n            self.is_dir = stat.S_ISDIR(statinfo.st_mode)\n            self.uid = statinfo.st_uid\n            self.gid = statinfo.st_gid\n            self.secontext = selinux.lgetfilecon(self.path)[1]\n        except Exception:\n            LOG.exception('Could not update metadata for %s', self.path)\n            raise\n\n    def __str__(self):\n        return \"uid: {} gid: {} path: {}{}\".format(\n            self.uid,\n            self.gid,\n            self.path,\n            '/' if self.is_dir else ''\n        )\n\n    def has_owner(self, uid, gid):\n        return self.uid == uid and self.gid == gid\n\n    def has_either(self, uid, gid):\n        return self.uid == uid or self.gid == gid\n\n    def chown(self, uid, gid):\n        target_uid = -1\n        target_gid = -1\n        if self.uid != uid:\n            target_uid = uid\n        if self.gid != gid:\n            target_gid = gid\n        if (target_uid, target_gid) != (-1, -1):\n            LOG.info('Changing ownership of %s from %d:%d to %d:%d',\n                     self.path,\n                     self.uid,\n                     self.gid,\n                     self.uid if target_uid == -1 else target_uid,\n                     self.gid if target_gid == -1 else target_gid)\n            try:\n                os.chown(self.path, target_uid, target_gid)\n                self._update()\n            except Exception:\n                LOG.exception('Could not change ownership of %s: ',\n                              self.path)\n                raise\n        else:\n            LOG.info('Ownership of %s already %d:%d',\n                     self.path,\n                     uid,\n                     gid)\n\n    def chcon(self, context):\n        # If dir returns whether to recusively set context\n        try:\n            try:\n                selinux.lsetfilecon(self.path, context)\n                LOG.info('Setting selinux context of %s to %s',\n                     self.path, context)\n                return True\n            except OSError as e:\n                if self.is_dir and e.errno == 95:\n                    # Operation not supported, assume NFS mount and skip\n                    LOG.info('Setting selinux context not supported for %s',\n                             self.path)\n                    return False\n                else:\n                    raise\n        except Exception:\n            LOG.exception('Could not set selinux context of %s to %s:',\n                          self.path, context)\n            raise\n\n\nclass NovaStatedirOwnershipManager(object):\n    \"\"\"Class to manipulate the ownership of the nova statedir (/var/lib/nova).\n\n       The nova uid/gid differ on the host and container images. An upgrade\n       that switches from host systemd services to docker requires a change in\n       ownership. Previously this was a naive recursive chown, however this\n       causes issues if nova instance are shared via an NFS mount: any open\n       filehandles in qemu/libvirt fail with an I/O error (LP1778465).\n\n       Instead the upgrade/FFU ansible tasks now lay down a marker file when\n       stopping and disabling the host systemd services. We use this file to\n       determine the host nova uid/gid. We then walk the tree and update any\n       files that have the host uid/gid to the docker nova uid/gid. As files\n       owned by root/qemu etc... are ignored this avoids the issues with open\n       filehandles. The marker is removed once the tree has been walked.\n\n       For subsequent runs, or for a new deployment, we simply ensure that the\n       docker nova user/group owns all directories. This is required as the\n       directories are created with root ownership in host_prep_tasks (the\n       docker nova uid/gid is not known in this context).\n    \"\"\"\n    def __init__(self, statedir, upgrade_marker='upgrade_marker',\n                 nova_user='nova', secontext_marker='../_nova_secontext',\n                 exclude_paths=None):\n        self.statedir = statedir\n        self.nova_user = nova_user\n\n        self.upgrade_marker_path = os.path.join(statedir, upgrade_marker)\n        self.secontext_marker_path = os.path.normpath(os.path.join(statedir, secontext_marker))\n        self.upgrade = os.path.exists(self.upgrade_marker_path)\n\n        self.exclude_paths = [self.upgrade_marker_path]\n        if exclude_paths is not None:\n            for p in exclude_paths:\n                if not p.startswith(os.path.sep):\n                    p = os.path.join(self.statedir, p)\n                self.exclude_paths.append(p)\n\n        self.target_uid, self.target_gid = self._get_nova_ids()\n        self.previous_uid, self.previous_gid = self._get_previous_nova_ids()\n        self.id_change = (self.target_uid, self.target_gid) != \\\n            (self.previous_uid, self.previous_gid)\n        self.target_secontext = self._get_secontext()\n\n    def _get_nova_ids(self):\n        nova_uid, nova_gid = pwd.getpwnam(self.nova_user)[2:4]\n        return nova_uid, nova_gid\n\n    def _get_previous_nova_ids(self):\n        if self.upgrade:\n            statinfo = os.stat(self.upgrade_marker_path)\n            return statinfo.st_uid, statinfo.st_gid\n        else:\n            return self._get_nova_ids()\n\n    def _get_secontext(self):\n        if os.path.exists(self.secontext_marker_path):\n            return selinux.lgetfilecon(self.secontext_marker_path)[1]\n        else:\n            return None\n\n    def _walk(self, top, chcon=True):\n        for f in os.listdir(top):\n            pathname = os.path.join(top, f)\n\n            if pathname in self.exclude_paths:\n                continue\n\n            try:\n                pathinfo = PathManager(pathname)\n                LOG.info(\"Checking %s\", pathinfo)\n                if pathinfo.is_dir:\n                    # Always chown the directories\n                    pathinfo.chown(self.target_uid, self.target_gid)\n                    chcon_r = chcon\n                    if chcon:\n                        chcon_r = pathinfo.chcon(self.target_secontext)\n                    self._walk(pathname, chcon_r)\n                elif self.id_change:\n                    # Only chown files if it's an upgrade and the file is owned by\n                    # the host nova uid/gid\n                    pathinfo.chown(\n                        self.target_uid if pathinfo.uid == self.previous_uid\n                        else pathinfo.uid,\n                        self.target_gid if pathinfo.gid == self.previous_gid\n                        else pathinfo.gid\n                    )\n                    if chcon:\n                        pathinfo.chcon(self.target_secontext)\n            except Exception:\n                # Likely to have been caused by external systems\n                # interacting with this directory tree,\n                # especially on NFS e.g snapshot dirs.\n                # Just ignore it and continue on to the next entry\n                continue\n\n    def run(self):\n        LOG.info('Applying nova statedir ownership')\n        LOG.info('Target ownership for %s: %d:%d',\n                 self.statedir,\n                 self.target_uid,\n                 self.target_gid)\n\n        pathinfo = PathManager(self.statedir)\n        LOG.info(\"Checking %s\", pathinfo)\n        pathinfo.chown(self.target_uid, self.target_gid)\n        chcon = self.target_secontext is not None\n\n        if chcon:\n            pathinfo.chcon(self.target_secontext)\n\n        self._walk(self.statedir, chcon)\n\n        if self.upgrade:\n            LOG.info('Removing upgrade_marker %s',\n                     self.upgrade_marker_path)\n            os.unlink(self.upgrade_marker_path)\n\n        LOG.info('Nova statedir ownership complete')\n\n\ndef get_exclude_paths():\n    exclude_paths = os.environ.get('NOVA_STATEDIR_OWNERSHIP_SKIP')\n    if exclude_paths is not None:\n        exclude_paths = exclude_paths.split(os.pathsep)\n    return exclude_paths\n\n\nif __name__ == '__main__':\n    NovaStatedirOwnershipManager('/var/lib/nova', exclude_paths=get_exclude_paths()).run()\n"}, "nova_wait_for_compute_service.py": {"mode": "0755", "content": "#!/usr/bin/python3\n#\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\nimport argparse\nimport logging\nimport os\nimport six\nimport socket\nimport sys\nimport time\n\n\nfrom keystoneauth1 import loading\nfrom keystoneauth1 import session\n\nfrom novaclient import client\n\n# In python3 SafeConfigParser was renamed to ConfigParser and the default\n# for duplicate options default to true. In case of nova it is valid to\n# have duplicate option lines, e.g. passthrough_whitelist which leads to\n# issues reading the nova.conf\n# https://bugs.launchpad.net/tripleo/+bug/1827775\nif six.PY3:\n    from six.moves.configparser import ConfigParser\n    config = ConfigParser(strict=False)\nelse:\n    from six.moves.configparser import SafeConfigParser\n    config = SafeConfigParser()\n\ndebug = os.getenv('__OS_DEBUG', 'false')\n\nif debug.lower() == 'true':\n    loglevel = logging.DEBUG\nelse:\n    loglevel = logging.INFO\n\nlogging.basicConfig(stream=sys.stdout, level=loglevel)\nLOG = logging.getLogger('nova_wait_for_compute_service')\n\niterations = 60\ntimeout = 10\nnova_cfg = '/etc/nova/nova.conf'\n\nif __name__ == '__main__':\n    parser = argparse.ArgumentParser(usage='%(prog)s [options]')\n    parser.add_argument('-k', '--insecure',\n                        action=\"store_false\",\n                        dest='insecure',\n                        default=True,\n                        help='Allow insecure connection when using SSL')\n\n    args = parser.parse_args()\n    LOG.debug('Running with parameter insecure = %s',\n              args.insecure)\n\n    if os.path.isfile(nova_cfg):\n        try:\n            config.read(nova_cfg)\n        except Exception:\n            LOG.exception('Error while reading nova.conf:')\n    else:\n        LOG.error('Nova configuration file %s does not exist', nova_cfg)\n        sys.exit(1)\n\n    my_host = config.get('DEFAULT', 'host')\n    if not my_host:\n        # If host isn't set nova defaults to this\n        my_host = socket.gethostname()\n\n    loader = loading.get_plugin_loader('password')\n    auth = loader.load_from_options(\n        auth_url=config.get('neutron',\n                            'auth_url'),\n        username=config.get('neutron',\n                            'username'),\n        password=config.get('neutron',\n                            'password'),\n        project_name=config.get('neutron',\n                                'project_name'),\n        project_domain_name=config.get('neutron',\n                                       'project_domain_name'),\n        user_domain_name=config.get('neutron',\n                                    'user_domain_name'))\n    sess = session.Session(auth=auth, verify=args.insecure)\n    nova = client.Client('2.11', session=sess, endpoint_type='internal',\n                         region_name=config.get('neutron', 'region_name'))\n\n    # Wait until this host is listed in the service list\n    for i in range(iterations):\n        try:\n            service_list = nova.services.list(binary='nova-compute')\n            for entry in service_list:\n                host = getattr(entry, 'host', '')\n                zone = getattr(entry, 'zone', '')\n                if host == my_host and zone != 'internal':\n                    LOG.info('Nova-compute service registered')\n                    sys.exit(0)\n            LOG.info('Waiting for nova-compute service to register')\n        except Exception:\n            LOG.exception(\n                'Error while waiting for nova-compute service to register')\n        time.sleep(timeout)\nsys.exit(1)\n\n# vim: set et ts=4 sw=4 :\n"}}, "docker_config": {"step_2": {"nova_compute_init_log": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-compute:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/nova:/var/log/nova:z"], "command": ["/bin/bash", "-c", "chown -R nova:nova /var/log/nova"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_3": {"nova_statedir_owner": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-compute:17.1", "net": "none", "user": "root", "security_opt": ["label=disable"], "privileged": false, "detach": false, "volumes": ["/var/lib/nova:/var/lib/nova:shared", "/var/lib/_nova_secontext:/var/lib/_nova_secontext:shared,z", "/var/lib/container-config-scripts:/container-config-scripts:z"], "command": "/container-config-scripts/pyshim.sh /container-config-scripts/nova_statedir_ownership.py", "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992", "__OS_DEBUG": "true", "NOVA_STATEDIR_OWNERSHIP_SKIP": "triliovault-mounts"}}}, "step_5": {"nova_compute": {"start_order": 3, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-compute:17.1", "ulimit": ["nofile=131072", "memlock=67108864"], "ipc": "host", "net": "host", "privileged": true, "user": "nova", "restart": "always", "depends_on": ["tripleo_nova_libvirt.target"], "healthcheck": {"test": "/openstack/healthcheck 5672"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova", "/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro", "/var/lib/kolla/config_files/nova_compute.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/var/lib/config-data/puppet-generated/iscsid/etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/dev:/dev", "/lib/modules:/lib/modules:ro", "/run:/run", "/run/nova:/run/nova:z", "/var/lib/iscsi:/var/lib/iscsi:z", "/var/lib/libvirt:/var/lib/libvirt:shared", "/sys/class/net:/sys/class/net", "/sys/bus/pci:/sys/bus/pci", "/boot:/boot:ro", "/var/lib/nova:/var/lib/nova:shared"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "LIBGUESTFS_BACKEND": "direct"}}, "nova_wait_for_compute_service": {"start_order": 4, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-compute:17.1", "net": "host", "detach": false, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/nova_compute_wait_for_compute_service.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/var/log/containers/nova:/var/log/nova", "/var/lib/container-config-scripts:/container-config-scripts"], "user": "nova", "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "__OS_DEBUG": "true"}}}}, "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "setype": "{{ item.setype }}", "state": "directory", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/nova", "setype": "container_file_t", "mode": "0750"}]}, {"name": "Check for /var/lib/nova/instances directory exitance", "stat": {"path": "/var/lib/nova/instances"}, "register": "check_nova_instances_result"}, {"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype | default(omit) }}"}, "with_items": [{"path": "/var/lib/nova", "setype": "container_file_t"}, {"path": "/var/lib/_nova_secontext", "setype": "container_file_t"}, {"path": "/var/lib/nova/instances", "setype": "container_file_t"}, {"path": "/var/lib/libvirt", "setype": "container_file_t"}, {"path": "/etc/tmpfiles.d"}]}, {"name": "ensure /run/nova is present upon reboot", "copy": {"dest": "/etc/tmpfiles.d/run-nova.conf", "content": "d /run/nova 0777 root root - -\n"}}, {"name": "create tempfiles", "command": "systemd-tmpfiles --create"}, {"name": "Mount Nova NFS Share", "vars": {"nfs_backend_enable": false, "nfs_share": "", "nfs_options": "context=system_u:object_r:nfs_t:s0", "nfs_vers": "4.2"}, "mount": {"name": "/var/lib/nova/instances", "state": "mounted", "fstype": "nfs4", "src": "{{nfs_share}}", "opts": "_netdev,bg,{{nfs_options}},nfsvers={{nfs_vers}}"}, "when": "nfs_backend_enable|bool"}, {"name": "delay installation of systemd-container", "when": "check_nova_instances_result.stat.exists", "block": [{"name": "check systemd-container package installed or not", "package": {"name": "systemd-container", "state": "present"}, "check_mode": true, "register": "is_systemd_container_package", "failed_when": false}, {"name": "get latest package from downloaded package versions", "shell": "ls -1t /var/tmp/systemd-container* | head -1", "register": "latest_systemd_container_package", "when": ["is_systemd_container_package is defined", "('changed' in is_systemd_container_package)", "is_systemd_container_package.changed"], "failed_when": false}, {"name": "Install systemd-container package on boot", "copy": {"dest": "/etc/systemd/system/install-systemd-container.service", "content": "[Unit]\nDescription=Install systemd container package on system boot\nBefore=tripleo_nova_virtqemud.service\n\n[Service]\nType=oneshot\nExecStart=/usr/bin/rpm -ivh {{ latest_systemd_container_package.stdout }}\nExecStartPost=rm -f /var/tmp/systemd-container*\n\n[Install]\nWantedBy=multi-user.target\n"}, "when": ["latest_systemd_container_package is defined", "('stdout' in latest_systemd_container_package)", "is_systemd_container_package is defined", "('changed' in is_systemd_container_package)", "is_systemd_container_package.changed"]}, {"name": "enable install-systemd-container on compute boot", "systemd": {"name": "install-systemd-container", "state": "stopped", "enabled": true, "daemon_reload": true}, "when": ["is_systemd_container_package is defined", "('changed' in is_systemd_container_package)", "is_systemd_container_package.changed"]}, {"name": "check if install-systemd-container service exists", "stat": {"path": "/etc/systemd/system/install-systemd-container.service"}, "register": "install_systemd_container_result"}, {"name": "disable and mask install-systemd-container on compute boot", "systemd": {"name": "install-systemd-container", "enabled": false, "masked": true, "daemon_reload": true}, "when": ["is_systemd_container_package is defined", "('changed' in is_systemd_container_package)", "not is_systemd_container_package.changed", "install_systemd_container_result.stat.exists"]}]}, {"name": "is Nova Resume Guests State On Host Boot enabled", "set_fact": {"resume_guests_state_on_host_boot_enabled": false}}, {"name": "install libvirt-guests systemd unit file (docker)", "when": ["resume_guests_state_on_host_boot_enabled|bool", "container_cli == 'docker'"], "block": [{"name": "libvirt-guests unit to stop nova_compute container before shutdown VMs", "copy": {"dest": "/etc/systemd/system/libvirt-guests.service", "content": "[Unit]\nDescription=Suspend/Resume Running libvirt Guests\nAfter=network.target\nAfter=time-sync.target\nAfter=virt-guest-shutdown.target\nAfter=docker.service\nAfter=tripleo-container-shutdown.service\nAfter=rhel-push-plugin.service\nDocumentation=man:libvirtd(8)\nDocumentation=https://libvirt.org\n\n[Service]\nEnvironmentFile=-/var/lib/config-data/puppet-generated/nova_libvirt/etc/sysconfig/libvirt-guests\n# Hack just call traditional service until we factor\n# out the code\nExecStart=/bin/{{container_cli}} exec nova_libvirt /bin/sh -x /usr/libexec/libvirt-guests.sh start\nExecStop=/bin/{{container_cli}} stop nova_compute\nExecStop=/bin/{{container_cli}}  exec nova_libvirt /bin/sh -x /usr/libexec/libvirt-guests.sh stop\nType=oneshot\nRemainAfterExit=yes\nStandardOutput=journal+console\nTimeoutStopSec=0\n\n[Install]\nWantedBy=multi-user.target\n"}}, {"name": "Making sure virt-guest-shutdown.target is present", "copy": {"dest": "/etc/systemd/system/virt-guest-shutdown.target", "content": "[Unit]\nDescription=Libvirt guests shutdown\nDocumentation=https://libvirt.org\nAfter=systemd-machined.service\nAfter=network-online.target\nAfter=tripleo_nova_libvirt.target\nBefore=tripleo_nova_compute.service\n"}}, {"name": "libvirt-guests enable VM shutdown on compute reboot/shutdown", "systemd": {"name": "libvirt-guests", "enabled": true, "daemon_reload": true}}]}, {"name": "install tripleo_nova_libvirt_guests systemd unit file (podman)", "when": ["resume_guests_state_on_host_boot_enabled|bool", "container_cli == 'podman'"], "block": [{"name": "libvirt-guests unit to stop nova_compute container before shutdown VMs (monolithic libvirt)", "copy": {"dest": "/etc/systemd/system/tripleo_nova_libvirt_guests.service", "content": "[Unit]\nDescription=Suspend libvirt Guests in tripleo\nRequires=virt-guest-shutdown.target\nAfter=systemd-machined.service\nAfter=network-online.target\nAfter=tripleo_nova_libvirt.target\nBefore=tripleo_nova_compute.service\nDocumentation=man:libvirtd(8)\nDocumentation=https://libvirt.org\n\n[Service]\nEnvironmentFile=-/etc/sysconfig/libvirt-guests\nExecStart=/bin/podman exec nova_libvirt /bin/rm -f /var/lib/libvirt/libvirt-guests\nExecStop=/bin/podman exec nova_libvirt /bin/sh -x /usr/libexec/libvirt-guests.sh shutdown\nType=oneshot\nRemainAfterExit=yes\nStandardOutput=journal+console\nTimeoutStopSec=0\n\n[Install]\nWantedBy=multi-user.target\n"}, "when": "tripleo_nova_libvirt_virsh_container | default('nova_libvirt') == 'nova_libvirt'"}, {"name": "libvirt-guests unit to stop nova_compute container before shutdown VMs (modular libvirt)", "copy": {"dest": "/etc/systemd/system/tripleo_nova_libvirt_guests.service", "content": "[Unit]\nDescription=Suspend libvirt Guests in tripleo\nRequires=virt-guest-shutdown.target\nAfter=systemd-machined.service\nAfter=network-online.target\nAfter=tripleo_nova_libvirt.target\nBefore=tripleo_nova_compute.service\nDocumentation=man:libvirtd(8)\nDocumentation=https://libvirt.org\n\n[Service]\nEnvironmentFile=-/etc/sysconfig/libvirt-guests\nExecStart=/bin/podman exec nova_virtproxyd /bin/rm -f /var/lib/libvirt/libvirt-guests\nExecStop=/bin/podman exec nova_virtproxyd /bin/sh -x /usr/libexec/libvirt-guests.sh shutdown\nType=oneshot\nRemainAfterExit=yes\nStandardOutput=journal+console\nTimeoutStopSec=0\n\n[Install]\nWantedBy=multi-user.target\n"}, "when": "tripleo_nova_libvirt_virsh_container | default('nova_libvirt') != 'nova_libvirt'"}, {"name": "Making sure virt-guest-shutdown.target is present", "copy": {"dest": "/etc/systemd/system/virt-guest-shutdown.target", "content": "[Unit]\nDescription=Libvirt guests shutdown\nDocumentation=https://libvirt.org\nAfter=systemd-machined.service\nAfter=network-online.target\nAfter=tripleo_nova_libvirt.target\nBefore=tripleo_nova_compute.service\n"}}, {"name": "tripleo_nova_libvirt_guests enable VM shutdown on compute reboot/shutdown", "systemd": {"name": "tripleo_nova_libvirt_guests", "enabled": true, "daemon_reload": true}}]}, {"name": "ensure ceph configurations exist", "file": {"path": "/var/lib/tripleo-config/ceph", "state": "directory"}}, {"name": "enable virt_sandbox_use_netlink for healthcheck", "seboolean": {"name": "virt_sandbox_use_netlink", "persistent": true, "state": true}, "when": ["ansible_facts.selinux is defined", "ansible_facts.selinux.status == \"enabled\""]}, {"name": "Do we prepend nova startup with a delay", "set_fact": {"nova_compute_delay": 0}}, {"name": "install nova-compute delay wrapper script", "copy": {"content": "#!/usr/libexec/platform-python\n\"\"\"\nThis wrapper was created to add an optional delay to the startup of nova-compute.\nWe know that instances will fail to boot, after a compute reboot, if ceph is not\nhealthy.\n\nIdeally, we would poll ceph to get its health, but it's not guaranteed that the\ncompute node will have access to the keys.\n\"\"\"\n\nimport os\nimport sys\nimport time\nimport logging\nimport argparse\n\nparser = argparse.ArgumentParser(description='Process some integers.')\nparser.add_argument('--config-file', dest='nova_config', action='store',\n                    default=\"/etc/nova/nova.conf\",\n                    help='path to nova configuration (default: /etc/nova/nova.conf)')\nparser.add_argument('--nova-binary', dest='nova_binary', action='store',\n                    default=\"/usr/bin/nova-compute\",\n                    help='path to nova compute binary (default: /usr/bin/nova-compute)')\nparser.add_argument('--delay', dest='delay', action='store',\n                    default=120, type=int,\n                    help='Number of seconds to wait until nova-compute is started')\nparser.add_argument('--state-dir', dest='state_dir', action='store',\n                    default=\"/run/nova/startup\", help='state-file directory')\nparser.add_argument('--state-file', dest='state_file', action='store',\n                    default=\"nova-compute-delayed\",\n                    help='file exists if we already delayed nova-compute startup')\n\n\nsections = {}\n(args, remaining) = parser.parse_known_args(sys.argv)\n\nreal_args = [args.nova_binary, '--config-file', args.nova_config]\nreal_args.extend(remaining[1:])\n\nif not os.path.isdir(args.state_dir):\n  os.makedirs(args.state_dir, mode = 0o751)\n\nstate_file_name = os.path.join(args.state_dir, args.state_file)\n\nif not os.path.isfile(state_file_name):\n    logging.info(\"Delaying nova-compute startup by %s seconds\" % args.delay)\n    time.sleep(args.delay)\n    open(state_file_name, 'a').close()\n\nlogging.info(\"Executing %s\" % real_args)\nos.execv(args.nova_binary, real_args)\n", "dest": "/var/lib/nova/delay-nova-compute", "mode": 493}, "when": "nova_compute_delay|int > 0"}, {"name": "Is irqbalance enabled", "set_fact": {"compute_irqbalance_disabled": false}}, {"name": "disable irqbalance service on compute", "when": "compute_irqbalance_disabled|bool", "service": {"name": "irqbalance.service", "state": "stopped", "enabled": false}}], "deploy_steps_tasks": [{"name": "install systemd-container for a greenfield", "when": ["step|int == 1", "not check_nova_instances_result.stat.exists"], "package": {"name": "systemd-container", "state": "present"}}, {"name": "validate nova-compute container state", "containers.podman.podman_container_info": {"name": "nova_compute"}, "register": "nova_compute_infos", "failed_when": ["nova_compute_infos.containers.0.Healthcheck.Status is defined", "'healthy' not in nova_compute_infos.containers.0.Healthcheck.Status"], "retries": 10, "delay": 30, "tags": ["opendev-validation", "opendev-validation-nova"], "when": ["container_cli == 'podman'", "not container_healthcheck_disabled", "step|int == 6"]}, {"name": "manage PMEM namespaces for vPMEM", "include_role": {"name": "tripleo_nvdimm"}, "vars": {"tripleo_nvdimm_pmem_namespaces": ""}, "when": ["step|int == 1", "tripleo_nvdimm_pmem_namespaces != ''"]}, {"name": "manage OS version 9 specific sysctls", "when": ["step|int == 1", "ansible_facts['os_family'] == 'RedHat'", "ansible_facts['distribution_major_version'] is version('9', '==')"], "block": [{"name": "Enable post-copy by setting unprivileged_userfaultfd", "sysctl": {"name": "vm.unprivileged_userfaultfd", "value": 1, "sysctl_set": true, "sysctl_file": "/etc/sysctl.d/99-tripleo-postcopy.conf", "state": "present", "reload": true}}]}, {"name": "enable/disable ksm", "when": ["step|int == 1"], "block": [{"name": "is KSM enabled", "set_fact": {"compute_ksm_enabled": false}}, {"name": "disable KSM on compute", "when": "not compute_ksm_enabled|bool", "block": [{"name": "Check for ksm", "shell": "systemctl is-active ksm.service || systemctl is-enabled ksm.service", "become": true, "failed_when": false, "register": "ksm_service_check"}, {"name": "disable KSM services", "service": {"name": "{{ item }}", "state": "stopped", "enabled": false}, "with_items": ["ksm.service", "ksmtuned.service"], "when": ["ksm_service_check.rc is defined", "ksm_service_check.rc == 0"], "register": "ksmdisabled"}, {"name": "delete PageKSM after disable ksm on compute", "command": "echo 2 >/sys/kernel/mm/ksm/run", "when": ["ksm_service_check.rc is defined", "ksm_service_check.rc == 0", "ksmdisabled is changed"]}]}, {"name": "enable KSM on compute", "when": "compute_ksm_enabled|bool", "block": [{"name": "make sure package providing ksmtuned is installed (RHEL8 or CentOS8)", "package": {"name": "qemu-kvm-common", "state": "present"}, "when": ["ansible_facts['distribution_major_version'] is version('8', '==')"]}, {"name": "make sure package providing ksmtuned is installed (RHEL9 or CentOS9)", "package": {"name": "ksmtuned", "state": "present"}, "when": ["ansible_facts['distribution_major_version'] is version('9', '==')"]}, {"name": "enable ksmtunded", "service": {"name": "{{ item }}", "state": "started", "enabled": true}, "with_items": ["ksm.service", "ksmtuned.service"]}]}]}], "external_post_deploy_tasks": [{"block": [{"name": "is additonal Cell?", "set_fact": {"nova_additional_cell": false}}, {"name": "check if discover hosts is required", "when": ["not nova_additional_cell|bool", "nova_cellv2_discovery_done is not defined"], "block": [{"name": "discover via nova_manager?", "set_fact": {"nova_cellv2_discovery_delegate_host": "{{ groups['nova_manager'][0] }}", "nova_cellv2_discovery_container": "nova_manager"}, "when": ["groups['nova_manager'] is defined and (groups['nova_manager']|length>0)"]}, {"name": "discover via nova_api?", "set_fact": {"nova_cellv2_discovery_delegate_host": "{{ groups['nova_api'][0] }}", "nova_cellv2_discovery_container": "nova_api"}, "when": ["nova_cellv2_discovery_delegate_host is not defined", "groups['nova_api'] is defined and (groups['nova_api']|length>0)"]}, {"name": "Warn if no discovery host available", "fail": {"msg": "No hosts available to run nova cell_v2 host discovery."}, "ignore_errors": true, "when": ["nova_cellv2_discovery_delegate_host is not defined"]}, {"name": "Discovering nova hosts", "command": "{{ container_cli }} exec {{ nova_cellv2_discovery_container }} nova-manage cell_v2 discover_hosts --by-service", "become": true, "changed_when": false, "delegate_to": "{{ nova_cellv2_discovery_delegate_host }}", "when": ["nova_cellv2_discovery_delegate_host is defined"]}, {"set_fact": {"nova_cellv2_discovery_done": true}}]}]}], "upgrade_tasks": [{"name": "Remove openstack-nova-compute and python-nova package during upgrade", "package": {"name": ["openstack-nova-compute", "python-nova"], "state": "removed"}, "failed_when": false, "when": "step|int == 2"}, {"name": "nova_compute pre system_upgrade tasks", "tags": ["never", "system_upgrade", "system_upgrade_run", "system_upgrade_nfsmounts"], "when": ["step|int == 3", "upgrade_leapp_enabled"], "block": [{"name": "Unmount Nova NFS Share", "vars": {"nfs_backend_enable": false}, "mount": {"name": "/var/lib/nova/instances", "state": "absent", "fstype": "nfs4"}, "when": "nfs_backend_enable|bool"}]}, {"name": "nova_compute post system_upgrade tasks", "tags": ["never", "system_upgrade", "system_upgrade_run", "system_upgrade_nfsmounts"], "when": ["step|int == 5", "upgrade_leapp_enabled"], "block": [{"name": "Mount Nova NFS Share", "vars": {"nfs_backend_enable": false, "nfs_share": "", "nfs_options": "context=system_u:object_r:nfs_t:s0", "nfs_vers": "4.2"}, "mount": {"name": "/var/lib/nova/instances", "state": "mounted", "fstype": "nfs4", "src": "{{nfs_share}}", "opts": "_netdev,bg,{{nfs_options}},nfsvers={{nfs_vers}}"}, "when": "nfs_backend_enable|bool"}]}], "update_tasks": [{"name": "Remove openstack-nova-compute and python-nova package during upgrade", "package": {"name": ["openstack-nova-compute", "python-nova"], "state": "removed"}, "failed_when": false, "when": "step|int == 2"}], "post_update_tasks": [{"name": "Ensure instanceha directory is purged", "when": "step|int == 1", "become": true, "file": {"path": "/var/lib/nova/instanceha", "state": "absent"}}], "post_upgrade_tasks": [{"name": "Ensure instanceha directory is purged", "when": "step|int == 1", "become": true, "file": {"path": "/var/lib/nova/instanceha", "state": "absent"}}], "scale_tasks": [{"when": ["step|int == 1", "container_cli == 'podman'"], "tags": "down", "become": true, "environment": {"OS_CLOUD": "standalone"}, "block": [{"name": "Get nova-compute service ID", "command": "openstack compute service list --service nova-compute --column ID --column Host --format yaml", "register": "nova_compute_service_result", "delegate_to": "{{ groups['Undercloud'] | first }}", "check_mode": false, "changed_when": false}, {"name": "is additional Cell?", "set_fact": {"is_additional_cell": false}}, {"name": "Set fact for nova_compute services", "set_fact": {"nova_compute_service": "{{ nova_compute_service_result.stdout | from_yaml | selectattr('Host', 'match', ansible_facts['fqdn'] ~ '.*') | list }}"}, "delegate_to": "{{ groups['Undercloud'] | first }}", "check_mode": false}, {"name": "Check search output", "fail": {"msg": "Found multiple `{{ ansible_facts['fqdn'] }}`, which is unexpected. This means that the FQDN of the selected device to disable is either wrong or is sharing a name with another host, which is also wrong. Please correct this issue before continuing. Nova service list return data can be found here -> {{ nova_compute_service }}."}, "when": ["(nova_compute_service | length) > 1"]}, {"name": "Disable nova services", "when": ["(nova_compute_service | length) == 1"], "block": [{"name": "Disable nova-compute service", "command": "openstack compute service set {{ nova_compute_service[0].Host }} nova-compute --disable", "delegate_to": "{{ groups['Undercloud'] | first }}", "check_mode": false, "when": ["not is_additional_cell|bool"]}, {"name": "Stop nova-compute container", "service": {"name": "tripleo_nova_compute", "state": "stopped", "enabled": false}, "become": true, "register": "stop_nova_compute", "failed_when": "('msg' in stop_nova_compute and 'Could not find the requested service tripleo_nova_compute' not in stop_nova_compute.msg) or ('rc' in stop_nova_compute and stop_nova_compute.rc != 0)"}, {"name": "Delete nova-compute service", "command": "openstack compute service delete {{ nova_compute_service[0].ID }}", "delegate_to": "{{ groups['Undercloud'] | first }}", "check_mode": false}]}]}]}, {"service_name": "nova_libvirt", "firewall_rules": {"200 nova_libvirt": {"dport": [16514, "61152-61215", "5900-6923"]}}, "ansible_group_vars": {"tripleo_nova_libvirt_virsh_container": "nova_virtproxyd"}, "monitoring_subscription": "overcloud-nova-libvirt", "config_settings": {"nova::my_ip": "%{hiera('internal_api')}", "nova::rabbit_use_ssl": "False", "nova::keystone::service_user::send_service_user_token": true, "nova::keystone::service_user::project_name": "service", "nova::keystone::service_user::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::service_user::auth_url": "http://172.17.0.2:5000", "nova::keystone::service_user::region_name": "regionOne", "nova::placement::project_name": "service", "nova::placement::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "nova::placement::auth_url": "http://172.17.0.2:5000", "nova::placement::region_name": "regionOne", "nova::placement::valid_interfaces": "internal", "nova::os_region_name": "regionOne", "nova::logging::debug": true, "nova::purge_config": false, "nova::network::neutron::project_name": "service", "nova::network::neutron::username": "neutron", "nova::network::neutron::region_name": "regionOne", "nova::dhcp_domain": "", "nova::network::neutron::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "nova::network::neutron::auth_url": "http://172.17.0.2:5000/v3", "nova::network::neutron::valid_interfaces": "internal", "nova::glance::valid_interfaces": "internal", "nova::rabbit_heartbeat_timeout_threshold": 60, "nova::cinder_catalog_info": "volumev3:cinderv3:internalURL", "nova::host": "%{hiera('fqdn_canonical')}", "nova::notify_on_state_change": "vm_and_task_state", "nova::notification_driver": ["noop"], "nova::notification_format": "unversioned", "nova::network::neutron::auth_type": "v3password", "nova::db::database_db_max_retries": -1, "nova::db::database_max_retries": -1, "nova::network::neutron::ovs_bridge": "br-int", "nova::compute::sync_power_state_interval": 600, "nova_is_additional_cell": false, "nova::cross_az_attach": true, "nova::cache::enabled": true, "nova::cache::tls_enabled": false, "nova::cache::backend": "dogpile.cache.memcached", "nova::rabbit_heartbeat_in_pthread": false, "nova::ovsdb_connection": "", "vhostuser_socket_group": "qemu", "nova_libvirt_opt_volumes": [], "ContainerNovaLibvirtImage": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "ContainerNovaLibvirtConfigImage": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "nova::compute::libvirt::log_outputs": "1:file:/var/log/libvirt/libvirtd.log", "nova::compute::libvirt::virtlogd::log_outputs": "1:file:/var/log/libvirt/virtlogd.log", "nova::compute::libvirt::virtsecretd::log_outputs": "1:file:/var/log/libvirt/virtsecretd.log", "nova::compute::libvirt::virtnodedevd::log_outputs": "1:file:/var/log/libvirt/virtnodedevd.log", "nova::compute::libvirt::virtqemud::log_outputs": "1:file:/var/log/libvirt/virtqemud.log", "nova::compute::libvirt::virtproxyd::log_outputs": "1:file:/var/log/libvirt/virtproxyd.log", "nova::compute::libvirt::virtstoraged::log_outputs": "1:file:/var/log/libvirt/virtstoraged.log", "nova::compute::libvirt::qemu::memory_backing_dir": "", "nova::compute::libvirt::manage_libvirt_services": false, "tripleo::profile::base::nova::virtlogd_wrapper::enable_wrapper": true, "tripleo::profile::base::nova::virtlogd_wrapper::virtlogd_process_wrapper": "/var/lib/container-config-scripts/virtlogd_wrapper", "tripleo::profile::base::nova::virtlogd_wrapper::virtlogd_image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "tripleo::profile::base::nova::virtlogd_wrapper::debug": true, "nova::compute::libvirt::migration_support": false, "nova::compute::rbd::libvirt_images_rbd_ceph_conf": "/etc/ceph/ceph.conf", "nova::compute::rbd::libvirt_rbd_user": "openstack", "tripleo::profile::base::nova::compute_libvirt_shared::nova_rbd_ceph_conf_path": "/var/lib/tripleo-config/ceph", "nova::compute::rbd::rbd_keyring": "client.openstack", "nova::compute::rbd::libvirt_rbd_secret_key": "AQBP7ytpAAAAABAAXQLQTQkMxLJ7uGigDaf94Q==", "nova::compute::rbd::libvirt_rbd_secret_uuid": "55dae5e5-7ec8-55ff-b761-b30fedcc71d6", "tripleo::profile::base::nova::migration::client::libvirt_enabled": true, "tripleo::profile::base::nova::migration::client::ssh_private_key": "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEApEW+iSDuQm4pBk/4gct8N21CUktJkYPZh42iMMBRmjvtdesa\nzByskHhSHzsdcZVXB6QMglFiP2qlX3OCmioW5N0jqTmpTbU/69vDerntqnzxNWpY\nJdycEi7LLztxn6UOlIQykb9KiEY+ou9NO4k8SAU72a5PplhZVPN0PTVs303ASP1r\nSV/mwd6ybKKqzEZm7bqeAAY/v34jevkPmPP0b7E31jDLDHMyWlEBrQQc9dzpnfMA\nzm4XJ7FoY/fxKYhzT1BFCpXJidTGvskPlta9yshXNigWBtbcQaJvt1bIEoofl5WD\n/BXa20TrltlD/nQ6VTbBHADz9uCbRQxaHI0e6QIDAQABAoIBADmks6CA6rpq14B8\njXJWSidU8qHYA1/JiYqUe6Zpxfi+MKxhLI0VPsSPRBnrb4CM6/YlbbJJBpXEQXWG\nHf0k6dWzd7EJJ8y53k1QV374QVrC7w1UXMwkZqvGucvAKzXfAIfC/8++9VNuespH\n+Kz8kieflonjLdXsSKSTrWVRGvwcRohAQ6k5YofMeOfke+Tk3/QNDmNTOOofyEN0\npeGouReKeh9N7FV1Uy0QzEi1Bq8odpcK7dHUNXPJM4NlISMXl3hsCu19OOenxLT4\nMPAcQodjIYO795S9hN9HL7tWNTwEReq5n3X6U3ZuDH5nedy5ngEATt9cEIboTjSh\nmNdbXuUCgYEAtobrDEiywQXXOjGTejM8l1G/cteml6aJPS5hc9qUJZr6UzW+HkfH\nBK3eh0ACoFxsiy1pP1wXCG79V1nROJu2josvXWqkkmJ5MUgIsg1NO4qhBppMhlEX\n1qhEbrf0k05ykrsp2ZLkAYkRbQeCyVbSud0flZxPWO7fTJqk5ZxC7Y8CgYEA5mW5\nUSYt3j5T8BhEkdHKj7DacnQE6CWBck1iMZ61W/gbhJgqUlb14v8fvf2guVgSI+nk\nZJwBo7A/O+yWxMCHVkzSiEvn7VQUA04cK8vvyA0sXq5XpR6smlh6kXYd8z2/Ke6U\n8M1iUsipNwV4QBOa1jvAkLPWEy2TvCBN24ytYAcCgYEAoZG0Jl+Du9ELv3t67ETG\ncbvvGB1EXiEQgpO3t/HYCaGxzE5aSDuJM98kh9xldsXZXymHbIp76P82Skg7JzzO\n0EbsWmi33MxGMxVsfQ/BOd9wwoN7Ebpeu6ifStKTMIHUcYYzeWDJa4KFZlmrr8Wn\nKtdNygVfR33LM+f/5y/3HokCgYAp7Rz6DW6PYNw5dtETkA/CcjZuWXqDHEAgIsDZ\nsr3iDJZFXMNm1HhOZIrMMhlLYSF785YiUNljhjEboIQXLcnzBGvTbay43XH+3/n0\nZlouF2eZmzRL0fcq5yr2yX3Nw9DDuqzT3SwlUGqqno4xat/kWF9EddVhDAw0UdSZ\nPVJXhQKBgBJuA1NrpGyERbKfwmTriaQ/4jXXZwcIZf+U0KOKchDtaR9PmY05qQm8\nbwH5USjLpLcfIZCxGRh+KPBusLOnvnKX4PZwv397Zfq9j5y8UcsWKab71RDqVY8V\nZj25fDMEw29QQyaxQ+SRV78MG20YJtsb9TieU1XvJkpFURUpypNX\n-----END RSA PRIVATE KEY-----\n", "tripleo::profile::base::nova::migration::client::ssh_port": 2022, "nova::compute::libvirt::services::libvirt_virt_type": "kvm", "nova::compute::libvirt::virt_type": "kvm", "nova::compute::libvirt::enabled_perf_events": [], "nova::compute::libvirt::qemu::configure_qemu": true, "nova::compute::libvirt::qemu::max_files": 32768, "nova::compute::libvirt::qemu::max_processes": 131072, "nova::migration::qemu::configure_qemu": true, "nova::migration::qemu::migration_port_min": 61152, "nova::migration::qemu::migration_port_max": 61215, "nova::compute::libvirt::vncserver_listen": "%{hiera('internal_api')}", "nova::compute::libvirt::log_filters": "1:libvirt 1:qemu 1:conf 1:security 3:event 3:json 3:file 3:object 1:util", "nova::compute::libvirt::virtlogd::log_filters": "1:logging 4:object 4:json 4:event 1:util", "rbd_persistent_storage": true, "rbd_disk_cachemodes": ["network=writeback"], "nova::compute::libvirt::virtsecretd::log_filters": "1:qemu 1:libvirt 4:object 4:json 4:event 1:util", "nova::compute::libvirt::virtnodedevd::log_filters": "1:qemu 1:libvirt 4:object 4:json 4:event 1:util", "nova::compute::libvirt::virtstoraged::log_filters": "1:qemu 1:libvirt 4:object 4:json 4:event 1:util", "nova::compute::libvirt::virtqemud::log_filters": "1:qemu 1:libvirt 4:object 4:json 4:event 1:util", "nova::compute::libvirt::virtproxyd::log_filters": "1:qemu 1:libvirt 4:object 4:json 4:event 1:util", "tripleo::profile::base::nova::libvirt::modular_libvirt": true, "nova::compute::libvirt::services::modular_libvirt": true, "nova::compute::libvirt::modular_libvirt": true, "nova::migration::libvirt::modular_libvirt": true, "nova::migration::libvirt::live_migration_inbound_addr": "%{hiera('fqdn_internal_api')}"}, "puppet_config": {"config_volume": "nova_libvirt", "puppet_tags": "libvirtd_config,virtlogd_config,virtproxyd_config,virtqemud_config,virtnodedevd_config,virtsecretd_config,virtstoraged_config,nova_config,file,libvirt_tls_password", "step_config": "include tripleo::profile::base::nova::libvirt\n", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/nova_virtlogd.json": {"command": "/usr/local/bin/virtlogd_wrapper", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-ceph/", "dest": "/etc/ceph/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/etc/ceph/ceph.client.openstack.keyring", "owner": "nova:nova", "perm": "0600"}]}, "/var/lib/kolla/config_files/nova_virtsecretd.json": {"command": "/usr/sbin/virtsecretd --config /etc/libvirt/virtsecretd.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-ceph/", "dest": "/etc/ceph/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/etc/ceph/ceph.client.openstack.keyring", "owner": "nova:nova", "perm": "0600"}]}, "/var/lib/kolla/config_files/nova_virtnodedevd.json": {"command": "/usr/sbin/virtnodedevd --config /etc/libvirt/virtnodedevd.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-ceph/", "dest": "/etc/ceph/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/etc/ceph/ceph.client.openstack.keyring", "owner": "nova:nova", "perm": "0600"}]}, "/var/lib/kolla/config_files/nova_virtstoraged.json": {"command": "/usr/sbin/virtstoraged --config /etc/libvirt/virtstoraged.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-ceph/", "dest": "/etc/ceph/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/etc/ceph/ceph.client.openstack.keyring", "owner": "nova:nova", "perm": "0600"}]}, "/var/lib/kolla/config_files/nova_virtqemud.json": {"command": "/usr/sbin/virtqemud --config /etc/libvirt/virtqemud.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-ceph/", "dest": "/etc/ceph/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/etc/ceph/ceph.client.openstack.keyring", "owner": "nova:nova", "perm": "0600"}]}, "/var/lib/kolla/config_files/nova_virtproxyd.json": {"command": "/usr/sbin/virtproxyd --config /etc/libvirt/virtproxyd.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-ceph/", "dest": "/etc/ceph/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/etc/ceph/ceph.client.openstack.keyring", "owner": "nova:nova", "perm": "0600"}]}}, "container_config_scripts": {"nova_libvirt_init_secret.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -e\n\nCEPH_INFO=($*)\n\nif [ -z \"$CEPH_INFO\" ]; then\n    echo \"error: At least one CLUSTER:CLIENT tuple must be specified\"\n    exit 1\nfi\n\necho \"------------------------------------------------\"\necho \"Initializing virsh secrets for: ${CEPH_INFO[@]}\"\n\nfor INFO in ${CEPH_INFO[@]}; do\n    IFS=: read CLUSTER CLIENT <<< $INFO\n    if [ ! -f /etc/ceph/${CLUSTER}.conf ]; then\n        echo \"Error: /etc/ceph/${CLUSTER}.conf was not found\"\n        echo \"Path to nova_libvirt_init_secret was ${CEPH_INFO}\"\n        exit 1\n    fi\n    FSID=$(awk '$1 == \"fsid\" {print $3}' /etc/ceph/${CLUSTER}.conf)\n    if [ -z \"${FSID}\" ]; then\n        echo \"Error: /etc/ceph/${CLUSTER}.conf contained an empty fsid definition\"\n        echo \"Check your ceph configuration\"\n        exit 1\n    fi\n\n    echo \"--------\"\n    echo \"Initializing the virsh secret for '$CLUSTER' cluster ($FSID) '$CLIENT' client\"\n\n    # Ensure the secret XML file exists. Puppet should have created a secret.xml\n    # file for the first cluster's secret, so detect when to use that file.\n    if grep -q $FSID /etc/nova/secret.xml; then\n        SECRET_FILE=\"/etc/nova/secret.xml\"\n        SECRET_NAME=\"client.${CLIENT} secret\"\n    else\n        SECRET_FILE=\"/etc/nova/${CLUSTER}-secret.xml\"\n        SECRET_NAME=\"${CLUSTER}.client.${CLIENT} secret\"\n    fi\n\n    if [ ! -f $SECRET_FILE ]; then\n        echo \"Creating $SECRET_FILE\"\n        cat <<EOF > $SECRET_FILE\n<secret ephemeral='no' private='no'>\n  <usage type='ceph'>\n    <name>${SECRET_NAME}</name>\n  </usage>\n  <uuid>${FSID}</uuid>\n</secret>\nEOF\n    else\n        echo \"The $SECRET_FILE file already exists\"\n    fi\n\n    # Ensure the libvirt secret is defined\n    if /usr/bin/virsh secret-list | grep -q $FSID; then\n        echo \"The virsh secret for $FSID has already been defined\"\n    else\n        /usr/bin/virsh secret-define --file $SECRET_FILE\n    fi\n\n    # Fetch the key from the keyring and ensure the secret is set\n    KEY=$(awk '$1 == \"key\" {print $3}' /etc/ceph/${CLUSTER}.client.${CLIENT}.keyring)\n    if /usr/bin/virsh secret-get-value $FSID 2>/dev/null | grep -q $KEY; then\n        echo \"The virsh secret for $FSID has already been set\"\n    else\n        /usr/bin/virsh secret-set-value --secret $FSID --base64 $KEY\n    fi\ndone\n"}}, "docker_config": {"step_2": {"nova_virtqemud_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "net": "none", "privileged": true, "security_opt": ["label=level:s0", "label=type:spc_t", "label=filetype:container_file_t"], "user": "root", "volumes": ["/var/log/containers/libvirt/swtpm:/var/log/swtpm:shared,z"], "command": ["/bin/bash", "-c", "chown -R tss:tss /var/log/swtpm"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}, "create_virtlogd_wrapper": {"start_order": 1, "detach": false, "cgroupns": "host", "net": "host", "pid": "host", "user": "root", "command": ["/container_puppet_apply.sh", "4", "file", "include ::tripleo::profile::base::nova::virtlogd_wrapper"], "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/var/lib/container-config-scripts/container_puppet_apply.sh:/container_puppet_apply.sh:ro", "/etc/puppet:/tmp/puppet-etc:ro", "/usr/share/openstack-puppet/modules:/usr/share/openstack-puppet/modules:ro", "/var/lib/container-config-scripts:/var/lib/container-config-scripts:shared,z"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_3": {"nova_virtlogd_wrapper": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "ulimit": ["nofile=131072", "nproc=126960"], "cgroupns": "host", "net": "host", "pid": "host", "privileged": true, "security_opt": ["label=level:s0", "label=type:spc_t", "label=filetype:container_file_t"], "restart": "always", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/libvirt:/var/log/libvirt:shared,z", "/lib/modules:/lib/modules:ro", "/dev:/dev", "/run:/run", "/sys/fs/cgroup:/sys/fs/cgroup", "/sys/fs/selinux:/sys/fs/selinux", "/etc/selinux/config:/etc/selinux/config:ro", "/etc/libvirt:/etc/libvirt:shared", "/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro", "/run/libvirt:/run/libvirt:shared,z", "/var/lib/nova:/var/lib/nova:shared", "/var/lib/libvirt:/var/lib/libvirt:shared", "/var/cache/libvirt:/var/cache/libvirt:shared", "/var/lib/vhost_sockets:/var/lib/vhost_sockets", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/kolla/config_files/nova_virtlogd.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/container-config-scripts/virtlogd_wrapper:/usr/local/bin/virtlogd_wrapper:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "nova_virtsecretd": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "ulimit": ["nofile=131072", "nproc=126960"], "cgroupns": "host", "net": "host", "pid": "host", "pids_limit": 65536, "privileged": true, "security_opt": ["label=level:s0", "label=type:spc_t", "label=filetype:container_file_t"], "restart": "always", "depends_on": ["tripleo_nova_virtlogd_wrapper.service"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/libvirt:/var/log/libvirt:shared,z", "/lib/modules:/lib/modules:ro", "/dev:/dev", "/run:/run", "/sys/fs/cgroup:/sys/fs/cgroup", "/sys/fs/selinux:/sys/fs/selinux", "/etc/selinux/config:/etc/selinux/config:ro", "/etc/libvirt:/etc/libvirt:shared", "/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro", "/run/libvirt:/run/libvirt:shared,z", "/var/lib/nova:/var/lib/nova:shared", "/var/lib/libvirt:/var/lib/libvirt:shared", "/var/cache/libvirt:/var/cache/libvirt:shared", "/var/lib/vhost_sockets:/var/lib/vhost_sockets", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/kolla/config_files/nova_virtsecretd.json:/var/lib/kolla/config_files/config.json:ro"]}, "nova_virtnodedevd": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "ulimit": ["nofile=131072", "nproc=126960"], "cgroupns": "host", "net": "host", "pid": "host", "pids_limit": 65536, "privileged": true, "security_opt": ["label=level:s0", "label=type:spc_t", "label=filetype:container_file_t"], "restart": "always", "depends_on": ["tripleo_nova_virtlogd_wrapper.service"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/libvirt:/var/log/libvirt:shared,z", "/lib/modules:/lib/modules:ro", "/dev:/dev", "/run:/run", "/sys/fs/cgroup:/sys/fs/cgroup", "/sys/fs/selinux:/sys/fs/selinux", "/etc/selinux/config:/etc/selinux/config:ro", "/etc/libvirt:/etc/libvirt:shared", "/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro", "/run/libvirt:/run/libvirt:shared,z", "/var/lib/nova:/var/lib/nova:shared", "/var/lib/libvirt:/var/lib/libvirt:shared", "/var/cache/libvirt:/var/cache/libvirt:shared", "/var/lib/vhost_sockets:/var/lib/vhost_sockets", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/kolla/config_files/nova_virtnodedevd.json:/var/lib/kolla/config_files/config.json:ro"]}, "nova_virtstoraged": {"start_order": 3, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "ulimit": ["nofile=131072", "nproc=126960"], "cgroupns": "host", "net": "host", "pid": "host", "pids_limit": 65536, "privileged": true, "security_opt": ["label=level:s0", "label=type:spc_t", "label=filetype:container_file_t"], "restart": "always", "depends_on": ["tripleo_nova_virtlogd_wrapper.service"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/libvirt:/var/log/libvirt:shared,z", "/lib/modules:/lib/modules:ro", "/dev:/dev", "/run:/run", "/sys/fs/cgroup:/sys/fs/cgroup", "/sys/fs/selinux:/sys/fs/selinux", "/etc/selinux/config:/etc/selinux/config:ro", "/etc/libvirt:/etc/libvirt:shared", "/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro", "/run/libvirt:/run/libvirt:shared,z", "/var/lib/nova:/var/lib/nova:shared", "/var/lib/libvirt:/var/lib/libvirt:shared", "/var/cache/libvirt:/var/cache/libvirt:shared", "/var/lib/vhost_sockets:/var/lib/vhost_sockets", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/kolla/config_files/nova_virtstoraged.json:/var/lib/kolla/config_files/config.json:ro"]}, "nova_virtqemud": {"start_order": 4, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "ulimit": ["nofile=131072", "nproc=126960"], "cgroupns": "host", "net": "host", "pid": "host", "pids_limit": 65536, "privileged": true, "security_opt": ["label=level:s0", "label=type:spc_t", "label=filetype:container_file_t"], "restart": "always", "depends_on": ["tripleo_nova_virtlogd_wrapper.service"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/libvirt:/var/log/libvirt:shared,z", "/lib/modules:/lib/modules:ro", "/dev:/dev", "/run:/run", "/sys/fs/cgroup:/sys/fs/cgroup", "/sys/fs/selinux:/sys/fs/selinux", "/etc/selinux/config:/etc/selinux/config:ro", "/etc/libvirt:/etc/libvirt:shared", "/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro", "/run/libvirt:/run/libvirt:shared,z", "/var/lib/nova:/var/lib/nova:shared", "/var/lib/libvirt:/var/lib/libvirt:shared", "/var/cache/libvirt:/var/cache/libvirt:shared", "/var/lib/vhost_sockets:/var/lib/vhost_sockets", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/kolla/config_files/nova_virtqemud.json:/var/lib/kolla/config_files/config.json:ro", "/var/log/containers/libvirt/swtpm:/var/log/swtpm:z"]}, "nova_virtproxyd": {"start_order": 5, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "ulimit": ["nofile=131072", "nproc=126960"], "cgroupns": "host", "net": "host", "pid": "host", "pids_limit": 65536, "privileged": true, "security_opt": ["label=level:s0", "label=type:spc_t", "label=filetype:container_file_t"], "restart": "always", "depends_on": ["tripleo_nova_virtlogd_wrapper.service"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/libvirt:/var/log/libvirt:shared,z", "/lib/modules:/lib/modules:ro", "/dev:/dev", "/run:/run", "/sys/fs/cgroup:/sys/fs/cgroup", "/sys/fs/selinux:/sys/fs/selinux", "/etc/selinux/config:/etc/selinux/config:ro", "/etc/libvirt:/etc/libvirt:shared", "/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro", "/run/libvirt:/run/libvirt:shared,z", "/var/lib/nova:/var/lib/nova:shared", "/var/lib/libvirt:/var/lib/libvirt:shared", "/var/cache/libvirt:/var/cache/libvirt:shared", "/var/lib/vhost_sockets:/var/lib/vhost_sockets", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/kolla/config_files/nova_virtproxyd.json:/var/lib/kolla/config_files/config.json:ro"]}}, "step_4": {"nova_libvirt_init_secret": {"detach": false, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "security_opt": ["label=disable"], "privileged": false, "user": "root", "cgroupns": "host", "net": "host", "environment": {"LIBVIRT_DEFAULT_URI": "qemu:///system"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/config-data/puppet-generated/nova_libvirt/etc/nova:/etc/nova", "/etc/libvirt:/etc/libvirt", "/run/libvirt:/run/libvirt:shared,z", "/var/lib/libvirt:/var/lib/libvirt:shared", "/var/lib/container-config-scripts/nova_libvirt_init_secret.sh:/nova_libvirt_init_secret.sh:ro", "/var/lib/tripleo-config/ceph:/etc/ceph:ro"], "command": "/nova_libvirt_init_secret.sh ceph:openstack"}}}, "deploy_steps_tasks": [{"name": "Ensure recovery of containerized virtqemud", "when": "step|int == 4", "become": true, "block": [{"name": "Create virtqemud recovery script", "copy": {"dest": "/usr/libexec/recover_tripleo_nova_virtqemud.sh", "mode": "0755", "content": "#!/bin/bash\npgrep -f /sbin/virtqemud\n[ $? -eq 0 ] && exit 0\nkill -9 $(cat /run/nova_virtqemud.pid)\n"}}, {"name": "Create virtqemud recovery trigger service", "copy": {"dest": "/etc/systemd/system/tripleo_nova_virtqemud_recover.service", "mode": "0644", "content": "[Unit]\nDescription=Check and recover tripleo_nova_virtqemud\nAfter=tripleo_nova_virtqemud.service\nRequisite=tripleo_nova_virtqemud.service\n[Service]\nType=oneshot\nExecStart=bash /usr/libexec/recover_tripleo_nova_virtqemud.sh\nSyslogIdentifier=recover_tripleo_nova_virtqemud\n[Install]\nWantedBy=multi-user.target\n"}}, {"name": "Create virtqemud recovery trigger timer", "copy": {"dest": "/etc/systemd/system/tripleo_nova_virtqemud_recover.timer", "mode": "0644", "content": "[Unit]\nDescription=Check and recover tripleo_nova_virtqemud every 10m\nPartOf=tripleo_nova_virtqemud.service\n[Timer]\nOnCalendar=*:5/10\nOnActiveSec=120\nOnUnitActiveSec=60\nRandomizedDelaySec=1m\n[Install]\nWantedBy=timers.target\n"}}, {"name": "Enable virtqemud recovery trigger service", "systemd": {"name": "tripleo_nova_virtqemud_recover.service", "enabled": true, "daemon_reload": true}}, {"name": "Enable virtqemud recovery trigger timer", "systemd": {"name": "tripleo_nova_virtqemud_recover.timer", "enabled": true, "state": "restarted", "daemon_reload": true}, "register": "virtqemud_recover_timer_result"}, {"name": "Really enable virtqemud recovery trigger timer", "shell": {"cmd": "systemctl enable --now tripleo_nova_virtqemud_recover.timer"}, "when": "virtqemud_recover_timer_result.status.UnitFileState != \"enabled\""}]}, {"name": "Set up systemd target for libvirt services", "when": "step|int == 4", "block": [{"name": "Create systemd file", "copy": {"dest": "/etc/systemd/system/tripleo_nova_libvirt.target", "mode": "0644", "owner": "root", "group": "root", "content": "[Unit]\nWants=tripleo_nova_virtsecretd.service\nWants=tripleo_nova_virtnodedevd.service\nWants=tripleo_nova_virtstoraged.service\nWants=tripleo_nova_virtproxyd.service\nWants=tripleo_nova_virtqemud.service\nAfter=tripleo_nova_virtsecretd.service\nAfter=tripleo_nova_virtnodedevd.service\nAfter=tripleo_nova_virtstoraged.service\nAfter=tripleo_nova_virtproxyd.service\nAfter=tripleo_nova_virtqemud.service\n"}, "become": true, "register": "libvirt_target_result"}, {"name": "Reload systemd", "systemd": {"name": "tripleo_nova_libvirt.target", "enabled": true, "state": "restarted", "daemon_reload": true}, "become": true, "when": "libvirt_target_result.changed"}]}], "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "setype": "{{ item.setype }}", "state": "directory", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/libvirt", "setype": "container_file_t", "mode": "0750"}]}, {"name": "create libvirt persistent data directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype | default(omit) }}", "mode": "{{ item.mode | default(omit) }}"}, "with_items": [{"path": "/etc/libvirt", "setype": "container_file_t"}, {"path": "/etc/libvirt/secrets", "setype": "container_file_t"}, {"path": "/etc/libvirt/qemu", "setype": "container_file_t"}, {"path": "/var/lib/libvirt", "setype": "container_file_t"}, {"path": "/var/cache/libvirt"}, {"path": "/var/lib/nova", "setype": "container_file_t"}, {"path": "/run/libvirt"}, {"path": "/var/log/containers/libvirt/swtpm", "setype": "container_file_t", "mode": "0770"}]}, {"name": "ensure qemu group is present on the host", "group": {"name": "qemu", "gid": 107, "state": "present"}}, {"name": "ensure qemu user is present on the host", "user": {"name": "qemu", "uid": 107, "group": "qemu", "state": "present", "shell": "/sbin/nologin", "comment": "qemu user"}}, {"name": "create directory for vhost-user sockets with qemu ownership", "file": {"path": "/var/lib/vhost_sockets", "state": "directory", "owner": "qemu", "group": "qemu", "setype": "virt_cache_t", "seuser": "system_u"}}, {"name": "ensure ceph configurations exist", "file": {"path": "/var/lib/tripleo-config/ceph", "state": "directory"}}, {"name": "check if libvirt is installed", "command": "/usr/bin/rpm -q libvirt-daemon", "failed_when": false, "register": "libvirt_installed", "check_mode": false}, {"name": "make sure libvirt services are disabled and masked", "service": {"name": "{{ item }}", "state": "stopped", "enabled": false, "masked": true, "daemon_reload": true}, "with_items": ["libvirtd.service", "virtlogd.socket"], "when": "libvirt_installed.rc == 0"}, {"name": "ensure /run/libvirt is present upon reboot", "copy": {"dest": "/etc/tmpfiles.d/run-libvirt.conf", "content": "d /run/libvirt 0755 root root - -\n"}}, {"name": "Enable os_enable_vtpm SELinux boolean for vTPM", "seboolean": {"name": "os_enable_vtpm", "persistent": true, "state": true}, "when": ["ansible_facts.selinux is defined", "ansible_facts.selinux.status == \"enabled\""]}, [{"name": "remove gnutls-qemu.config", "file": {"path": "/etc/crypto-policies/local.d/gnutls-qemu.config", "state": "absent"}, "register": "update_gnutls_config"}], {"name": "update crypto policies", "command": "update-crypto-policies", "when": "update_gnutls_config.changed"}], "metadata_settings": [], "upgrade_tasks": [{"name": "Fail if we are on RHEL8 as this is monolithic libvirt deployment", "when": ["step|int == 0", "'system_upgrade' not in ansible_run_tags", "ansible_facts['distribution_major_version'] is version('8', '==')"], "fail": {"msg": "Use of the Modular Libvirt deployment on RHEL8 results in loss of workloads. Use NovaLibvirtLegacy in roles data."}}, {"name": "nova_libvirt_container_tmpfile_cleanup", "when": "step|int == 1", "block": [{"name": "Remove old tmpfiles.d config", "file": {"path": "/etc/tmpfiles.d/var-run-libvirt.conf", "state": "absent"}}]}, {"name": "Get container virtlogd image", "when": "step|int == 1", "tags": "always", "set_fact": {"virtlogd_image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1"}}, {"name": "Force pull image novalibvirt", "when": ["step|int == 1"], "containers.podman.podman_image": {"name": "{{ virtlogd_image }}", "force": true}}, {"name": "Diff OS release in container image and baremetal", "when": "step|int == 1", "shell": "diff /etc/redhat-release <(podman run {{ virtlogd_image }} cat /etc/redhat-release)\n", "args": {"executable": "/bin/bash"}, "failed_when": false, "register": "libvirt_ubi_diff"}, {"name": "Fail if UBI of the container does not match RHEL of running host", "when": ["'system_upgrade' not in ansible_run_tags", "step|int == 1", "libvirt_ubi_diff.rc == 1"], "fail": {"msg": "The host operating system versus container UBI does not match.\n{{ libvirt_ubi_diff.stdout }}\nMost likely container passed to NovaLibvirt was EL8 one.\nYou have to pass the correct container for container-image-prepare.\nIf you would continue all running workloads would get destroyed.\n"}}, {"name": "Switch virtlogd image before system upgrade", "when": "step|int == 1", "tags": ["never", "system_upgrade"], "block": [{"name": "Get running image version", "shell": "podman ps --filter name=^nova_virtlogd$ --format \"{% raw %}{{ .Image }}{% endraw %}\"\n", "register": "old_virtlogd_image"}, {"name": "Switch running image to new image in wrapper script", "shell": "set -o pipefail\nsed -i -e 's|{{ old_virtlogd_image.stdout }}|{{ virtlogd_image }}|' /var/lib/container-config-scripts/virtlogd_wrapper\n", "when": ["old_virtlogd_image.stdout != virtlogd_image", "old_virtlogd_image.stdout != ''"]}]}], "update_tasks": [{"name": "check systemd-container package present or not", "package": {"name": "systemd-container", "state": "present"}, "check_mode": true, "register": "is_systemd_container_package"}, {"name": "download systemd-container package if not present", "yum": {"name": "systemd-container", "state": "present", "download_only": true, "download_dir": "/var/tmp/"}, "when": "is_systemd_container_package.changed"}, {"name": "nova_libvirt_container_tmpfile_cleanup", "when": "step|int == 1", "block": [{"name": "Remove old tmpfiles.d config", "file": {"path": "/etc/tmpfiles.d/var-run-libvirt.conf", "state": "absent"}}]}]}, {"service_name": "nova_migration_target", "firewall_rules": {"113 nova_migration_target accept libvirt subnet 172.17.0.0/24": {"source": "172.17.0.0/24", "proto": "tcp", "dport": 2022}, "113 nova_migration_target accept api subnet 172.17.0.0/24": {"source": "172.17.0.0/24", "proto": "tcp", "dport": 2022}}, "config_settings": {"tripleo::profile::base::sshd::bannertext": "", "tripleo::profile::base::sshd::motd": "", "tripleo::profile::base::sshd::options": {"HostKey": ["/etc/ssh/ssh_host_rsa_key", "/etc/ssh/ssh_host_ecdsa_key", "/etc/ssh/ssh_host_ed25519_key"], "SyslogFacility": "AUTHPRIV", "AuthorizedKeysFile": ".ssh/authorized_keys", "ChallengeResponseAuthentication": "no", "GSSAPIAuthentication": "no", "GSSAPICleanupCredentials": "no", "UsePAM": "yes", "UseDNS": "no", "PrintMotd": "no", "X11Forwarding": "yes", "AcceptEnv": ["LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES", "LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT", "LC_IDENTIFICATION LC_ALL LANGUAGE", "XMODIFIERS"], "Subsystem": "sftp  /usr/libexec/openssh/sftp-server"}, "tripleo::profile::base::sshd::password_authentication": "no", "tripleo::profile::base::nova::migration::target::ssh_authorized_keys": ["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCkRb6JIO5CbikGT/iBy3w3bUJSS0mRg9mHjaIwwFGaO+116xrMHKyQeFIfOx1xlVcHpAyCUWI/aqVfc4KaKhbk3SOpOalNtT/r28N6ue2qfPE1algl3JwSLssvO3GfpQ6UhDKRv0qIRj6i7007iTxIBTvZrk+mWFlU83Q9NWzfTcBI/WtJX+bB3rJsoqrMRmbtup4ABj+/fiN6+Q+Y8/RvsTfWMMsMczJaUQGtBBz13Omd8wDObhcnsWhj9/EpiHNPUEUKlcmJ1Ma+yQ+W1r3KyFc2KBYG1txBom+3VsgSih+XlYP8FdrbROuW2UP+dDpVNsEcAPP24JtFDFocjR7p Generated by TripleO"], "tripleo::profile::base::nova::migration::target::ssh_localaddrs": ["%{hiera('cold_migration_ssh_inbound_addr')}", "%{hiera('live_migration_ssh_inbound_addr')}"], "live_migration_ssh_inbound_addr": "%{hiera('internal_api')}", "cold_migration_ssh_inbound_addr": "%{hiera('internal_api')}", "tripleo::profile::base::sshd::port": [22]}, "puppet_config": {"config_volume": "nova_libvirt", "step_config": "include tripleo::profile::base::sshd\ninclude tripleo::profile::base::nova::migration::target", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/nova-migration-target.json": {"command": "dumb-init --single-child -- /usr/sbin/sshd -D -p 2022", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/host-ssh/ssh_host_*_key", "dest": "/etc/ssh/", "owner": "root", "perm": "0600"}]}}, "host_prep_tasks": [{"name": "Create libvirt persistent data directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}"}, "loop": [{"path": "/run/libvirt", "setype": "virt_var_run_t"}]}, {"name": "ensure /run/libvirt is present upon reboot", "copy": {"dest": "/etc/tmpfiles.d/run-libvirt.conf", "content": "d /run/libvirt 0755 root root - -\n"}}], "docker_config": {"step_4": {"nova_migration_target": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-compute:17.1", "net": "host", "privileged": true, "user": "root", "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/nova-migration-target.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/etc/ssh:/host-ssh:ro", "/run/libvirt:/run/libvirt:shared,z", "/var/lib/nova:/var/lib/nova:shared"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "deploy_steps_tasks": [{"name": "validate nova-migration-target container state", "containers.podman.podman_container_info": {"name": "nova_migration_target"}, "register": "nova_migration_target_infos", "failed_when": ["nova_migration_target_infos.containers.0.Healthcheck.Status is defined", "'healthy' not in nova_migration_target_infos.containers.0.Healthcheck.Status"], "retries": 10, "delay": 30, "tags": ["opendev-validation", "opendev-validation-nova"], "when": ["container_cli == 'podman'", "not container_healthcheck_disabled", "step|int == 5"]}], "update_tasks": [{"name": "nova_migration_target_tmpfile_cleanup", "when": "step|int == 1", "block": [{"name": "Remove old tmpfiles.d config", "file": {"path": "/etc/tmpfiles.d/var-run-libvirt.conf", "state": "absent"}}]}], "upgrade_tasks": [{"name": "nova_migration_target_tmpfile_cleanup", "when": "step|int == 1", "block": [{"name": "Remove old tmpfiles.d config", "file": {"path": "/etc/tmpfiles.d/var-run-libvirt.conf", "state": "absent"}}]}]}, {"service_name": "nova_vnc_proxy", "firewall_rules": {"137 nova_vnc_proxy": {"dport": [6080]}}, "firewall_frontend_rules": {"100 nova_vncproxy_haproxy_frontend": {"dport": [6080]}}, "firewall_ssl_frontend_rules": {"100 nova_vncproxy_haproxy_frontend_ssl": {"dport": [13080]}}, "config_settings": {"nova::my_ip": "%{hiera('internal_api')}", "nova::rabbit_use_ssl": "False", "nova::keystone::service_user::send_service_user_token": true, "nova::keystone::service_user::project_name": "service", "nova::keystone::service_user::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::keystone::service_user::auth_url": "http://172.17.0.2:5000", "nova::keystone::service_user::region_name": "regionOne", "nova::placement::project_name": "service", "nova::placement::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "nova::placement::auth_url": "http://172.17.0.2:5000", "nova::placement::region_name": "regionOne", "nova::placement::valid_interfaces": "internal", "nova::os_region_name": "regionOne", "nova::logging::debug": true, "nova::purge_config": false, "nova::network::neutron::project_name": "service", "nova::network::neutron::username": "neutron", "nova::network::neutron::region_name": "regionOne", "nova::dhcp_domain": "", "nova::network::neutron::password": "Y7cJplS04DjBDFMUWgbx3kdiC", "nova::network::neutron::auth_url": "http://172.17.0.2:5000/v3", "nova::network::neutron::valid_interfaces": "internal", "nova::glance::valid_interfaces": "internal", "nova::rabbit_heartbeat_timeout_threshold": 60, "nova::cinder_catalog_info": "volumev3:cinderv3:internalURL", "nova::host": "%{hiera('fqdn_canonical')}", "nova::notify_on_state_change": "vm_and_task_state", "nova::notification_driver": ["noop"], "nova::notification_format": "unversioned", "nova::network::neutron::auth_type": "v3password", "nova::db::database_db_max_retries": -1, "nova::db::database_max_retries": -1, "nova::network::neutron::ovs_bridge": "br-int", "nova::compute::sync_power_state_interval": 600, "nova_is_additional_cell": false, "nova::cross_az_attach": true, "nova::cache::enabled": true, "nova::cache::tls_enabled": false, "nova::cache::backend": "dogpile.cache.memcached", "nova::rabbit_heartbeat_in_pthread": false, "nova::ovsdb_connection": "", "nova::database_connection": "mysql+pymysql://nova:nad1A8ZvKDsT60yawIjN1cH4K@172.17.0.2/nova?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "nova::vncproxy::enabled": true, "nova::vncproxy::common::vncproxy_protocol": "http", "nova::vncproxy::common::vncproxy_host": "172.21.0.2", "nova::vncproxy::common::vncproxy_port": "6080", "nova::vncproxy::host": "%{hiera('internal_api')}"}, "service_config_settings": {"rabbitmq": {"get_attr": ["NovaBase", "role_data", "service_config_settings"], "rabbitmq": null}, "mysql": {"nova::db::mysql::password": "nad1A8ZvKDsT60yawIjN1cH4K", "nova::db::mysql::user": "nova", "nova::db::mysql::host": "%", "nova::db::mysql::dbname": "nova"}, "rsyslog": {"tripleo_logging_sources_nova_vnc_proxy": [{"tag": "openstack.nova.vncproxy", "file": "/var/log/containers/nova/nova-novncproxy.log"}]}}, "puppet_config": {"config_volume": "nova", "puppet_tags": "nova_config", "step_config": "include tripleo::profile::base::nova::vncproxy\ninclude tripleo::profile::base::database::mysql::client", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/nova_vnc_proxy.json": {"command": "/usr/bin/nova-novncproxy --web /usr/share/novnc/ ", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-tls/*", "dest": "/", "merge": true, "preserve_properties": true, "optional": true}], "permissions": [{"path": "/var/log/nova", "owner": "nova:nova", "recurse": true}, {"path": "/etc/pki/tls/certs/novnc-proxy.crt", "owner": "root:root", "perm": "0644"}, {"path": "/etc/pki/tls/private/novnc-proxy.key", "owner": "root:qemu", "perm": "0640"}, {"path": "/etc/pki/tls/certs/libvirt-vnc-client-cert.crt", "owner": "root:root", "perm": "0644"}, {"path": "/etc/pki/tls/private/libvirt-vnc-client-cert.key", "owner": "root:qemu", "perm": "0640"}]}}, "docker_config": {"step_4": {"nova_vnc_proxy": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-novncproxy:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova", "/var/lib/kolla/config_files/nova_vnc_proxy.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "deploy_steps_tasks": [{"name": "validate nova-vnc-proxy container state", "containers.podman.podman_container_info": {"name": "nova_vnc_proxy"}, "register": "nova_vnc_proxy_infos", "failed_when": ["nova_vnc_proxy_infos.containers.0.Healthcheck.Status is defined", "'healthy' not in nova_vnc_proxy_infos.containers.0.Healthcheck.Status"], "retries": 10, "delay": 30, "tags": ["opendev-validation", "opendev-validation-nova"], "when": ["container_cli == 'podman'", "not container_healthcheck_disabled", "step|int == 5"]}], "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "setype": "{{ item.setype }}", "state": "directory", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/nova", "setype": "container_file_t", "mode": "0750"}]}, {"name": "ensure qemu group is present on the host", "group": {"name": "qemu", "gid": 107, "state": "present"}}, {"name": "ensure qemu user is present on the host", "user": {"name": "qemu", "uid": 107, "group": "qemu", "state": "present", "shell": "/sbin/nologin", "comment": "qemu user"}}], "post_upgrade_tasks": [{"when": ["step|int == 3"], "become": true, "environment": {"OS_CLOUD": "standalone"}, "block": [{"name": "Get nova-consoleauth service ID", "command": "openstack compute service list --service nova-consoleauth --column ID  --column Host --format yaml", "register": "nova_compute_service_result", "check_mode": false, "run_once": true, "changed_when": false, "failed_when": false, "delegate_to": "{{ groups['Undercloud'] | first }}"}, {"name": "Deleting nova-consoleauth", "command": "openstack compute service delete {{ item.ID }}", "loop": "{{ nova_compute_service_result.stdout | from_yaml | default([]) }}", "run_once": true, "delegate_to": "{{ groups['Undercloud'] | first }}", "failed_when": false}]}], "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_transfer_data", "system_upgrade_stop_services"], "block": [{"name": "Stop nova vnc proxy container", "import_role": {"name": "tripleo_container_stop"}, "vars": {"tripleo_containers_to_stop": ["nova_vnc_proxy"], "tripleo_delegate_to": "{{ groups['nova_vnc_proxy'] | difference(groups['excluded_overcloud']) }}"}}]}]}, {"service_name": "ovn_controller", "firewall_rules": {"118 neutron vxlan networks": {"proto": "udp", "dport": 4789, "state": []}, "119 neutron geneve networks": {"proto": "udp", "dport": 6081, "state": []}, "120 neutron geneve networks no conntrack": {"proto": "udp", "dport": 6081, "table": "raw", "chain": "OUTPUT", "jump": "NOTRACK", "action": "append", "state": ["INVALID"]}, "121 neutron geneve networks no conntrack": {"proto": "udp", "dport": 6081, "table": "raw", "chain": "PREROUTING", "jump": "NOTRACK", "action": "append", "state": ["INVALID"]}}, "config_settings": {"ovn::controller::ovn_bridge_mappings": ["datacentre:br-ctlplane"], "ovn::controller::ovn_cms_options": "enable-chassis-as-gw", "ovn::controller::ovn_encap_tos": "0", "vswitch::ovs::enable_hw_offload": false, "container_cpuset_cpus": "", "ContainerOvnControllerImage": "registry.redhat.io/rhosp-rhel9/openstack-ovn-controller:17.1", "ContainerOvnControllerConfigImage": "registry.redhat.io/rhosp-rhel9/openstack-ovn-controller:17.1", "ovn::southbound::port": 6642, "ovn::controller::ovn_encap_ip": "%{hiera('tenant')}", "ovn::controller::ovn_encap_type": "geneve", "ovn::controller::ovn_bridge": "br-int", "ovn::controller::hostname": "%{hiera('fqdn_canonical')}", "ovn::controller::ovn_remote_probe_interval": 60000, "ovn::controller::ovn_openflow_probe_interval": 60, "ovn::controller::ovn_ofctrl_wait_before_clear": 8000, "ovn::controller::garp_max_timeout_sec": 0, "ovn::controller::ovn_monitor_all": true}, "service_config_settings": {}, "puppet_config": {"puppet_tags": "vs_config,exec", "config_volume": "ovn_controller", "step_config": "include tripleo::profile::base::neutron::agents::ovn\n", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-controller:17.1", "volumes": ["/lib/modules:/lib/modules:ro", "/run/openvswitch:/run/openvswitch:shared,z", "/etc/sysconfig/modules:/etc/sysconfig/modules"]}, "kolla_config": {"/var/lib/kolla/config_files/ovn_controller.json": {"command": "/usr/bin/ovn-controller --pidfile --log-file unix:/run/openvswitch/db.sock ", "permissions": [{"path": "/var/log/openvswitch", "owner": "root:root", "recurse": true}, {"path": "/var/log/ovn", "owner": "root:root", "recurse": true}]}}, "metadata_settings": null, "docker_config": {"step_4": {"configure_cms_options": {"start_order": 0, "detach": false, "net": "host", "privileged": true, "user": "root", "command": ["/bin/bash", "-c", "CMS_OPTS=$(hiera ovn::controller::ovn_cms_options -c /etc/puppet/hiera.yaml);  if [ X\"$CMS_OPTS\" !=  X ]; then ovs-vsctl set open . external_ids:ovn-cms-options=$CMS_OPTS;else ovs-vsctl remove open . external_ids ovn-cms-options; fi"], "image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-controller:17.1", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/lib/modules:/lib/modules:ro", "/run/openvswitch:/run/openvswitch:shared,z"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}, "ovn_controller": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-controller:17.1", "net": "host", "privileged": true, "user": "root", "restart": "always", "depends_on": ["openvswitch.service"], "healthcheck": {"test": "/openstack/healthcheck 6642"}, "volumes": ["/var/lib/kolla/config_files/ovn_controller.json:/var/lib/kolla/config_files/config.json:ro", "/lib/modules:/lib/modules:ro", "/run:/run", "/var/lib/openvswitch/ovn:/run/ovn:shared,z", "/var/log/containers/openvswitch:/var/log/openvswitch:z", "/var/log/containers/openvswitch:/var/log/ovn:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "deploy_steps_tasks": [{"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "ovn_controller", "dns": "{{fqdn_internal_api}}", "principal": "ovn_controller/{{fqdn_internal_api}}@{{idm_realm}}", "key_size": "2048", "ca": "ipa"}]}}]}], "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode|default(omit) }}"}, "with_items": [{"path": "/var/log/containers/openvswitch", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/lib/openvswitch/ovn", "setype": "container_file_t"}]}, {"name": "enable virt_sandbox_use_netlink for healthcheck", "seboolean": {"name": "virt_sandbox_use_netlink", "persistent": true, "state": true}, "when": ["ansible_facts.selinux is defined", "ansible_facts.selinux.status == \"enabled\""]}, {"name": "Copy in cleanup script", "copy": {"content": "#!/usr/bin/bash\n# Cleanup neutron OVS bridges. To be called on startup to avoid\n# \"difficult-to-debug\" issues with partially configured resources.\n\nNEUTRON_OVS_CONF=/var/lib/config-data/puppet-generated/neutron/etc/neutron/plugins/ml2/openvswitch_agent.ini\n\nif [ -e ${NEUTRON_OVS_CONF} ];\nthen\n    INT_BRIDGE=`crudini --get ${NEUTRON_OVS_CONF} ovs integration_bridge`\n    TUN_BRIDGE=`crudini --get ${NEUTRON_OVS_CONF} ovs tunnel_bridge`\nfi\n\nfor port in `ovs-vsctl list-ports ${INT_BRIDGE:-\"br-int\"}`;\ndo\n    skip_cleanup=`ovs-vsctl --if-exists get Interface $port external_ids:skip_cleanup`\n    if ! [[ \"x$skip_cleanup\" == \"x\\\"true\\\"\" ]];\n    then\n        ovs-vsctl del-port ${INT_BRIDGE:-\"br-int\"} $port\n    fi\ndone\n\novs-vsctl --if-exists del-br ${TUN_BRIDGE:-\"br-tun\"}\n\n# Clean up trunk port bridges\nfor br in $(ovs-vsctl list-br | egrep 'tbr-[0-9a-f\\-]+'); do\n    ovs-vsctl --if-exists del-br $br\ndone\n", "dest": "/usr/libexec/neutron-cleanup", "force": true, "mode": "0755"}}, {"name": "Copy in cleanup service", "copy": {"content": "[Unit]\nDescription=Neutron cleanup on startup\nAfter=openvswitch.service network.target\nBefore=tripleo_neutron_ovs_agent.service tripleo_neutron_dhcp.service tripleo_neutron_l3_agent.service tripleo_nova_compute.service\nRefuseManualStop=yes\n\n[Service]\nType=oneshot\nExecStart=/usr/libexec/neutron-cleanup\n\n[Install]\nWantedBy=multi-user.target\n", "dest": "/usr/lib/systemd/system/neutron-cleanup.service", "force": true}}, {"name": "Enabling the cleanup service", "service": {"name": "neutron-cleanup", "enabled": true}, "when": "not (ansible_check_mode|bool)"}], "external_deploy_tasks": [{"when": ["step|int == 0"], "name": "ovn_controller_external_deploy_init", "block": [{"name": "create ovn mac address for Standalone role nodes", "tripleo_ovn_mac_addresses": {"playbook_dir": "{{ playbook_dir }}", "stack_name": "standalone", "role_name": "Standalone", "server_resource_names": "{{ groups[\"Standalone\"] }}", "ovn_bridge_mappings": ["datacentre:br-ctlplane"], "ovn_static_bridge_mac_mappings": {"standalone": {"datacentre": "fa:16:3a:00:53:00"}}}}]}], "external_update_tasks": [{"name": "Force pull image in case image name doesn't change.", "when": "step|int == 1", "tags": ["ovn", "ovn_image"], "become": true, "loop": "{{ groups['ovn_controller'] | difference(groups['excluded_overcloud']) }}", "delegate_to": "{{ item }}", "async": 600, "poll": 0, "register": "ovn_controller_image_update", "containers.podman.podman_image": {"name": "registry.redhat.io/rhosp-rhel9/openstack-ovn-controller:17.1", "validate_certs": false, "force": true}}, {"name": "Was the ovn_controller image pull successful.", "when": ["step|int == 1", "'results' in ovn_controller_image_update"], "tags": ["ovn", "ovn_image"], "become": true, "delegate_to": "{{ async_result_item.item }}", "async_status": {"jid": "{{ async_result_item.ansible_job_id }}"}, "loop": "{{ovn_controller_image_update.results }}", "loop_control": {"loop_var": "async_result_item"}, "register": "async_poll_results", "until": "async_poll_results.finished", "retries": 600, "delay": 1}, {"name": "OVN Container image used", "debug": {"msg": "ovn container will be using {{ image }}"}, "vars": {"image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-controller:17.1"}, "when": "step|int == 1", "tags": "ovn"}, {"name": "Update OVN OVS related parameters before update.", "when": ["step|int == 1"], "tags": ["ovn"], "become": true, "vars": {"timeout": 8000}, "shell": "set -e\novs-vsctl set Open_vSwitch . external_ids:ovn-ofctrl-wait-before-clear={{ timeout }}\novs-vsctl set Open_vSwitch . external_ids:ovn-monitor-all=true\novs-vsctl set Open_vSwitch . external_ids:ovn-match-northd-version=false\n", "async": 600, "poll": 0, "register": "ovs_vsctl", "loop": "{{ groups['ovn_controller'] | difference(groups['excluded_overcloud']) }}", "delegate_to": "{{ item }}"}, {"name": "Was the update of OVN OVS related parameter successful.", "when": ["step|int == 1", "'results' in ovs_vsctl"], "become": true, "tags": ["ovn"], "delegate_to": "{{ async_result_item.item }}", "async_status": {"jid": "{{ async_result_item.ansible_job_id }}"}, "loop": "{{ovs_vsctl.results }}", "loop_control": {"loop_var": "async_result_item"}, "register": "async_poll_results", "until": "async_poll_results.finished", "retries": 600, "delay": 1}, {"set_fact": {"any_ovn_host": "{{groups['ovn_controller'] | difference(groups['excluded_overcloud']) | first }}"}, "tags": "ovn", "when": "step|int == 1"}, {"name": "Find ovn_controller configs in container-startup-configs", "find": {"paths": "/var/lib/tripleo-config/container-startup-config/", "patterns": "*ovn_controller.json", "recurse": true}, "register": "ovn_cont_17_0", "delegate_to": "{{ any_ovn_host }}", "when": "(step|int == 1) and (any_ovn_host is defined) and (any_ovn_host|length > 0)", "tags": ["ovn"], "become": true}, {"name": "get directory path from the ovn_cont_17_0", "set_fact": {"ovn_config_path": "{{ ovn_cont_17_0.files.0.path | dirname }}"}, "when": "step|int == 1", "tags": "ovn"}, {"name": "Get PIDfile used by systemd on each ovn node", "when": ["step|int == 1"], "tags": ["ovn"], "become": true, "shell": "set -e\ngrep PID /etc/systemd/system/tripleo_ovn_controller.service | cut -d= -f2\n", "register": "pidfile", "loop": "{{ groups['ovn_controller'] | difference(groups['excluded_overcloud']) }}", "delegate_to": "{{ item }}"}, {"name": "Update ovn_controller.", "vars": {"config_step": "{{ ('step_4' in ovn_config_path) | ternary('4', '3')}}", "query": "results[?item == '{{item}}'].stdout"}, "when": "step|int == 1", "tags": "ovn", "become": true, "loop": "{{ groups['ovn_controller'] | difference(groups['excluded_overcloud']) }}", "delegate_to": "{{ item }}", "async": 600, "poll": 0, "register": "ovn_controller_update", "tripleo_container_manage": {"config_dir": "{{ ovn_config_path }}", "config_patterns": "*ovn_controller.json", "config_id": ["tripleo_step{{config_step}}"], "log_base_path": "{{ container_log_stdout_path }}", "debug": "{{ enable_debug | bool }}", "config_overrides": {".*ovn_controller": {"image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-controller:17.1", "conmon_pidfile": "{{ pidfile | json_query(query) | first }}", "name": "ovn_controller"}}}}, {"name": "Was the ovn_controller successful.", "when": ["step|int == 1", "'results' in ovn_controller_update"], "tags": "ovn", "become": true, "delegate_to": "{{ async_result_item.item }}", "async_status": {"jid": "{{ async_result_item.ansible_job_id }}"}, "loop": "{{ovn_controller_update.results }}", "loop_control": {"loop_var": "async_result_item"}, "register": "async_poll_results", "until": "async_poll_results.finished", "retries": 600, "delay": 1}, {"name": "Pause for 30s to give ovn_controllers time to reconnect to dbs", "when": ["step|int == 1"], "tags": "ovn", "wait_for": {"timeout": 30}}], "upgrade_tasks": [{"name": "Fetch running ovn_controller image", "shell": "set -e\npodman inspect --format \"{{'{{'}}.ImageName{{'}}'}}\" ovn_controller\n", "register": "running_ovn_image", "when": ["step|int == 4"], "tags": ["ovn", "ovn_image"]}, {"name": "Run ovn_controller upgrade", "tags": ["ovn", "ovn_image"], "vars": {"ovn_controller_image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-controller:17.1"}, "when": ["step|int == 4", "running_ovn_image.stdout != ovn_controller_image"], "block": [{"name": "Force pull image in case image name doesn't change.", "when": "step|int == 4", "tags": ["ovn", "ovn_image"], "containers.podman.podman_image": {"name": "registry.redhat.io/rhosp-rhel9/openstack-ovn-controller:17.1", "validate_certs": false, "force": true}}, {"name": "Update OVN OVS related parameters before update.", "when": ["step|int == 4"], "tags": ["ovn"], "vars": {"timeout": 8000}, "shell": "set -e\novs-vsctl set Open_vSwitch . external_ids:ovn-ofctrl-wait-before-clear={{ timeout }}\novs-vsctl set Open_vSwitch . external_ids:ovn-monitor-all=true\novs-vsctl set Open_vSwitch . external_ids:ovn-match-northd-version=false\n"}, {"name": "Find ovn_controller configs in container-startup-configs", "find": {"paths": "/var/lib/tripleo-config/container-startup-config/", "patterns": "*ovn_controller.json", "recurse": true}, "register": "ovn_cont_17_0", "when": ["step|int == 4"], "tags": ["ovn"]}, {"name": "get directory path from the ovn_cont_17_0", "set_fact": {"ovn_config_path": "{{ ovn_cont_17_0.files.0.path | dirname }}"}, "when": "step|int == 4", "tags": "ovn"}, {"name": "Get PIDfile used by systemd on each ovn node", "when": ["step|int == 4"], "tags": ["ovn"], "shell": "set -e\ngrep PID /etc/systemd/system/tripleo_ovn_controller.service | cut -d= -f2\n", "register": "pidfile"}, {"name": "Update ovn_controller.", "vars": {"config_step": "{{ ('step_4' in ovn_config_path) | ternary('4', '3')}}"}, "when": "step|int == 4", "tags": "ovn", "tripleo_container_manage": {"config_dir": "{{ ovn_config_path }}", "config_patterns": "*ovn_controller.json", "config_id": ["tripleo_step{{config_step}}"], "log_base_path": "{{ container_log_stdout_path }}", "debug": "{{ enable_debug | bool }}", "config_overrides": {".*ovn_controller": {"image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-controller:17.1", "conmon_pidfile": "{{ pidfile.stdout }}", "name": "ovn_controller"}}}}, {"name": "Pause for 30s to give ovn_controllers time to reconnect to dbs", "when": ["step|int == 4"], "tags": "ovn", "wait_for": {"timeout": 30}}]}], "scale_tasks": [{"when": ["step|int == 1", "container_cli == 'podman'"], "tags": "down", "become": true, "environment": {"OS_CLOUD": "standalone"}, "block": [{"name": "Get neutron agents ID", "command": "openstack network agent list --column ID --column Host --column Binary --format yaml", "register": "neutron_agents_result", "delegate_to": "{{ groups['Undercloud'] | first }}", "check_mode": false, "changed_when": false}, {"name": "Filter only current host", "set_fact": {"neutron_agents": "{{ neutron_agents_result.stdout | from_yaml | selectattr('Host', 'match', ansible_facts['fqdn'] ~ '.*') | list }}"}, "delegate_to": "{{ groups['Undercloud'] | first }}", "check_mode": false}, {"name": "Deleting OVN agents", "block": [{"name": "Stop OVN containers", "loop": ["tripleo_ovn_controller", "tripleo_ovn_metadata_agent"], "service": {"name": "{{ item }}", "state": "stopped", "enabled": false}, "become": true, "register": "stop_containers", "failed_when": "('msg' in stop_containers and 'Could not find the requested service' not in stop_containers.msg) or ('rc' in stop_containers and stop_containers.rc != 0)"}, {"name": "Delete neutron agents", "loop": "{{ neutron_agents }}", "loop_control": {"loop_var": "agent", "label": "{{ agent.Host }}/{{ agent.Binary }}"}, "command": "openstack network agent delete {{ agent.ID }}", "delegate_to": "{{ groups['Undercloud'] | first }}", "check_mode": false}]}]}]}, {"service_name": "ovn_dbs", "firewall_rules": {"121 OVN DB server and cluster ports for 172.17.0.0/24": {"proto": "tcp", "source": "172.17.0.0/24", "dport": [6641, 6642, 6643, 6644]}}, "kolla_config": {"/var/lib/kolla/config_files/ovn_cluster_north_db_server.json": {"command": "bash -c $* -- eval source /etc/sysconfig/ovn_cluster; exec /usr/local/bin/start-nb-db-server ${OVN_NB_DB_OPTS}", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-tls/*", "dest": "/", "merge": true, "preserve_properties": true, "optional": true}], "permissions": [{"path": "/var/log/openvswitch", "owner": "root:root", "recurse": true}, {"path": "/var/log/ovn", "owner": "root:root", "recurse": true}]}, "/var/lib/kolla/config_files/ovn_cluster_south_db_server.json": {"command": "bash -c $* -- eval source /etc/sysconfig/ovn_cluster; exec /usr/local/bin/start-sb-db-server ${OVN_SB_DB_OPTS}", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-tls/*", "dest": "/", "merge": true, "preserve_properties": true, "optional": true}], "permissions": [{"path": "/var/log/openvswitch", "owner": "root:root", "recurse": true}, {"path": "/var/log/ovn", "owner": "root:root", "recurse": true}]}, "/var/lib/kolla/config_files/ovn_cluster_northd.json": {"command": "bash -c $* -- eval source /etc/sysconfig/ovn_cluster; exec /usr/bin/ovn-northd ${OVN_NORTHD_OPTS}", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-tls/*", "dest": "/", "merge": true, "preserve_properties": true, "optional": true}], "permissions": [{"path": "/var/log/openvswitch", "owner": "root:root", "recurse": true}, {"path": "/var/log/ovn", "owner": "root:root", "recurse": true}]}}, "docker_config": {"step_0": {"ovn_cluster_north_db_server": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-nb-db-server:17.1", "net": "host", "privileged": false, "restart": "always", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/ovn_cluster_north_db_server.json:/var/lib/kolla/config_files/config.json:ro", "/lib/modules:/lib/modules:ro", "/var/lib/openvswitch/ovn:/var/lib/openvswitch:shared,z", "/var/lib/openvswitch/ovn:/run/openvswitch:shared,z", "/var/log/containers/openvswitch:/var/log/openvswitch:z", "/var/lib/openvswitch/ovn:/var/lib/ovn:shared,z", "/var/lib/openvswitch/ovn:/etc/openvswitch:shared,z", "/var/lib/openvswitch/ovn:/etc/ovn:shared,z", "/var/lib/openvswitch/ovn:/run/ovn:shared,z", "/var/log/containers/openvswitch:/var/log/ovn:z", "/var/lib/config-data/ansible-generated/ovn:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "ovn_cluster_south_db_server": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-sb-db-server:17.1", "net": "host", "privileged": false, "restart": "always", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/ovn_cluster_south_db_server.json:/var/lib/kolla/config_files/config.json:ro", "/lib/modules:/lib/modules:ro", "/var/lib/openvswitch/ovn:/var/lib/openvswitch:shared,z", "/var/lib/openvswitch/ovn:/run/openvswitch:shared,z", "/var/log/containers/openvswitch:/var/log/openvswitch:z", "/var/lib/openvswitch/ovn:/var/lib/ovn:shared,z", "/var/lib/openvswitch/ovn:/etc/openvswitch:shared,z", "/var/lib/openvswitch/ovn:/etc/ovn:shared,z", "/var/lib/openvswitch/ovn:/run/ovn:shared,z", "/var/log/containers/openvswitch:/var/log/ovn:z", "/var/lib/config-data/ansible-generated/ovn:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "ovn_cluster_northd": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-northd:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/ovn_cluster_northd.json:/var/lib/kolla/config_files/config.json:ro", "/lib/modules:/lib/modules:ro", "/var/lib/openvswitch/ovn:/run/openvswitch:shared,z", "/var/log/containers/openvswitch:/var/log/openvswitch:z", "/var/lib/openvswitch/ovn:/run/ovn:shared,z", "/var/log/containers/openvswitch:/var/log/ovn:z", "/var/lib/config-data/ansible-generated/ovn:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "global_config_settings": {"ovn_db_clustered": true}, "metadata_settings": null, "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode|default(omit) }}"}, "loop": [{"path": "/var/log/containers/openvswitch", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/lib/openvswitch/ovn", "setype": "container_file_t"}]}], "deploy_steps_tasks": [{"name": "set is_ovn_dbs_bootstrap_node fact", "set_fact": "is_ovn_dbs_bootstrap_node={{ovn_dbs_short_bootstrap_node_name|lower == ansible_facts['hostname']|lower}}", "when": ["step|int == 1"]}, {"name": "Populate ansible service facts so we can check for pacemaker service", "ansible.builtin.service_facts": null, "when": ["is_ovn_dbs_bootstrap_node | bool", "step|int == 1"]}, {"name": "Remove OVNDBs from Pacemaker", "when": ["step|int == 1", "is_ovn_dbs_bootstrap_node | bool", "ansible_facts['services']['pacemaker.service']['state']  | default('unknown') == 'running'"], "block": [{"name": "Fetch ovn-dbs-bundle", "shell": "pcs status |grep ovn-dbs-bundle ||true\n", "register": "ovn_dbs"}, {"name": "Fetch ovn VIP", "shell": "pcs constraint config |grep \"with ovn-dbs-bundle\" |awk '{print $1}'\n", "register": "ovn_vip"}, {"name": "Remove OVNDBs from pacemaker", "pacemaker_resource": {"resource": "ovn-dbs-bundle", "state": "delete"}, "when": "ovn_dbs.stdout |length  > 0"}, {"name": "Remove OVNDBs VIP from pacemaker", "pacemaker_resource": {"resource": "{{ ovn_vip.stdout }}", "state": "delete"}, "when": "ovn_vip.stdout |length > 0"}, {"name": "Clean up pacemaker remote nodes cache", "shell": "pcs cluster node clear ovn-dbs-bundle-0\npcs cluster node clear ovn-dbs-bundle-1\npcs cluster node clear ovn-dbs-bundle-2\ncrm_attribute --name OVN_REPL_INFO --delete\n"}, {"name": "Remove pacemaker attributes", "shell": "pcs node attribute \"{{ item }}\" ovn-dbs-role= || true\n", "loop": "{{ ovn_dbs_short_node_names }}"}]}, {"name": "Prepare OVN cluster", "when": "step|int == 1", "block": [{"name": "Certificate generation", "when": "enable_internal_tls | bool", "block": [{"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "ovn_dbs", "dns": "{{fqdn_internal_api}}", "principal": "ovn_dbs/{{fqdn_internal_api}}@{{idm_realm}}", "key_size": "2048", "ca": "ipa"}]}}]}, {"block": [{"name": "Configure OVN DBs and northd", "include_role": {"name": "tripleo_ovn_cluster"}, "vars": {"tripleo_ovn_cluster_dbs_protocol": "{{ enable_internal_tls | ternary('ssl', 'tcp', 'tcp') }}", "tripleo_ovn_cluster_network": "internal_api", "tripleo_ovn_cluster_nb_db_port": 6641, "tripleo_ovn_cluster_sb_db_port": 6642, "tripleo_ovn_cluster_nb_local_port": 6643, "tripleo_ovn_cluster_nb_remote_port": 6643, "tripleo_ovn_cluster_sb_local_port": 6644, "tripleo_ovn_cluster_sb_remote_port": 6644, "tripleo_ovn_cluster_northd_ssl_ca_cert": "/etc/ipa/ca.crt", "tripleo_ovn_cluster_sb_ssl_ca_cert": "/etc/ipa/ca.crt", "tripleo_ovn_cluster_nb_ssl_ca_cert": "/etc/ipa/ca.crt"}}]}]}, {"name": "Start OVN DBs and northd containers (bootstrap node)", "when": ["step|int == 3", "is_ovn_dbs_bootstrap_node | bool"], "block": [{"block": [{"name": "Start OVN container", "include_role": {"name": "tripleo_container_manage"}, "vars": {"tripleo_container_manage_config": "/var/lib/tripleo-config/container-startup-config/step_0", "tripleo_container_manage_config_id": "{{ ovn_container }}", "tripleo_container_manage_config_patterns": "{{ ovn_container }}.json"}, "loop": ["ovn_cluster_north_db_server", "ovn_cluster_south_db_server", "ovn_cluster_northd"], "loop_control": {"loop_var": "ovn_container"}}]}, {"name": "Set NB connection", "become": true, "shell": "podman exec ovn_cluster_north_db_server bash -c \"ovn-nbctl --no-leader-only --inactivity-probe={{ tripleo_ovn_cluster_probe_interval }} set-connection p{{ tripleo_ovn_cluster_dbs_protocol }}:{{ tripleo_ovn_cluster_nb_db_port }}:{{ tripleo_ovn_cluster_dbs_addr }}\"\n", "register": "nb_result", "retries": 20, "delay": 5, "until": "nb_result.rc == 0", "when": ["is_ovn_dbs_bootstrap_node | bool"], "vars": {"tripleo_ovn_cluster_network": "internal_api", "tripleo_ovn_cluster_nb_db_port": 6641, "tripleo_ovn_cluster_dbs_protocol": "{{ enable_internal_tls | ternary('ssl', 'tcp', 'tcp') }}", "tripleo_ovn_cluster_dbs_addr": "0.0.0.0", "tripleo_ovn_cluster_probe_interval": 60000}}, {"name": "Set SB connection", "become": true, "shell": "podman exec ovn_cluster_south_db_server bash -c \"ovn-sbctl --no-leader-only --inactivity-probe={{ tripleo_ovn_cluster_probe_interval }} set-connection p{{ tripleo_ovn_cluster_dbs_protocol }}:{{ tripleo_ovn_cluster_sb_db_port }}:{{ tripleo_ovn_cluster_dbs_addr }}\"\n", "register": "sb_result", "retries": 20, "delay": 5, "until": "sb_result.rc == 0", "when": ["is_ovn_dbs_bootstrap_node | bool"], "vars": {"tripleo_ovn_cluster_network": "internal_api", "tripleo_ovn_cluster_sb_db_port": 6642, "tripleo_ovn_cluster_dbs_protocol": "{{ enable_internal_tls | ternary('ssl', 'tcp', 'tcp') }}", "tripleo_ovn_cluster_dbs_addr": "0.0.0.0", "tripleo_ovn_cluster_probe_interval": 60000}}]}, {"name": "Start OVN DBs and northd containers (non-bootstrap nodes)", "when": ["step|int == 4", "not is_ovn_dbs_bootstrap_node | bool"], "block": [{"block": [{"name": "Start OVN container", "include_role": {"name": "tripleo_container_manage"}, "vars": {"tripleo_container_manage_config": "/var/lib/tripleo-config/container-startup-config/step_0", "tripleo_container_manage_config_id": "{{ ovn_container }}", "tripleo_container_manage_config_patterns": "{{ ovn_container }}.json"}, "loop": ["ovn_cluster_north_db_server", "ovn_cluster_south_db_server", "ovn_cluster_northd"], "loop_control": {"loop_var": "ovn_container"}}]}, {"name": "Set NB connection", "become": true, "shell": "podman exec ovn_cluster_north_db_server bash -c \"ovn-nbctl --no-leader-only --inactivity-probe={{ tripleo_ovn_cluster_probe_interval }} set-connection p{{ tripleo_ovn_cluster_dbs_protocol }}:{{ tripleo_ovn_cluster_nb_db_port }}:{{ tripleo_ovn_cluster_dbs_addr }}\"\n", "register": "nb_result", "retries": 20, "delay": 5, "until": "nb_result.rc == 0", "when": ["is_ovn_dbs_bootstrap_node | bool"], "vars": {"tripleo_ovn_cluster_network": "internal_api", "tripleo_ovn_cluster_nb_db_port": 6641, "tripleo_ovn_cluster_dbs_protocol": "{{ enable_internal_tls | ternary('ssl', 'tcp', 'tcp') }}", "tripleo_ovn_cluster_dbs_addr": "0.0.0.0", "tripleo_ovn_cluster_probe_interval": 60000}}, {"name": "Set SB connection", "become": true, "shell": "podman exec ovn_cluster_south_db_server bash -c \"ovn-sbctl --no-leader-only --inactivity-probe={{ tripleo_ovn_cluster_probe_interval }} set-connection p{{ tripleo_ovn_cluster_dbs_protocol }}:{{ tripleo_ovn_cluster_sb_db_port }}:{{ tripleo_ovn_cluster_dbs_addr }}\"\n", "register": "sb_result", "retries": 20, "delay": 5, "until": "sb_result.rc == 0", "when": ["is_ovn_dbs_bootstrap_node | bool"], "vars": {"tripleo_ovn_cluster_network": "internal_api", "tripleo_ovn_cluster_sb_db_port": 6642, "tripleo_ovn_cluster_dbs_protocol": "{{ enable_internal_tls | ternary('ssl', 'tcp', 'tcp') }}", "tripleo_ovn_cluster_dbs_addr": "0.0.0.0", "tripleo_ovn_cluster_probe_interval": 60000}}]}], "external_update_tasks": [{"name": "Run ovndbs update", "tags": ["ovn", "ovn_image"], "when": "step|int == 2", "block": [{"name": "Get OVN_Northbound cluster leader", "become": true, "shell": "podman exec -u root ovn_cluster_north_db_server ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound | grep 'Leader: '\n", "register": "ovn_nb_dbs_cluster_leader", "loop": "{{groups['ovn_dbs'] | difference(groups['excluded_overcloud']) }}", "delegate_to": "{{ item }}"}, {"set_fact": {"ovn_nb_leader": "{{ ovn_nb_dbs_cluster_leader| json_query(\"results[?contains(stdout,'Leader: self')].item\") }}"}}, {"name": "Graceful stop OVN NB leader container", "delegate_to": "{{ ovn_nb_leader[0] }}", "become": true, "shell": "podman exec -u root ovn_cluster_north_db_server ovs-appctl -t /var/run/ovn/ovnnb_db.ctl exit\n"}, {"name": "Start OVN NB leader container", "delegate_to": "{{ ovn_nb_leader[0] }}", "become": true, "tripleo_container_manage": {"config_dir": "/var/lib/tripleo-config/container-startup-config/step_0", "config_id": ["ovn_cluster_north_db_server"], "config_patterns": "*ovn_cluster_north_db_server.json", "config_overrides": {".*ovn_cluster_north_db_server": {"image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-nb-db-server:17.1", "name": "ovn_cluster_north_db_server"}}}}, {"name": "Get OVN_Southbound cluster leader", "become": true, "shell": "sudo podman exec -u root ovn_cluster_south_db_server ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound | grep 'Leader: '\n", "register": "ovn_sb_dbs_cluster_leader", "loop": "{{groups['ovn_dbs'] | difference(groups['excluded_overcloud']) }}", "delegate_to": "{{ item }}"}, {"set_fact": {"ovn_sb_leader": "{{ ovn_sb_dbs_cluster_leader | json_query(\"results[?contains(stdout,'Leader: self')].item\") }}"}}, {"name": "Graceful stop OVN SB leader container", "delegate_to": "{{ ovn_sb_leader[0] }}", "become": true, "shell": "sudo podman exec -u root ovn_cluster_south_db_server ovs-appctl -t /var/run/ovn/ovnsb_db.ctl exit\n"}, {"name": "Start OVN SB leader container", "delegate_to": "{{ ovn_sb_leader[0] }}", "become": true, "tripleo_container_manage": {"config_dir": "/var/lib/tripleo-config/container-startup-config/step_0", "config_id": ["ovn_cluster_south_db_server"], "config_patterns": "*ovn_cluster_south_db_server.json", "config_overrides": {".*ovn_cluster_south_db_server": {"image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-sb-db-server:17.1", "name": "ovn_cluster_south_db_server"}}}}, {"name": "Loop on OVN_Northbound cluster for leader election", "become": true, "shell": "podman exec -u root ovn_cluster_north_db_server ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound | grep 'Leader: '\n", "register": "wait_leader", "delegate_to": "{{ groups['ovn_dbs'] | difference(groups['excluded_overcloud']) |  difference(ovn_nb_leader[0]) | first }}", "failed_when": "'Leader: unknown' in wait_leader.stdout or wait_leader.stderr != ''", "until": "'Leader: unknown' not in wait_leader.stdout and wait_leader.stderr == ''", "retries": 10, "delay": 5}, {"name": "Loop on OVN_Southbound cluster for leader election", "become": true, "shell": "podman exec -u root ovn_cluster_south_db_server ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound | grep 'Leader: '\n", "register": "wait_leader", "delegate_to": "{{ groups['ovn_dbs'] | difference(groups['excluded_overcloud']) |  difference(ovn_nb_leader[0]) | first }}", "failed_when": "'Leader: unknown' in wait_leader.stdout or wait_leader.stderr != ''", "until": "'Leader: unknown' not in wait_leader.stdout and wait_leader.stderr == ''", "retries": 10, "delay": 5}, {"name": "Pause for 30 seconds to ensure ovn db leader knows it was elected and clients reconnect", "wait_for": {"timeout": 30}}, {"name": "Get OVN_Northbound cluster new leader", "become": true, "shell": "podman exec -u root ovn_cluster_north_db_server ovs-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound | grep 'Leader: '\n", "register": "ovn_nb_dbs_cluster_leader_new", "loop": "{{groups['ovn_dbs'] | difference(groups['excluded_overcloud']) }}", "until": "'Leader: unknown' not in ovn_nb_dbs_cluster_leader_new.stdout", "retries": 10, "delay": 5, "delegate_to": "{{ item }}"}, {"set_fact": {"ovn_nb_leader_new": "{{ ovn_nb_dbs_cluster_leader_new| json_query(\"results[?contains(stdout,'Leader: self')].item\") }}"}}, {"name": "Get OVN_Southbound cluster new leader", "become": true, "shell": "sudo podman exec -u root ovn_cluster_south_db_server ovs-appctl -t /var/run/ovn/ovnsb_db.ctl cluster/status OVN_Southbound | grep 'Leader: '\n", "register": "ovn_sb_dbs_cluster_leader_new", "loop": "{{groups['ovn_dbs'] | difference(groups['excluded_overcloud']) }}", "until": "'Leader: unknown' not in ovn_sb_dbs_cluster_leader_new.stdout", "retries": 10, "delay": 5, "delegate_to": "{{ item }}"}, {"set_fact": {"ovn_sb_leader_new": "{{ ovn_sb_dbs_cluster_leader_new | json_query(\"results[?contains(stdout,'Leader: self')].item\") }}"}}, {"name": "Start OVN NB containers (non-leader nodes)", "delegate_to": "{{ item }}", "become": true, "tripleo_container_manage": {"config_dir": "/var/lib/tripleo-config/container-startup-config/step_0", "config_id": ["ovn_cluster_north_db_server"], "config_patterns": "*ovn_cluster_north_db_server.json", "config_overrides": {".*ovn_cluster_north_db_server": {"image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-nb-db-server:17.1", "name": "ovn_cluster_north_db_server"}}}, "loop": "{{groups['ovn_dbs'] | difference(groups['excluded_overcloud']) | difference(ovn_nb_leader[0])| difference(ovn_nb_leader_new[0])}}", "loop_control": {"pause": 15}}, {"name": "Start OVN SB containers (non-leader nodes)", "delegate_to": "{{ item }}", "become": true, "tripleo_container_manage": {"config_dir": "/var/lib/tripleo-config/container-startup-config/step_0", "config_id": ["ovn_cluster_south_db_server"], "config_patterns": "*ovn_cluster_south_db_server.json", "config_overrides": {".*ovn_cluster_south_db_server": {"image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-sb-db-server:17.1", "name": "ovn_cluster_south_db_server"}}}, "loop": "{{groups['ovn_dbs'] | difference(groups['excluded_overcloud']) | difference(ovn_sb_leader[0])| difference(ovn_sb_leader_new[0])}}", "loop_control": {"pause": 15}}, {"name": "Pause for 50 seconds before restarting new leaders of OVN", "wait_for": {"timeout": 50}}, {"name": "Graceful stop OVN NB new-leader container", "delegate_to": "{{ ovn_nb_leader_new[0] }}", "become": true, "shell": "podman exec -u root ovn_cluster_north_db_server ovs-appctl -t /var/run/ovn/ovnnb_db.ctl exit\n", "when": "ovn_nb_leader_new[0] != ovn_nb_leader[0]"}, {"name": "Start OVN NB containers (new-leader nodes)", "delegate_to": "{{ ovn_nb_leader_new[0] }}", "become": true, "tripleo_container_manage": {"config_dir": "/var/lib/tripleo-config/container-startup-config/step_0", "config_id": ["ovn_cluster_north_db_server"], "config_patterns": "*ovn_cluster_north_db_server.json", "config_overrides": {".*ovn_cluster_north_db_server": {"image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-nb-db-server:17.1", "name": "ovn_cluster_north_db_server"}}}, "when": "ovn_nb_leader_new[0] != ovn_nb_leader[0]"}, {"name": "Graceful stop OVN SB new-leader container", "delegate_to": "{{ ovn_sb_leader_new[0] }}", "become": true, "shell": "sudo podman exec -u root ovn_cluster_south_db_server ovs-appctl -t /var/run/ovn/ovnsb_db.ctl exit\n", "when": "ovn_sb_leader_new[0] != ovn_sb_leader[0]"}, {"name": "Start OVN SB containers (new-leader nodes)", "delegate_to": "{{ ovn_sb_leader_new[0] }}", "become": true, "tripleo_container_manage": {"config_dir": "/var/lib/tripleo-config/container-startup-config/step_0", "config_id": ["ovn_cluster_south_db_server"], "config_patterns": "*ovn_cluster_south_db_server.json", "config_overrides": {".*ovn_cluster_south_db_server": {"image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-sb-db-server:17.1", "name": "ovn_cluster_south_db_server"}}}, "when": "ovn_sb_leader_new[0] != ovn_sb_leader[0]"}, {"name": "Start OVN northd container", "delegate_to": "{{ item }}", "become": true, "tripleo_container_manage": {"config_dir": "/var/lib/tripleo-config/container-startup-config/step_0", "config_id": ["ovn_cluster_northd"], "config_patterns": "*ovn_cluster_northd.json", "config_overrides": {".*ovn_cluster_northd": {"image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-northd:17.1", "name": "ovn_cluster_northd"}}}, "loop": "{{groups['ovn_dbs'] | difference(groups['excluded_overcloud'])}}", "loop_control": {"pause": 15}}]}], "update_tasks": [{"name": "Configure OVN cluster during update", "when": "step|int == 4", "block": [{"name": "Configure OVN DBs and northd", "include_role": {"name": "tripleo_ovn_cluster"}, "vars": {"tripleo_ovn_cluster_dbs_protocol": "{{ enable_internal_tls | ternary('ssl', 'tcp', 'tcp') }}", "tripleo_ovn_cluster_network": "internal_api", "tripleo_ovn_cluster_nb_db_port": 6641, "tripleo_ovn_cluster_sb_db_port": 6642, "tripleo_ovn_cluster_nb_local_port": 6643, "tripleo_ovn_cluster_nb_remote_port": 6643, "tripleo_ovn_cluster_sb_local_port": 6644, "tripleo_ovn_cluster_sb_remote_port": 6644, "tripleo_ovn_cluster_northd_ssl_ca_cert": "/etc/ipa/ca.crt", "tripleo_ovn_cluster_sb_ssl_ca_cert": "/etc/ipa/ca.crt", "tripleo_ovn_cluster_nb_ssl_ca_cert": "/etc/ipa/ca.crt"}}]}, {"name": "Update OVN cluster containers", "when": "step|int == 4", "block": [{"name": "Start OVN container", "include_role": {"name": "tripleo_container_manage"}, "vars": {"tripleo_container_manage_config": "/var/lib/tripleo-config/container-startup-config/step_0", "tripleo_container_manage_config_id": "{{ ovn_container }}", "tripleo_container_manage_config_patterns": "{{ ovn_container }}.json"}, "loop": ["ovn_cluster_north_db_server", "ovn_cluster_south_db_server", "ovn_cluster_northd"], "loop_control": {"loop_var": "ovn_container"}}]}]}, {"service_name": "ovn_metadata", "monitoring_subscription": "overcloud-ovn-metadata", "config_settings": {"neutron::notification_driver": ["noop"], "neutron::core_plugin": "ml2", "neutron::service_plugins": ["qos", "ovn-router", "trunk", "segments", "port_forwarding", "log"], "neutron::report_interval": 300, "neutron::logging::debug": true, "neutron::purge_config": false, "neutron::allow_overlapping_ips": true, "neutron::dhcp_agent_notification": true, "neutron::dns_domain": "localdomain", "neutron::rabbit_heartbeat_timeout_threshold": 60, "neutron::host": "%{hiera('fqdn_canonical')}", "neutron::db::database_db_max_retries": -1, "neutron::db::database_max_retries": -1, "neutron::db::sync::db_sync_timeout": 900, "neutron::global_physnet_mtu": 1500, "neutron::db::sync::extra_params": "", "neutron::vlan_transparent": false, "neutron::rabbit_heartbeat_in_pthread": false, "tripleo::profile::base::neutron::ovn_metadata_agent_wrappers::enable_haproxy_wrapper": true, "tripleo::profile::base::neutron::ovn_metadata_agent_wrappers::haproxy_process_wrapper": "/var/lib/neutron/ovn_metadata_haproxy_wrapper", "tripleo::profile::base::neutron::ovn_metadata_agent_wrappers::haproxy_image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-metadata-agent-ovn:17.1", "tripleo::profile::base::neutron::ovn_metadata_agent_wrappers::debug": true, "tripleo::profile::base::neutron::container_cli": "podman", "neutron::agents::ovn_metadata::shared_secret": "mfZEPZHNi0vmUaK2wU20kjgr2", "neutron::agents::ovn_metadata::auth_password": "Y7cJplS04DjBDFMUWgbx3kdiC", "neutron::agents::ovn_metadata::auth_url": "http://172.17.0.2:5000", "neutron::agents::ovn_metadata::auth_tenant": "service", "neutron::agents::ovn_metadata::metadata_host": "172.17.0.2", "neutron::agents::ovn_metadata::ovsdb_connection_timeout": 180, "neutron::agents::ovn_metadata::ovsdb_probe_interval": 60000, "neutron::agents::ovn_metadata::hwol_qos_enabled": false, "ovn::northbound::port": 6641, "ovn::southbound::port": 6642, "neutron::agents::ovn_metadata::debug": true, "neutron::agents::ovn_metadata::metadata_protocol": "http", "neutron::agents::ovn_metadata::metadata_workers": "1"}, "puppet_config": {"puppet_tags": "neutron_config,ovn_metadata_agent_config", "config_volume": "neutron", "step_config": "include tripleo::profile::base::neutron::ovn_metadata\n", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-server:17.1", "volumes": ["/lib/modules:/lib/modules:ro", "/run/openvswitch:/run/openvswitch:shared,z"]}, "kolla_config": {"/var/lib/kolla/config_files/ovn_metadata_agent.json": {"command": "/usr/bin/networking-ovn-metadata-agent --config-file /etc/neutron/neutron.conf --config-file /etc/neutron/plugins/networking-ovn/networking-ovn-metadata-agent.ini --log-file=/var/log/neutron/ovn-metadata-agent.log", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/neutron", "owner": "neutron:neutron", "recurse": true}, {"path": "/var/lib/neutron", "owner": "neutron:neutron", "recurse": true}, {"path": "/etc/pki/tls/certs/ovn_metadata.crt", "owner": "neutron:neutron", "optional": true, "perm": "0644"}, {"path": "/etc/pki/tls/private/ovn_metadata.key", "owner": "neutron:neutron", "optional": true, "perm": "0644"}]}}, "container_config_scripts": {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}}, "docker_config": {"step_2": {"create_haproxy_wrapper": {"start_order": 1, "detach": false, "net": "host", "pid": "host", "user": "root", "command": ["/container_puppet_apply.sh", "4", "file", "include ::tripleo::profile::base::neutron::ovn_metadata_agent_wrappers"], "image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-metadata-agent-ovn:17.1", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/var/lib/container-config-scripts/container_puppet_apply.sh:/container_puppet_apply.sh:ro", "/etc/puppet:/tmp/puppet-etc:ro", "/usr/share/openstack-puppet/modules:/usr/share/openstack-puppet/modules:ro", "/run/openvswitch:/run/openvswitch:shared,z", "/var/lib/neutron:/var/lib/neutron:shared,z"]}}, "step_4": {"setup_ovs_manager": {"start_order": 0, "detach": false, "net": "host", "privileged": true, "user": "root", "command": ["/container_puppet_apply.sh", "4", "exec", "include tripleo::profile::base::neutron::ovn_metadata"], "image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-metadata-agent-ovn:17.1", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/var/lib/container-config-scripts/container_puppet_apply.sh:/container_puppet_apply.sh:ro", "/etc/puppet:/tmp/puppet-etc:ro", "/usr/share/openstack-puppet/modules:/usr/share/openstack-puppet/modules:ro", "/lib/modules:/lib/modules:ro", "/run/openvswitch:/run/openvswitch:shared,z"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}, "ovn_metadata_agent": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-metadata-agent-ovn:17.1", "net": "host", "pid": "host", "cgroupns": "host", "privileged": true, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/neutron:/var/log/neutron:z", "/var/lib/kolla/config_files/ovn_metadata_agent.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/neutron:/var/lib/kolla/config_files/src:ro", "/lib/modules:/lib/modules:ro", "/run/openvswitch:/run/openvswitch:shared,z", "/var/lib/neutron:/var/lib/neutron:shared,z", "/run/netns:/run/netns:shared", "/var/lib/neutron/kill_scripts:/etc/neutron/kill_scripts:shared,z", "/var/lib/neutron/ovn_metadata_haproxy_wrapper:/usr/local/bin/haproxy:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "metadata_settings": [], "deploy_steps_tasks": [{"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "ovn_metadata", "dns": "{{fqdn_internal_api}}", "principal": "ovn_metadata/{{fqdn_internal_api}}@{{idm_realm}}", "key_size": "2048", "ca": "ipa"}]}}]}], "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/neutron", "setype": "container_file_t", "mode": "0750"}]}, {"name": "create /run/netns with temp namespace", "command": "ip netns add ns_temp", "register": "ipnetns_add_result", "failed_when": false}, {"name": "remove temp namespace", "command": "ip netns delete ns_temp", "failed_when": false, "when": ["ipnetns_add_result.rc is defined", "ipnetns_add_result.rc == 0"]}, {"name": "create /var/lib/neutron", "file": {"path": "/var/lib/neutron", "state": "directory", "setype": "container_file_t"}}, {"name": "set conditions", "set_fact": {"haproxy_wrapper_enabled": true, "debug_enabled": true, "docker_additional_sockets": ["/var/lib/openstack/docker.sock"]}}, {"name": "create kill_scripts directory within /var/lib/neutron", "file": {"state": "directory", "path": "/var/lib/neutron/kill_scripts"}}, {"name": "create haproxy kill script", "when": "haproxy_wrapper_enabled|bool", "copy": {"dest": "/var/lib/neutron/kill_scripts/haproxy-kill", "mode": 493, "content": "#!/bin/bash\n{% if debug_enabled|bool -%}\nset -x\n{% endif -%}\nadd_date() {\n  echo \"$(date) $@\"\n}\n\n# Set up script logging for debugging purpose.\n# It will be taken care of by logrotate since there is the .log\n# suffix.\nexec 3>&1 4>&2\ntrap 'exec 2>&4 1>&3' 0 1 2 3\nexec 1>>/var/log/neutron/kill-script.log 2>&1\n\nSIG=$1\nPID=$2\nNETNS=$(ip netns identify ${PID})\n\nif [ \"x${NETNS}\" == \"x\" ]; then\n  CLI=\"nsenter --all --preserve-credentials -t 1 podman\"\n  SIG=9\nelse\n  CLI=\"nsenter --net=/run/netns/${NETNS} --preserve-credentials -m -t 1 podman\"\nfi\n\nkill_container() {\n  add_date \"Killing container $1 ($2)\"\n  signal_container $1 $2 9\n}\n\nstop_container() {\n  add_date \"Stopping container $1 ($2)\"\n  $CLI stop $2\n}\n\nsignal_container() {\n  SIGNAL=$3\n  if [ -z \"$SIGNAL\" ]; then\n      SIGNAL=\"HUP\"\n  fi\n  add_date \"Sending signal '$SIGNAL' to $1 ($2)\"\n  $CLI kill --signal $SIGNAL $2\n}\n\ndelete_container() {\n  add_date \"Deleting container $1 ($2)\"\n  $CLI rm $2 || echo \"Deleting container $1 ($2) failed\"\n}\n\n\n{% raw -%}\nif [ -f /proc/$PID/cgroup ]; then\n  # Get container ID based on process cgroups\n  CT_ID=$(awk 'BEGIN {FS=\".scope|-\"} /\\/libpod-/ {if ($(NF-1)) print $(NF-1);exit}' /proc/$PID/cgroup)\n  CT_NAME=$($CLI inspect -f '{{.Name}}' $CT_ID)\n\n  case $SIG in\n    HUP)\n      signal_container $CT_NAME $CT_ID\n      ;;\n    9)\n      kill_container $CT_NAME $CT_ID\n      delete_container $CT_NAME $CT_ID\n      ;;\n    15)\n      stop_container $CT_NAME $CT_ID\n      delete_container $CT_NAME $CT_ID\n      ;;\n    *)\n      add_date \"Unknown action ${SIG} for ${CT_NAME} ${CT_ID}\"\n      exit 1\n      ;;\n  esac\n\nelse\n  add_date \"No such PID: ${PID}\"\n  exit 1\nfi\n{% endraw %}\n"}}], "upgrade_tasks": [], "external_update_tasks": [{"name": "Check if python3-openvswitch3.1 workaround is needed", "delegate_to": "{{ item }}", "become": true, "tags": ["ovn", "ovn_image"], "when": "step|int == 1", "shell": "if podman exec ovn_metadata_agent rpm -q python3-openvswitch3.1 > /dev/null ; then echo REQUIRED; else echo OK ; fi\n", "register": "metadata_python_ovs_3_1", "failed_when": false, "loop": "{{ groups['ovn_metadata'] | difference(groups['excluded_overcloud']) }}"}, {"name": "Update ovn_metadata_agents", "vars": {"hosts_to_update_metadata": "{{ metadata_python_ovs_3_1.results | selectattr('stdout', 'eq', 'REQUIRED') | map(attribute='item') | list }}"}, "tags": ["ovn", "ovn_image"], "when": ["step|int == 1", "hosts_to_update_metadata | length > 0"], "block": [{"name": "Force pull image in case image name doesn't change.", "become": true, "loop": "{{ hosts_to_update_metadata }}", "delegate_to": "{{ item }}", "async": 600, "poll": 0, "register": "ovn_metadata_image_update", "containers.podman.podman_image": {"name": "registry.redhat.io/rhosp-rhel9/openstack-neutron-metadata-agent-ovn:17.1", "validate_certs": false, "force": true}}, {"name": "Was the ovn_metadata_agent image pull successful.", "when": ["'results' in ovn_metadata_image_update"], "become": true, "delegate_to": "{{ async_result_item.item }}", "async_status": {"jid": "{{ async_result_item.ansible_job_id }}"}, "loop": "{{ovn_metadata_image_update.results }}", "loop_control": {"loop_var": "async_result_item"}, "register": "async_poll_results", "until": "async_poll_results.finished", "retries": 600, "delay": 1}, {"name": "Now rotate all ovn_metadata_agents", "tags": ["ovn_metadata_parallel"], "delegate_to": "{{ item }}", "become": true, "tripleo_container_manage": {"config_dir": "/var/lib/tripleo-config/container-startup-config/step_4", "config_id": ["ovn_metadata_agent"], "config_patterns": "*ovn_metadata_agent.json", "config_overrides": {".*ovn_metadata_agent": {"image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-metadata-agent-ovn:17.1", "name": "ovn_metadata_agent"}}}, "loop": "{{ hosts_to_update_metadata }}", "async": 600, "poll": 0}]}, {"name": "Now rotate all ovn_metadata_agents", "vars": {"hosts_to_update_metadata": "{{ metadata_python_ovs_3_1.results | selectattr('stdout', 'eq', 'REQUIRED') | map(attribute='item') | list }}"}, "tags": ["never", "ovn_metadata_serial"], "when": ["step|int == 1", "hosts_to_update_metadata | length > 0"], "delegate_to": "{{ item }}", "become": true, "tripleo_container_manage": {"config_dir": "/var/lib/tripleo-config/container-startup-config/step_4", "config_id": ["ovn_metadata_agent"], "config_patterns": "*ovn_metadata_agent.json", "config_overrides": {".*ovn_metadata_agent": {"image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-metadata-agent-ovn:17.1", "name": "ovn_metadata_agent"}}}, "loop": "{{ hosts_to_update_metadata }}", "loop_control": {"pause": 1}}]}, {"service_name": "openstack_clients", "host_prep_tasks": [{"include_role": {"name": "tripleo_clients_install"}}]}, {"service_name": "oslo_messaging_notify", "global_config_settings": {"oslo_messaging_notify_scheme": "rabbit", "oslo_messaging_notify_user_name": "guest", "oslo_messaging_notify_password": "orxoJKQXPaAVXVpoTxqAR2c6v", "oslo_messaging_notify_use_ssl": "False", "oslo_messaging_notify_port": 5672}}, {"service_name": "oslo_messaging_rpc", "firewall_rules": {"109 rabbitmq-bundle": {"dport": [3122, 4369, 5672, 25672, "25673-25683"]}}, "global_config_settings": {"oslo_messaging_rpc_scheme": "rabbit", "oslo_messaging_rpc_user_name": "guest", "oslo_messaging_rpc_password": "orxoJKQXPaAVXVpoTxqAR2c6v", "oslo_messaging_rpc_use_ssl": "False", "oslo_messaging_rpc_port": 5672}, "config_settings": {"rabbitmq::file_limit": 65536, "rabbitmq::default_user": "guest", "rabbitmq::default_pass": "orxoJKQXPaAVXVpoTxqAR2c6v", "rabbit_ipv6": false, "rabbitmq::delete_guest_user": false, "rabbitmq::wipe_db_on_cookie_change": true, "rabbitmq::port": 5672, "rabbitmq::loopback_users": [], "rabbitmq::tcp_backlog": 4096, "rabbitmq::repos_ensure": false, "rabbitmq::tcp_keepalive": true, "rabbitmq_environment": {"LANG": "en_US.UTF-8", "LC_ALL": "en_US.UTF-8", "NODE_PORT": "", "NODE_IP_ADDRESS": "", "RABBITMQ_NODENAME": "rabbit@%{hiera('fqdn_internal_api')}", "RABBITMQ_USE_LONGNAME": "true", "RABBITMQ_CTL_DIST_PORT_MIN": "25673", "RABBITMQ_CTL_DIST_PORT_MAX": "25683", "export ERL_EPMD_ADDRESS": "%{hiera('rabbitmq::interface')}"}, "tripleo::profile::base::rabbitmq::additional_erl_args": "'+sbwt none +sbwtdcpu none +sbwtdio none'", "rabbitmq_kernel_variables": {"inet_dist_listen_min": "25672", "inet_dist_listen_max": "25672", "net_ticktime": 15}, "rabbitmq_config_variables": {"cluster_partition_handling": "ignore", "queue_master_locator": "<<\"min-masters\">>"}, "rabbitmq::config_additional_variables": {"lager": "[{crash_log_count, 0}, { crash_log_date, \"\"}, { crash_log_size, 0 }]"}, "rabbitmq::erlang_cookie": "XNOQDfh9VshBYfsmJBif", "rabbitmq::interface": "%{hiera('internal_api')}", "rabbitmq::nr_ha_queues": 0, "rabbitmq::ssl": false, "rabbitmq::ssl_erl_dist": false, "rabbitmq::ssl_port": 5672, "rabbitmq::ssl_depth": 1, "rabbitmq::ssl_only": false, "rabbitmq::ssl_interface": "%{hiera('internal_api')}", "tripleo::profile::base::rabbitmq::enable_internal_tls": false, "tripleo::profile::base::rabbitmq::fips_mode": false, "rabbitmq::collect_statistics_interval": 30000, "rabbitmq::admin_enable": false, "rabbitmq::management_enable": true, "rabbitmq::use_config_file_for_plugins": true, "rabbitmq::management_ip_address": "127.0.0.1", "rabbitmq::management_port": 15672, "rabbitmq::config_management_variables": {"rates_mode": "none"}, "rabbitmq::ssl_cacert": "/etc/ipa/ca.crt", "rabbitmq::ssl_management_cacert": "/etc/ipa/ca.crt", "rabbitmq::service_manage": false, "tripleo::profile::pacemaker::rabbitmq_bundle::rabbitmq_docker_image": "cluster.common.tag/rabbitmq:pcmklatest", "tripleo::profile::pacemaker::rabbitmq_bundle::control_port": 3122, "tripleo::profile::pacemaker::rabbitmq_bundle::container_backend": "podman", "tripleo::profile::pacemaker::rabbitmq_bundle::force_ocf": true}, "service_config_settings": null, "puppet_config": {"config_volume": "rabbitmq", "puppet_tags": "file,file_line", "step_config": "['Rabbitmq_policy', 'Rabbitmq_user'].each |String $val| { noop_resource($val) }\ninclude tripleo::profile::pacemaker::rabbitmq_bundle", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/rabbitmq.json": {"command": "/usr/sbin/pacemaker_remoted", "config_files": [{"dest": "/etc/libqb/force-filesystem-sockets", "source": "/dev/null", "owner": "root", "perm": "0644"}, {"dest": "/var/log/btmp", "source": "/dev/null", "owner": "root:utmp", "perm": "0600"}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src-tls/*", "dest": "/", "merge": true, "optional": true, "preserve_properties": true}], "permissions": [{"path": "/var/lib/rabbitmq", "owner": "rabbitmq:rabbitmq", "recurse": true}, {"path": "/var/log/rabbitmq", "owner": "rabbitmq:rabbitmq", "recurse": true}, {"path": "/etc/pki/tls/certs/rabbitmq.crt", "owner": "rabbitmq:rabbitmq", "perm": "0600", "optional": true}, {"path": "/etc/pki/tls/private/rabbitmq.key", "owner": "rabbitmq:rabbitmq", "perm": "0600", "optional": true}]}}, "container_config_scripts": {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}}, "docker_config": {"step_1": {"rabbitmq_bootstrap": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.1", "net": "host", "privileged": false, "user": "root", "command": ["bash", "-ec", "kolla_set_configs\nif [[ -e \"/var/lib/rabbitmq/.erlang.cookie\" ]]; then rm -f /var/lib/rabbitmq/.erlang.cookie; fi\nhiera 'rabbitmq::erlang_cookie' > /var/lib/rabbitmq/.erlang.cookie\nchown rabbitmq:rabbitmq /var/lib/rabbitmq/.erlang.cookie\nchmod 400 /var/lib/rabbitmq/.erlang.cookie"], "volumes": ["/var/lib/kolla/config_files/rabbitmq.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/rabbitmq:/var/lib/kolla/config_files/src:ro", "/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/var/lib/rabbitmq:/var/lib/rabbitmq:z", "/etc/puppet:/etc/puppet:ro,z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_2": {"rabbitmq_wait_bundle": {"start_order": 0, "detach": false, "net": "host", "ipc": "host", "user": "root", "command": ["/container_puppet_apply.sh", "2", "file,file_line,concat,augeas,rabbitmq_policy,rabbitmq_user,rabbitmq_ready", "include tripleo::profile::pacemaker::rabbitmq_bundle", ""], "image": "registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.1", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/var/lib/container-config-scripts/container_puppet_apply.sh:/container_puppet_apply.sh:ro", "/etc/puppet:/tmp/puppet-etc:ro", "/usr/share/openstack-puppet/modules:/usr/share/openstack-puppet/modules:ro", "/bin/true:/bin/epmd"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "KOLLA_BOOTSTRAP": true, "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}}, "metadata_settings": null, "host_prep_tasks": [{"name": "create fcontext for rabbitmq data", "community.general.sefcontext": {"target": "/var/lib/rabbitmq(/.*)?", "setype": "container_file_t", "state": "present"}}, {"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode|default(omit) }}"}, "with_items": [{"path": "/var/lib/rabbitmq", "setype": "container_file_t"}, {"path": "/var/log/containers/rabbitmq", "setype": "container_file_t", "mode": "0750"}]}, {"name": "stop the Erlang port mapper on the host and make sure it cannot bind to the port used by container", "shell": "echo 'export ERL_EPMD_ADDRESS=127.0.0.1' > /etc/rabbitmq/rabbitmq-env.conf\necho 'export ERL_EPMD_PORT=4370' >> /etc/rabbitmq/rabbitmq-env.conf\nfor pid in $(pgrep epmd --ns 1 --nslist pid); do kill $pid; done\n"}], "deploy_steps_tasks": [{"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "rabbitmq", "dns": "{{fqdn_internal_api}}", "principal": "rabbitmq/{{fqdn_internal_api}}@{{idm_realm}}", "run_after": "container_name=$({{container_cli}} ps --format=\\{\\{.Names\\}\\} | grep -w -E 'rabbitmq(-bundle-.*-[0-9]+)?')\nservice_crt=\"/etc/pki/tls/certs/rabbitmq.crt\"\nservice_key=\"/etc/pki/tls/private/rabbitmq.key\"\nif echo \"$container_name\" | grep -q \"^rabbitmq-bundle\"; then\n  # lp#1917868: Do not use podman cp with HA containers as they get\n  # frozen temporarily and that can make pacemaker operation fail.\n  tar -c \"$service_crt\" \"$service_key\" | {{container_cli}} exec -i \"$container_name\" tar -C / -xv\n  # no need to update the mount point, because pacemaker\n  # recreates the container when it's restarted\nelse\n  # Refresh the cert at the mount-point\n  {{container_cli}} cp $service_crt \"$container_name:/var/lib/kolla/config_files/src-tls/$service_crt\"\n  # Refresh the key at the mount-point\n  {{container_cli}} cp $service_key \"$container_name:/var/lib/kolla/config_files/src-tls/$service_key\"\n  # Copy the new cert from the mount-point to the real path\n  {{container_cli}} exec -u root \"$container_name\" cp \"/var/lib/kolla/config_files/src-tls$service_crt\" \"$service_crt\"\n  # Copy the new key from the mount-point to the real path\n  {{container_cli}} exec -u root \"$container_name\" cp \"/var/lib/kolla/config_files/src-tls$service_key\" \"$service_key\"\nfi\n# Set appropriate permissions\n{{container_cli}} exec -u root \"$container_name\" chown rabbitmq:rabbitmq \"$service_crt\"\n{{container_cli}} exec -u root \"$container_name\" chown rabbitmq:rabbitmq \"$service_key\"\n# Trigger a pem cache clear in RabbitMQ to read the new certificates\n{{container_cli}} exec \"$container_name\" rabbitmqctl eval \"ssl:clear_pem_cache().\"\n", "key_size": "2048", "ca": "ipa"}]}}]}, {"name": "RabbitMQ tag container image for pacemaker", "when": "step|int == 1", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.1", "container_image_latest": "cluster.common.tag/rabbitmq:pcmklatest"}}, {"name": "RabbitMQ RPC HA Wrappers Step", "when": "step|int == 2", "block": [{"name": "Rabbitmq rpc puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "oslo_messaging_rpc", "tripleo_ha_wrapper_resource_name": "rabbitmq", "tripleo_ha_wrapper_bundle_name": "rabbitmq-bundle", "tripleo_ha_wrapper_resource_state": "Started", "tripleo_ha_wrapper_puppet_config_volume": "rabbitmq", "tripleo_ha_wrapper_puppet_execute": "[\"Rabbitmq_policy\", \"Rabbitmq_user\"].each |String $val| { noop_resource($val) }; include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::rabbitmq_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ip,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".previous_run"}}]}], "external_update_tasks": [{"when": "step|int == 1", "tags": "ha_image_update", "block": [{"block": [{"name": "Get rabbitmq image from pacemaker", "become": true, "register": "xmllint_pcmk_rabbitmq_rpc_image", "shell": "xmllint --xpath \"string(//bundle[@id='rabbitmq-bundle']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container rabbitmq image", "set_fact": {"rabbitmq_rpc_image": "registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.1", "rabbitmq_rpc_image_latest": "cluster.common.tag/rabbitmq:pcmklatest", "pcmk_rabbitmq_rpc_image": "{{xmllint_pcmk_rabbitmq_rpc_image.stdout}}"}}], "delegate_to": "{{ (groups[\"oslo_messaging_rpc\"] | difference(groups[\"excluded_overcloud\"]))[0] }}"}, {"name": "rabbitmq temporary pacemaker container tag in case of image switch", "become": true, "import_role": {"name": "tripleo_ha_image_update"}, "vars": {"tripleo_ha_image_update_node_names": "{{ groups[\"oslo_messaging_rpc\"] | difference(groups[\"excluded_overcloud\"]) }}", "tripleo_ha_image_update_bundle": "rabbitmq-bundle", "tripleo_ha_image_update_new_image": "{{rabbitmq_rpc_image_latest}}", "tripleo_ha_image_update_old_image": "{{pcmk_rabbitmq_rpc_image}}"}, "when": ["(pcmk_rabbitmq_rpc_image != \"\")", "(pcmk_rabbitmq_rpc_image != rabbitmq_rpc_image_latest)"]}]}], "update_tasks": [{"block": [{"name": "Get rabbitmq image from pacemaker", "become": true, "register": "xmllint_pcmk_rabbitmq_rpc_image", "shell": "xmllint --xpath \"string(//bundle[@id='rabbitmq-bundle']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container rabbitmq image", "set_fact": {"rabbitmq_rpc_image": "registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.1", "rabbitmq_rpc_image_latest": "cluster.common.tag/rabbitmq:pcmklatest", "pcmk_rabbitmq_rpc_image": "{{xmllint_pcmk_rabbitmq_rpc_image.stdout}}"}}], "when": "(step|int == 0 or step|int == 2)"}, {"name": "Check for update of rabbitmq container image name", "when": "step|int == 0", "assert": {"that": "pcmk_rabbitmq_rpc_image == rabbitmq_rpc_image_latest", "fail_msg": "rabbitmq image change detected, run overcloud external-update --tags ha_image_update first"}}, {"name": "Tear-down non-HA rabbitmq container", "when": ["step|int == 1"], "block": [{"name": "Remove non-HA rabbitmq container", "include_role": {"name": "tripleo_container_rm"}, "vars": {"tripleo_container_cli": "{{ container_cli }}", "tripleo_containers_to_rm": ["rabbitmq"]}}]}, {"name": "Rabbitmq fetch and retag container image for pacemaker", "when": "step|int == 2", "block": [{"name": "Retag pcmklatest to latest rabbitmq image", "include_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{rabbitmq_rpc_image}}", "container_image_latest": "{{rabbitmq_rpc_image_latest}}"}}]}], "post_update_tasks": [{"name": "Rabbitmq rpc bundle post update", "when": "step|int == 1", "block": [{"name": "Rabbitmq rpc puppet bundle", "import_role": {"name": "tripleo_ha_wrapper"}, "vars": {"tripleo_ha_wrapper_service_name": "oslo_messaging_rpc", "tripleo_ha_wrapper_resource_name": "rabbitmq", "tripleo_ha_wrapper_bundle_name": "rabbitmq-bundle", "tripleo_ha_wrapper_resource_state": "Started", "tripleo_ha_wrapper_puppet_config_volume": "rabbitmq", "tripleo_ha_wrapper_puppet_execute": "[\"Rabbitmq_policy\", \"Rabbitmq_user\"].each |String $val| { noop_resource($val) }; include ::tripleo::profile::base::pacemaker; include ::tripleo::profile::pacemaker::rabbitmq_bundle", "tripleo_ha_wrapper_puppet_tags": "pacemaker::resource::bundle,pacemaker::property,pacemaker::resource::ip,pacemaker::resource::ocf,pacemaker::constraint::order,pacemaker::constraint::colocation", "tripleo_ha_wrapper_puppet_debug": false, "tripleo_ha_wrapper_config_suffix": ".previous_run"}}], "vars": {"tripleo_ha_wrapper_minor_update": true}}], "upgrade_tasks": [{"name": "Tear-down non-HA rabbitmq container", "when": ["step|int == 0"], "block": [{"name": "Remove non-HA rabbitmq container", "include_role": {"name": "tripleo_container_rm"}, "vars": {"tripleo_container_cli": "{{ container_cli }}", "tripleo_containers_to_rm": ["rabbitmq"]}}]}, {"name": "Prepare switch of rabbitmq image name", "when": ["step|int == 0"], "block": [{"name": "Get rabbitmq image id currently used by pacemaker", "shell": "pcs resource config rabbitmq-bundle | grep -Eo 'image=[^ ]+' | awk -F= '{print $2;}'", "register": "rabbitmq_rpc_image_current_res", "failed_when": false}, {"name": "Image facts for rabbitmq", "set_fact": {"rabbitmq_rpc_image_latest": "cluster.common.tag/rabbitmq:pcmklatest", "rabbitmq_rpc_image_current": "{{rabbitmq_rpc_image_current_res.stdout}}"}}, {"name": "Temporarily tag the current rabbitmq image id with the upgraded image name", "import_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{rabbitmq_rpc_image_current}}", "container_image_latest": "{{rabbitmq_rpc_image_latest}}", "pull_image": false}, "when": ["rabbitmq_rpc_image_current != ''", "rabbitmq_rpc_image_current != rabbitmq_rpc_image_latest"]}, {"name": "Create rabbitmq_rpc retag statefile", "file": {"path": "/var/lib/tripleo/rabbitmq_rpc_needs_retag", "state": "touch"}, "when": ["rabbitmq_rpc_image_current != ''", "rabbitmq_rpc_image_current != rabbitmq_rpc_image_latest"]}]}, {"name": "Update rabbitmq-bundle pcs resource bundle for new container image", "when": ["step|int == 1"], "block": [{"name": "set is_rpc_rabbitmq_bootstrap_node fact", "set_fact": "is_rpc_rabbitmq_bootstrap_node={{oslo_messaging_rpc_short_bootstrap_node_name|lower == ansible_facts['hostname']|lower}}"}, {"name": "Check for rabbitmq_rpc retag statefile", "stat": {"path": "/var/lib/tripleo/rabbitmq_rpc_needs_retag"}, "register": "rabbitmq_rpc_retag_state_file"}]}, {"name": "Update rabbitmq-bundle pcs resource bundle for new container image", "when": ["step|int == 1", "is_rpc_rabbitmq_bootstrap_node|bool", "rabbitmq_rpc_retag_state_file.stat.exists|bool"], "block": [{"name": "Disable the rabbitmq cluster resource before container upgrade", "pacemaker_resource": {"resource": "rabbitmq-bundle", "state": "disable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}, {"name": "Check rabbitmq logging configuration in pacemaker", "command": "cibadmin --query --xpath \"//storage-mapping[@id='rabbitmq-log']\"", "failed_when": false, "register": "rabbitmq_logs_moved"}, {"name": "Add a bind mount for logging in the rabbitmq bundle", "when": "rabbitmq_logs_moved.rc == 6", "command": "pcs resource bundle update rabbitmq-bundle storage-map add id=rabbitmq-log source-dir=/var/log/containers/rabbitmq target-dir=/var/log/rabbitmq options=rw"}, {"name": "Update the rabbitmq bundle to use the new container image name", "command": "pcs resource bundle update rabbitmq-bundle container image={{rabbitmq_rpc_image_latest}}"}, {"name": "Enable the rabbitmq cluster resource", "pacemaker_resource": {"resource": "rabbitmq-bundle", "state": "enable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}]}, {"name": "Check for rabbitmq_rpc retag statefile", "when": ["step|int == 3"], "stat": {"path": "/var/lib/tripleo/rabbitmq_rpc_needs_retag"}, "register": "rabbitmq_rpc_retag_state_file"}, {"name": "Retag rabbitmq-bundle container image", "when": ["step|int == 3", "rabbitmq_rpc_retag_state_file.stat.exists|bool"], "block": [{"name": "Disable the rabbitmq cluster resource before container upgrade", "pacemaker_resource": {"resource": "rabbitmq-bundle", "state": "disable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}, {"name": "Retag the pacemaker image if containerized", "block": [{"block": [{"name": "Get rabbitmq image from pacemaker", "become": true, "register": "xmllint_pcmk_rabbitmq_rpc_image", "shell": "xmllint --xpath \"string(//bundle[@id='rabbitmq-bundle']/podman/@image)\" /var/lib/pacemaker/cib/cib.xml"}, {"name": "Get container rabbitmq image", "set_fact": {"rabbitmq_rpc_image": "registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.1", "rabbitmq_rpc_image_latest": "cluster.common.tag/rabbitmq:pcmklatest", "pcmk_rabbitmq_rpc_image": "{{xmllint_pcmk_rabbitmq_rpc_image.stdout}}"}}]}, {"block": [{"name": "Retag pcmklatest to latest rabbitmq image", "include_role": {"name": "tripleo_container_tag"}, "vars": {"container_image": "{{rabbitmq_rpc_image}}", "container_image_latest": "{{rabbitmq_rpc_image_latest}}"}}]}]}, {"name": "Enable the rabbitmq cluster resource", "pacemaker_resource": {"resource": "rabbitmq-bundle", "state": "enable", "wait_for_resource": true}, "register": "output", "retries": 5, "until": "output.rc == 0"}, {"name": "Remove rabbitmq_rpc retag statefile", "file": {"path": "/var/lib/tripleo/rabbitmq_rpc_needs_retag", "state": "absent"}}]}]}, {"service_name": "pacemaker", "monitoring_subscription": "overcloud-pacemaker", "firewall_rules": {"130 pacemaker tcp": {"proto": "tcp", "dport": [2224, 3121, 21064]}, "131 pacemaker udp": {"proto": "udp", "dport": 5405}}, "config_settings": {"pacemaker::corosync::cluster_name": "tripleo_cluster", "pacemaker::corosync::manage_fw": false, "pacemaker::resource_defaults::defaults": {"resource-stickiness": {"value": "INFINITY"}}, "corosync_token_timeout": 10000, "pacemaker::corosync::settle_tries": 360, "pacemaker::resource::bundle::deep_compare": true, "pacemaker::resource::ip::deep_compare": true, "pacemaker::resource::ocf::deep_compare": true, "corosync_ipv6": false, "tripleo::fencing::config": {}, "tripleo::fencing::deep_compare": true, "enable_fencing": false, "hacluster_pwd": "V7RyZXR24nKo1fUk", "tripleo::profile::base::pacemaker::remote_authkey": "[encoded data, 4096 characters]", "tripleo::profile::base::pacemaker::pcsd_bind_addr": "%{hiera('internal_api')}", "tripleo::profile::base::pacemaker::resource_op_defaults": {"bundle": {"name": "timeout", "value": "120s"}}}, "service_config_settings": {"rsyslog": {"tripleo_logging_sources_pacemaker": [{"tag": "system.pacemaker", "file": "/var/log/host/pacemaker/pacemaker.log", "startmsg.regex": "^[a-zA-Z]{3} [0-9]{2} [:0-9]{8}"}]}}, "step_config": "include tripleo::profile::base::pacemaker\n", "host_prep_tasks": [{"name": "Make sure python3-novaclient is installed when IHA is enabled", "package": {"name": "python3-novaclient", "state": "present"}, "when": false}, {"name": "Remove existing entries from logind conf", "ansible.builtin.lineinfile": {"path": "/etc/systemd/logind.conf", "regexp": "^\\s*#?\\s*HandlePowerKey\\s*=.*", "state": "absent"}}, {"name": "Make sure systemd-logind ignores power off", "ansible.builtin.lineinfile": {"path": "/etc/systemd/logind.conf", "regexp": "^#?HandlePowerKey", "line": "HandlePowerKey=ignore"}}, {"name": "Restart systemd-logind", "ansible.builtin.service": {"name": "systemd-logind", "state": "restarted"}}, {"name": "Gather service_facts on pacemaker_bootstrap_node", "ansible.builtin.service_facts": null, "when": ["{{(pacemaker_short_bootstrap_node_name|lower == ansible_facts['hostname']|lower)|bool}}", false]}, {"name": "Check and eventually delete duplicate constraints (bootstrap node)", "become": true, "shell": "COUNT=$(cibadmin --query | xmllint --xpath '//rsc_location[@rsc=\"stonith-fence_compute-fence-nova\"]/@id' - |grep -oP '(?<=[\"])[^\"]*' -c)\nif [[ $COUNT > 1 ]]; then\n    echo \"Detected $COUNT duplicate constraints, deleting them\"\n    # assemble string with current list of computes\n    GOOD=$(echo location-stonith-fence_compute-fence-nova-$(hiera pacemaker_remote_short_node_names |grep -oP '\".*?\"' |tr -d '\"'|tr -d '\\n' )--10000)\n    # delete old constraints\n    for i in $(cibadmin --query | xmllint --xpath '//rsc_location[@rsc=\"stonith-fence_compute-fence-nova\"]/@id' - |grep -oP '(?<=[\"])[^\"]*' |grep -v $GOOD ); do pcs constraint delete $i ; done\nelse\n    echo \"No duplicate constraint found\"\nfi\n", "when": ["{{(pacemaker_short_bootstrap_node_name|lower == ansible_facts['hostname']|lower)|bool}}", false, "ansible_facts['services']['pacemaker.service']['state'] == 'running'"]}], "scale_tasks": [{"when": ["step|int == 1", "container_cli == 'podman'", "\"pacemaker_remote\" in enabled_services|list", "inventory_hostname_short in compute_instanceha_short_node_names"], "tags": "down", "become": true, "block": [{"name": "Getting Nova compute hostname", "command": "crm_node -n", "register": "nova_compute_hostname"}, {"name": "Check if pacemaker_short_bootstrap_node_name is a defined variable", "when": "pacemaker_short_bootstrap_node_name is not defined", "fail": {"msg": "Cannot delegate pacemaker cleanup to the bootstrap node. Please delete the resources manually."}, "ignore_errors": true}, {"name": "Clean up Pacemaker remote and STONITH resources for Compute node", "when": ["nova_compute_hostname.stdout in compute_instanceha_short_node_names", "pacemaker_short_bootstrap_node_name is defined"], "block": [{"name": "List STONITH resource for the Compute node", "command": "stonith_admin -l {{ nova_compute_hostname.stdout }}", "register": "stonith_service_results", "delegate_to": "{{ pacemaker_short_bootstrap_node_name }}"}, {"name": "Disable the STONITH resources for the Compute node", "command": "pcs stonith disable \"{{ item }}\"", "loop": "{{ stonith_service_results.stdout | regex_findall('^(stonith-fence(?!_compute-fence-nova).*)', multiline=True)}}", "delegate_to": "{{ pacemaker_short_bootstrap_node_name }}", "register": "disable_output", "failed_when": "disable_output.rc != 0", "retries": 3, "delay": 5, "until": "disable_output.rc == 0"}, {"name": "Delete the STONITH resources for the Compute node", "command": "pcs stonith delete \"{{ item }}\"", "loop": "{{ stonith_service_results.stdout | regex_findall('^(stonith-fence(?!_compute-fence-nova).*)', multiline=True)}}", "delegate_to": "{{ pacemaker_short_bootstrap_node_name }}", "register": "delete_output", "failed_when": "delete_output.rc != 0", "retries": 3, "delay": 5, "until": "delete_output.rc == 0"}, {"name": "Clear the stonith level hierarchy for the Compute node target", "command": "pcs stonith level clear target {{ nova_compute_hostname.stdout }}", "delegate_to": "{{ pacemaker_short_bootstrap_node_name }}"}, {"name": "Delete Compute node from cluster", "command": "pcs cluster node remove-remote {{ nova_compute_hostname.stdout }}", "delegate_to": "{{ pacemaker_short_bootstrap_node_name }}"}, {"name": "Remove compute node in pcmk_host_list parameter from stonith-fence_compute-fence-nova", "command": "crm_resource --set-parameter=pcmk_host_list -r stonith-fence_compute-fence-nova -v \"{{ pacemaker_remote_short_node_names | reject('search', nova_compute_hostname.stdout) | join(',') }}\"", "delegate_to": "{{ pacemaker_short_bootstrap_node_name }}"}]}]}], "upgrade_tasks": [{"name": "Ensure redis is removed", "when": ["step|int == 5", "\"redis\" not in enabled_services|list", "{{ (pacemaker_short_bootstrap_node_name|lower == ansible_facts['hostname']|lower)|bool}}"], "become": true, "shell": "if crm_resource -r redis-bundle -q &>/dev/null; then\n   pcs resource delete redis-bundle || true\n   pcs resource delete ip-$(hiera redis_vip) || true\nfi\n"}, {"name": "Clean up cluster node cache", "when": ["step|int == 5", "\"redis\" not in enabled_services|list", "{{ (pacemaker_short_bootstrap_node_name|lower == ansible_facts['hostname']|lower)|bool}}"], "become": true, "shell": "pcs cluster node clear redis-bundle-0\npcs cluster node clear redis-bundle-1\npcs cluster node clear redis-bundle-2\ncrm_attribute --name redis_REPL_INFO --delete\n"}, {"name": "Clean up redis attribute", "when": ["step|int == 5", "\"redis\" not in enabled_services|list", "{{ (pacemaker_short_bootstrap_node_name|lower == ansible_facts['hostname']|lower)|bool}}"], "become": true, "shell": "pcs node attribute \"{{ item }}\" redis-role= || true\n", "loop": "{{ pacemaker_short_node_names }}"}], "external_upgrade_tasks": [{"when": ["step|int == 1"], "tags": ["never", "system_upgrade_stop_services", "system_upgrade_transfer_data"], "block": [{"name": "Stop cluster", "become": true, "shell": "set -eu\nFILE=/usr/sbin/pcs\nif test -f \"$FILE\"; then\n  /usr/sbin/pcs cluster stop --force\nfi\n", "delegate_to": "{{ item }}", "with_items": "{{ groups['pacemaker'] | difference(groups['excluded_overcloud']) }}"}]}], "update_tasks": [{"name": "Check pacemaker cluster running before the minor update", "when": "step|int == 0", "pacemaker_cluster": "state=online check_and_fail=true", "async": 30, "poll": 4}, {"name": "Acquire the cluster shutdown lock to stop pacemaker cluster", "when": "step|int == 1", "command": "systemd-cat -t ha-shutdown /var/lib/container-config-scripts/pacemaker_mutex_shutdown.sh --acquire"}, {"name": "Stop pacemaker cluster", "when": "step|int == 1", "pacemaker_cluster": "state=offline"}, {"name": "Start pacemaker cluster", "when": "step|int == 4", "pacemaker_cluster": "state=online"}, {"name": "Release the cluster shutdown lock", "when": "step|int == 4", "command": "systemd-cat -t ha-shutdown /var/lib/container-config-scripts/pacemaker_mutex_shutdown.sh --release"}]}, {"service_name": "placement", "firewall_rules": {"138 placement": {"dport": [8778]}}, "firewall_frontend_rules": {"100 placement_haproxy_frontend": {"dport": [8778]}}, "firewall_ssl_frontend_rules": {"100 placement_haproxy_frontend_ssl": {"dport": [13778]}}, "keystone_resources": {"placement": {"endpoints": {"public": "http://172.21.0.2:8778/placement", "internal": "http://172.17.0.2:8778/placement", "admin": "http://172.17.0.2:8778/placement"}, "users": {"placement": {"password": "P6qVEsGa91uUJyG6cL2sqYpFU"}}, "region": "regionOne", "service": "placement"}}, "config_settings": {"apache::default_vhost": false, "placement::keystone::authtoken::project_name": "service", "placement::keystone::authtoken::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "placement::keystone::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "placement::keystone::authtoken::auth_url": "http://172.17.0.2:5000", "placement::keystone::authtoken::region_name": "regionOne", "placement::keystone::authtoken::interface": "internal", "placement::keystone::authtoken::memcache_use_advanced_pool": true, "placement::logging::debug": true, "placement::policy::enforce_new_defaults": false, "placement::policy::enforce_scope": false, "placement::policy::policies": {}, "placement::wsgi::apache::api_port": "8778", "placement::wsgi::apache::ssl": false, "placement::wsgi::apache::bind_host": "%{hiera('internal_api')}", "placement::wsgi::apache::servername": "%{hiera('fqdn_internal_api')}", "placement::db::database_connection": "mysql+pymysql://placement:P6qVEsGa91uUJyG6cL2sqYpFU@172.17.0.2/placement?read_default_file=/etc/my.cnf.d/tripleo.cnf&read_default_group=tripleo", "placement::wsgi::apache::workers": 1}, "service_config_settings": {"rsyslog": {"tripleo_logging_sources_placement": [{"tag": "openstack.placement", "file": "/var/log/containers/placement/placement.log"}]}, "mysql": {"placement::db::mysql::password": "P6qVEsGa91uUJyG6cL2sqYpFU", "placement::db::mysql::user": "placement", "placement::db::mysql::host": "%", "placement::db::mysql::dbname": "placement"}}, "puppet_config": {"config_volume": "placement", "puppet_tags": "placement_config", "step_config": "include tripleo::profile::base::database::mysql::client\ninclude tripleo::profile::base::placement::api", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-placement-api:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/placement_api.json": {"command": "/usr/sbin/httpd -DFOREGROUND", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/placement", "owner": "placement:placement", "recurse": true}]}, "/var/lib/kolla/config_files/placement_api_db_sync.json": {"command": "/usr/bin/bootstrap_host_exec placement su placement -s /bin/bash -c '/usr/bin/placement-manage db sync'", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/placement", "owner": "placement:placement", "recurse": true}]}, "/var/lib/kolla/config_files/placement_api_wait_for_service.json": {"command": "/usr/bin/bootstrap_host_exec placement su placement -s /bin/bash -c '/container-config-scripts/pyshim.sh /container-config-scripts/placement_wait_for_service.py'", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/log/placement", "owner": "placement:placement", "recurse": true}]}}, "container_config_scripts": {"container_puppet_apply.sh": {"mode": "0700", "content": "#!/bin/bash\nset -eux\nSTEP=$1\nTAGS=$2\nCONFIG=$3\nEXTRA_ARGS=${4:-''}\nif [ -d /tmp/puppet-etc ]; then\n  # ignore copy failures as these may be the same file depending on docker mounts\n  cp -a /tmp/puppet-etc/* /etc/puppet || true\nfi\necho \"{\\\"step\\\": ${STEP}}\" > /etc/puppet/hieradata/docker_puppet.json\n# $::deployment_type in puppet-tripleo\nexport FACTER_deployment_type=containers\nset +e\npuppet apply $EXTRA_ARGS \\\n    --verbose \\\n    --detailed-exitcodes \\\n    --summarize \\\n    --color=false \\\n    --modulepath /etc/puppet/modules:/opt/stack/puppet-modules:/usr/share/openstack-puppet/modules \\\n    --tags $TAGS \\\n    -e \"noop_resource('package'); ${CONFIG}\"\nrc=$?\nset -e\nset +ux\nif [ $rc -eq 2 -o $rc -eq 0 ]; then\n    exit 0\nfi\nexit $rc\n"}, "pyshim.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n#\n# Usage: pyshim.sh <script and/or arguments>\n#\n# Unfortunately THT doesn't know which version of python might be in a\n# container so we need this script to be able to try python3 or python2\n# depending on availability.  Since this is a temporary shim until we've\n# fully cut over to python3, we check for the existence of python3 first\n# before falling back to python2. This will help in the transition from\n# python2 based containers to python3.\n\nshow_usage() {\n    echo \"Usage: pyshim.sh <script and/or arguments>\"\n}\n\nif [ $# -lt 1 ]\nthen\n    show_usage\n    exit 1\nfi\n\nset -x\nif command -v python3 >/dev/null; then\n    python3 \"$@\"\nelif command -v python2 >/dev/null; then\n    python2 \"$@\"\nelif command -v python >/dev/null; then\n    python \"$@\"\nelse\n    echo \"ERROR: python is not available!\"\n    exit 1\nfi\n"}, "pacemaker_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nset -u\n\n# ./pacemaker_restart_bundle.sh mysql galera galera-bundle Master _\n# ./pacemaker_restart_bundle.sh redis redis redis-bundle Slave Master\n# ./pacemaker_restart_bundle.sh ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\nRESTART_SCRIPTS_DIR=$(dirname $0)\nTRIPLEO_SERVICE=$1\nRESOURCE_NAME=$2\nBUNDLE_NAME=$3\nWAIT_TARGET_LOCAL=$4\nWAIT_TARGET_ANYWHERE=${5:-_}\nTRIPLEO_MINOR_UPDATE=\"${TRIPLEO_MINOR_UPDATE:-false}\"\nTRIPLEO_HA_WRAPPER_RESOURCE_EXISTS=\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS:-false}\"\n\nbundle_can_be_restarted() {\n    local bundle=$1\n    # As long as the resource bundle is managed by pacemaker and is\n    # not meant to stay stopped, no matter the state of any inner\n    # pcmk_remote or ocf resource, we should restart it to give it a\n    # chance to read the new config.\n    [ \"$(crm_resource --meta -r $1 -g is-managed 2>/dev/null)\" != \"false\" ] && \\\n    [ \"$(crm_resource --meta -r $1 -g target-role 2>/dev/null)\" != \"Stopped\" ]\n}\n\nlog() {\n    local msg=$1\n    logger -t pcmkrestart \"$1\"\n}\n\nHOSTNAME=$(/bin/hostname -s)\nif [ x\"${TRIPLEO_MINOR_UPDATE,,}\" != x\"true\" ]; then\n    if [ x\"${TRIPLEO_HA_WRAPPER_RESOURCE_EXISTS,,}\" = x\"false\" ]; then\n        # Do not restart during initial deployment, as the resource\n        # has just been created.\n        SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n        if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n            log \"Initial deployment, skipping the restart of ${BUNDLE_NAME}\"\n\tfi\n        exit 0\n    else\n        # During a stack update, this script is called in parallel on\n        # every node the resource runs on, after the service's configs\n        # have been updated on all nodes. So we need to run pcs only\n        # once (e.g. on the service's boostrap node).\n        if bundle_can_be_restarted ${BUNDLE_NAME}; then\n            SERVICE_NODEID=$(/bin/hiera -c /etc/puppet/hiera.yaml \"${TRIPLEO_SERVICE}_short_bootstrap_node_name\")\n            if [[ \"${HOSTNAME,,}\" == \"${SERVICE_NODEID,,}\" ]]; then\n                replicas_running=$(crm_resource -Q -r $BUNDLE_NAME --locate 2>&1 | wc -l)\n                if [ \"$replicas_running\" != \"0\" ]; then\n                    log \"Restarting ${BUNDLE_NAME} globally. Stopping:\"\n                    /sbin/pcs resource disable --wait=600 $BUNDLE_NAME\n                    log \"Restarting ${BUNDLE_NAME} globally. Starting:\"\n                    /sbin/pcs resource enable --wait=600 $BUNDLE_NAME\n                else\n                    log \"${BUNDLE_NAME} is not running anywhere,\" \\\n                         \"cleaning up to restart it globally if necessary\"\n                    /sbin/pcs resource cleanup $BUNDLE_NAME\n                fi\n            else\n                log \"Skipping global restart of ${BUNDLE_NAME} on ${HOSTNAME} it will be restarted by node ${SERVICE_NODEID}\"\n            fi\n\n        else\n            log \"No global restart needed for ${BUNDLE_NAME}.\"\n        fi\n    fi\nelse\n    # During a minor update workflow however, a host gets fully\n    # updated before updating the next one. So unlike stack\n    # update, at the time this script is called, the service's\n    # configs aren't updated on all nodes yet. So only restart the\n    # resource locally, where it's guaranteed that the config is\n    # up to date.\n    HOST=$(facter hostname)\n\n    if bundle_can_be_restarted ${BUNDLE_NAME}; then\n\t# if the resource is running locally, restart it\n\tif crm_resource -r $BUNDLE_NAME --locate 2>&1 | grep -w -q \"${HOST}\"; then\n            log \"Restarting ${BUNDLE_NAME} locally on '${HOST}'\"\n            /sbin/pcs resource restart $BUNDLE_NAME \"${HOST}\"\n\n\telse\n\t    # At this point, if no resource is running locally, it's\n\t    # either because a) it has failed previously, or b) because\n\t    # it's an A/P resource running elsewhere.\n\t    # By cleaning up resource, we ensure that a) it will try to\n\t    # restart, or b) it won't do anything if the resource is\n\t    # already running elsewhere.\n            log \"${BUNDLE_NAME} is currently not running on '${HOST}',\" \\\n                 \"cleaning up its state to restart it if necessary\"\n            /sbin/pcs resource cleanup $BUNDLE_NAME node=\"${HOST}\"\n\tfi\n\n\t# Wait until the resource is in the expected target state\n\t$RESTART_SCRIPTS_DIR/pacemaker_wait_bundle.sh \\\n            $RESOURCE_NAME $BUNDLE_NAME \\\n            \"$WAIT_TARGET_LOCAL\" \"$WAIT_TARGET_ANYWHERE\" \\\n\t    \"${HOST}\" 600\n    else\n        log \"No restart needed for ${BUNDLE_NAME}.\"\n    fi\nfi\n"}, "pacemaker_wait_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# ----\n# Wait for an OCF resource or a bundle to be restarted\n# ----\n# e.g.:\n# M/S OCF:      $0 galera galera-bundle Master\n# clone OCF:    $0 rabbitmq rabbitmq-bundle Started\n# A/P M/S OCF:  $0 redis redis-bundle Slave Master\n# A/P bundle:   $0 openstack-cinder-volume openstack-cinder-volume _ Started\n# clone bundle: $0 haproxy-bundle haproxy-bundle Started\n\n# design note 1:\n#  - this script is called during a minor update; it is called\n#    once per node that hosts a service replica.\n#  - the purpose of this script is to ensure that restarting the\n#    service replica locally won't disrupt the service availability\n#    for the end user. To reach that goal, the script waits until the\n#    service is restarted locally or globallu and reaches a given\n#    target state (i.e. Started, Slave or Master).\n# design note 2:\n#   - we don't want to track restart error: our only job is to ensure\n#     service restart synchronization, not service health.\n#   - In particular, we don't want to error out in case the resource\n#     cannot be restarted locally, because that would make the minor\n#     update fail, even if potentially other replicas still provide\n#     the service.\n# design note 3:\n#   - we can bail out early if we determine that the resource can't\n#     be restarted automatically by pacemaker (e.g. its \"blocked\",\n#     unmanaged or disabled).\n\nlog() {\n    local msg=$1\n    echo \"$(date -u): $1\"\n}\n\nusage() {\n    echo 2>&1 \"Usage: $0 NAME BUNDLE_NAME ROLE_LOCAL [ROLE_ANYWHERE] [HOST] [TIMEOUT]\"\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\n#\n# Utility functions to detect stuck resources\n#\n\nbundle_failures_locally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local replica_name\n    for i in $(seq 0 $last); do\n\treplica_name=${BUNDLE_NAME}-${engine}-${i}\n\tcrm_failcount -q -G -r $replica_name -N $HOST\n    done\n}\n\nbundle_failures_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    for i in $(seq 0 $last); do\n\tcrm_failcount -q -G -r ${BUNDLE_NAME}-${engine}-${i}\n    done\n}\n\nbundle_running_globally() {\n    local engine=$BUNDLE_CONTAINER_ENGINE\n    # return the number of running bundles replica, i.e. the number of\n    # docker/podman resource replicas currently running in the cluster\n    crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:heartbeat:${engine}']/node)\" -\n}\n\nocf_failures_globally() {\n    local replicas=$BUNDLE_REPLICAS\n    local last=$(($replicas - 1))\n    local bundle_node\n    for i in $(seq 0 $last); do\n\tbundle_node=${BUNDLE_NAME}-${i}\n\tcrm_failcount -q -G -r $NAME -N $bundle_node\n    done\n}\n\ndid_resource_failed_locally() {\n    local failures\n    local running\n    local remotehost\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# if we're dealing with an ocf resource, it is running on a\n\t# pacemaker_remote rather that on the real host, and the\n\t# failcounts are thus associated to the pcmk remote. Replace\n\t# the host's name with the pcmk remote's name.\n        remotehost=$(crm_mon --as-xml | xmllint --xpath \"string(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource/node[@name='${HOST}']/../../resource[@resource_agent='${OCF}:pacemaker:remote']/@id)\" -)\n\tif [ -n \"${remotehost}\" ]; then\n\t    crm_failcount -q -G -r $NAME -N $remotehost | grep -q -w INFINITY\n\t    return $?\n\tfi\n\t# If no pcmk remote is currently running, the failcount from\n\t# the ocf resource is useless, compute the failcount from the\n\t# bundle case instead (computed below).\n    fi\n\n    # for bundles, pacemaker can run any bundle replica locally\n    # (e.g. galera-bundle-docker-{0,1,2}), and a failure happens when\n    # there are no more replica to try.\n    # That is, when _at least_ one replica failed locally, and all the\n    # others either failed or are currently running elsewhere.\n    failures=$(bundle_failures_locally $HOST | grep -c -w INFINITY)\n    running=$(bundle_running_globally)\n    test $failures -gt 0 && \\\n    test $(( $failures + $running )) -ge $BUNDLE_REPLICAS\n}\n\ndid_resource_failed_globally() {\n    local remotecount\n    local failures\n    if [ \"${NAME}\" != \"${BUNDLE_NAME}\" ]; then\n\t# we check the state of an ocf resource only if the\n\t# pcmkremotes are started\n        remotecount=$(crm_mon --as-xml | xmllint --xpath \"count(//resources/bundle[@id='${BUNDLE_NAME}']/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node)\" -)\n\tif [ \"${remotecount}\" = \"0\" ]; then\n\t    # no pcmkremote is running, so check the bundle state\n\t    # instead of checking the ocf resource\n\t    # bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\t    failures=$(bundle_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\telse\n\t    # ocf resource failed if it failed to start on\n\t    # all $BUNDLE_REPLICAS bundle nodes\n\t    failures=$(ocf_failures_globally | grep -c -w INFINITY)\n\t    test $failures -eq $BUNDLE_REPLICAS\n\tfi\n    else\n\t# bundle failed if all ${BUNDLE_REPLICAS} replicas failed\n\tfailures=$(bundle_failures_globally | grep -c -w INFINITY)\n\ttest $failures -eq $BUNDLE_REPLICAS\n    fi\n}\n\n\n# Input validation\n#\n\nNAME=$1\nif [ -z \"${NAME}\" ]; then\n    echo 2>&1 \"Error: argument NAME must not be empty\"\n    exit 1\nfi\n\nBUNDLE_NAME=$2\nif [ -z \"${BUNDLE_NAME}\" ]; then\n    echo 2>&1 \"Error: argument BUNDLE_NAME must not be empty\"\n    exit 1\nfi\n\nROLE_LOCAL=$3\nif [ \"${ROLE_LOCAL}\" = \"_\" ]; then\n    ROLE_LOCAL=\"\"\nfi\n\nROLE_ANYWHERE=$4\nif [ \"${ROLE_ANYWHERE}\" = \"_\" ]; then\n    ROLE_ANYWHERE=\"\"\nfi\n\nif [ -z \"${ROLE_LOCAL}\" ]; then\n    if [ -z \"${ROLE_ANYWHERE}\" ]; then\n        echo 2>&1 \"Error: either ROLE_LOCAL or ROLE_ANYWHERE must be non empty\"\n        exit 1\n    fi\nelse\n    if !(echo \"${ROLE_LOCAL}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n        echo 2>&1 \"Error: argument ROLE_LOCAL must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n        exit 1\n    fi\nfi\n\nif [ -n \"${ROLE_ANYWHERE}\" ] && !(echo \"${ROLE_ANYWHERE}\" | grep -q -x -E \"(Started|Slave|Master|Unpromoted|Promoted)\"); then\n    echo 2>&1 \"Error: argument ROLE_ANYWHERE must be either 'Started' 'Slave' 'Master' 'Unpromoted' or 'Promoted'\"\n    exit 1\nfi\n\n# Ensure compatibility with pacemaker 2.1\nif pacemaker_supports_promoted; then\n    ROLE_LOCAL=$(echo \"$ROLE_LOCAL\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    ROLE_ANYWHERE=$(echo \"$ROLE_ANYWHERE\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    OCF=\"ocf\"\nelse\n    OCF=\"ocf:\"\nfi\n\nHOST=${5:-$(facter hostname)}\nTIMEOUT=${6:-600}\n\n\n# Configure the search\n# ----\n# Note: we can't use crm_resource in all searches because we can't\n# easily extract the host the OCF resources run on (crm_resource\n# returns the pcmk-remote nodes rather than the hosts)\n# So instead, we implement various searches with XPath directly.\n\nif [ \"${BUNDLE_NAME}\" != \"${NAME}\" ]; then\n# ocf resource\nlocal_resource_xpath=\"//bundle/replica/resource[@resource_agent='${OCF}:pacemaker:remote']/node[@name='${HOST}']/../../resource[@id='${NAME}']\"\nany_resource_xpath=\"//bundle//resource[@id='${NAME}']\"\nreplicas_xpath=\"//bundle/primitive[@id='${BUNDLE_NAME}']/../*[boolean(@image) and boolean(@replicas)]\"\nelse\n# bundle resource\nlocal_resource_xpath=\"//bundle[@id='${NAME}']/replica/resource/node[@name='${HOST}']/../../resource\"\nany_resource_xpath=\"//bundle[@id='${NAME}']//resource\"\nreplicas_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nfi\n\nbundle_def_xpath=\"//bundle[@id='${BUNDLE_NAME}']/*[boolean(@image) and boolean(@replicas)]\"\nBUNDLE_CONTAINER_ENGINE=$(cibadmin -Q | xmllint --xpath \"name(${bundle_def_xpath})\" -)\nBUNDLE_REPLICAS=$(cibadmin -Q | xmllint --xpath \"string(${bundle_def_xpath}/@replicas)\" -)\n\n\n# The wait algorithm follows a two-stage approach\n#  1. Depending on how the script is called, we first check whether\n#     the resource is restarted locally. An A/P resource may be\n#     restarted elsewhere in the cluster.\n#  2. If needed, check whether the A/P resource has restarted\n#     elsewhere. For A/P M/S resources, in case the resource is\n#     restarted as Slave locally, ensure a Master is available.\n\nsuccess=1\nbailout=1\ntimeout=$TIMEOUT\nrole=\"\"\n\n# Stage 1: local check\nif [ -n \"$ROLE_LOCAL\" ]; then\n    log \"Waiting until ${NAME} has restarted on ${HOST} and is in state ${ROLE_LOCAL}\"\n    log \"Will probe resource state with the following XPath pattern: ${local_resource_xpath}\"\n\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resource=$(crm_mon -r --as-xml | xmllint --xpath \"${local_resource_xpath}\" - 2>/dev/null)\n        role=$(echo \"${resource}\" | sed -ne 's/.*\\Wrole=\"\\([^\"]*\\)\".*/\\1/p')\n\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resource}\" | grep -q -w \"\\Wblocked=\\\"true\\\"\"; then\n            log \"${NAME} is blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_locally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif [ \"$role\" = \"$ROLE_LOCAL\" ]; then\n            success=0\n            continue\n        elif [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" = \"$ROLE_ANYWHERE\" ]; then\n            # A/P: we are restarted in the expected state\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_LOCAL} on ${HOST}\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\n# Stage 2: global check\nif [ $timeout -gt 0 ] && [ -n \"$ROLE_ANYWHERE\" ] && [ \"$role\" != \"$ROLE_ANYWHERE\" ]; then\n    log \"Wait until ${NAME} is restarted anywhere in the cluster in state ${ROLE_ANYWHERE}\"\n    log \"Will probe resource state with the following XPath pattern: ${any_resource_xpath}\"\n\n    success=1\n    bailout=1\n    while [ $timeout -gt 0 ] && [ $bailout -ne 0 ] && [ $success -ne 0 ]; do\n        resources=$(crm_mon -r --as-xml | xmllint --xpath \"${any_resource_xpath}\" - 2>/dev/null)\n\tif [ \"$(crm_resource --meta -r ${NAME} -g is-managed 2>/dev/null)\" = \"false\" ]; then\n            log \"${NAME} is unmanaged, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif [ \"$(crm_resource --meta -r ${NAME} -g target-role 2>/dev/null)\" = \"Stopped\" ]; then\n            log \"${NAME} is disabled, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif ! (echo \"${resources}\" | grep -q -w \"\\Wblocked=\\\"false\\\"\"); then\n            log \"${NAME} blocked, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n\telif did_resource_failed_globally; then\n            log \"${NAME} is in failed state, will never reach target role. Bailing out\"\n            bailout=0\n            continue\n        elif echo \"${resources}\" | grep -q -w \"\\Wrole=\\\"${ROLE_ANYWHERE}\\\"\"; then\n            success=0\n            continue\n        else\n            log \"Waiting for ${NAME} to transition to role ${ROLE_ANYWHERE} anywhere in the cluster\"\n        fi\n\n        if [ $bailout -ne 0 ] && [ $success -ne 0 ]; then\n            sleep 4\n            timeout=$((timeout-4))\n        fi\n    done\nfi\n\nif [ $timeout -le 0 ]; then\n    log \"Timeout reached after ${TIMEOUT}s while waiting for ${NAME} to be restarted\"\nelif [ $bailout -le 0 ]; then\n    log \"Restart monitoring for ${NAME} cancelled\"\nfi\n\nif [ $success -eq 0 ]; then\n    log \"${NAME} successfully restarted\"\nelse\n    log \"${NAME} was not restarted properly\"\nfi\n\n# Don't block minor update or stack update if the wait was unsuccessful\nexit 0\n"}, "wait-port-and-run.sh": {"mode": "0755", "content": "#!/usr/bin/bash\nset -eu\n\nHOST=$1\nPORT=$2\n\necho \"$(date -u): Checking whether we can bind to ${HOST}:${PORT}\"\nwhile (ss -Htnl src \"${HOST}\" \"sport = :${PORT}\" | grep -wq \"${PORT}\"); do\n    echo \"$(date -u): ${HOST}:${PORT} still in use, waiting...\";\n    sleep 10;\ndone\n\nshift 2\nCOMMAND=\"$*\"\nif [ -z \"${COMMAND}\" ]; then\n    COMMAND=\"true\"\nfi\nexec $COMMAND\n"}, "pacemaker_resource_lock.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\nMAX_RETRIES=10\nCIB_ENOTFOUND=105\n\nusage() {\n   echo \"Set a global property in the cluster with a validity timestamp.\"\n   echo \"Usage:\"\n   echo \"   $0 --acquire <lock_name> <lock_owner> <lock_ttl_in_seconds>\"\n   echo \"   $0 --release <lock_name> <lock_owner>\"\n   echo\n}\n\nlog() {\n    echo \"$(date -u): $1\" 1>&2\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\nlock_create() {\n    local name=$1\n    local data=$2\n    # cibadmin won't overwrite a key if someone else succeeded to create it concurrently\n    cibadmin --sync-call --scope crm_config --create --xml-text \"<cluster_property_set id='${name}'><nvpair id='${name}-pair' name='${name}' value='${data}'/></cluster_property_set>\" &>/dev/null\n    return $?\n}\n\nlock_update() {\n    local name=$1\n    local expected_data=$2\n    local new_data=$3\n    # we only update the lock we expect to see, so we can't update someone else's lock\n    cibadmin --sync-call --scope crm_config --modify --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" --xml-text \"<nvpair id='${name}-pair' name='${name}' value='${new_data}'/>\" &>/dev/null\n    return $?\n}\n\nlock_delete() {\n    local name=$1\n    local expected_data=$2\n    # we only delete the lock we expect to see, so we can't delete someone else's lock\n    cibadmin --sync-call --scope crm_config --delete --xpath \"//cluster_property_set/nvpair[@name='${name}' and @value='${expected_data}']/..\" &>/dev/null\n    return $?\n}\n\nlock_get() {\n    local lockname=$1\n    local res\n    local rc\n    res=$(cibadmin --query --scope crm_config --xpath \"//cluster_property_set/nvpair[@name='$lockname']\" 2>/dev/null)\n    rc=$?\n    if [ $rc -eq 0 ]; then\n        echo \"$res\" | sed -n 's/.*value=\"\\([^\"]*\\)\".*/\\1/p'\n    fi\n    return $rc\n}\n\nlock_owner() {\n    local lock=$1\n    echo \"$lock\" | cut -d':' -f1\n}\n\nlock_has_expired() {\n    local lock=$1\n    local expiry=$(echo \"$lock\" | cut -d':' -f2)\n    local now=$(date +%s)\n    test $now -ge $expiry\n}\n\n\n# Perform a lock action and restart if the CIB has been modified before\n# committing the lock action\ntry_action() {\n    local fun=$1\n    local lock=$2\n    local requester=$3\n    local args=${4:-}\n    local tries=$MAX_RETRIES\n    local rc=1\n    if [ \"$fun\" = \"lock_acquire\" ] || [ \"$fun\" = \"lock_release\" ]; then\n        log \"Try running $fun\"\n    else\n        return 2\n    fi\n    while [ $rc -ne 0 ]; do\n        $fun $lock $requester $args\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Operation $1 succeeded\"\n            return 0\n        elif [ $rc -eq 3 ]; then\n            # rc == 3 -> CIB changed before push\n            if [ $tries -eq 0 ]; then\n                log \"Failed to commit after $MAX_RETRIES retries. Bailing out.\"\n                return 2\n            else\n                log \"Failed to commit. Retrying operation.\"\n                tries=$(($tries - 1))\n            fi\n        elif [ $rc -eq 2 ]; then\n            # rc == 2 -> unrecoverable cib error (e.g. pacemaker down)\n            log \"Unexpected failure. Bailing out\"\n            return $rc\n        else\n            # rc == 1 -> lock error (not owner, lock doesn't exists)\n            return $rc\n        fi\n    done\n}\n\n# The lock mechanism uses cibadmin's atomic creation so cluster-wide\n# state coherency is guaranteed by pacemaker\nlock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc\n    local lock\n    local expiry\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -n \"$lock\" ]; then\n        lock_has_expired $lock\n        rc=$?\n        if [ $rc -eq 0 ]; then\n            log \"Lock has expired, now available for being held\"\n        else\n            # lock is still held. check whether we're the owner\n            owner=$(lock_owner $lock)\n            if [ \"$owner\" = \"$requester\" ];then\n                log \"Requester already owns the lock, acquiring attempt will just reconfigure the TTL\"\n            else\n                log \"Lock is held by someone else ($owner)\"\n                return 1\n            fi\n        fi\n    else\n        log \"Lock is not held yet\"\n    fi\n\n    # prepare the lock info\n    expiry=$(($(date +%s) + $ttl))\n\n    if [ -n \"$lock\" ]; then\n        log \"Attempting to update the lock\"\n        lock_update $lockname \"$lock\" \"$requester:$expiry\"\n        rc=$?\n    else\n        log \"Attempting to acquire the lock\"\n        lock_create $lockname \"$requester:$expiry\"\n        rc=$?\n    fi\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' acquired by '$requester', valid until $(date -d @$expiry)\"\n        return 0\n    else\n        log \"CIB changed, lock cannot be acquired\"\n        return 3\n    fi\n}\n\n\n# The lock mechanism uses the CIB's num_updates tag to implement\n# a conditional store. Cluster-wide locking is guaranteed by pacemaker\nlock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n    local lock\n    local owner\n\n    log \"Check whether the lock is already held in the CIB\"\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        log \"Could not retrieve info from the CIB\"\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        log \"Lock doesn't exist. Nothing to release\"\n        return 0\n    else\n        log \"Lock exists, check whether we're the owner\"\n        owner=$(lock_owner $lock)\n        if [ \"$owner\" != \"$requester\" ];then\n            log \"Lock is held by someone else ($owner), will not unlock\"\n            return 1\n        fi\n    fi\n\n    lock_delete $lockname \"$lock\"\n    rc=$?\n\n    if [ $rc -eq 0 ]; then\n        log \"Lock '$lockname' released by '$requester'\"\n        return 0\n    else\n        log \"CIB deletion error, lock cannot be released\"\n        return 3\n    fi\n}\n\n\n# Retrieve the owner of a lock from the CIB\n# this is a read-only operation, so no need to log debug info\nlock_get_owner() {\n    local lockname=$1\n    local rc\n    local lock\n    local owner\n\n    lock=$(lock_get $lockname)\n    rc=$?\n    if [ $rc -ne 0 ] && [ $rc -ne $CIB_ENOTFOUND ]; then\n        return 2\n    fi\n\n    if [ -z \"$lock\" ]; then\n        return 1\n    else\n        lock_owner $lock\n        return 0\n    fi\n}\n\n\nACTION=$1\nLOCKNAME=$2\nREQUESTER=$3\nTTL=${4:-60}\n\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nif [ $ACTION != \"--help\" ]; then\n    if [ -z \"$LOCKNAME\" ]; then\n        error \"You must specific a lock name\"\n    fi\n    if [ $ACTION != \"--owner\" ] && [ $ACTION != \"-o\" ]; then\n        if [ -z \"$REQUESTER\" ]; then\n            error \"You must specific a lock requester\"\n        fi\n    fi\nfi\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) try_action lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --release|-r) try_action lock_release $LOCKNAME $REQUESTER;;\n    --acquire-once|-A) lock_acquire $LOCKNAME $REQUESTER $TTL;;\n    --owner|-o) lock_get_owner $LOCKNAME;;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "pacemaker_mutex_restart_bundle.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_restart_bundle.sh --lock mysql galera galera-bundle Master _\n# pacemaker_mutex_restart_bundle.sh --lock ovn_dbs ovndb_servers ovn-dbs-bundle Slave Master\n\nset -u\n\nusage() {\n    echo \"Restart a clustered resource in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --lock <tripleo-service> <pcmk-resource> <pcmk-bundle> <target-state-local> <target-state-cluster>\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\npacemaker_supports_promoted() {\n    # The Promoted token is only matched in recent pacemaker versions\n    grep -wq \"<value>Promoted</value>\" /usr/share/pacemaker/resources-*.rng\n}\n\nACTION=$1\ncase $ACTION in\n    --help) usage; exit 0;;\n    --lock) ;;\n    *) error \"Unknown action '$ACTION'\";;\nesac\n\nTRIPLEO_SERVICE=$2\nLOCK_NAME=${TRIPLEO_SERVICE}-restart-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to restart resource $TRIPLEO_SERVICE\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\nRESOURCE_NAME=$3\nBUNDLE_NAME=$4\nWAIT_TARGET_LOCAL=$5\nWAIT_TARGET_ANYWHERE=${6:-_}\n\nif pacemaker_supports_promoted; then\n    WAIT_TARGET_LOCAL=$(echo \"$5\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    WAIT_TARGET_ANYWHERE=$(echo \"${6:-_}\" | sed -e 's/Master/Promoted/' -e 's/Slave/Unpromoted/')\n    promoted_role=\"Promoted\"\nelse\n    promoted_role=\"Master\"\nfi\n\n# The lock TTL should accomodate for the resource start/promote timeout\nif [ \"$RESOURCE_NAME\" != \"$BUNDLE_NAME\" ]; then\n    if [ \"$WAIT_TARGET_LOCAL\" = \"$promoted_role\" ] || [ \"$WAIT_TARGET_ANYWHERE\" = \"$promoted_role\" ]; then\n        rsc_op=\"promote\"\n    else\n        rsc_op=\"start\"\n    fi\n    # <op id=\"galera-promote-interval-0s\" interval=\"0s\" name=\"promote\" on-fail=\"block\" timeout=\"300s\"/>\n    PCMK_TTL=$(cibadmin -Q | xmllint -xpath \"string(//primitive[@id='${RESOURCE_NAME}']/operations/op[@name='${rsc_op}']/@timeout)\" - | sed 's/s$//')\n    LOCK_TTL=$((PCMK_TTL + 30))\nelse\n    # The podman RA's default start timeout\n    LOCK_TTL=90\nfi\n\nlog \"Acquire a ${LOCK_TTL}s restart lock for service $TRIPLEO_SERVICE before restarting it\"\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nrc=1\nwhile [ $rc -ne 0 ]; do\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire $LOCK_NAME $LOCK_OWNER $LOCK_TTL\n    rc=$?\n    if [ $rc != 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not acquire lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Could not acquire lock, retrying\"\n            sleep 10\n        fi\n    fi\ndone\n\nlog \"Restart the service $TRIPLEO_SERVICE locally\"\n# Reuse the local restart script in t-h-t (driven by env var TRIPLEO_MINOR_UPDATE)\nTRIPLEO_MINOR_UPDATE=true /var/lib/container-config-scripts/pacemaker_restart_bundle.sh $TRIPLEO_SERVICE $RESOURCE_NAME $BUNDLE_NAME $WAIT_TARGET_LOCAL $WAIT_TARGET_ANYWHERE\n\n# If we reached this point, always try to release the lock\nlog \"Release the restart lock for service $TRIPLEO_SERVICE\"\n/var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $LOCK_NAME $LOCK_OWNER\nrc=$?\nif [ $rc -ne 0 ] && [ $rc -ne 1 ]; then\n    error \"Could not release held lock (rc: $rc)\"\nfi\n"}, "pacemaker_mutex_shutdown.sh": {"mode": "0755", "content": "#!/usr/bin/bash\n\n# pacemaker_mutex_shutdown.sh --acquire\n# pacemaker_mutex_shutdown.sh --release\n\nset -u\n\nusage() {\n    echo \"Shutdown a cluster node in a coordinated way across the cluster\"\n    echo \"Usage:\"\n    echo \"   $0 --acquire # prevent other node from shutting down until we hold the lock\"\n    echo \"   $0 --release # release the lock, other node can compete for the shutdown lock\"\n    echo\n}\n\nlog() {\n    echo \"$(date -u): $1\"\n}\n\nerror() {\n    echo \"$(date -u): $1\" 1>&2\n    exit 1\n}\n\n# Loop until we hold the lock. The lock has a TTL, so we're guaranteed to get it eventually\nshutdown_lock_acquire() {\n    local lockname=$1\n    local requester=$2\n    local ttl=$3\n    local rc=1\n    local current_owner\n    local owner_stopped\n    local owner_rc\n\n    log \"Acquiring the shutdown lock\"\n    while [ $rc -ne 0 ]; do\n        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --acquire-once $lockname $requester $ttl\n        rc=$?\n        if [ $rc -ne 0 ]; then\n            if [ $rc -eq 2 ]; then\n                error \"Could not acquire the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n            else\n                # The lock is held by another node.\n                current_owner=$(/var/lib/container-config-scripts/pacemaker_resource_lock.sh --owner $lockname)\n                owner_rc=$?\n                if [ $owner_rc -eq 2 ]; then\n                    error \"Could not get the shutdown lock owner due to unrecoverable error (rc: $owner_rc), bailing out\"\n                fi\n                if [ $owner_rc -eq 0 ]; then\n                    # If the owner is marked as offline, that means it has shutdown and\n                    # we can clean the lock preemptively and try to acquire it.\n                    owner_stopped=$(crm_mon -1X | xmllint --xpath 'count(//nodes/node[@name=\"'${current_owner}'\" and @online=\"false\" and @unclean=\"false\"])' -)\n                    if [ \"${owner_stopped}\" = \"1\" ]; then\n                        log \"Shutdown lock held by stopped node '${current_owner}', lock can be released\"\n                        /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $current_owner\n                        continue\n                    fi\n                fi\n                log \"Shutdown lock held by another node (rc: $rc), retrying\"\n                sleep 10\n            fi\n        fi\n    done\n    log \"Shutdown lock acquired\"\n    return 0\n}\n\n\n# Release the lock if we still own it. Not owning it anymore is not fatal\nshutdown_lock_release() {\n    local lockname=$1\n    local requester=$2\n    local rc\n\n    log \"Releasing the shutdown lock\"\n    /var/lib/container-config-scripts/pacemaker_resource_lock.sh --release $lockname $requester\n    rc=$?\n    if [ $rc -ne 0 ]; then\n        if [ $rc -gt 1 ]; then\n            error \"Could not release the shutdown lock due to unrecoverable error (rc: $rc), bailing out\"\n        else\n            log \"Shutdown lock no longer held, nothing to do\"\n        fi\n    else\n        log \"Shutdown lock released\"\n    fi\n    return 0\n}\n\n\nACTION=$1\nif [ -z \"$ACTION\" ]; then\n    error \"Action must be specified\"\nfi\n\nLOCK_NAME=tripleo-shutdown-lock\nLOCK_OWNER=$(crm_node -n 2>/dev/null)\nrc=$?\nif [ $rc -ne 0 ]; then\n    if [ $rc -eq 102 ]; then\n        log \"Cluster is not running locally, no need to aquire the shutdown lock\"\n        exit 0\n    else\n        error \"Unexpected error while connecting to the cluster (rc: $rc), bailing out\"\n    fi\nfi\n\n# We start with a very high TTL, that long enough to accomodate a cluster stop.\n# As soon as the node will get offline, the other competing node will be entitled\n# to steal the lock, so they should never wait that long in practice.\nLOCK_TTL=600\n\n\ncase $ACTION in\n    --help) usage; exit 0;;\n    --acquire|-a) shutdown_lock_acquire ${LOCK_NAME} ${LOCK_OWNER} ${LOCK_TTL};;\n    --release|-r) shutdown_lock_release ${LOCK_NAME} ${LOCK_OWNER};;\n    *) error \"Invalid action\";;\nesac\nexit $?\n"}, "placement_wait_for_service.py": {"mode": "0755", "content": "#!/usr/bin/python3\n#\n# Copyright 2018 Red Hat Inc.\n#\n# Licensed under the Apache License, Version 2.0 (the \"License\"); you may\n# not use this file except in compliance with the License. You may obtain\n# a copy of the License at\n#\n#      http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT\n# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the\n# License for the specific language governing permissions and limitations\n# under the License.\n\n# shell script to check if placement API is up after X attempts.\n# Default max is 60 iterations with 10s (default) timeout in between.\n\nimport logging\nimport os\nimport re\nimport six\nimport sys\nimport time\n\nfrom keystoneauth1.identity import v3\nfrom keystoneauth1 import session\nfrom keystoneclient.v3 import client\nimport requests\n\n# In python3 SafeConfigParser was renamed to ConfigParser and the default\n# for duplicate options default to true. In case of nova it is valid to\n# have duplicate option lines, e.g. passthrough_whitelist which leads to\n# issues reading the nova.conf\n# https://bugs.launchpad.net/tripleo/+bug/1827775\nif six.PY3:\n    from six.moves.configparser import ConfigParser\n    config = ConfigParser(strict=False)\nelse:\n    from six.moves.configparser import SafeConfigParser\n    config = SafeConfigParser()\n\n\ndebug = os.getenv('__OS_DEBUG', 'false')\n\nif debug.lower() == 'true':\n    loglevel = logging.DEBUG\nelse:\n    loglevel = logging.INFO\n\nlogging.basicConfig(stream=sys.stdout, level=loglevel)\nLOG = logging.getLogger('placement_wait_for_service')\n\niterations = 60\ntimeout = 10\nplacement_cfg = '/etc/placement/placement.conf'\n\nif __name__ == '__main__':\n    if os.path.isfile(placement_cfg):\n        try:\n            config.read(placement_cfg)\n        except Exception:\n            LOG.exception('Error while reading placement.conf:')\n    else:\n        LOG.error('Placement configuration file %s does not exist',\n                  placement_cfg)\n        sys.exit(1)\n\n    # get keystone client with details from [keystone_authtoken] section\n    auth = v3.Password(\n        user_domain_name=config.get('keystone_authtoken', 'user_domain_name'),\n        username=config.get('keystone_authtoken', 'username'),\n        password=config.get('keystone_authtoken', 'password'),\n        project_name=config.get('keystone_authtoken', 'project_name'),\n        project_domain_name=config.get('keystone_authtoken',\n                                       'project_domain_name'),\n        auth_url=config.get('keystone_authtoken', 'auth_url') + '/v3')\n    sess = session.Session(auth=auth, verify=False)\n    keystone = client.Client(session=sess, interface='internal')\n\n    iterations_endpoint = iterations\n    placement_endpoint_url = None\n    while iterations_endpoint > 1:\n        iterations_endpoint -= 1\n        try:\n            # get placement service id\n            placement_service_id = keystone.services.list(\n                name='placement')[0].id\n\n            # get placement endpoint\n            # Note: puppet-placement does not support setting the interface\n            #       until we have https://review.opendev.org/688862.\n            #       Lets hard code 'internal' for now.\n            placement_endpoint_url = keystone.endpoints.list(\n                service=placement_service_id,\n                region=config.get('keystone_authtoken', 'region_name'),\n                interface='internal')[0].url\n            if not placement_endpoint_url:\n                LOG.error('Failed to get placement service endpoint!')\n            else:\n                break\n        except Exception:\n            LOG.exception('Retry - Failed to get placement service endpoint:')\n        time.sleep(timeout)\n\n    if not placement_endpoint_url:\n        LOG.error('Failed to get placement service endpoint!')\n        sys.exit(1)\n\n    # we should have CURRENT in the request response from placement:\n    # {\"versions\": [{\"status\": \"CURRENT\", \"min_version\": \"1.0\", \"max_version\":\n    # \"1.29\", \"id\": \"v1.0\", \"links\": [{\"href\": \"\", \"rel\": \"self\"}]}]}\n    response_reg = re.compile('.*CURRENT,*')\n\n    while iterations > 1:\n        iterations -= 1\n        try:\n            r = requests.get(placement_endpoint_url + '/', verify=False)\n            if r.status_code == 200 and response_reg.match(r.text):\n                LOG.info('Placement service up! - %s', r.text)\n                sys.exit(0)\n                break\n            else:\n                LOG.info('response - %r', r)\n                LOG.info('Placement service not up - %s, %s',\n                         r.status_code,\n                         r.text)\n        except Exception:\n            LOG.exception('Error query the placement endpoint:')\n        time.sleep(timeout)\n\n    sys.exit(1)\n\n# vim: set et ts=4 sw=4 :\n"}}, "docker_config": {"step_2": {"placement_init_log": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-placement-api:17.1", "net": "none", "user": "root", "command": ["/bin/bash", "-c", "chown -R placement:placement /var/log/placement"], "volumes": ["/var/log/containers/placement:/var/log/placement:z", "/var/log/containers/httpd/placement:/var/log/httpd:z"]}}, "step_3": {"placement_api_db_sync": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-placement-api:17.1", "cap_add": ["AUDIT_WRITE"], "net": "host", "detach": false, "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/placement:/var/log/placement:z", "/var/log/containers/httpd/placement:/var/log/httpd:z", "/var/lib/kolla/config_files/placement_api_db_sync.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/placement:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_4": {"placement_api": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-placement-api:17.1", "net": "host", "user": "root", "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/placement:/var/log/placement:z", "/var/log/containers/httpd/placement:/var/log/httpd:z", "/var/lib/kolla/config_files/placement_api.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/placement:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "placement_wait_for_service": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-placement-api:17.1", "user": "root", "net": "host", "privileged": false, "detach": false, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/placement:/var/log/placement:z", "/var/log/containers/httpd/placement:/var/log/httpd:z", "/var/lib/kolla/config_files/placement_api_wait_for_service.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/placement:/var/lib/kolla/config_files/src:ro", "/var/lib/container-config-scripts:/container-config-scripts:z"], "environment": {"__OS_DEBUG": "true", "KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "host_prep_tasks": [{"name": "create persistent logs directory", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode }}"}, "with_items": [{"path": "/var/log/containers/placement", "setype": "container_file_t", "mode": "0750"}, {"path": "/var/log/containers/httpd/placement", "setype": "container_file_t", "mode": "0750"}]}], "deploy_steps_tasks": [{"name": "Certificate generation", "when": ["step|int == 1", "enable_internal_tls"], "block": [{"name": "Create dirs for certificates and keys", "file": {"path": "{{ item }}", "state": "directory", "serole": "object_r", "setype": "cert_t", "seuser": "system_u"}, "with_items": ["/etc/pki/tls/certs/httpd", "/etc/pki/tls/private/httpd"]}, {"include_role": {"name": "linux-system-roles.certificate"}, "vars": {"certificate_requests": [{"name": "httpd-ctlplane", "dns": "{{ fqdn_ctlplane }}", "principal": "HTTP/{{ fqdn_ctlplane }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-ctlplane.crt /etc/pki/tls/certs/httpd/httpd-ctlplane.crt\ncp /etc/pki/tls/private/httpd-ctlplane.key /etc/pki/tls/private/httpd/httpd-ctlplane.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage", "dns": "{{ fqdn_storage }}", "principal": "HTTP/{{ fqdn_storage }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage.crt /etc/pki/tls/certs/httpd/httpd-storage.crt\ncp /etc/pki/tls/private/httpd-storage.key /etc/pki/tls/private/httpd/httpd-storage.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-storage_mgmt", "dns": "{{ fqdn_storage_mgmt }}", "principal": "HTTP/{{ fqdn_storage_mgmt }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-storage_mgmt.crt /etc/pki/tls/certs/httpd/httpd-storage_mgmt.crt\ncp /etc/pki/tls/private/httpd-storage_mgmt.key /etc/pki/tls/private/httpd/httpd-storage_mgmt.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-internal_api", "dns": "{{ fqdn_internal_api }}", "principal": "HTTP/{{ fqdn_internal_api }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-internal_api.crt /etc/pki/tls/certs/httpd/httpd-internal_api.crt\ncp /etc/pki/tls/private/httpd-internal_api.key /etc/pki/tls/private/httpd/httpd-internal_api.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}, {"name": "httpd-external", "dns": "{{ fqdn_external }}", "principal": "HTTP/{{ fqdn_external }}@{{ idm_realm }}", "run_after": "cp /etc/pki/tls/certs/httpd-external.crt /etc/pki/tls/certs/httpd/httpd-external.crt\ncp /etc/pki/tls/private/httpd-external.key /etc/pki/tls/private/httpd/httpd-external.key\npkill -USR1 httpd\n", "key_size": "2048", "ca": "ipa"}]}}]}], "upgrade_tasks": []}, {"service_name": "podman", "config_settings": {}, "step_config": "", "host_prep_tasks": [{"name": "Install and configure Podman", "block": [{"name": "Set login facts", "no_log": true, "set_fact": {"container_registry_insecure_registries": ["192.168.122.100:8787"], "container_registry_login": false, "container_registry_logins": {}, "container_registry_logins_json": {}, "container_default_pids_limit": 4096, "container_events_logger_mechanism": "journald"}}, {"name": "Convert logins json to dict", "no_log": true, "set_fact": {"container_registry_logins": "{{ container_registry_logins_json | from_json }}"}, "when": ["container_registry_logins_json is string", "container_registry_login | bool", "(container_registry_logins_json | length) > 0"]}, {"name": "Set registry logins", "no_log": true, "set_fact": {"container_registry_logins": "{{ container_registry_logins_json }}"}, "when": ["container_registry_logins_json is mapping", "container_registry_login | bool", "(container_registry_logins_json | length) > 0"]}, {"name": "Run podman install", "include_role": {"name": "tripleo_podman", "tasks_from": "tripleo_podman_install.yml"}, "vars": {"tripleo_container_registry_insecure_registries": "{{ container_registry_insecure_registries }}", "tripleo_container_default_pids_limit": "{{ container_default_pids_limit }}", "tripleo_container_events_logger_mechanism": "{{ container_events_logger_mechanism }}"}}, {"name": "Run podman login", "include_role": {"name": "tripleo_podman", "tasks_from": "tripleo_podman_login.yml"}, "vars": {"tripleo_container_registry_logins": "{{ container_registry_logins }}", "tripleo_container_registry_login": "{{ container_registry_login | bool }}"}}]}, [{"name": "Configure tripleo_container_manage to generate systemd drop-in dependencies", "copy": {"dest": "/etc/sysconfig/podman_drop_in", "content": "This file makes tripleo_container_manage generate additional systemd\ndependencies for containers that have special\nstart/stop ordering constraints. It ensures that\nthose constraints are enforced on reboot/shutdown.\n"}}]], "deploy_steps_tasks": [{"when": ["(step|int) == 1"], "block": [{"name": "Pre-fetch all the containers", "become": true, "containers.podman.podman_image": {"name": "{{ prefetch_image }}", "validate_certs": false, "force": true}, "register": "result", "until": "result is succeeded", "retries": 5, "delay": 5, "loop_control": {"loop_var": "prefetch_image"}, "loop": "{{ lookup('template', tripleo_role_name + '/docker_config.yaml', errors='ignore') | default('{}', True) | from_yaml | recursive_get_key_from_dict(key='image') | unique }}"}]}], "service_config_settings": {}, "workaround_upgrade_tasks": [{"name": "Configure Podman registry", "when": ["step|int == 0"], "tags": ["setup_packages"], "block": [{"name": "Make sure the Undercloud hostname is included in /etc/hosts", "when": ["undercloud_hosts_entries is defined"], "lineinfile": {"dest": "/etc/hosts", "line": "{{ undercloud_hosts_entries | join('') }}", "state": "present"}}]}, {"name": "Run podman install", "tags": ["system_upgrade", "system_upgrade_run", "setup_packages"], "when": ["step|int == 0"], "block": [{"name": "Set login facts", "no_log": true, "set_fact": {"container_registry_insecure_registries": ["192.168.122.100:8787"], "container_registry_login": false, "container_registry_logins": {}, "container_registry_logins_json": {}, "container_default_pids_limit": 4096, "container_events_logger_mechanism": "journald"}}, {"name": "Convert logins json to dict", "no_log": true, "set_fact": {"container_registry_logins": "{{ container_registry_logins_json | from_json }}"}, "when": ["container_registry_logins_json is string", "container_registry_login | bool", "(container_registry_logins_json | length) > 0"]}, {"name": "Set registry logins", "no_log": true, "set_fact": {"container_registry_logins": "{{ container_registry_logins_json }}"}, "when": ["container_registry_logins_json is mapping", "container_registry_login | bool", "(container_registry_logins_json | length) > 0"]}, {"name": "Run podman install", "include_role": {"name": "tripleo_podman", "tasks_from": "tripleo_podman_install.yml"}, "vars": {"tripleo_container_registry_insecure_registries": "{{ container_registry_insecure_registries }}", "tripleo_container_default_pids_limit": "{{ container_default_pids_limit }}", "tripleo_container_events_logger_mechanism": "{{ container_events_logger_mechanism }}"}}, {"name": "Run podman login", "include_role": {"name": "tripleo_podman", "tasks_from": "tripleo_podman_login.yml"}, "vars": {"tripleo_container_registry_logins": "{{ container_registry_logins }}", "tripleo_container_registry_login": "{{ container_registry_login | bool }}"}}]}, {"name": "Comment infra_image in container.conf", "lineinfile": {"dest": "/usr/share/containers/containers.conf", "regexp": "(?i)^(infra_image.*)", "line": "#\\1", "backrefs": true, "state": "present"}, "tags": ["setup_packages"], "when": ["step|int == 0"]}], "upgrade_tasks": [{"name": "Configure Podman registry", "when": ["step|int == 1"], "block": [{"name": "Make sure the Undercloud hostname is included in /etc/hosts", "when": ["undercloud_hosts_entries is defined"], "lineinfile": {"dest": "/etc/hosts", "line": "{{ undercloud_hosts_entries | join('') }}", "state": "present"}}]}, {"name": "Run podman install", "tags": ["system_upgrade", "system_upgrade_run"], "when": ["step|int == 1"], "block": [{"name": "Set login facts", "no_log": true, "set_fact": {"container_registry_insecure_registries": ["192.168.122.100:8787"], "container_registry_login": false, "container_registry_logins": {}, "container_registry_logins_json": {}, "container_default_pids_limit": 4096, "container_events_logger_mechanism": "journald"}}, {"name": "Convert logins json to dict", "no_log": true, "set_fact": {"container_registry_logins": "{{ container_registry_logins_json | from_json }}"}, "when": ["container_registry_logins_json is string", "container_registry_login | bool", "(container_registry_logins_json | length) > 0"]}, {"name": "Set registry logins", "no_log": true, "set_fact": {"container_registry_logins": "{{ container_registry_logins_json }}"}, "when": ["container_registry_logins_json is mapping", "container_registry_login | bool", "(container_registry_logins_json | length) > 0"]}, {"name": "Run podman install", "include_role": {"name": "tripleo_podman", "tasks_from": "tripleo_podman_install.yml"}, "vars": {"tripleo_container_registry_insecure_registries": "{{ container_registry_insecure_registries }}", "tripleo_container_default_pids_limit": "{{ container_default_pids_limit }}", "tripleo_container_events_logger_mechanism": "{{ container_events_logger_mechanism }}"}}, {"name": "Run podman login", "include_role": {"name": "tripleo_podman", "tasks_from": "tripleo_podman_login.yml"}, "vars": {"tripleo_container_registry_logins": "{{ container_registry_logins }}", "tripleo_container_registry_login": "{{ container_registry_login | bool }}"}}]}], "update_tasks": [{"name": "Force pre-fetch of container images", "when": ["(step|int) == 5"], "block": [{"name": "Pre-fetch all the containers", "become": true, "containers.podman.podman_image": {"name": "{{ prefetch_image }}", "validate_certs": false, "force": true}, "register": "result", "until": "result is succeeded", "retries": 5, "delay": 5, "loop_control": {"loop_var": "prefetch_image"}, "loop": "{{ lookup('template', tripleo_role_name + '/docker_config.yaml', errors='ignore') | default('{}', True) | from_yaml | recursive_get_key_from_dict(key='image') | unique }}"}]}], "post_upgrade_tasks": [{"name": "Purge Podman", "when": ["(step | int) == 3", "container_cli == 'podman'", "deployment_source_hosts not in group_names"], "include_role": {"name": "tripleo_podman", "tasks_from": "tripleo_podman_purge.yml"}, "vars": {"deployment_source_hosts": "Undercloud"}}], "post_update_tasks": [{"name": "Purge Podman", "when": ["(step | int) == 3", "container_cli == 'podman'", "deployment_source_hosts not in group_names"], "include_role": {"name": "tripleo_podman", "tasks_from": "tripleo_podman_purge.yml"}, "vars": {"deployment_source_hosts": "Undercloud"}}]}, {"service_name": "rsyslog_baremetal_ansible", "config_settings": {}, "deploy_steps_tasks": [], "docker_config": {}, "kolla_config": {}, "puppet_config": {}, "host_prep_tasks": [{"name": "Ensure rsyslog is installed", "package": {"name": "rsyslog", "state": "installed"}}, {"name": "Ensure rsyslog is enabled and started", "systemd": {"state": "started", "name": "rsyslog", "enabled": true}}]}, {"service_name": "snmp", "firewall_rules": {"124 snmp 192.168.122.0/24": {"dport": 161, "proto": "udp", "source": "192.168.122.0/24"}}, "config_settings": {"tripleo::profile::base::snmp::snmpd_user": "ro_snmp_user", "tripleo::profile::base::snmp::snmpd_password": "KY25uL2U94jHHfJNHHl8m07r", "snmp::agentaddress": ["udp:161", "udp6:[::1]:161"], "snmp::snmpd_options": "-LS0-5d"}, "step_config": "include tripleo::profile::base::snmp\n", "upgrade_tasks": [{"when": "step|int == 0", "block": [{"name": "Check if snmpd is enabled", "command": "systemctl is-enabled --quiet snmpd", "failed_when": false, "register": "snmpd_enabled_result"}, {"name": "Set fact snmpd_enabled", "set_fact": {"snmpd_enabled": "{{ snmpd_enabled_result.rc == 0 }}"}}]}, {"name": "Stop snmp service", "when": ["step|int == 1", "snmpd_enabled|bool"], "service": "name=snmpd state=stopped"}]}, {"service_name": "sshd", "firewall_rules": {"003 accept ssh from all": {"proto": "tcp", "dport": 22, "extras": {"ensure": "present"}}}, "host_prep_tasks": [{"include_role": {"name": "tripleo_sshd"}, "vars": {"tripleo_sshd_server_options": {"HostKey": ["/etc/ssh/ssh_host_rsa_key", "/etc/ssh/ssh_host_ecdsa_key", "/etc/ssh/ssh_host_ed25519_key"], "SyslogFacility": "AUTHPRIV", "AuthorizedKeysFile": ".ssh/authorized_keys", "ChallengeResponseAuthentication": "no", "GSSAPIAuthentication": "no", "GSSAPICleanupCredentials": "no", "UsePAM": "yes", "UseDNS": "no", "PrintMotd": "no", "X11Forwarding": "yes", "AcceptEnv": ["LANG LC_CTYPE LC_NUMERIC LC_TIME LC_COLLATE LC_MONETARY LC_MESSAGES", "LC_PAPER LC_NAME LC_ADDRESS LC_TELEPHONE LC_MEASUREMENT", "LC_IDENTIFICATION LC_ALL LANGUAGE", "XMODIFIERS"], "Subsystem": "sftp  /usr/libexec/openssh/sftp-server"}, "tripleo_sshd_password_authentication": "no", "tripleo_sshd_banner_enabled": false, "tripleo_sshd_banner_text": "", "tripleo_sshd_motd_enabled": false, "tripleo_sshd_message_of_the_day": ""}}]}, {"service_name": "swift_proxy", "firewall_rules": {"122 swift proxy": {"dport": [8080]}}, "firewall_frontend_rules": {"100 swift_proxy_server_haproxy_frontend": {"dport": [8080]}}, "firewall_ssl_frontend_rules": {"100 swift_proxy_server_haproxy_frontend_ssl": {"dport": [13808]}}, "keystone_resources": {"swift": {"endpoints": {"public": "http://172.21.0.2:8080/v1/AUTH_%(tenant_id)s", "internal": "http://172.18.0.2:8080/v1/AUTH_%(tenant_id)s", "admin": "http://172.18.0.2:8080"}, "users": {"swift": {"password": "8dzPI93pJLWng5a7IqjhGTq4v"}}, "region": "regionOne", "service": "object-store", "roles": ["swiftoperator", "ResellerAdmin"]}}, "monitoring_subscription": "overcloud-swift-proxy", "config_settings": {"swift::swift_hash_path_suffix": "9DcVlAPcWbp0ySxp5PbD0EM2K", "swift::swift_hash_path_prefix": "", "swift::proxy::authtoken::www_authenticate_uri": "http://172.21.0.2:5000", "swift::proxy::authtoken::auth_url": "http://172.17.0.2:5000", "swift::proxy::authtoken::password": "8dzPI93pJLWng5a7IqjhGTq4v", "swift::proxy::authtoken::project_name": "service", "swift::proxy::authtoken::region_name": "regionOne", "swift::proxy::authtoken::interface": "internal", "swift::proxy::s3token::auth_uri": "http://172.17.0.2:5000/v3", "swift::proxy::node_timeout": 60, "swift::proxy::recoverable_node_timeout": 30, "swift::proxy::workers": "1", "swift::proxy::staticweb::url_base": "http://172.21.0.2:8080", "tripleo::profile::base::swift::proxy::ceilometer_enabled": false, "swift::proxy::keystone::operator_roles": ["admin", "swiftoperator", "ResellerAdmin"], "swift::proxy::versioned_writes::allow_versioned_writes": true, "swift::proxy::keystone::system_reader_roles": ["SwiftSystemReader"], "swift::proxy::pipeline": ["catch_errors", "gatekeeper", "healthcheck", "proxy-logging", "cache", "listing_formats", "ratelimit", "bulk", "tempurl", "formpost", "authtoken", "s3api", "s3token", "keystone", "staticweb", "copy", "container_quotas", "account_quotas", "slo", "dlo", "versioned_writes", "proxy-logging", "proxy-server"], "swift::proxy::account_autocreate": true, "tripleo::profile::base::swift::proxy::tls_proxy_bind_ip": "%{hiera('storage')}", "tripleo::profile::base::swift::proxy::tls_proxy_fqdn": "%{hiera('fqdn_storage')}", "tripleo::profile::base::swift::proxy::tls_proxy_port": "8080", "swift::proxy::port": "8080", "swift::proxy::proxy_local_net_ip": "%{hiera('storage')}", "swift::proxy::cache::tls_enabled": false}, "puppet_config": {"config_volume": "swift", "puppet_tags": "swift_config,swift_proxy_config,swift_keymaster_config", "step_config": "include tripleo::profile::base::swift::proxy\n", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-swift-proxy-server:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/swift_proxy.json": {"command": "/usr/bin/swift-proxy-server /etc/swift/proxy-server.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/swift_proxy_tls_proxy.json": {"command": "/usr/sbin/httpd -DFOREGROUND", "config_files": [{"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.d", "dest": "/etc/httpd/conf.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/etc/httpd/conf.modules.d", "dest": "/etc/httpd/conf.modules.d", "merge": false, "preserve_properties": true}, {"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}}, "container_config_scripts": {"create_swift_secret.sh": {"mode": "0700", "content": "#!/bin/bash\nexport OS_PROJECT_DOMAIN_ID=$(crudini --get /etc/swift/keymaster.conf kms_keymaster project_domain_id)\nexport OS_USER_DOMAIN_ID=$(crudini --get /etc/swift/keymaster.conf kms_keymaster user_domain_id)\nexport OS_PROJECT_NAME=$(crudini --get /etc/swift/keymaster.conf kms_keymaster project_name)\nexport OS_USERNAME=$(crudini --get /etc/swift/keymaster.conf kms_keymaster username)\nexport OS_PASSWORD=$(crudini --get /etc/swift/keymaster.conf kms_keymaster password)\nexport OS_AUTH_URL=$(crudini --get /etc/swift/keymaster.conf kms_keymaster auth_endpoint)\nexport OS_AUTH_TYPE=password\nexport OS_IDENTITY_API_VERSION=3\n\necho \"Check if secret already exists\"\nsecret_href=$(openstack secret list --name swift_root_secret_uuid)\nrc=$?\nif [[ $rc != 0 ]]; then\n  echo \"Failed to check secrets, check if Barbican in enabled and responding properly\"\n  exit $rc;\nfi\nif [ -z \"$secret_href\" ]; then\n  echo \"Create new secret\"\n  order_href=$(openstack secret order create --name swift_root_secret_uuid --payload-content-type=\"application/octet-stream\" --algorithm aes --bit-length 256 --mode ctr key -f value -c \"Order href\")\nfi\n"}, "set_swift_keymaster_key_id.sh": {"mode": "0700", "content": "#!/bin/bash\nexport OS_PROJECT_DOMAIN_ID=$(crudini --get /etc/swift/keymaster.conf kms_keymaster project_domain_id)\nexport OS_USER_DOMAIN_ID=$(crudini --get /etc/swift/keymaster.conf kms_keymaster user_domain_id)\nexport OS_PROJECT_NAME=$(crudini --get /etc/swift/keymaster.conf kms_keymaster project_name)\nexport OS_USERNAME=$(crudini --get /etc/swift/keymaster.conf kms_keymaster username)\nexport OS_PASSWORD=$(crudini --get /etc/swift/keymaster.conf kms_keymaster password)\nexport OS_AUTH_URL=$(crudini --get /etc/swift/keymaster.conf kms_keymaster auth_endpoint)\nexport OS_AUTH_TYPE=password\nexport OS_IDENTITY_API_VERSION=3\necho \"retrieve key_id\"\nloop_wait=2\nfor i in {0..5}; do\n  #TODO update uuid from mistral here too\n  secret_href=$(openstack secret list --name swift_root_secret_uuid)\n  if [ \"$secret_href\" ]; then\n    echo \"set key_id in keymaster.conf\"\n    secret_href=$(openstack secret list --name swift_root_secret_uuid -f value -c \"Secret href\")\n    crudini --set /etc/swift/keymaster.conf kms_keymaster key_id ${secret_href##*/}\n    exit 0\n  else\n    echo \"no key, wait for $loop_wait and check again\"\n    sleep $loop_wait\n    ((loop_wait++))\n  fi\ndone\necho \"Failed to set secret in keymaster.conf, check if Barbican is enabled and responding properly\"\nexit 1\n"}}, "docker_config": {"step_4": {"swift_proxy": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-proxy-server:17.1", "start_order": 2, "net": "host", "user": "swift", "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/swift_proxy.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:ro", "/srv/node:/srv/node", "/dev:/dev", "/var/log/containers/swift:/var/log/swift:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode|default(omit) }}"}, "with_items": [{"path": "/srv/node", "setype": "container_file_t"}, {"path": "/var/log/swift", "setype": "container_file_t"}, {"path": "/var/log/containers/swift", "setype": "container_file_t", "mode": "0750"}]}], "deploy_steps_tasks": [{"name": "Configure rsyslog for swift", "when": ["step|int == 1"], "block": [{"name": "Check if rsyslog exists", "shell": "systemctl is-active rsyslog", "register": "swift_rsyslog_config"}, {"name": "Forward logging to swift.log file", "copy": {"content": "# Fix for https://bugs.launchpad.net/tripleo/+bug/1776180\nlocal2.*                 /var/log/containers/swift/swift.log\n&                        stop\n", "dest": "/etc/rsyslog.d/openstack-swift.conf"}, "register": "swift_logconfig", "when": ["swift_rsyslog_config is defined", "swift_rsyslog_config.rc == 0"]}, {"name": "Restart rsyslogd service after logging conf change", "service": {"name": "rsyslog", "state": "restarted"}, "when": ["swift_logconfig is defined", "swift_logconfig is changed"]}]}, {"name": "Run kolla_set_configs to copy ring files", "when": "step|int == 5", "shell": "{{ container_cli }} exec -u root {{ item }} /usr/local/bin/kolla_set_configs ", "become": true, "register": "kolla_set_configs_result", "failed_when": ["kolla_set_configs_result.rc is defined", "kolla_set_configs_result.rc not in [0, 125]"], "with_items": ["swift_proxy"]}], "update_tasks": [{"name": "Check swift containers log folder/symlink exists", "stat": {"path": "/var/log/containers/swift"}, "register": "swift_log_link"}, {"name": "Delete if symlink", "file": {"path": "/var/log/containers/swift", "state": "absent"}, "when": "swift_log_link.stat.islnk is defined and swift_log_link.stat.islnk"}], "metadata_settings": null}, {"service_name": "swift_ringbuilder", "config_settings": {"tripleo::profile::base::swift::ringbuilder::skip_consistency_check": true, "tripleo::profile::base::swift::ringbuilder::swift_ring_get_tempurl": "", "tripleo::profile::base::swift::ringbuilder::swift_ring_put_tempurl": "", "tripleo::profile::base::swift::ringbuilder::build_ring": true, "tripleo::profile::base::swift::ringbuilder::replicas": 1, "tripleo::profile::base::swift::ringbuilder::part_power": 10, "tripleo::profile::base::swift::ringbuilder::min_part_hours": 1, "tripleo::profile::base::swift::ringbuilder::raw_disk_prefix": "r1z1-", "tripleo::profile::base::swift::ringbuilder::raw_disks": [":%PORT%/d1"]}, "service_config_settings": {}, "puppet_config": {"config_volume": "swift_ringbuilder", "puppet_tags": "swift_config,exec,fetch_swift_ring_tarball,extract_swift_ring_tarball,ring_object_device,swift::ringbuilder::create,tripleo::profile::base::swift::add_devices,swift::ringbuilder::rebalance,create_swift_ring_tarball,upload_swift_ring_tarball", "step_config": "include tripleo::profile::base::swift::ringbuilder\n", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-swift-proxy-server:17.1"}, "kolla_config": {}, "docker_config": {"step_3": {"swift_copy_rings": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-proxy-server:17.1", "net": "none", "user": "root", "detach": false, "command": ["/bin/bash", "-c", "cp -v -dR --preserve -t /etc/swift /swift_ringbuilder/etc/swift/*.gz /swift_ringbuilder/etc/swift/*.builder /swift_ringbuilder/etc/swift/backups"], "volumes": ["/var/lib/config-data/puppet-generated/swift/etc/swift:/etc/swift:rw,z", "/var/lib/config-data/puppet-generated/swift_ringbuilder:/swift_ringbuilder:ro"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}}}, {"service_name": "swift_storage", "firewall_rules": {"123 swift storage": {"dport": [873, 6000, 6001, 6002]}}, "config_settings": {"swift::swift_hash_path_suffix": "9DcVlAPcWbp0ySxp5PbD0EM2K", "swift::swift_hash_path_prefix": "", "swift::storage::all::mount_check": false, "tripleo::profile::base::swift::storage::use_local_dir": true, "swift::storage::all::incoming_chmod": "Du=rwx,g=rx,o=rx,Fu=rw,g=r,o=r", "swift::storage::all::outgoing_chmod": "Du=rwx,g=rx,o=rx,Fu=rw,g=r,o=r", "swift::storage::all::account_port": 6002, "swift::storage::all::container_port": 6001, "swift::storage::all::object_port": 6000, "swift::storage::all::object_pipeline": ["healthcheck", "recon", "object-server"], "swift::storage::all::container_pipeline": ["healthcheck", "recon", "container-server"], "swift::storage::all::account_pipeline": ["healthcheck", "recon", "account-server"], "swift::storage::disks::args": {}, "swift::storage::all::storage_local_net_ip": "%{hiera('storage_mgmt')}", "rsync::server::pid_file": "UNSET", "swift::objectexpirer::cache_tls_enabled": false}, "service_config_settings": {}, "puppet_config": {"config_volume": "swift", "puppet_tags": "swift_config,swift_container_config,swift_container_sync_realms_config,swift_account_config,swift_object_config,swift_object_expirer_config,rsync::server", "step_config": "class xinetd() {}\ndefine xinetd::service($bind='',$port='',$server='',$server_args='') {}\nnoop_resource('service')\ninclude tripleo::profile::base::swift::storage", "config_image": "registry.redhat.io/rhosp-rhel9/openstack-swift-proxy-server:17.1"}, "kolla_config": {"/var/lib/kolla/config_files/swift_account_auditor.json": {"command": "/usr/bin/swift-account-auditor /etc/swift/account-server.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/swift_account_reaper.json": {"command": "/usr/bin/swift-account-reaper /etc/swift/account-server.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/swift_account_replicator.json": {"command": "/usr/bin/swift-account-replicator /etc/swift/account-server.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/swift_account_server.json": {"command": "/usr/bin/swift-account-server /etc/swift/account-server.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/swift_container_auditor.json": {"command": "/usr/bin/swift-container-auditor /etc/swift/container-server.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/swift_container_replicator.json": {"command": "/usr/bin/swift-container-replicator /etc/swift/container-server.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/swift_container_updater.json": {"command": "/usr/bin/swift-container-updater /etc/swift/container-server.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/swift_container_server.json": {"command": "/usr/bin/swift-container-server /etc/swift/container-server.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/swift_container_sharder.json": {"command": "/usr/bin/swift-container-sharder /etc/swift/container-server.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/swift_object_auditor.json": {"command": "/usr/bin/swift-object-auditor /etc/swift/object-server.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/swift_object_expirer.json": {"command": "/usr/bin/swift-object-expirer /etc/swift/object-expirer.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/swift_object_replicator.json": {"command": "/usr/bin/swift-object-replicator /etc/swift/object-server.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/swift_object_updater.json": {"command": "/usr/bin/swift-object-updater /etc/swift/object-server.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}, "/var/lib/kolla/config_files/swift_object_server.json": {"command": "/usr/bin/swift-object-server /etc/swift/object-server.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}], "permissions": [{"path": "/var/cache/swift", "owner": "swift:swift", "recurse": true}]}, "/var/lib/kolla/config_files/swift_rsync.json": {"command": "/usr/bin/rsync --daemon --no-detach --config=/etc/rsyncd.conf", "config_files": [{"source": "/var/lib/kolla/config_files/src/*", "dest": "/", "merge": true, "preserve_properties": true}]}}, "docker_config": {"step_3": {"swift_setup_srv": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-account:17.1", "net": "none", "user": "root", "command": ["find", "/srv/node", "-maxdepth", "1", "-type", "d", "-exec", "chown", "swift:", "{}", ";"], "volumes": ["/srv/node:/srv/node:z"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_4": {"swift_account_reaper": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-account:17.1", "net": "host", "user": "swift", "restart": "always", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/swift_account_reaper.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:ro", "/srv/node:/srv/node", "/dev:/dev", "/var/cache/swift:/var/cache/swift:z", "/var/log/containers/swift:/var/log/swift:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "swift_account_server": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-account:17.1", "net": "host", "user": "swift", "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/swift_account_server.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:ro", "/srv/node:/srv/node", "/dev:/dev", "/var/cache/swift:/var/cache/swift", "/var/log/containers/swift:/var/log/swift:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "swift_container_updater": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-container:17.1", "net": "host", "user": "swift", "restart": "always", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/swift_container_updater.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:ro", "/srv/node:/srv/node", "/dev:/dev", "/var/cache/swift:/var/cache/swift", "/var/log/containers/swift:/var/log/swift:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "swift_container_server": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-container:17.1", "net": "host", "user": "swift", "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/swift_container_server.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:ro", "/srv/node:/srv/node", "/dev:/dev", "/var/cache/swift:/var/cache/swift", "/var/log/containers/swift:/var/log/swift:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "swift_object_expirer": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-proxy-server:17.1", "net": "host", "user": "swift", "restart": "always", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/swift_object_expirer.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:ro", "/srv/node:/srv/node", "/dev:/dev", "/var/cache/swift:/var/cache/swift", "/var/log/containers/swift:/var/log/swift:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "swift_object_updater": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-object:17.1", "net": "host", "user": "swift", "restart": "always", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/swift_object_updater.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:ro", "/srv/node:/srv/node", "/dev:/dev", "/var/cache/swift:/var/cache/swift", "/var/log/containers/swift:/var/log/swift:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "swift_object_server": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-object:17.1", "net": "host", "user": "swift", "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/swift_object_server.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:ro", "/srv/node:/srv/node", "/dev:/dev", "/var/cache/swift:/var/cache/swift", "/var/log/containers/swift:/var/log/swift:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, "host_prep_tasks": [{"name": "create persistent directories", "file": {"path": "{{ item.path }}", "state": "directory", "setype": "{{ item.setype }}", "mode": "{{ item.mode|default(omit) }}"}, "with_items": [{"path": "/srv/node", "setype": "container_file_t"}, {"path": "/var/cache/swift", "setype": "container_file_t"}, {"path": "/var/log/containers/swift", "setype": "container_file_t", "mode": "0750"}]}, {"name": "Set swift_use_local_disks fact", "set_fact": {"swift_use_local_disks": true}}, {"name": "Set use_node_data_lookup fact", "set_fact": {"use_node_data_lookup": false}}, {"name": "Create Swift d1 directory if needed", "file": {"path": "/srv/node/d1", "state": "directory"}, "when": "swift_use_local_disks"}, {"name": "Set fact for SwiftRawDisks", "set_fact": {"swift_raw_disks": {}}, "when": "not use_node_data_lookup|bool"}, {"name": "Get hiera swift::storage::disks::args", "shell": "hiera -c /etc/puppet/hiera.yaml swift::storage::disks::args | sed =e 's/=>/:/g'", "register": "hiera_output", "when": "use_node_data_lookup|bool"}, {"name": "Set fact for swift_raw_disks", "set_fact": {"swift_raw_disks": "{{ hiera_output.stdout | from_json }}"}, "when": "use_node_data_lookup|bool"}, {"name": "Format SwiftRawDisks", "community.general.filesystem": {"fstype": "xfs", "dev": "{{ swift_raw_disks[item]['base_dir']|default('/dev') }}/{{ item }}", "opts": "-f -i size=1024"}, "with_items": "{{ swift_raw_disks }}", "when": "swift_raw_disks|length > 0"}, {"name": "Refresh facts if SwiftRawDisks is set to get uuids if newly created partitions", "setup": {"gather_subset": ["!all", "!min", "hardware"], "filter": "ansible_device_links"}, "when": "swift_raw_disks|length > 0"}, {"name": "Mount devices defined in SwiftRawDisks", "mount": {"name": "/srv/node/{{ item }}", "src": "{% if lsblk.results['uuids'][item] is defined %}UUID={{ ansible_facts['device_links']['uuids'][item][0] }}{% else %}{{ swift_raw_disks[item]['base_dir']|default('/dev') }}/{{ item }}{% endif %}", "fstype": "xfs", "opts": "noatime", "state": "mounted"}, "with_items": "{{ swift_raw_disks }}", "when": "swift_raw_disks|length > 0"}], "deploy_steps_tasks": [{"name": "Configure rsyslog for swift", "when": ["step|int == 1"], "block": [{"name": "Check if rsyslog exists", "shell": "systemctl is-active rsyslog", "register": "swift_rsyslog_config"}, {"name": "Forward logging to swift.log file", "copy": {"content": "# Fix for https://bugs.launchpad.net/tripleo/+bug/1776180\nlocal2.*                 /var/log/containers/swift/swift.log\n&                        stop\n", "dest": "/etc/rsyslog.d/openstack-swift.conf"}, "register": "swift_logconfig", "when": ["swift_rsyslog_config is defined", "swift_rsyslog_config.rc == 0"]}, {"name": "Restart rsyslogd service after logging conf change", "service": {"name": "rsyslog", "state": "restarted"}, "when": ["swift_logconfig is defined", "swift_logconfig is changed"]}]}, {"name": "Run kolla_set_configs to copy ring files", "when": "step|int == 5", "shell": "{{ container_cli }} exec -u root {{ item }} /usr/local/bin/kolla_set_configs", "become": true, "register": "kolla_set_configs_result", "failed_when": ["kolla_set_configs_result.rc is defined", "kolla_set_configs_result.rc not in [0, 125]"], "with_items": ["swift_account_auditor", "swift_account_reaper", "swift_account_replicator", "swift_account_server", "swift_container_auditor", "swift_container_replicator", "swift_container_server", "swift_container_updater", "swift_object_auditor", "swift_object_expirer", "swift_object_replicator", "swift_object_server", "swift_object_updater"]}], "update_tasks": [{"name": "Check swift containers log folder/symlink exists", "stat": {"path": "/var/log/containers/swift"}, "register": "swift_log_link"}, {"name": "Delete if symlink", "file": {"path": "/var/log/containers/swift", "state": "absent"}, "when": "swift_log_link.stat.islnk is defined and swift_log_link.stat.islnk"}]}, {"service_name": "chrony", "firewall_rules": {"105 ntp": {"dport": 123, "proto": "udp"}}, "host_prep_tasks": [{"name": "Check for NTP service", "shell": "systemctl is-active ntpd.service || systemctl is-enabled ntpd.service", "failed_when": false, "become": true, "register": "ntp_service_check"}, {"name": "Disable NTP before configuring Chrony", "service": {"name": "ntpd", "state": "stopped", "enabled": false}, "when": ["ntp_service_check.rc is defined", "ntp_service_check.rc == 0"]}, {"name": "Install, Configure and Run Chrony", "include_role": {"name": "chrony"}}, {"name": "Force NTP sync", "command": "chronyc makestep"}, {"name": "Ensure system is NTP time synced", "command": "chronyc waitsync 30"}], "ansible_group_vars": {"chrony_role_action": "all", "chrony_ntp_servers": ["pool.ntp.org"], "chrony_ntp_pools": [], "chrony_global_server_settings": "iburst minpoll 6 maxpoll 10", "chrony_global_pool_settings": "iburst minpoll 6 maxpoll 10", "chrony_manage_package": true, "chrony_acl_rules": ["deny all"]}}, {"service_name": "timezone", "host_prep_tasks": [{"name": "Run timezone role", "include_role": {"name": "tripleo_timezone"}, "vars": {"tripleo_timezone": "UTC"}}]}, {"service_name": "logrotate_tmpwatch", "host_prep_tasks": [{"name": "install tmpwatch on the host", "package": {"name": "tmpwatch", "state": "installed"}}]}, {"service_name": "tripleo_firewall", "config_settings": {"tripleo::firewall::manage_firewall": false, "tripleo::firewall::purge_firewall_rules": false}, "ansible_group_vars": {"tripleo_nftables_debug": true}, "firewall_rules": {"003 accept ssh from ctlplane subnet 192.168.122.0/24": {"source": "192.168.122.0/24", "proto": "tcp", "dport": 22}}, "host_prep_tasks": [{"name": "Notice - ctlplane subnet is set", "debug": {"msg": "CIDRs found in the ctlplane network tags.\n"}}], "host_firewall_tasks": [{"name": "Run firewall role", "include_role": {"name": "tripleo_firewall"}}], "update_tasks": [{"name": "Cleanup tripleo-iptables services", "when": ["(step | int) == 1"], "block": [{"name": "Disable tripleo-iptables.service", "systemd": {"name": "tripleo-iptables.service", "state": "stopped", "enabled": false}, "register": "systemd_tripleo_iptables", "failed_when": false}, {"name": "Cleanup tripleo-iptables.services", "file": {"path": "/etc/systemd/system/tripleo-iptables.service", "state": "absent"}}, {"name": "Disable tripleo-ip6tables.service", "systemd": {"name": "tripleo-ip6tables.service", "state": "stopped", "enabled": false}, "register": "systemd_tripleo_ip6tables", "failed_when": false}, {"name": "Cleanup tripleo-ip6tables.services", "file": {"path": "/etc/systemd/system/tripleo-ip6tables.service", "state": "absent"}}, {"name": "Reload systemd", "systemd": {"daemon_reload": true}, "when": ["(systemd_tripleo_iptables is changed or systemd_tripleo_ip6tables is changed)"]}]}], "upgrade_tasks": [{"name": "Cleanup tripleo-iptables services", "when": ["(step | int) == 1"], "block": [{"name": "Disable tripleo-iptables.service", "systemd": {"name": "tripleo-iptables.service", "state": "stopped", "enabled": false}, "register": "systemd_tripleo_iptables", "failed_when": false}, {"name": "Cleanup tripleo-iptables.services", "file": {"path": "/etc/systemd/system/tripleo-iptables.service", "state": "absent"}}, {"name": "Disable tripleo-ip6tables.service", "systemd": {"name": "tripleo-ip6tables.service", "state": "stopped", "enabled": false}, "register": "systemd_tripleo_ip6tables", "failed_when": false}, {"name": "Cleanup tripleo-ip6tables.services", "file": {"path": "/etc/systemd/system/tripleo-ip6tables.service", "state": "absent"}}, {"name": "Reload systemd", "systemd": {"daemon_reload": true}, "when": ["(systemd_tripleo_iptables is changed or systemd_tripleo_ip6tables is changed)"]}]}, {"when": ["(step | int) == 3"], "block": [{"name": "blank ipv6 rule before activating ipv6 firewall.", "shell": "cat /etc/sysconfig/ip6tables > /etc/sysconfig/ip6tables.n-o-upgrade; cat</dev/null>/etc/sysconfig/ip6tables", "args": {"creates": "/etc/sysconfig/ip6tables.n-o-upgrade"}}, {"name": "cleanup unmanaged rules pushed by iptables-services", "shell": "iptables -C INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT &>/dev/null && \\\n  iptables -D INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\niptables -C INPUT -p icmp -j ACCEPT &>/dev/null && \\\n  iptables -D INPUT -p icmp -j ACCEPT\niptables -C INPUT -i lo -j ACCEPT &>/dev/null && \\\n  iptables -D INPUT -i lo -j ACCEPT\niptables -C INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT &>/dev/null && \\\n  iptables -D INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT\niptables -C INPUT -j REJECT --reject-with icmp-host-prohibited &>/dev/null && \\\n  iptables -D INPUT -j REJECT --reject-with icmp-host-prohibited\niptables -C FORWARD -j REJECT --reject-with icmp-host-prohibited &>/dev/null && \\\n  iptables -D FORWARD -j REJECT --reject-with icmp-host-prohibited\n\nsed -i '/^-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT$/d' /etc/sysconfig/iptables\nsed -i '/^-A INPUT -p icmp -j ACCEPT$/d' /etc/sysconfig/iptables\nsed -i '/^-A INPUT -i lo -j ACCEPT$/d' /etc/sysconfig/iptables\nsed -i '/^-A INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT$/d' /etc/sysconfig/iptables\nsed -i '/^-A INPUT -j REJECT --reject-with icmp-host-prohibited$/d' /etc/sysconfig/iptables\nsed -i '/^-A FORWARD -j REJECT --reject-with icmp-host-prohibited$/d' /etc/sysconfig/iptables\n\nip6tables -C INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT &>/dev/null && \\\n  ip6tables -D INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT\nip6tables -C INPUT -p ipv6-icmp -j ACCEPT &>/dev/null && \\\n  ip6tables -D INPUT -p ipv6-icmp -j ACCEPT\nip6tables -C INPUT -i lo -j ACCEPT &>/dev/null && \\\n  ip6tables -D INPUT -i lo -j ACCEPT\nip6tables -C INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT &>/dev/null && \\\n  ip6tables -D INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT\nip6tables -C INPUT -d fe80::/64 -p udp -m udp --dport 546 -m state --state NEW -j ACCEPT &>/dev/null && \\\n  ip6tables -D INPUT -d fe80::/64 -p udp -m udp --dport 546 -m state --state NEW -j ACCEPT\nip6tables -C INPUT -j REJECT --reject-with icmp6-adm-prohibited &>/dev/null && \\\n  ip6tables -D INPUT -j REJECT --reject-with icmp6-adm-prohibited\nip6tables -C FORWARD -j REJECT --reject-with icmp6-adm-prohibited &>/dev/null && \\\n  ip6tables -D FORWARD -j REJECT --reject-with icmp6-adm-prohibited\n\nsed -i '/^-A INPUT -m state --state RELATED,ESTABLISHED -j ACCEPT$/d' /etc/sysconfig/ip6tables\nsed -i '/^-A INPUT -p ipv6-icmp -j ACCEPT$/d' /etc/sysconfig/ip6tables\nsed -i '/^-A INPUT -i lo -j ACCEPT$/d' /etc/sysconfig/ip6tables\nsed -i '/^-A INPUT -p tcp -m state --state NEW -m tcp --dport 22 -j ACCEPT$/d' /etc/sysconfig/ip6tables\nsed -i '/^-A INPUT -d fe80::\\/64 -p udp -m udp --dport 546 -m state --state NEW -j ACCEPT$/d' /etc/sysconfig/ip6tables\nsed -i '/^-A INPUT -j REJECT --reject-with icmp6-adm-prohibited$/d' /etc/sysconfig/ip6tables\nsed -i '/^-A FORWARD -j REJECT --reject-with icmp6-adm-prohibited$/d' /etc/sysconfig/ip6tables"}]}]}, {"service_name": "tripleo_packages", "config_settings": {"tripleo::packages::enable_install": true}, "step_config": "include ::tripleo::packages\n", "upgrade_tasks": [{"name": "Gather missing facts", "setup": {"gather_subset": ["!all", "!min", "distribution"]}, "tags": ["always"]}, {"name": "Set leapp facts", "set_fact": {"upgrade_leapp_enabled": "{{ _upgradeLeappEnabled | bool and\n   ansible_facts['distribution'] == 'RedHat' and\n   ansible_facts['distribution_major_version'] is version('8', '==') }}", "upgrade_leapp_debug": false, "upgrade_leapp_devel_skip": "", "upgrade_leapp_command_options": "", "upgrade_leapp_reboot_timeout": 3600, "upgrade_leapp_post_reboot_delay": 120}, "vars": {"_upgradeLeappEnabled": false}, "tags": ["always"]}, {"name": "system_upgrade_prepare step 3", "tags": ["never", "system_upgrade", "system_upgrade_prepare"], "when": ["step|int == 3", "upgrade_leapp_enabled"], "block": [{"name": "Run LeappRepoInitCommand", "shell": "#!/bin/bash\n\n"}, {"name": "Remove firewalld bindings", "package": {"name": "python3-firewall", "state": "absent"}}, {"name": "install leapp", "package": {"name": "leapp-repository-openstack", "state": "latest"}}, {"name": "Run LeappInitCommand", "shell": "#!/bin/bash\n\n"}, {"name": "Remove vdo", "package": {"name": "vdo", "state": "absent"}}, {"name": "add packages into Leapp's to_remove file", "vars": {"pkg_to_remove": []}, "lineinfile": {"path": "/etc/leapp/transaction/to_remove", "line": "{{ item }}"}, "loop": "{{ pkg_to_remove }}"}, {"name": "add packages into Leapp's to_install file", "vars": {"pkg_to_install": []}, "lineinfile": {"path": "/etc/leapp/transaction/to_install", "line": "{{ item }}"}, "loop": "{{ pkg_to_install }}"}, {"name": "check sshd_config file", "stat": {"path": "/etc/ssh/sshd_config"}, "register": "sshd_config_result"}, {"name": "add PermitRootLogin option for leapp", "lineinfile": {"path": "/etc/ssh/sshd_config", "regexp": "^(# *)?PermitRootLogin", "line": "PermitRootLogin without-password"}}, {"name": "Remove paunch-services", "package": {"name": "paunch-services", "state": "absent"}}, {"name": "tripleo_container_manage reconfiguration", "import_role": {"name": "tripleo_container_manage", "tasks_from": "shutdown.yml"}}]}, {"name": "system_upgrade_prepare step 4", "tags": ["never", "system_upgrade", "system_upgrade_prepare"], "when": ["step|int == 4", "upgrade_leapp_enabled"], "block": [{"name": "Delete the kernel modules in Leapp database (device_driver_deprecation_data.json)", "vars": {"modules_to_unload": []}, "shell": "set -o pipefail && jq '. | del(.data[] | select(.driver_name == \"{{ item }}\"))' /etc/leapp/files/device_driver_deprecation_data.json > /etc/leapp/files/device_driver_deprecation_data.json_modified && mv /etc/leapp/files/device_driver_deprecation_data.json_modified /etc/leapp/files/device_driver_deprecation_data.json\n", "loop": "{{ modules_to_unload }}"}, {"name": "Remove leapp actors to prevent them inhibiting the upgrade", "vars": {"actors_to_remove": []}, "command": {"cmd": "find /usr/share/leapp-repository/repositories/ -name {{ item }} -type d -print -exec rm -rf \"{}\" +"}, "loop": "{{ actors_to_remove }}"}, {"name": "set leapp required answers", "shell": "leapp answer --add --section check_vdo.confirm=True\n"}, {"name": "Replace EFI grub.cfg with redirect to /boot/grub2/grub.cfg", "import_role": {"name": "tripleo_kernel", "tasks_from": "efigrub.yml"}}, {"name": "Keep nics with prefix in NICsPrefixesToUdev from renaming", "vars": {"nics_prefixes_to_keep": []}, "shell": "ip -j link show | \\ jq -r --arg prefix \"{{ item }}\" '.[] |\n  select((.ifname | startswith($prefix)) and\n  (.ifname | test(\"^.*v[0-9]*$\") | not) and\n  (.ifname | test(\"^.*_[0-9]*$\") | not) and\n  (.ifname | test(\"^.*\\\\..*$\") | not)) |\n  if .permaddr? then .address=.permaddr else . end |\n  \"SUBSYSTEM==\\\"net\\\",ACTION==\\\"add\\\",DRIVERS==\\\"?*\\\",\" + \"NAME=\\\"\" + .ifname +\"\\\" ,ATTR{address}==\\\"\" + .address + \"\\\"\"' >> /etc/udev/rules.d/70-rhosp-persistent-net.rules\n", "loop": "{{ nics_prefixes_to_keep|list }}"}, {"name": "run leapp upgrade (download packages)", "shell": "{% if upgrade_leapp_devel_skip|default(false) %}{{ upgrade_leapp_devel_skip }}{% endif %} leapp upgrade {% if upgrade_leapp_debug|default(true) %}--debug{% endif %} {% if upgrade_leapp_command_options|default(false) %}{{ upgrade_leapp_command_options }}{% endif %}\n", "when": "upgrade_leapp_enabled"}]}, {"name": "system_upgrade_run step 4", "tags": ["never", "system_upgrade", "system_upgrade_run", "system_upgrade_reboot"], "when": ["step|int == 4", "upgrade_leapp_enabled", "'Undercloud' not in group_names"], "block": [{"name": "Run LeappPreRebootCommand", "shell": "#!/bin/bash\n\n"}, {"name": "Check that nova_libvirt is running", "shell": "podman ps --filter name=^nova_virtlogd$ --format \"{% raw %}{{ .Image }}{% endraw %}\"\n", "register": "is_virtlogd_image_running"}, {"name": "Remove systemd files to disable them", "file": {"path": "/etc/systemd/system/{{ item }}", "state": "absent"}, "with_items": ["tripleo_nova_libvirt.service", "tripleo_nova_virtlogd_wrapper.service", "tripleo_nova_libvirt.target"], "when": "is_virtlogd_image_running.stdout != ''"}, {"name": "reboot to perform the upgrade", "reboot": {"reboot_timeout": "{{upgrade_leapp_reboot_timeout}}", "test_command": "source /etc/os-release; [ \"${VERSION_ID%.*}\" -ge \"8\" ] && systemctl is-system-running | grep -qE \"running|degraded\" || exit 1", "post_reboot_delay": "{{ upgrade_leapp_post_reboot_delay }}"}}, {"name": "Set selinux back to enforcing after leapp reboot", "selinux": {"policy": "targeted", "state": "enforcing"}}, {"name": "Run LeappPostRebootCommand", "shell": "#!/bin/bash\n\n"}]}, {"name": "Host packages setup step0", "tags": "setup_packages", "when": "step|int == 0", "block": [{"name": "Package and repo update tasks", "block": [{"name": "Run UpgradeInitCommand", "shell": "#!/bin/bash\n\nif [[ -f /etc/resolv.conf.save ]] ; then rm /etc/resolv.conf.save; fi\n\n"}, {"name": "Run UpgradeInitCommonCommand", "shell": "#!/bin/bash\n\n"}, {"name": "Ensure DNF modules have the right stream", "vars": {"dnf_module_list": []}, "dnf": {"name": "@{{ item.module }}:{{ item.stream }}/{{ item.profile|default('common') }}", "state": "present"}, "loop": "{{ dnf_module_list|list }}", "when": ["dnf_module_list|length > 0", "item.distribution_version is defined", "ansible_facts['distribution_major_version'] is version(item.distribution_version, '==')"]}, {"name": "Ensure TripleO prerequisite packages are installed", "package": {"name": ["jq", "lvm2", "openstack-selinux", "os-net-config", "puppet-tripleo", "python3-heat-agent*", "rsync"], "state": "present"}, "when": "ansible_facts['distribution_major_version'] is version('8', '==')"}, {"name": "Ensure TripleO prerequisite packages are installed and use role based heat variable to provide specific list of packages", "vars": {"base_tripleo_packages": []}, "package": {"name": "{{ base_tripleo_packages }}", "state": "present"}, "when": ["ansible_facts['distribution_major_version'] is version('8', '==')", "base_tripleo_packages|length > 0"]}, {"name": "WA for 2240185 - If the image is schema 1 and lacks signatures than add empty signatures", "shell": "for manifest_file in `find /var/lib/containers/storage/overlay-images/ -name 'manifest'`\ndo\n    cat <<< $( jq  'if .schemaVersion == 1 then if has(\"signatures\") then . else .signatures=[] end else . end' $manifest_file ) > $manifest_file\ndone\n", "when": "ansible_facts['distribution_major_version'] is version('8', '==')"}]}, {"name": "check if libvirt is installed", "command": "/usr/bin/rpm -q libvirt-daemon", "failed_when": false, "register": "libvirt_installed", "check_mode": false}, {"name": "make sure libvirt services are disabled and masked", "service": {"name": "{{ item }}", "state": "stopped", "enabled": false, "masked": true, "daemon_reload": true}, "loop": ["libvirtd.service", "virtlogd.socket"], "when": ["libvirt_installed.rc == 0"]}]}, {"name": "Host packages setup step2", "tags": "setup_packages", "when": "step|int == 2", "block": [{"name": "Special treatment for OpenvSwitch", "tripleo_ovs_upgrade": null, "register": "ovs_upgrade"}, {"name": "Always ensure the openvswitch service is enabled and running after upgrades", "service": {"name": "openvswitch", "enabled": true, "state": "started"}, "when": ["ovs_upgrade.changed|bool"]}]}, {"name": "Host packages setup step3", "tags": "setup_packages", "when": "step|int == 3", "block": [{"name": "Check for os-net-config upgrade", "shell": "yum check-upgrade | awk '/os-net-config/{print}'", "register": "os_net_config_need_upgrade"}, {"name": "Check that os-net-config has legacy configuration", "stat": {"path": "/etc/os-net-config/config.json", "get_attributes": false, "get_checksum": false, "get_mime": false}, "register": "stat_config_json"}, {"name": "Check that os-net-config has new configuration", "stat": {"path": "/etc/os-net-config/config.yaml", "get_attributes": false, "get_checksum": false, "get_mime": false}, "register": "stat_config_yaml"}, {"name": "Slurp the os-net-config config.json", "slurp": {"src": "/etc/os-net-config/config.json"}, "register": "os_config_json", "when": ["stat_config_json.stat.exists", "not stat_config_yaml.stat.exists"]}, {"name": "Write updated /etc/os-net-config/config.yaml", "copy": {"content": "{{ os_config_json.content | b64decode | from_json | to_yaml }}", "dest": "/etc/os-net-config/config.yaml"}, "when": ["stat_config_json.stat.exists", "not stat_config_yaml.stat.exists"]}, {"name": "Remove legacy os-net-config configuration", "command": "mv /etc/os-net-config/config.json /etc/os-net-config/deprecated_config.json", "when": ["stat_config_json.stat.exists"]}, {"block": [{"name": "Upgrade os-net-config", "package": "name=os-net-config state=latest"}, {"name": "take new os-net-config parameters into account now", "command": "os-net-config --no-activate -c /etc/os-net-config/config.yaml -v --detailed-exit-codes", "register": "os_net_config_upgrade", "failed_when": "os_net_config_upgrade.rc not in [0,2]", "changed_when": "os_net_config_upgrade.rc == 2"}], "when": ["os_net_config_need_upgrade.stdout", "stat_config_yaml.stat.exists or stat_config_json.stat.exists"]}, {"name": "Update all packages", "when": ["not skip_package_update|bool"], "yum": {"name": "*", "state": "latest", "exclude": "ansible-core"}, "vars": {"skip_package_update": false}}, {"name": "Always ensure the openvswitch service is enabled and running after upgrades rhbz#2329821", "service": {"name": "openvswitch", "enabled": true, "state": "started"}}]}], "external_upgrade_tasks": [{"name": "Clean up upgrade artifacts", "when": "step|int == 1", "tags": ["never", "system_upgrade_cleanup"], "block": [{"name": "cleanup tripleo_persist", "include_role": {"name": "tripleo_persist", "tasks_from": "cleanup.yml"}}]}], "update_tasks": [{"name": "Enforce RHOSP rules regarding subscription.", "include_role": {"name": "tripleo_redhat_enforce"}, "vars": {"skip_rhel_enforcement": false}, "when": ["step|int == 0", "ansible_facts['distribution'] == 'RedHat'", "not (skip_rhel_enforcement | bool)"]}, {"name": "Ensure DNF modules have the right stream enabled", "vars": {"dnf_module_list": []}, "tripleo_dnf_stream": {"name": "{{ item.module }}:{{ item.stream }}", "state": "enabled"}, "loop": "{{ dnf_module_list|list }}", "when": ["step|int == 0", "dnf_module_list|length > 0", "item.distribution_version is defined", "ansible_facts['distribution_major_version'] is version(item.distribution_version, '==')"]}, {"name": "Check for existing yum.pid", "stat": "path=/run/yum.pid", "register": "yum_pid_file", "when": "step|int == 0 or step|int == 3"}, {"name": "Exit if existing yum process", "fail": "msg=\"ERROR existing yum.pid detected - can't continue! Please ensure there is no other package update process for the duration of the minor update worfklow. Exiting.\"", "when": "(step|int == 0 or step|int == 3) and yum_pid_file.stat.exists"}, {"name": "Special treatment for OpenvSwitch", "tripleo_ovs_upgrade": null, "when": ["step|int == 2"], "register": "ovs_upgrade"}, {"name": "Always ensure the openvswitch service is enabled and running after upgrades", "service": {"name": "openvswitch", "enabled": true, "state": "started"}, "when": ["step|int == 2", "ovs_upgrade.changed|bool"]}, {"name": "Update packages and EFI grub.cfg", "when": ["step|int == 3", "not skip_package_update|bool"], "block": [{"name": "Update all packages", "yum": {"name": "*", "state": "latest", "exclude": "ansible-core"}}, {"name": "Replace EFI grub.cfg with redirect to /boot/grub2/grub.cfg", "import_role": {"name": "tripleo_kernel", "tasks_from": "efigrub.yml"}}], "vars": {"skip_package_update": false}}, {"name": "Ensure openvswitch is running after update", "when": "step|int == 3", "service": {"name": "openvswitch", "enabled": true, "state": "started"}, "ignore_errors": true}]}, {"service_name": "tuned", "host_prep_tasks": [{"include_role": {"name": "tuned"}}], "ansible_group_vars": {"tuned_profile": "throughput-performance", "tuned_custom_profile": ""}}], "tuple:('role_data', 'step_config')": [null, null, null, "include tripleo::trusted_cas", null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, "include tripleo::profile::base::database::mysql::client", null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, "include tripleo::profile::base::pacemaker\n", null, "", null, "include tripleo::profile::base::snmp\n", null, null, null, null, null, null, null, null, "include ::tripleo::packages\n", null], "tuple:('role_data', 'service_names')": [null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null], "tuple:('role_data', 'docker_config')": [null, {"step_2": {"barbican_init_log": {"image": "registry.redhat.io/rhosp-rhel9/openstack-barbican-api:17.1", "net": "none", "user": "root", "volumes": ["/var/log/containers/barbican:/var/log/barbican:z", "/var/log/containers/httpd/barbican-api:/var/log/httpd:z"], "command": ["/bin/bash", "-c", "chown -R barbican:barbican /var/log/barbican"]}}, "step_3": {"barbican_api_db_sync": {"start_order": 3, "image": "registry.redhat.io/rhosp-rhel9/openstack-barbican-api:17.1", "cap_add": ["AUDIT_WRITE"], "net": "host", "detach": false, "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/barbican:/var/log/barbican:z", "/var/log/containers/httpd/barbican-api:/var/log/httpd:z", "/var/lib/config-data/puppet-generated/barbican:/var/lib/kolla/config_files/src:ro", "/var/lib/kolla/config_files/barbican_api_db_sync.json:/var/lib/kolla/config_files/config.json:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "barbican_api_secret_store_sync": {"start_order": 4, "image": "registry.redhat.io/rhosp-rhel9/openstack-barbican-api:17.1", "net": "host", "detach": false, "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/barbican:/var/log/barbican:z", "/var/log/containers/httpd/barbican-api:/var/log/httpd:z", "/var/lib/config-data/puppet-generated/barbican:/var/lib/kolla/config_files/src:ro", "/var/lib/kolla/config_files/barbican_api_secret_store_sync.json:/var/lib/kolla/config_files/config.json:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "barbican_api": {"start_order": 5, "image": "registry.redhat.io/rhosp-rhel9/openstack-barbican-api:17.1", "net": "host", "privileged": false, "restart": "always", "user": "root", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/barbican:/var/log/barbican:z", "/var/log/containers/httpd/barbican-api:/var/log/httpd:z", "/var/lib/kolla/config_files/barbican_api.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/barbican:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "barbican_keystone_listener": {"start_order": 6, "image": "registry.redhat.io/rhosp-rhel9/openstack-barbican-keystone-listener:17.1", "net": "host", "privileged": false, "restart": "always", "user": "barbican", "healthcheck": {"test": "/openstack/healthcheck 5672"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/barbican:/var/log/barbican:z", "/var/log/containers/httpd/barbican-api:/var/log/httpd:z", "/var/lib/kolla/config_files/barbican_keystone_listener.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/barbican:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "barbican_worker": {"start_order": 7, "image": "registry.redhat.io/rhosp-rhel9/openstack-barbican-worker:17.1", "net": "host", "privileged": false, "restart": "always", "user": "barbican", "healthcheck": {"test": "/openstack/healthcheck 5672"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/barbican:/var/log/barbican:z", "/var/log/containers/httpd/barbican-api:/var/log/httpd:z", "/var/lib/kolla/config_files/barbican_worker.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/barbican:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, null, null, {}, {}, {}, {}, {}, {"step_2": {"cinder_api_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-api:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/cinder:/var/log/cinder:z", "/var/log/containers/httpd/cinder-api:/var/log/httpd:z"], "command": ["/bin/bash", "-c", "chown -R cinder:cinder /var/log/cinder"]}}, "step_3": {"cinder_api_db_sync": {"image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-api:17.1", "cap_add": ["AUDIT_WRITE"], "net": "host", "privileged": false, "detach": false, "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/cinder_api_db_sync.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro", "/var/log/containers/cinder:/var/log/cinder:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_4": {"cinder_api": {"image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-api:17.1", "net": "host", "privileged": false, "restart": "always", "user": "root", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/cinder_api.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro", "/var/log/containers/cinder:/var/log/cinder:z", "/var/log/containers/httpd/cinder-api:/var/log/httpd:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "cinder_api_cron": {"image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-api:17.1", "net": "host", "user": "root", "privileged": false, "restart": "always", "healthcheck": {"test": "/usr/share/openstack-tripleo-common/healthcheck/cron cinder"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/cinder_api_cron.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro", "/var/log/containers/cinder:/var/log/cinder:z", "/var/log/containers/httpd/cinder-api:/var/log/httpd:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_3": {"cinder_backup_init_logs": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-backup:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/cinder:/var/log/cinder:z"], "command": ["/bin/bash", "-c", "chown -R cinder:cinder /var/log/cinder"]}}}, {"step_2": {"cinder_scheduler_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-scheduler:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/cinder:/var/log/cinder:z"], "command": ["/bin/bash", "-c", "chown -R cinder:cinder /var/log/cinder"]}}, "step_4": {"cinder_scheduler": {"image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-scheduler:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck 5672"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/config-data/puppet-generated/cinder:/var/lib/kolla/config_files/src:ro", "/var/log/containers/cinder:/var/log/cinder:z", "/var/lib/kolla/config_files/cinder_scheduler.json:/var/lib/kolla/config_files/config.json:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_3": {"cinder_volume_init_logs": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-cinder-volume:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/cinder:/var/log/cinder"], "command": ["/bin/bash", "-c", "chown -R cinder:cinder /var/log/cinder"]}}}, {"step_2": {"clustercheck": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-mariadb:17.1", "restart": "always", "net": "host", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/clustercheck.json:/var/lib/kolla/config_files/config.json", "/var/lib/config-data/puppet-generated/clustercheck:/var/lib/kolla/config_files/src:ro", "/var/lib/mysql:/var/lib/mysql"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, null, {"step_4": {"logrotate_crond": {"image": "registry.redhat.io/rhosp-rhel9/openstack-cron:17.1", "net": "none", "pid": "host", "privileged": true, "user": "root", "restart": "always", "healthcheck": {"test": "/usr/share/openstack-tripleo-common/healthcheck/cron"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/logrotate-crond.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/crond:/var/lib/kolla/config_files/src:ro", "/var/log/containers:/var/log/containers:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, null, {"step_2": {"glance_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-glance-api:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/glance:/var/log/glance:z", "/var/log/containers/httpd/glance:/var/log/httpd:z"], "command": ["/bin/bash", "-c", "chown -R glance:glance /var/log/glance"]}}, "step_3": {"glance_api_db_sync": {"image": "registry.redhat.io/rhosp-rhel9/openstack-glance-api:17.1", "cap_add": ["AUDIT_WRITE"], "net": "host", "privileged": false, "detach": false, "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/glance:/var/log/glance:z", "/var/log/containers/httpd/glance:/var/log/httpd:z", "/var/lib/kolla/config_files/glance_api.json:/var/lib/kolla/config_files/config.json", "/var/lib/config-data/puppet-generated/glance_api:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/config-data/puppet-generated/iscsid/etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro", "/var/lib/glance:/var/lib/glance:shared"], "environment": {"KOLLA_BOOTSTRAP": true, "KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}, "command": "/usr/bin/bootstrap_host_exec glance_api su glance -s /bin/bash -c '/usr/local/bin/kolla_start'"}}, "step_4": {"glance_api": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-glance-api:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/glance:/var/log/glance:z", "/var/log/containers/httpd/glance:/var/log/httpd:z", "/var/lib/kolla/config_files/glance_api.json:/var/lib/kolla/config_files/config.json", "/var/lib/config-data/puppet-generated/glance_api:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/config-data/puppet-generated/iscsid/etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro", "/var/lib/glance:/var/lib/glance:shared"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "glance_api_cron": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-glance-api:17.1", "net": "host", "user": "root", "privileged": false, "restart": "always", "healthcheck": {"test": "/usr/share/openstack-tripleo-common/healthcheck/cron glance"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/glance:/var/log/glance:z", "/var/log/containers/httpd/glance:/var/log/httpd:z", "/var/lib/kolla/config_files/glance_api_cron.json:/var/lib/kolla/config_files/config.json", "/var/lib/config-data/puppet-generated/glance_api:/var/lib/kolla/config_files/src:ro", "/var/lib/glance:/var/lib/glance:shared"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_2": {"glance_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-glance-api:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/glance:/var/log/glance:z", "/var/log/containers/httpd/glance:/var/log/httpd:z"], "command": ["/bin/bash", "-c", "chown -R glance:glance /var/log/glance"]}}, "step_4": {"glance_api_internal": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-glance-api:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/glance:/var/log/glance:z", "/var/log/containers/httpd/glance:/var/log/httpd:z", "/var/lib/kolla/config_files/glance_api.json:/var/lib/kolla/config_files/config.json", "/var/lib/config-data/puppet-generated/glance_api_internal:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/config-data/puppet-generated/iscsid/etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro", "/var/lib/glance:/var/lib/glance:shared"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, null, {"step_2": {}, "step_4": {"heat_api": {"image": "registry.redhat.io/rhosp-rhel9/openstack-heat-api:17.1", "net": "host", "privileged": false, "restart": "always", "user": "root", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/heat:/var/log/heat:z", "/var/log/containers/httpd/heat-api:/var/log/httpd:z", "/var/lib/kolla/config_files/heat_api.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/heat_api:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "heat_api_cron": {"image": "registry.redhat.io/rhosp-rhel9/openstack-heat-api:17.1", "net": "host", "user": "root", "privileged": false, "restart": "always", "healthcheck": {"test": "/usr/share/openstack-tripleo-common/healthcheck/cron heat"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/heat:/var/log/heat:z", "/var/log/containers/httpd/heat-api:/var/log/httpd:z", "/var/lib/kolla/config_files/heat_api_cron.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/heat_api:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_2": {}, "step_4": {"heat_api_cfn": {"image": "registry.redhat.io/rhosp-rhel9/openstack-heat-api-cfn:17.1", "net": "host", "privileged": false, "restart": "always", "user": "root", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/heat:/var/log/heat:z", "/var/log/containers/httpd/heat-api-cfn:/var/log/httpd:z", "/var/lib/kolla/config_files/heat_api_cfn.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/heat_api_cfn:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_2": {"heat_init_log": {"image": "registry.redhat.io/rhosp-rhel9/openstack-heat-engine:17.1", "net": "none", "user": "root", "volumes": ["/var/log/containers/heat:/var/log/heat:z"], "command": ["/bin/bash", "-c", "chown -R heat:heat /var/log/heat"]}}, "step_3": {"heat_engine_db_sync": {"image": "registry.redhat.io/rhosp-rhel9/openstack-heat-engine:17.1", "cap_add": ["AUDIT_WRITE"], "net": "host", "privileged": false, "detach": false, "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/heat:/var/log/heat:z", "/var/lib/kolla/config_files/heat_engine_db_sync.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/heat:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_4": {"heat_engine": {"image": "registry.redhat.io/rhosp-rhel9/openstack-heat-engine:17.1", "net": "host", "privileged": false, "stop_grace_period": 60, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck 5672"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/heat:/var/log/heat:z", "/var/lib/kolla/config_files/heat_engine.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/heat:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_2": {"horizon_fix_perms": {"image": "registry.redhat.io/rhosp-rhel9/openstack-horizon:17.1", "net": "none", "user": "root", "command": ["/bin/bash", "-c", "touch /var/log/horizon/horizon.log ; chown -R apache:apache /var/log/horizon && chmod -R a+rx /etc/openstack-dashboard"], "volumes": ["/var/log/containers/horizon:/var/log/horizon:z", "/var/log/containers/httpd/horizon:/var/log/httpd:z", "/var/lib/config-data/puppet-generated/horizon/etc/openstack-dashboard:/etc/openstack-dashboard"]}}, "step_3": {"horizon": {"image": "registry.redhat.io/rhosp-rhel9/openstack-horizon:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/horizon.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/horizon:/var/lib/kolla/config_files/src:ro", "/var/log/containers/horizon:/var/log/horizon:z", "/var/log/containers/httpd/horizon:/var/log/httpd:z", "/var/tmp/horizon:/var/tmp:z", "/var/www:/var/www:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "ENABLE_DESIGNATE": "yes", "ENABLE_HEAT": "yes", "ENABLE_IRONIC": "yes", "ENABLE_MANILA": "yes", "ENABLE_OCTAVIA": "yes"}}}}, {"step_3": {"iscsid": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-iscsid:17.1", "net": "host", "privileged": true, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/iscsid.json:/var/lib/kolla/config_files/config.json:ro", "/dev:/dev", "/run:/run", "/sys:/sys", "/lib/modules:/lib/modules:ro", "/var/lib/config-data/puppet-generated/iscsid/etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro", "/etc/target:/etc/target:z", "/var/lib/iscsi:/var/lib/iscsi:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, null, {"step_2": {"keystone_init_log": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-keystone:17.1", "net": "none", "user": "root", "command": ["/bin/bash", "-c", "chown -R keystone:keystone /var/log/keystone"], "volumes": ["/var/log/containers/keystone:/var/log/keystone:z", "/var/log/containers/httpd/keystone:/var/log/httpd:z"]}}, "step_3": {"keystone_db_sync": {"image": "registry.redhat.io/rhosp-rhel9/openstack-keystone:17.1", "net": "host", "user": "root", "privileged": false, "detach": false, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/keystone:/var/log/keystone:z", "/var/log/containers/httpd/keystone:/var/log/httpd:z", "/etc/openldap:/etc/openldap:ro", "/var/lib/kolla/config_files/keystone.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/keystone:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_BOOTSTRAP": true, "KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}, "command": ["/usr/bin/bootstrap_host_exec", "keystone", "/usr/local/bin/kolla_start"]}, "keystone": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-keystone:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/keystone:/var/log/keystone:z", "/var/log/containers/httpd/keystone:/var/log/httpd:z", "/etc/openldap:/etc/openldap:ro", "/var/lib/kolla/config_files/keystone.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/keystone:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "keystone_bootstrap": {"start_order": 3, "action": "exec", "user": "root", "command": ["keystone", "/usr/bin/bootstrap_host_exec", "keystone", "keystone-manage", "bootstrap"], "environment": {"KOLLA_BOOTSTRAP": true, "OS_BOOTSTRAP_PASSWORD": "79Ijk5YjIkKxsXqqoJYsL9qZ5", "OS_BOOTSTRAP_USERNAME": "admin", "OS_BOOTSTRAP_PROJECT_NAME": "admin", "OS_BOOTSTRAP_ROLE_NAME": "admin", "OS_BOOTSTRAP_SERVICE_NAME": "keystone", "OS_BOOTSTRAP_ADMIN_URL": "http://192.168.122.99:35357", "OS_BOOTSTRAP_PUBLIC_URL": "http://172.21.0.2:5000", "OS_BOOTSTRAP_INTERNAL_URL": "http://172.17.0.2:5000", "OS_BOOTSTRAP_REGION_ID": "regionOne"}}, "keystone_cron": {"start_order": 4, "image": "registry.redhat.io/rhosp-rhel9/openstack-keystone:17.1", "user": "root", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/usr/share/openstack-tripleo-common/healthcheck/cron keystone"}, "command": ["/bin/bash", "-c", "/usr/local/bin/kolla_set_configs && /usr/sbin/crond -n"], "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/keystone:/var/log/keystone:z", "/var/log/containers/httpd/keystone:/var/log/httpd:z", "/var/lib/kolla/config_files/keystone_cron.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/keystone:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}, "step_4": {"keystone_refresh": {"start_order": 1, "action": "exec", "user": "root", "command": ["keystone", "pkill", "--signal", "USR1", "httpd"]}}}, {"step_2": {"manila_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-manila-api:17.1", "net": "none", "user": "root", "volumes": ["/var/log/containers/manila:/var/log/manila:z", "/var/log/containers/httpd/manila-api:/var/log/httpd:z"], "command": ["/bin/bash", "-c", "chown -R manila:manila /var/log/manila"]}}, "step_3": {"manila_api_db_sync": {"user": "root", "image": "registry.redhat.io/rhosp-rhel9/openstack-manila-api:17.1", "cap_add": ["AUDIT_WRITE"], "net": "host", "detach": false, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/manila_api_db_sync.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/manila:/var/lib/kolla/config_files/src:ro", "/var/log/containers/manila:/var/log/manila:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}, "step_4": {"manila_api": {"image": "registry.redhat.io/rhosp-rhel9/openstack-manila-api:17.1", "net": "host", "restart": "always", "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/manila_api.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/manila:/var/lib/kolla/config_files/src:ro", "/var/log/containers/manila:/var/log/manila:z", "/var/log/containers/httpd/manila-api:/var/log/httpd:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "manila_api_cron": {"image": "registry.redhat.io/rhosp-rhel9/openstack-manila-api:17.1", "net": "host", "user": "root", "privileged": false, "restart": "always", "healthcheck": {"test": "/usr/share/openstack-tripleo-common/healthcheck/cron manila"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/manila_api_cron.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/manila:/var/lib/kolla/config_files/src:ro", "/var/log/containers/manila:/var/log/manila:z", "/var/log/containers/httpd/manila-api:/var/log/httpd:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, null, {"step_4": {"manila_scheduler": {"image": "registry.redhat.io/rhosp-rhel9/openstack-manila-scheduler:17.1", "net": "host", "restart": "always", "healthcheck": {"test": "/openstack/healthcheck 5672"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/manila_scheduler.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/manila:/var/lib/kolla/config_files/src:ro", "/var/log/containers/manila:/var/log/manila:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_3": {"manila_share_init_logs": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-manila-share:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/manila:/var/log/manila:z"], "command": ["/bin/bash", "-c", "chown -R manila:manila /var/log/manila"]}}}, {"step_1": {"memcached": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-memcached:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/memcached.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/memcached:/var/lib/kolla/config_files/src:rw,z", "/var/log/containers/memcached:/var/log/memcached:rw"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_1": {"mysql_data_ownership": {"start_order": 0, "detach": false, "image": "registry.redhat.io/rhosp-rhel9/openstack-mariadb:17.1", "net": "host", "user": "root", "command": ["chown", "-R", "mysql:", "/var/lib/mysql"], "volumes": ["/var/lib/mysql:/var/lib/mysql:z"]}, "mysql_bootstrap": {"start_order": 1, "detach": false, "image": "registry.redhat.io/rhosp-rhel9/openstack-mariadb:17.1", "net": "host", "user": "root", "command": ["bash", "-ec", "if [ -e /var/lib/mysql/mysql ]; then exit 0; fi\necho -e \"\\n[mysqld]\\nwsrep_provider=none\" >> /etc/my.cnf\nexport DB_ROOT_PASSWORD=$(hiera 'mysql::server::root_password')\nkolla_set_configs\nsudo -u mysql -E kolla_extend_start\ntimeout ${DB_MAX_TIMEOUT} /bin/bash -c 'while pgrep -af /usr/bin/mysqld_safe | grep -q -v grep; do sleep 1; done'\nmysqld_safe --skip-networking --wsrep-on=OFF &\ntimeout ${DB_MAX_TIMEOUT} /bin/bash -c 'until mysqladmin -uroot -p\"$(hiera 'mysql::server::root_password')\" ping 2>/dev/null; do sleep 1; done'\nmysql -uroot -p\"$(hiera 'mysql::server::root_password')\" -e \"CREATE USER 'clustercheck'@'localhost' IDENTIFIED BY '$(hiera mysql_clustercheck_password)';\"\nmysql -uroot -p\"$(hiera 'mysql::server::root_password')\" -e \"GRANT PROCESS ON *.* TO 'clustercheck'@'localhost' WITH GRANT OPTION;\"\nmysql -uroot -p\"$(hiera 'mysql::server::root_password')\" -e \"DELETE FROM mysql.user WHERE user = 'root' AND host NOT IN ('%','localhost');\"\ntimeout ${DB_MAX_TIMEOUT} mysqladmin -uroot -p\"$(hiera 'mysql::server::root_password')\" shutdown"], "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/mysql.json:/var/lib/kolla/config_files/config.json:rw,z", "/var/lib/config-data/puppet-generated/mysql:/var/lib/kolla/config_files/src:ro,z", "/var/lib/mysql:/var/lib/mysql:rw,z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "KOLLA_BOOTSTRAP": true, "DB_MAX_TIMEOUT": 60, "DB_MARIABACKUP_USER": "mariabackup", "DB_MARIABACKUP_PASSWORD": "3Qi03Zmdc69WCw00okVFEQzzM"}}}, "step_2": {"mysql_wait_bundle": {"start_order": 0, "detach": false, "net": "host", "ipc": "host", "user": "root", "command": ["/container_puppet_apply.sh", "2", "file,file_line,concat,augeas,galera_ready,mysql_database,mysql_grant,mysql_user", "include tripleo::profile::pacemaker::database::mysql_bundle"], "image": "registry.redhat.io/rhosp-rhel9/openstack-mariadb:17.1", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/var/lib/container-config-scripts/container_puppet_apply.sh:/container_puppet_apply.sh:ro", "/etc/puppet:/tmp/puppet-etc:ro", "/usr/share/openstack-puppet/modules:/usr/share/openstack-puppet/modules:ro", "/var/lib/mysql:/var/lib/mysql:rw,z", "/var/lib/config-data/puppet-generated/mysql/root:/root:rw"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}}, null, {"step_2": {"neutron_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-server:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/neutron:/var/log/neutron:z", "/var/log/containers/httpd/neutron-api:/var/log/httpd:z"], "command": ["/bin/bash", "-c", "chown -R neutron:neutron /var/log/neutron"]}}, "step_3": {"neutron_db_sync": {"image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-server:17.1", "cap_add": ["AUDIT_WRITE"], "net": "host", "privileged": false, "detach": false, "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/neutron:/var/log/neutron:z", "/var/log/containers/httpd/neutron-api:/var/log/httpd:z", "/var/lib/kolla/config_files/neutron_api_db_sync.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/neutron:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_4": {"neutron_api": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-server:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/neutron:/var/log/neutron:z", "/var/log/containers/httpd/neutron-api:/var/log/httpd:z", "/var/lib/kolla/config_files/neutron_api.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/neutron:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {}, {"step_4": {"neutron_sriov_agent": {"start_order": 10, "image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-sriov-agent:17.1", "net": "host", "pid": "host", "privileged": true, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck 5672"}, "ulimit": ["nofile=16384"], "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/neutron:/var/log/neutron:z", "/var/lib/kolla/config_files/neutron_sriov_agent.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/neutron:/var/lib/kolla/config_files/src:ro", "/lib/modules:/lib/modules:ro", "/run:/run", "/sys/class/net:/sys/class/net:rw"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_2": {"create_dnsmasq_wrapper": {"start_order": 1, "detach": false, "net": "host", "pid": "host", "user": "root", "command": ["/container_puppet_apply.sh", "4", "file", "include ::tripleo::profile::base::neutron::dhcp_agent_wrappers"], "image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-dhcp-agent:17.1", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/var/lib/container-config-scripts/container_puppet_apply.sh:/container_puppet_apply.sh:ro", "/etc/puppet:/tmp/puppet-etc:ro", "/usr/share/openstack-puppet/modules:/usr/share/openstack-puppet/modules:ro", "/run/openvswitch:/run/openvswitch:shared,z", "/var/lib/neutron:/var/lib/neutron:shared,z"]}}, "step_4": {"neutron_dhcp": {"start_order": 10, "image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-dhcp-agent:17.1", "net": "host", "pid": "host", "cgroupns": "host", "privileged": true, "restart": "always", "security_opt": ["label=disable"], "depends_on": ["openvswitch.service"], "healthcheck": {"test": "/openstack/healthcheck 5672"}, "ulimit": ["nofile=16384"], "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/neutron:/var/log/neutron:z", "/var/lib/kolla/config_files/neutron_dhcp.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/neutron:/var/lib/kolla/config_files/src:ro", "/lib/modules:/lib/modules:ro", "/run/openvswitch:/run/openvswitch:shared,z", "/var/lib/neutron:/var/lib/neutron:shared,z", "/run/netns:/run/netns:shared", "/var/lib/neutron/kill_scripts:/etc/neutron/kill_scripts:shared,z", "/var/lib/neutron/dnsmasq_wrapper:/usr/local/bin/dnsmasq:ro", "/var/lib/neutron/dhcp_haproxy_wrapper:/usr/local/bin/haproxy:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_2": {"nova_api_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/nova:/var/log/nova:z", "/var/log/containers/httpd/nova-api:/var/log/httpd:z"], "command": ["/bin/bash", "-c", "chown -R nova:nova /var/log/nova"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_3": {"nova_api_db_sync": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1", "cap_add": ["AUDIT_WRITE"], "net": "host", "detach": false, "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova:z", "/var/log/containers/httpd/nova-api:/var/log/httpd:z", "/var/lib/kolla/config_files/nova_api_db_sync.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}, "nova_api_ensure_default_cells": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1", "user": "root", "net": "host", "privileged": false, "detach": false, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova:z", "/var/log/containers/httpd/nova-api:/var/log/httpd:z", "/var/lib/kolla/config_files/nova_api_ensure_default_cells.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro", "/var/lib/container-config-scripts:/container-config-scripts:z"], "environment": {"__OS_DEBUG": "true", "KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}, "step_4": {"nova_api": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1", "net": "host", "user": "root", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova:z", "/var/log/containers/httpd/nova-api:/var/log/httpd:z", "/var/lib/kolla/config_files/nova_api.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "nova_wait_for_api_service": {"start_order": 3, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1", "user": "root", "net": "host", "privileged": false, "detach": false, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova:z", "/var/log/containers/httpd/nova-api:/var/log/httpd:z", "/var/lib/kolla/config_files/nova_wait_for_api_service.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro", "/var/lib/container-config-scripts:/container-config-scripts:z"], "environment": {"__OS_DEBUG": "true", "KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "nova_api_cron": {"start_order": 4, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1", "net": "host", "user": "root", "privileged": false, "restart": "always", "healthcheck": {"test": "/usr/share/openstack-tripleo-common/healthcheck/cron nova"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova:z", "/var/log/containers/httpd/nova-api:/var/log/httpd:z", "/var/lib/kolla/config_files/nova_api_cron.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_2": {"nova_conductor_init_log": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-conductor:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/nova:/var/log/nova:z"], "command": ["/bin/bash", "-c", "chown -R nova:nova /var/log/nova"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_3": {"nova_db_sync": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-conductor:17.1", "cap_add": ["AUDIT_WRITE"], "start_order": 3, "net": "host", "detach": false, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova", "/var/lib/kolla/config_files/nova_conductor_db_sync.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro"], "user": "root", "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_4": {"nova_conductor": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-conductor:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck 5672"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova", "/var/lib/kolla/config_files/nova_conductor.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_2": {"nova_metadata_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/nova:/var/log/nova:z", "/var/log/containers/httpd/nova-metadata:/var/log/httpd:z"], "command": ["/bin/bash", "-c", "chown -R nova:nova /var/log/nova"]}}, "step_4": {"nova_metadata": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-api:17.1", "net": "host", "user": "root", "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova:z", "/var/log/containers/httpd/nova-metadata:/var/log/httpd:z", "/var/lib/kolla/config_files/nova_metadata.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova_metadata:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_4": {"nova_scheduler": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-scheduler:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck 5672"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova", "/var/lib/kolla/config_files/nova_scheduler.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_2": {"nova_compute_init_log": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-compute:17.1", "net": "none", "privileged": false, "user": "root", "volumes": ["/var/log/containers/nova:/var/log/nova:z"], "command": ["/bin/bash", "-c", "chown -R nova:nova /var/log/nova"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_3": {"nova_statedir_owner": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-compute:17.1", "net": "none", "user": "root", "security_opt": ["label=disable"], "privileged": false, "detach": false, "volumes": ["/var/lib/nova:/var/lib/nova:shared", "/var/lib/_nova_secontext:/var/lib/_nova_secontext:shared,z", "/var/lib/container-config-scripts:/container-config-scripts:z"], "command": "/container-config-scripts/pyshim.sh /container-config-scripts/nova_statedir_ownership.py", "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992", "__OS_DEBUG": "true", "NOVA_STATEDIR_OWNERSHIP_SKIP": "triliovault-mounts"}}}, "step_5": {"nova_compute": {"start_order": 3, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-compute:17.1", "ulimit": ["nofile=131072", "memlock=67108864"], "ipc": "host", "net": "host", "privileged": true, "user": "nova", "restart": "always", "depends_on": ["tripleo_nova_libvirt.target"], "healthcheck": {"test": "/openstack/healthcheck 5672"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova", "/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro", "/var/lib/kolla/config_files/nova_compute.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/var/lib/config-data/puppet-generated/iscsid/etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/dev:/dev", "/lib/modules:/lib/modules:ro", "/run:/run", "/run/nova:/run/nova:z", "/var/lib/iscsi:/var/lib/iscsi:z", "/var/lib/libvirt:/var/lib/libvirt:shared", "/sys/class/net:/sys/class/net", "/sys/bus/pci:/sys/bus/pci", "/boot:/boot:ro", "/var/lib/nova:/var/lib/nova:shared"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "LIBGUESTFS_BACKEND": "direct"}}, "nova_wait_for_compute_service": {"start_order": 4, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-compute:17.1", "net": "host", "detach": false, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/nova_compute_wait_for_compute_service.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/var/log/containers/nova:/var/log/nova", "/var/lib/container-config-scripts:/container-config-scripts"], "user": "nova", "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "__OS_DEBUG": "true"}}}}, {"step_2": {"nova_virtqemud_init_logs": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "net": "none", "privileged": true, "security_opt": ["label=level:s0", "label=type:spc_t", "label=filetype:container_file_t"], "user": "root", "volumes": ["/var/log/containers/libvirt/swtpm:/var/log/swtpm:shared,z"], "command": ["/bin/bash", "-c", "chown -R tss:tss /var/log/swtpm"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}, "create_virtlogd_wrapper": {"start_order": 1, "detach": false, "cgroupns": "host", "net": "host", "pid": "host", "user": "root", "command": ["/container_puppet_apply.sh", "4", "file", "include ::tripleo::profile::base::nova::virtlogd_wrapper"], "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/var/lib/container-config-scripts/container_puppet_apply.sh:/container_puppet_apply.sh:ro", "/etc/puppet:/tmp/puppet-etc:ro", "/usr/share/openstack-puppet/modules:/usr/share/openstack-puppet/modules:ro", "/var/lib/container-config-scripts:/var/lib/container-config-scripts:shared,z"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_3": {"nova_virtlogd_wrapper": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "ulimit": ["nofile=131072", "nproc=126960"], "cgroupns": "host", "net": "host", "pid": "host", "privileged": true, "security_opt": ["label=level:s0", "label=type:spc_t", "label=filetype:container_file_t"], "restart": "always", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/libvirt:/var/log/libvirt:shared,z", "/lib/modules:/lib/modules:ro", "/dev:/dev", "/run:/run", "/sys/fs/cgroup:/sys/fs/cgroup", "/sys/fs/selinux:/sys/fs/selinux", "/etc/selinux/config:/etc/selinux/config:ro", "/etc/libvirt:/etc/libvirt:shared", "/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro", "/run/libvirt:/run/libvirt:shared,z", "/var/lib/nova:/var/lib/nova:shared", "/var/lib/libvirt:/var/lib/libvirt:shared", "/var/cache/libvirt:/var/cache/libvirt:shared", "/var/lib/vhost_sockets:/var/lib/vhost_sockets", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/kolla/config_files/nova_virtlogd.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/container-config-scripts/virtlogd_wrapper:/usr/local/bin/virtlogd_wrapper:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "nova_virtsecretd": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "ulimit": ["nofile=131072", "nproc=126960"], "cgroupns": "host", "net": "host", "pid": "host", "pids_limit": 65536, "privileged": true, "security_opt": ["label=level:s0", "label=type:spc_t", "label=filetype:container_file_t"], "restart": "always", "depends_on": ["tripleo_nova_virtlogd_wrapper.service"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/libvirt:/var/log/libvirt:shared,z", "/lib/modules:/lib/modules:ro", "/dev:/dev", "/run:/run", "/sys/fs/cgroup:/sys/fs/cgroup", "/sys/fs/selinux:/sys/fs/selinux", "/etc/selinux/config:/etc/selinux/config:ro", "/etc/libvirt:/etc/libvirt:shared", "/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro", "/run/libvirt:/run/libvirt:shared,z", "/var/lib/nova:/var/lib/nova:shared", "/var/lib/libvirt:/var/lib/libvirt:shared", "/var/cache/libvirt:/var/cache/libvirt:shared", "/var/lib/vhost_sockets:/var/lib/vhost_sockets", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/kolla/config_files/nova_virtsecretd.json:/var/lib/kolla/config_files/config.json:ro"]}, "nova_virtnodedevd": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "ulimit": ["nofile=131072", "nproc=126960"], "cgroupns": "host", "net": "host", "pid": "host", "pids_limit": 65536, "privileged": true, "security_opt": ["label=level:s0", "label=type:spc_t", "label=filetype:container_file_t"], "restart": "always", "depends_on": ["tripleo_nova_virtlogd_wrapper.service"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/libvirt:/var/log/libvirt:shared,z", "/lib/modules:/lib/modules:ro", "/dev:/dev", "/run:/run", "/sys/fs/cgroup:/sys/fs/cgroup", "/sys/fs/selinux:/sys/fs/selinux", "/etc/selinux/config:/etc/selinux/config:ro", "/etc/libvirt:/etc/libvirt:shared", "/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro", "/run/libvirt:/run/libvirt:shared,z", "/var/lib/nova:/var/lib/nova:shared", "/var/lib/libvirt:/var/lib/libvirt:shared", "/var/cache/libvirt:/var/cache/libvirt:shared", "/var/lib/vhost_sockets:/var/lib/vhost_sockets", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/kolla/config_files/nova_virtnodedevd.json:/var/lib/kolla/config_files/config.json:ro"]}, "nova_virtstoraged": {"start_order": 3, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "ulimit": ["nofile=131072", "nproc=126960"], "cgroupns": "host", "net": "host", "pid": "host", "pids_limit": 65536, "privileged": true, "security_opt": ["label=level:s0", "label=type:spc_t", "label=filetype:container_file_t"], "restart": "always", "depends_on": ["tripleo_nova_virtlogd_wrapper.service"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/libvirt:/var/log/libvirt:shared,z", "/lib/modules:/lib/modules:ro", "/dev:/dev", "/run:/run", "/sys/fs/cgroup:/sys/fs/cgroup", "/sys/fs/selinux:/sys/fs/selinux", "/etc/selinux/config:/etc/selinux/config:ro", "/etc/libvirt:/etc/libvirt:shared", "/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro", "/run/libvirt:/run/libvirt:shared,z", "/var/lib/nova:/var/lib/nova:shared", "/var/lib/libvirt:/var/lib/libvirt:shared", "/var/cache/libvirt:/var/cache/libvirt:shared", "/var/lib/vhost_sockets:/var/lib/vhost_sockets", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/kolla/config_files/nova_virtstoraged.json:/var/lib/kolla/config_files/config.json:ro"]}, "nova_virtqemud": {"start_order": 4, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "ulimit": ["nofile=131072", "nproc=126960"], "cgroupns": "host", "net": "host", "pid": "host", "pids_limit": 65536, "privileged": true, "security_opt": ["label=level:s0", "label=type:spc_t", "label=filetype:container_file_t"], "restart": "always", "depends_on": ["tripleo_nova_virtlogd_wrapper.service"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/libvirt:/var/log/libvirt:shared,z", "/lib/modules:/lib/modules:ro", "/dev:/dev", "/run:/run", "/sys/fs/cgroup:/sys/fs/cgroup", "/sys/fs/selinux:/sys/fs/selinux", "/etc/selinux/config:/etc/selinux/config:ro", "/etc/libvirt:/etc/libvirt:shared", "/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro", "/run/libvirt:/run/libvirt:shared,z", "/var/lib/nova:/var/lib/nova:shared", "/var/lib/libvirt:/var/lib/libvirt:shared", "/var/cache/libvirt:/var/cache/libvirt:shared", "/var/lib/vhost_sockets:/var/lib/vhost_sockets", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/kolla/config_files/nova_virtqemud.json:/var/lib/kolla/config_files/config.json:ro", "/var/log/containers/libvirt/swtpm:/var/log/swtpm:z"]}, "nova_virtproxyd": {"start_order": 5, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "ulimit": ["nofile=131072", "nproc=126960"], "cgroupns": "host", "net": "host", "pid": "host", "pids_limit": 65536, "privileged": true, "security_opt": ["label=level:s0", "label=type:spc_t", "label=filetype:container_file_t"], "restart": "always", "depends_on": ["tripleo_nova_virtlogd_wrapper.service"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/libvirt:/var/log/libvirt:shared,z", "/lib/modules:/lib/modules:ro", "/dev:/dev", "/run:/run", "/sys/fs/cgroup:/sys/fs/cgroup", "/sys/fs/selinux:/sys/fs/selinux", "/etc/selinux/config:/etc/selinux/config:ro", "/etc/libvirt:/etc/libvirt:shared", "/etc/ssh/ssh_known_hosts:/etc/ssh/ssh_known_hosts:ro", "/run/libvirt:/run/libvirt:shared,z", "/var/lib/nova:/var/lib/nova:shared", "/var/lib/libvirt:/var/lib/libvirt:shared", "/var/cache/libvirt:/var/cache/libvirt:shared", "/var/lib/vhost_sockets:/var/lib/vhost_sockets", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/var/lib/tripleo-config/ceph:/var/lib/kolla/config_files/src-ceph:z", "/var/lib/kolla/config_files/nova_virtproxyd.json:/var/lib/kolla/config_files/config.json:ro"]}}, "step_4": {"nova_libvirt_init_secret": {"detach": false, "image": "registry.redhat.io/rhosp-rhel9/openstack-nova-libvirt:17.1", "security_opt": ["label=disable"], "privileged": false, "user": "root", "cgroupns": "host", "net": "host", "environment": {"LIBVIRT_DEFAULT_URI": "qemu:///system"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/config-data/puppet-generated/nova_libvirt/etc/nova:/etc/nova", "/etc/libvirt:/etc/libvirt", "/run/libvirt:/run/libvirt:shared,z", "/var/lib/libvirt:/var/lib/libvirt:shared", "/var/lib/container-config-scripts/nova_libvirt_init_secret.sh:/nova_libvirt_init_secret.sh:ro", "/var/lib/tripleo-config/ceph:/etc/ceph:ro"], "command": "/nova_libvirt_init_secret.sh ceph:openstack"}}}, {"step_4": {"nova_migration_target": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-compute:17.1", "net": "host", "privileged": true, "user": "root", "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/nova-migration-target.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro", "/etc/ssh:/host-ssh:ro", "/run/libvirt:/run/libvirt:shared,z", "/var/lib/nova:/var/lib/nova:shared"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_4": {"nova_vnc_proxy": {"image": "registry.redhat.io/rhosp-rhel9/openstack-nova-novncproxy:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/nova:/var/log/nova", "/var/lib/kolla/config_files/nova_vnc_proxy.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/nova:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_4": {"configure_cms_options": {"start_order": 0, "detach": false, "net": "host", "privileged": true, "user": "root", "command": ["/bin/bash", "-c", "CMS_OPTS=$(hiera ovn::controller::ovn_cms_options -c /etc/puppet/hiera.yaml);  if [ X\"$CMS_OPTS\" !=  X ]; then ovs-vsctl set open . external_ids:ovn-cms-options=$CMS_OPTS;else ovs-vsctl remove open . external_ids ovn-cms-options; fi"], "image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-controller:17.1", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/lib/modules:/lib/modules:ro", "/run/openvswitch:/run/openvswitch:shared,z"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}, "ovn_controller": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-controller:17.1", "net": "host", "privileged": true, "user": "root", "restart": "always", "depends_on": ["openvswitch.service"], "healthcheck": {"test": "/openstack/healthcheck 6642"}, "volumes": ["/var/lib/kolla/config_files/ovn_controller.json:/var/lib/kolla/config_files/config.json:ro", "/lib/modules:/lib/modules:ro", "/run:/run", "/var/lib/openvswitch/ovn:/run/ovn:shared,z", "/var/log/containers/openvswitch:/var/log/openvswitch:z", "/var/log/containers/openvswitch:/var/log/ovn:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_0": {"ovn_cluster_north_db_server": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-nb-db-server:17.1", "net": "host", "privileged": false, "restart": "always", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/ovn_cluster_north_db_server.json:/var/lib/kolla/config_files/config.json:ro", "/lib/modules:/lib/modules:ro", "/var/lib/openvswitch/ovn:/var/lib/openvswitch:shared,z", "/var/lib/openvswitch/ovn:/run/openvswitch:shared,z", "/var/log/containers/openvswitch:/var/log/openvswitch:z", "/var/lib/openvswitch/ovn:/var/lib/ovn:shared,z", "/var/lib/openvswitch/ovn:/etc/openvswitch:shared,z", "/var/lib/openvswitch/ovn:/etc/ovn:shared,z", "/var/lib/openvswitch/ovn:/run/ovn:shared,z", "/var/log/containers/openvswitch:/var/log/ovn:z", "/var/lib/config-data/ansible-generated/ovn:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "ovn_cluster_south_db_server": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-sb-db-server:17.1", "net": "host", "privileged": false, "restart": "always", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/ovn_cluster_south_db_server.json:/var/lib/kolla/config_files/config.json:ro", "/lib/modules:/lib/modules:ro", "/var/lib/openvswitch/ovn:/var/lib/openvswitch:shared,z", "/var/lib/openvswitch/ovn:/run/openvswitch:shared,z", "/var/log/containers/openvswitch:/var/log/openvswitch:z", "/var/lib/openvswitch/ovn:/var/lib/ovn:shared,z", "/var/lib/openvswitch/ovn:/etc/openvswitch:shared,z", "/var/lib/openvswitch/ovn:/etc/ovn:shared,z", "/var/lib/openvswitch/ovn:/run/ovn:shared,z", "/var/log/containers/openvswitch:/var/log/ovn:z", "/var/lib/config-data/ansible-generated/ovn:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "ovn_cluster_northd": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-ovn-northd:17.1", "net": "host", "privileged": false, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/ovn_cluster_northd.json:/var/lib/kolla/config_files/config.json:ro", "/lib/modules:/lib/modules:ro", "/var/lib/openvswitch/ovn:/run/openvswitch:shared,z", "/var/log/containers/openvswitch:/var/log/openvswitch:z", "/var/lib/openvswitch/ovn:/run/ovn:shared,z", "/var/log/containers/openvswitch:/var/log/ovn:z", "/var/lib/config-data/ansible-generated/ovn:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_2": {"create_haproxy_wrapper": {"start_order": 1, "detach": false, "net": "host", "pid": "host", "user": "root", "command": ["/container_puppet_apply.sh", "4", "file", "include ::tripleo::profile::base::neutron::ovn_metadata_agent_wrappers"], "image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-metadata-agent-ovn:17.1", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/var/lib/container-config-scripts/container_puppet_apply.sh:/container_puppet_apply.sh:ro", "/etc/puppet:/tmp/puppet-etc:ro", "/usr/share/openstack-puppet/modules:/usr/share/openstack-puppet/modules:ro", "/run/openvswitch:/run/openvswitch:shared,z", "/var/lib/neutron:/var/lib/neutron:shared,z"]}}, "step_4": {"setup_ovs_manager": {"start_order": 0, "detach": false, "net": "host", "privileged": true, "user": "root", "command": ["/container_puppet_apply.sh", "4", "exec", "include tripleo::profile::base::neutron::ovn_metadata"], "image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-metadata-agent-ovn:17.1", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/var/lib/container-config-scripts/container_puppet_apply.sh:/container_puppet_apply.sh:ro", "/etc/puppet:/tmp/puppet-etc:ro", "/usr/share/openstack-puppet/modules:/usr/share/openstack-puppet/modules:ro", "/lib/modules:/lib/modules:ro", "/run/openvswitch:/run/openvswitch:shared,z"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}, "ovn_metadata_agent": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-neutron-metadata-agent-ovn:17.1", "net": "host", "pid": "host", "cgroupns": "host", "privileged": true, "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/neutron:/var/log/neutron:z", "/var/lib/kolla/config_files/ovn_metadata_agent.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/neutron:/var/lib/kolla/config_files/src:ro", "/lib/modules:/lib/modules:ro", "/run/openvswitch:/run/openvswitch:shared,z", "/var/lib/neutron:/var/lib/neutron:shared,z", "/run/netns:/run/netns:shared", "/var/lib/neutron/kill_scripts:/etc/neutron/kill_scripts:shared,z", "/var/lib/neutron/ovn_metadata_haproxy_wrapper:/usr/local/bin/haproxy:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, null, null, {"step_1": {"rabbitmq_bootstrap": {"start_order": 0, "image": "registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.1", "net": "host", "privileged": false, "user": "root", "command": ["bash", "-ec", "kolla_set_configs\nif [[ -e \"/var/lib/rabbitmq/.erlang.cookie\" ]]; then rm -f /var/lib/rabbitmq/.erlang.cookie; fi\nhiera 'rabbitmq::erlang_cookie' > /var/lib/rabbitmq/.erlang.cookie\nchown rabbitmq:rabbitmq /var/lib/rabbitmq/.erlang.cookie\nchmod 400 /var/lib/rabbitmq/.erlang.cookie"], "volumes": ["/var/lib/kolla/config_files/rabbitmq.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/rabbitmq:/var/lib/kolla/config_files/src:ro", "/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/var/lib/rabbitmq:/var/lib/rabbitmq:z", "/etc/puppet:/etc/puppet:ro,z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_2": {"rabbitmq_wait_bundle": {"start_order": 0, "detach": false, "net": "host", "ipc": "host", "user": "root", "command": ["/container_puppet_apply.sh", "2", "file,file_line,concat,augeas,rabbitmq_policy,rabbitmq_user,rabbitmq_ready", "include tripleo::profile::pacemaker::rabbitmq_bundle", ""], "image": "registry.redhat.io/rhosp-rhel9/openstack-rabbitmq:17.1", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/var/lib/container-config-scripts/container_puppet_apply.sh:/container_puppet_apply.sh:ro", "/etc/puppet:/tmp/puppet-etc:ro", "/usr/share/openstack-puppet/modules:/usr/share/openstack-puppet/modules:ro", "/bin/true:/bin/epmd"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "KOLLA_BOOTSTRAP": true, "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}}, null, {"step_2": {"placement_init_log": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-placement-api:17.1", "net": "none", "user": "root", "command": ["/bin/bash", "-c", "chown -R placement:placement /var/log/placement"], "volumes": ["/var/log/containers/placement:/var/log/placement:z", "/var/log/containers/httpd/placement:/var/log/httpd:z"]}}, "step_3": {"placement_api_db_sync": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-placement-api:17.1", "cap_add": ["AUDIT_WRITE"], "net": "host", "detach": false, "user": "root", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/placement:/var/log/placement:z", "/var/log/containers/httpd/placement:/var/log/httpd:z", "/var/lib/kolla/config_files/placement_api_db_sync.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/placement:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS", "TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_4": {"placement_api": {"start_order": 1, "image": "registry.redhat.io/rhosp-rhel9/openstack-placement-api:17.1", "net": "host", "user": "root", "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/placement:/var/log/placement:z", "/var/log/containers/httpd/placement:/var/log/httpd:z", "/var/lib/kolla/config_files/placement_api.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/placement:/var/lib/kolla/config_files/src:ro"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "placement_wait_for_service": {"start_order": 2, "image": "registry.redhat.io/rhosp-rhel9/openstack-placement-api:17.1", "user": "root", "net": "host", "privileged": false, "detach": false, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/log/containers/placement:/var/log/placement:z", "/var/log/containers/httpd/placement:/var/log/httpd:z", "/var/lib/kolla/config_files/placement_api_wait_for_service.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/placement:/var/lib/kolla/config_files/src:ro", "/var/lib/container-config-scripts:/container-config-scripts:z"], "environment": {"__OS_DEBUG": "true", "KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, null, {}, null, null, {"step_4": {"swift_proxy": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-proxy-server:17.1", "start_order": 2, "net": "host", "user": "swift", "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/swift_proxy.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:ro", "/srv/node:/srv/node", "/dev:/dev", "/var/log/containers/swift:/var/log/swift:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, {"step_3": {"swift_copy_rings": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-proxy-server:17.1", "net": "none", "user": "root", "detach": false, "command": ["/bin/bash", "-c", "cp -v -dR --preserve -t /etc/swift /swift_ringbuilder/etc/swift/*.gz /swift_ringbuilder/etc/swift/*.builder /swift_ringbuilder/etc/swift/backups"], "volumes": ["/var/lib/config-data/puppet-generated/swift/etc/swift:/etc/swift:rw,z", "/var/lib/config-data/puppet-generated/swift_ringbuilder:/swift_ringbuilder:ro"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}}, {"step_3": {"swift_setup_srv": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-account:17.1", "net": "none", "user": "root", "command": ["find", "/srv/node", "-maxdepth", "1", "-type", "d", "-exec", "chown", "swift:", "{}", ";"], "volumes": ["/srv/node:/srv/node:z"], "environment": {"TRIPLEO_DEPLOY_IDENTIFIER": "1764486992"}}}, "step_4": {"swift_account_reaper": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-account:17.1", "net": "host", "user": "swift", "restart": "always", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/swift_account_reaper.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:ro", "/srv/node:/srv/node", "/dev:/dev", "/var/cache/swift:/var/cache/swift:z", "/var/log/containers/swift:/var/log/swift:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "swift_account_server": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-account:17.1", "net": "host", "user": "swift", "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/swift_account_server.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:ro", "/srv/node:/srv/node", "/dev:/dev", "/var/cache/swift:/var/cache/swift", "/var/log/containers/swift:/var/log/swift:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "swift_container_updater": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-container:17.1", "net": "host", "user": "swift", "restart": "always", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/swift_container_updater.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:ro", "/srv/node:/srv/node", "/dev:/dev", "/var/cache/swift:/var/cache/swift", "/var/log/containers/swift:/var/log/swift:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "swift_container_server": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-container:17.1", "net": "host", "user": "swift", "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/swift_container_server.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:ro", "/srv/node:/srv/node", "/dev:/dev", "/var/cache/swift:/var/cache/swift", "/var/log/containers/swift:/var/log/swift:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "swift_object_expirer": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-proxy-server:17.1", "net": "host", "user": "swift", "restart": "always", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/swift_object_expirer.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:ro", "/srv/node:/srv/node", "/dev:/dev", "/var/cache/swift:/var/cache/swift", "/var/log/containers/swift:/var/log/swift:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "swift_object_updater": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-object:17.1", "net": "host", "user": "swift", "restart": "always", "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/swift_object_updater.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:ro", "/srv/node:/srv/node", "/dev:/dev", "/var/cache/swift:/var/cache/swift", "/var/log/containers/swift:/var/log/swift:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}, "swift_object_server": {"image": "registry.redhat.io/rhosp-rhel9/openstack-swift-object:17.1", "net": "host", "user": "swift", "restart": "always", "healthcheck": {"test": "/openstack/healthcheck"}, "volumes": ["/etc/hosts:/etc/hosts:ro", "/etc/localtime:/etc/localtime:ro", "/etc/pki/ca-trust/extracted:/etc/pki/ca-trust/extracted:ro", "/etc/pki/ca-trust/source/anchors:/etc/pki/ca-trust/source/anchors:ro", "/etc/pki/tls/certs/ca-bundle.crt:/etc/pki/tls/certs/ca-bundle.crt:ro", "/etc/pki/tls/certs/ca-bundle.trust.crt:/etc/pki/tls/certs/ca-bundle.trust.crt:ro", "/etc/pki/tls/cert.pem:/etc/pki/tls/cert.pem:ro", "/dev/log:/dev/log", "/etc/puppet:/etc/puppet:ro", "/var/lib/kolla/config_files/swift_object_server.json:/var/lib/kolla/config_files/config.json:ro", "/var/lib/config-data/puppet-generated/swift:/var/lib/kolla/config_files/src:ro", "/srv/node:/srv/node", "/dev:/dev", "/var/cache/swift:/var/cache/swift", "/var/log/containers/swift:/var/log/swift:z"], "environment": {"KOLLA_CONFIG_STRATEGY": "COPY_ALWAYS"}}}}, null, null, null, null, null, null]}


