Last active
December 13, 2016 15:25
-
-
Save holser/8a41dd38c3cea6f3092cd1016287564e to your computer and use it in GitHub Desktop.
Test galera for ccp
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| #!/bin/bash | |
| TIMESTAMP=`date "+%Y-%m-%d %H:%M:%S"` | |
| #trap "cleanup; exit 1" 2 | |
| function prepare_topology() { | |
| cat <<EOF > ~/.ccp.yaml | |
| builder: | |
| push: True | |
| no_cache: true | |
| registry: | |
| address: "127.0.0.1:31500" | |
| repositories: | |
| skip_empty: True | |
| configs: | |
| private_interface: eth0 | |
| public_interface: eth1 | |
| neutron: | |
| external_interface: eth2 | |
| nodes: | |
| node1: | |
| roles: | |
| - percona | |
| node[2-3]: | |
| roles: | |
| - percona | |
| roles: | |
| percona: | |
| - etcd | |
| - galera | |
| EOF | |
| } | |
| function deploy() { | |
| echo "${TIMESTAMP} Starting deploy()" | |
| ccp deploy -c etcd galera > /dev/null 2>&1 | |
| } | |
| function cleanup() { | |
| echo "${TIMESTAMP} Starting cleanup()" | |
| ccp cleanup --skip-os-cleanup > /dev/null 2>&1 | |
| for i in 1 2 3; do | |
| ssh vagrant@node${i} sudo 'rm -rf /var/lib/mysql /var/log/ccp/mysql/mysql.log' | |
| done | |
| } | |
| get_galera_nodes() { | |
| kubectl get pods \ | |
| -o jsonpath='{.items[?(@.metadata.labels.app == "galera")].metadata.name}' | |
| } | |
| function wsrep_status() { | |
| for pod in $(get_galera_nodes); do | |
| echo "${TIMESTAMP} Getting wsrep status on ${pod}" | |
| kubectl exec -it ${pod} -c galera -- mysql -uroot -ppassword -e "SHOW STATUS LIKE 'wsrep%';" | egrep 'wsrep_evs_state|wsrep_cluster_status|wsrep_connected|wsrep_ready|wsrep_incoming_addresses|wsrep_local_state|wsrep_cluster_state_uuid|wsrep_last_committed' | |
| done | |
| } | |
| function populate_data() { | |
| echo "${TIMESTAMP} Starting populate_data()" | |
| local first_node=$(get_galera_nodes | awk '{print $1}') | |
| kubectl exec -it ${first_node} -c galera -- mysql -uroot -ppassword -e "CREATE DATABASE test_data; | |
| USE test_data; | |
| CREATE TABLE test_data (id int NOT NULL PRIMARY KEY AUTO_INCREMENT, val int); | |
| DELIMITER \$\$ | |
| CREATE PROCEDURE prepare_data() | |
| BEGIN | |
| DECLARE i INT DEFAULT 1; | |
| INSERT INTO test_data (val) VALUES (RAND()); | |
| WHILE i < 25 DO | |
| INSERT INTO test_data (val) SELECT val from test_data; | |
| SET i = i + 1; | |
| END WHILE; | |
| COMMIT; | |
| END\$\$ | |
| DELIMITER ; | |
| CALL prepare_data;" | |
| } | |
| cluster_status() { | |
| local timeout=$1 | |
| local ccp_status | |
| local synced_nodes | |
| while [[ ${timeout} -gt 0 ]]; do | |
| ccp_status=$(ccp status -s -f value) | |
| if [[ "${ccp_status}" == "ok" ]]; then | |
| synced_nodes=$(wsrep_status | awk '/wsrep_cluster_state_uuid/ {print $4}' | sort | uniq -c | awk '{print $1}') | |
| if [[ ${synced_nodes} -eq 3 ]]; then | |
| echo "${TIMESTAMP} All nodes are synced" | |
| return 0 | |
| fi | |
| else | |
| sleep 1 | |
| ((timeout--)) | |
| fi | |
| done | |
| echo "${TIMESTAMP} Not all nodes are synced during $1 seconds" | |
| return 1 | |
| } | |
| delete_random_pod() { | |
| local -a pods | |
| IFS=$'\n' read -rd '' -a pods <<< $(kubectl get pods -o jsonpath='{range .items[?(@.metadata.labels.app == "galera")]}{@.metadata.name} {@.status.hostIP}{"\n"}{end}') | |
| local pod=${pods[$(($RANDOM % ${#pods[@]}))]} | |
| echo "${TIMESTAMP} Deleting a pod ${pod%%\ *} on ${pod##*\ }" | |
| kubectl delete pod ${pod%%\ *} | |
| if [[ $1 -eq 1 ]]; then | |
| echo "${TIMESTAMP} Deleting data on node ${pod##*\ }" | |
| ssh vagrant@${pod##*\ } sudo "rm -rf /var/lib/mysql/* /var/log/ccp/mysql/mysql.log" | |
| fi | |
| } | |
| cleanup_etcd() { | |
| for node in $(etcdctl --endpoints http://etcd.ccp:2379 ls -r --sort -p /galera/k8scluster/nodes/ 2>/dev/null); do | |
| echo "${TIMESTAMP} Deleting etcd entry: ${node}" | |
| etcdctl --endpoints http://etcd.ccp:2379 rm ${node} > /dev/null 2>&1 | |
| done | |
| } | |
| write_logs() { | |
| local directory=$1 | |
| for pod in $(get_galera_nodes); do | |
| kubectl logs -f ${pod} -c galera > /tmp/logs/${directory}/${pod}-galera.log 2>&1 | |
| kubectl logs -f ${pod} -c galera-checker > /tmp/logs/${directory}/${pod}-checker.log 2>&1 | |
| done | |
| } | |
| BOOTSTRAP_FAILURE=0 | |
| RECOVERY_FAILURE=0 | |
| SST_RECOVERY_FAILURE=0 | |
| prepare_topology | |
| for attempt in {1..10}; do | |
| cleanup | |
| mkdir -p /tmp/logs/${attempt} | |
| echo "${TIMESTAMP} Attempt: ${attempt} ===========" | |
| echo "${TIMESTAMP} BOOTSTRAP_FAILURES: ${BOOTSTRAP_FAILURE}" | |
| echo "${TIMESTAMP} RECOVERY_FAILURES: ${RECOVERY_FAILURE}" | |
| echo "${TIMESTAMP} SST_RECOVERY_STATUS: ${SST_RECOVERY_FAILURE}" | |
| deploy | |
| cluster_status 120 | |
| if [[ $? -ne 0 ]]; then | |
| ((BOOTSTRAP_FAILURE++)) | |
| write_logs ${attempt} | |
| continue | |
| fi | |
| echo "${TIMESTAMP} Bootstrap completed" | |
| delete_random_pod | |
| cleanup_etcd | |
| cluster_status 300 | |
| if [[ $? -ne 0 ]]; then | |
| ((RECOVERY_FAILURE++)) | |
| write_logs ${attempt} | |
| continue | |
| fi | |
| populate_data | |
| delete_random_pod 1 | |
| cleanup_etcd | |
| cluster_status 600 | |
| if [[ $? -ne 0 ]]; then | |
| ((SST_RECOVERY_FAILURE++)) | |
| write_logs ${attempt} | |
| continue | |
| fi | |
| write_logs ${attempt} | |
| done | |
| echo "${TIMESTAMP} Final Results: ==========" | |
| echo "${TIMESTAMP} BOOTSTRAP_FAILURES: ${BOOTSTRAP_FAILURE}" | |
| echo "${TIMESTAMP} RECOVERY_FAILURES: ${RECOVERY_FAILURE}" | |
| echo "${TIMESTAMP} SST_RECOVERY_FAILURES: ${SST_RECOVERY_FAILURE}" | |
| cleanup |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment