#!/bin/bash # exit setup set -eo pipefail # [-e] - immediately exit if any command has a non-zero exit status # [-x] - all executed commands are printed to the terminal [not secure] # [-o pipefail] - if any command in a pipeline fails, that return code will be used as the return code of the whole pipeline SCRIPT_DIR=$(dirname -- "$(readlink -f -- "$0")") IS_CLEAR_DELETED="false" IS_CLEAR_E2E="false" IS_CLEAR_REVISIONS="false" # parse args for _ in "$@"; do case ${1} in --clear-deleted) IS_CLEAR_DELETED="true" shift # past argument with no value ;; --clear-e2e) IS_CLEAR_E2E="true" shift # past argument with no value ;; --clear-revisions) IS_CLEAR_REVISIONS="true" shift # past argument with no value ;; -*) echo "unknown arg: ${1}" exit 1 ;; *) ;; esac done echo "" echo "Start dump UnitedStorage entries..." echo " - workbooks" echo " - collections" echo " - entries" echo " - revisions" echo " - links" if [ "${IS_CLEAR_DELETED}" = "true" ]; then echo "+ clear deleted entries automatically..." fi if [ "${IS_CLEAR_E2E}" = "true" ]; then echo "+ clear e2e entries automatically..." fi if [ "${IS_CLEAR_REVISIONS}" = "true" ]; then echo "+ clear not actual entries revisions automatically..." fi COMPOSE_FILE=$(readlink -f "${SCRIPT_DIR}/../../tests/docker-compose.e2e.yml") DUMP_FILE=$(readlink -f "${SCRIPT_DIR}/../../tests/data/us-e2e-data.sql") echo "" echo "========================" echo "BEGIN;" >"${DUMP_FILE}" docker --log-level error compose -f "${COMPOSE_FILE}" exec \ --env "POSTGRES_DUMP_CLEAR_META=true" \ --env "POSTGRES_DUMP_SKIP_CONFLICT=false" \ -T postgres \ /init/us-dump.sh | sed -E 's|"cypher_text": "[^"]+"|"cypher_text": "{{POSTGRES_PASSWORD}}"|' | sed -E 's|"host": "[^"]+"|"host": "{{POSTGRES_HOST}}"|' | sed -E 's|"port": [^,]+,|"port": {{POSTGRES_PORT}},|' | sed -E 's|"db_name": "[^"]+"|"db_name": "{{POSTGRES_DB}}"|' | sed -E 's|"username": "[^"]+"|"username": "{{POSTGRES_USER}}"|' \ >>"${DUMP_FILE}" EXIT="$?" echo "COMMIT;" >>"${DUMP_FILE}" echo "" echo "========================" if [ "${IS_CLEAR_DELETED}" = "true" ]; then echo "" echo "Clear deleted entries..." DUMP=$(cat "${DUMP_FILE}") DELETED_ENTRIES=$( echo "${DUMP}" | { grep ' public.entries ' || true; } | { grep '__trash/' || true; } | { grep -oE '__trash/[0-9]+_' || true; } | sed 's|__trash/||' | sed 's|_||' | tr -d ' ' | sort | uniq ) IFS=$'\n' for DELETED_ENTRY in ${DELETED_ENTRIES}; do echo " clear deleted entry: ${DELETED_ENTRY}" DUMP=$(echo "${DUMP}" | { grep -v ", ${DELETED_ENTRY}, " || true; } | { grep -v "(${DELETED_ENTRY}, " || true; }) done unset IFS echo "${DUMP}" >"${DUMP_FILE}" fi if [ "${IS_CLEAR_E2E}" = "true" ]; then echo "" echo "Clear e2e entries..." DUMP=$(cat "${DUMP_FILE}") E2E_ENTRIES=$( echo "${DUMP}" | { grep ' public.entries ' || true; } | { grep 'e2e-entry-' || true; } | { grep -oE ", '[0-9]+/" || true; } | sed "s|, '||" | sed 's|/||' | tr -d ' ' | sort | uniq ) IFS=$'\n' for E2E_ENTRY in ${E2E_ENTRIES}; do echo " clear e2e entry: ${E2E_ENTRY}" DUMP=$(echo "${DUMP}" | { grep -v ", ${E2E_ENTRY}, " || true; } | { grep -v "(${E2E_ENTRY}, " || true; }) done unset IFS echo "${DUMP}" >"${DUMP_FILE}" fi if [ "${IS_CLEAR_REVISIONS}" = "true" ]; then echo "" echo "Clear not actual revisions..." DUMP=$(cat "${DUMP_FILE}") ENTRIES=$( echo "${DUMP}" | grep ' public.entries ' | grep -oE ", '[0-9]+/" | sed "s|, '||" | sed 's|/||' | tr -d ' ' | sort | uniq ) IFS=$'\n' for ENTRY in ${ENTRIES}; do echo " entry: ${ENTRY}" REVISIONS=$(echo "${DUMP}" | { grep " public.revisions " || true; } | { grep ", ${ENTRY}, " || true; } | sed 's|INSERT INTO public.revisions .*VALUES|VALUES|' | sed "s|''||g" | sed -E "s|VALUES \('[^']+',||") REVISIONS_COUNT=$(echo "${REVISIONS}" | wc -l | tr -d ' ') echo " revisions count: ${REVISIONS_COUNT}" if [ "${REVISIONS_COUNT}" == "0" ] || [ "${REVISIONS_COUNT}" == "1" ]; then continue fi echo " clear revisions..." for REVISION in ${REVISIONS}; do REVISION_ID=$(echo "${REVISION}" | { grep -oE ", [0-9]+," || true; } | sed 's|,||g' | tr -d ' ') ENTRY_REVISION=$(echo "${DUMP}" | { grep ' public.entries ' || true; } | { grep ", ${REVISION_ID}, " || true; } | tr -d ' ') if [ ! -z "${ENTRY_REVISION}" ]; then echo " actual revision id: ${REVISION_ID}" continue fi echo " clear revision id: ${REVISION_ID}" DUMP=$(echo "${DUMP}" | { grep -v ", ${REVISION_ID}, " || true; }) done done unset IFS echo "${DUMP}" >"${DUMP_FILE}" fi E2E_ENTRIES=$(cat "${DUMP_FILE}" | { grep "e2e-entry-" || true; }) DELETED_ENTRIES=$(cat "${DUMP_FILE}" | { grep "__trash/" || true; }) if [ -n "${E2E_ENTRIES}" ]; then ENTRIES_KEYS=$(echo "${E2E_ENTRIES}" | sed -E "s|INSERT INTO ([^ ]+) .*'([^']*e2e-entry-[^']*)'.*|\1 - \2|" | sed 's|^| - |') echo "" echo "⚠️ WARNING: Found entries with 'e2e-entry-' in key:" >&2 echo "${ENTRIES_KEYS}" >&2 echo "These entries might be test entries and should be reviewed..." >&2 fi if [ -n "${DELETED_ENTRIES}" ]; then ENTRIES_KEYS=$(echo "${DELETED_ENTRIES}" | sed -E "s|INSERT INTO ([^ ]+) .*'([^']*__trash/[^']*)'.*|\1 - \2|" | sed 's|^| - |') echo "" echo "⚠️ WARNING: Found deleted entries:" >&2 echo "${ENTRIES_KEYS}" >&2 fi # remove empty lines sed '/^$/N;/^\n$/D' "${DUMP_FILE}" >"${DUMP_FILE}.tmp" && mv "${DUMP_FILE}.tmp" "${DUMP_FILE}" if [ "${EXIT}" != "0" ]; then echo "" echo "Dump error, exit..." exit "${EXIT}" else echo "" echo "Dump done, saved at [${DUMP_FILE}]" exit 0 fi