diff --git a/bin/dbImport b/bin/dbImport index 1fbcf9db..21aa4626 100755 --- a/bin/dbImport +++ b/bin/dbImport @@ -1694,7 +1694,7 @@ optionHelpCallback() { # shellcheck disable=SC2317,SC2329 # if function is overridden optionVersionCallback() { # shellcheck disable=SC2154 - echo "${SCRIPT_NAME} version 3.0" + echo "${SCRIPT_NAME} version 4.0" exit 0 } @@ -1906,6 +1906,12 @@ longDescriptionFunction() { echo echo -e " 2. import from S3" echo -e " ${__HELP_EXAMPLE}dbImport --from-aws awsFile.tar.gz --target-dsn localhost fromDb toDb${__HELP_NORMAL}" + echo + echo -e " ${__HELP_TITLE}Changelog${__HELP_NORMAL}" + echo -e " ${__HELP_EXAMPLE}4.0 (2026-04-14)${__HELP_NORMAL}" + echo -e " - add support for tar.gz and tar.individual.sql.gz dump files" + echo -e " ${__HELP_EXAMPLE}3.0${__HELP_NORMAL}" + echo -e " - initial version" Db::checkRequirements } @@ -2070,7 +2076,7 @@ fromDsnOptionLongDescription() { optionVersionCallback() { # shellcheck disable=SC2154 - echo "${SCRIPT_NAME} version 3.0" + echo "${SCRIPT_NAME} version 4.0" Db::checkRequirements exit 0 } @@ -2102,6 +2108,8 @@ declare optionTables="" declare optionSkipSchema="0" declare optionFromDsn="" declare optionFromAws="" +declare optionReset="0" +declare optionDumpKind="auto" declare optionHelp="0" declare optionConfig="0" declare optionBashFrameworkConfig="" @@ -2146,6 +2154,12 @@ dbImportCommandParse() { optionFromAws="" local -i options_parse_optionParsedCountOptionFromAws ((options_parse_optionParsedCountOptionFromAws = 0)) || true + optionReset="0" + local -i options_parse_optionParsedCountOptionReset + ((options_parse_optionParsedCountOptionReset = 0)) || true + optionDumpKind="auto" + local -i options_parse_optionParsedCountOptionDumpKind + ((options_parse_optionParsedCountOptionDumpKind = 0)) || true optionHelp="0" local -i options_parse_optionParsedCountOptionHelp ((options_parse_optionParsedCountOptionHelp = 0)) || true @@ -2203,7 +2217,7 @@ dbImportCommandParse() { local options_parse_arg="$1" local argOptDefaultBehavior=0 case "${options_parse_arg}" in - # Option 1/21 + # Option 1/23 # optionCollationName alts --collation-name|-o # type: String min 0 max 1 --collation-name | -o) @@ -2222,7 +2236,7 @@ dbImportCommandParse() { optionCollationName="$1" ;; - # Option 2/21 + # Option 2/23 # optionTargetDsn alts --target-dsn|-t # type: String min 0 max 1 --target-dsn | -t) @@ -2241,7 +2255,7 @@ dbImportCommandParse() { optionTargetDsn="$1" ;; - # Option 3/21 + # Option 3/23 # optionCharacterSet alts --character-set|-c # type: String min 0 max 1 --character-set | -c) @@ -2260,7 +2274,7 @@ dbImportCommandParse() { optionCharacterSet="$1" ;; - # Option 4/21 + # Option 4/23 # optionProfile alts --profile|-p # type: String min 0 max 1 --profile | -p) @@ -2279,7 +2293,7 @@ dbImportCommandParse() { optionProfile="$1" ;; - # Option 5/21 + # Option 5/23 # optionTables alts --tables # type: String min 0 max 1 --tables) @@ -2300,7 +2314,7 @@ dbImportCommandParse() { ;; - # Option 6/21 + # Option 6/23 # optionSkipSchema alts --skip-schema|-s # type: Boolean min 0 max 1 --skip-schema | -s) @@ -2314,7 +2328,7 @@ dbImportCommandParse() { ((++options_parse_optionParsedCountOptionSkipSchema)) ;; - # Option 7/21 + # Option 7/23 # optionFromDsn alts --from-dsn|-f # type: String min 0 max 1 --from-dsn | -f) @@ -2333,7 +2347,7 @@ dbImportCommandParse() { optionFromDsn="$1" ;; - # Option 8/21 + # Option 8/23 # optionFromAws alts --from-aws|-a # type: String min 0 max 1 --from-aws | -a) @@ -2352,7 +2366,45 @@ dbImportCommandParse() { optionFromAws="$1" ;; - # Option 9/21 + # Option 9/23 + # optionReset alts --reset + # type: Boolean min 0 max 1 + --reset) + # shellcheck disable=SC2034 + optionReset="1" + + if ((options_parse_optionParsedCountOptionReset >= 1 )); then + Log::displayError "Command ${SCRIPT_NAME} - Option ${options_parse_arg} - Maximum number of option occurrences reached(1)" + return 1 + fi + ((++options_parse_optionParsedCountOptionReset)) + ;; + + # Option 10/23 + # optionDumpKind alts --dump-kind|-k + # type: String min 0 max 1 + # authorizedValues: auto|sql|tar|tar.gz|gz|tar.individual.sql.gz + --dump-kind | -k) + shift + if (($# == 0)); then + Log::displayError "Command ${SCRIPT_NAME} - Option ${options_parse_arg} - a value needs to be specified" + return 1 + fi + if [[ ! "$1" =~ auto|sql|tar|tar.gz|gz|tar.individual.sql.gz ]]; then + Log::displayError "Command ${SCRIPT_NAME} - Option ${options_parse_arg} - value '$1' is not part of authorized values(auto, sql, tar, tar.gz, gz, tar.individual.sql.gz)" + return 1 + fi + + if ((options_parse_optionParsedCountOptionDumpKind >= 1 )); then + Log::displayError "Command ${SCRIPT_NAME} - Option ${options_parse_arg} - Maximum number of option occurrences reached(1)" + return 1 + fi + ((++options_parse_optionParsedCountOptionDumpKind)) + # shellcheck disable=SC2034 + optionDumpKind="$1" + ;; + + # Option 11/23 # optionHelp alts --help|-h # type: Boolean min 0 max 1 --help | -h) @@ -2368,7 +2420,7 @@ dbImportCommandParse() { ;; - # Option 10/21 + # Option 12/23 # optionConfig alts --config # type: Boolean min 0 max 1 --config) @@ -2382,7 +2434,7 @@ dbImportCommandParse() { ((++options_parse_optionParsedCountOptionConfig)) ;; - # Option 11/21 + # Option 13/23 # optionBashFrameworkConfig alts --bash-framework-config # type: String min 0 max 1 --bash-framework-config) @@ -2403,7 +2455,7 @@ dbImportCommandParse() { ;; - # Option 12/21 + # Option 14/23 # optionInfoVerbose alts --verbose|-v # type: Boolean min 0 max 1 --verbose | -v) @@ -2421,7 +2473,7 @@ dbImportCommandParse() { ;; - # Option 13/21 + # Option 15/23 # optionDebugVerbose alts -vv # type: Boolean min 0 max 1 -vv) @@ -2439,7 +2491,7 @@ dbImportCommandParse() { ;; - # Option 14/21 + # Option 16/23 # optionTraceVerbose alts -vvv # type: Boolean min 0 max 1 -vvv) @@ -2457,7 +2509,7 @@ dbImportCommandParse() { ;; - # Option 15/21 + # Option 17/23 # optionLogLevel alts --log-level # type: String min 0 max 1 # authorizedValues: OFF|ERR|ERROR|WARN|WARNING|INFO|DEBUG|TRACE @@ -2485,7 +2537,7 @@ dbImportCommandParse() { ;; - # Option 16/21 + # Option 18/23 # optionLogFile alts --log-file # type: String min 0 max 1 --log-file) @@ -2508,7 +2560,7 @@ dbImportCommandParse() { ;; - # Option 17/21 + # Option 19/23 # optionDisplayLevel alts --display-level # type: String min 0 max 1 # authorizedValues: OFF|ERR|ERROR|WARN|WARNING|INFO|DEBUG|TRACE @@ -2536,7 +2588,7 @@ dbImportCommandParse() { ;; - # Option 18/21 + # Option 20/23 # optionNoColor alts --no-color # type: Boolean min 0 max 1 --no-color) @@ -2554,7 +2606,7 @@ dbImportCommandParse() { ;; - # Option 19/21 + # Option 21/23 # optionTheme alts --theme # type: String min 0 max 1 # authorizedValues: default|default-force|noColor @@ -2582,7 +2634,7 @@ dbImportCommandParse() { ;; - # Option 20/21 + # Option 22/23 # optionVersion alts --version # type: Boolean min 0 max 1 --version) @@ -2598,7 +2650,7 @@ dbImportCommandParse() { ;; - # Option 21/21 + # Option 23/23 # optionQuiet alts --quiet|-q # type: Boolean min 0 max 1 --quiet | -q) @@ -2696,6 +2748,8 @@ dbImportCommandParse() { + + @@ -2727,7 +2781,7 @@ dbImportCommandHelp() { # ------------------------------------------ # usage/options section # ------------------------------------------ - optionsAltList=("[--collation-name|-o ]" "[--target-dsn|-t ]" "[--character-set|-c ]" "[--profile|-p ]" "[--tables ]" "[--skip-schema|-s]" "[--from-dsn|-f ]" "[--from-aws|-a ]" "[--help|-h]" "[--config]" "[--bash-framework-config ]" "[--verbose|-v]" "[-vv]" "[-vvv]" "[--log-level ]" "[--log-file ]" "[--display-level ]" "[--no-color]" "[--theme ]" "[--version]" "[--quiet|-q]" + optionsAltList=("[--collation-name|-o ]" "[--target-dsn|-t ]" "[--character-set|-c ]" "[--profile|-p ]" "[--tables ]" "[--skip-schema|-s]" "[--from-dsn|-f ]" "[--from-aws|-a ]" "[--reset]" "[--dump-kind|-k ]" "[--help|-h]" "[--config]" "[--bash-framework-config ]" "[--verbose|-v]" "[-vv]" "[-vvv]" "[--log-level ]" "[--log-file ]" "[--display-level ]" "[--no-color]" "[--theme ]" "[--version]" "[--quiet|-q]" ) Array::wrap2 " " 80 2 "${__HELP_TITLE_COLOR}USAGE:${__RESET_COLOR}" \ "dbImport" "${optionsAltList[@]}" @@ -2798,6 +2852,36 @@ dbImportCommandHelp() { Array::wrap2 ' ' 76 4 " " "db dump will be downloaded from s3 instead of using remote db." "The value is the name of the file without s3 location" "(Only .gz or tar.gz file are supported)." "This option is incompatible with -f|--from-dsn option." "" echo + + echo -e " ${__HELP_OPTION_COLOR}--reset${__HELP_NORMAL} {single}" + Array::wrap2 ' ' 76 4 " " "Reset the table before importing data using TRUNCATE TABLE or DROP TABLE before CREATE TABLE." + echo + + + echo -e " ${__HELP_OPTION_COLOR}--dump-kind${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-k ${__HELP_NORMAL} {single}" + Array::wrap2 ' ' 76 4 " " "Kind of dump to import." "" + echo + + echo " Possible values: " + Array::wrap2 ' ' 76 8 " - ${__OPTION_COLOR}auto:${__RESET_COLOR} Deduce dump kind from filename: + ${__HELP_EXAMPLE}.sql${__HELP_NORMAL} -> sql + ${__HELP_EXAMPLE}.tar${__HELP_NORMAL} -> tar + ${__HELP_EXAMPLE}.tar.gz or .tgz${__HELP_NORMAL} -> tar.gz + ${__HELP_EXAMPLE}.gz${__HELP_NORMAL} -> gz" + echo + Array::wrap2 ' ' 76 8 " - ${__OPTION_COLOR}sql:${__RESET_COLOR} dump file is a .sql file." + echo + Array::wrap2 ' ' 76 8 " - ${__OPTION_COLOR}tar:${__RESET_COLOR} dump file is a .tar file containing multiple .sql files, one per table." + echo + Array::wrap2 ' ' 76 8 " - ${__OPTION_COLOR}tar.gz:${__RESET_COLOR} dump file is a .tar file gzipped containing multiple .sql files, one per table." + echo + Array::wrap2 ' ' 76 8 " - ${__OPTION_COLOR}gz:${__RESET_COLOR} dump file is a gz file containing a single .sql file." + echo + Array::wrap2 ' ' 76 8 " - ${__OPTION_COLOR}tar.individual.sql.gz:${__RESET_COLOR} dump file is a tar file containing multiple .sql.gz files, one per table." + echo + + Array::wrap2 ' ' 76 6 " Default value: " "auto" + echo echo echo -e "${__HELP_TITLE_COLOR}GLOBAL OPTIONS:${__RESET_COLOR}" echo -e " ${__HELP_OPTION_COLOR}--help${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-h${__HELP_NORMAL} {single}" @@ -2910,7 +2994,7 @@ dbImportCommandHelp() { # ------------------------------------------ echo echo -n -e "${__HELP_TITLE_COLOR}VERSION: ${__RESET_COLOR}" - echo "3.0" + echo "4.0" # ------------------------------------------ # author section # ------------------------------------------ @@ -3117,16 +3201,24 @@ if [[ -z "${optionFromAws}" ]]; then fi fi Log::displayInfo "import remote to local from file ${remoteDbDumpTempFile}" +# shellcheck disable=SC2154 declare -a dbImportStreamOptions=( --profile "${optionProfile}" --target-dsn "${optionTargetDsn}" --character-set "${targetCharacterSet}" + --dump-kind "${optionDumpKind}" ) if [[ -n "${optionTables:-}" ]]; then dbImportStreamOptions+=( --tables "${optionTables}" ) fi +# shellcheck disable=SC2154 +if [[ "${optionReset}" = "1" ]]; then + dbImportStreamOptions+=( + --reset + ) +fi time ( "${CURRENT_DIR}/dbImportStream" \ "${dbImportStreamOptions[@]}" \ diff --git a/bin/dbImportStream b/bin/dbImportStream index bcd91859..e1f7f055 100755 --- a/bin/dbImportStream +++ b/bin/dbImportStream @@ -986,6 +986,19 @@ Framework::createTempFile() { } +# @description retrieve linux distributor id +# @noargs +# @exitcode 1 if /etc/os-release not found +# @exitcode 2 if ID field not found in /etc/os-release +# @stdout the linux distributor id +Linux::getDistributorId() { + if [[ ! -f /etc/os-release ]]; then + return 1 + fi + awk -F= '/^ID=/ { gsub(/"/, "", $2); print $2; exit }' /etc/os-release || return 2 +} + + # @description ensure running user is not root # @exitcode 1 if current user is root # @stderr diagnostics information is displayed @@ -1601,7 +1614,7 @@ optionHelpCallback() { # shellcheck disable=SC2317,SC2329 # if function is overridden optionVersionCallback() { # shellcheck disable=SC2154 - echo "${SCRIPT_NAME} version 3.0" + echo "${SCRIPT_NAME} version 4.0" exit 0 } @@ -1801,6 +1814,12 @@ longDescriptionFunction() { fromDsnOptionLongDescription echo profileOptionLongDescription + echo + echo -e " ${__HELP_TITLE}Changelog${__HELP_NORMAL}" + echo -e " ${__HELP_EXAMPLE}4.0 (2026-04-14)${__HELP_NORMAL}" + echo -e " - add support for tar.gz and tar.individual.sql.gz dump files" + echo -e " ${__HELP_EXAMPLE}3.0${__HELP_NORMAL}" + echo -e " - initial version" } dbImportStreamCommandCallback() { @@ -1934,7 +1953,7 @@ fromDsnOptionLongDescription() { optionVersionCallback() { # shellcheck disable=SC2154 - echo "${SCRIPT_NAME} version 3.0" + echo "${SCRIPT_NAME} version 4.0" Db::checkRequirements exit 0 } @@ -1947,6 +1966,8 @@ optionVersionCallback() { # options variables initialization declare optionProfile="" declare optionTables="" +declare optionReset="0" +declare optionDumpKind="auto" declare optionTargetDsn="default.local" declare optionCharacterSet="" declare optionHelp="0" @@ -1975,6 +1996,12 @@ dbImportStreamCommandParse() { optionTables="" local -i options_parse_optionParsedCountOptionTables ((options_parse_optionParsedCountOptionTables = 0)) || true + optionReset="0" + local -i options_parse_optionParsedCountOptionReset + ((options_parse_optionParsedCountOptionReset = 0)) || true + optionDumpKind="auto" + local -i options_parse_optionParsedCountOptionDumpKind + ((options_parse_optionParsedCountOptionDumpKind = 0)) || true optionTargetDsn="default.local" local -i options_parse_optionParsedCountOptionTargetDsn ((options_parse_optionParsedCountOptionTargetDsn = 0)) || true @@ -2038,7 +2065,7 @@ dbImportStreamCommandParse() { local options_parse_arg="$1" local argOptDefaultBehavior=0 case "${options_parse_arg}" in - # Option 1/17 + # Option 1/19 # optionProfile alts --profile|-p # type: String min 0 max 1 --profile | -p) @@ -2057,7 +2084,7 @@ dbImportStreamCommandParse() { optionProfile="$1" ;; - # Option 2/17 + # Option 2/19 # optionTables alts --tables # type: String min 0 max 1 --tables) @@ -2078,7 +2105,45 @@ dbImportStreamCommandParse() { ;; - # Option 3/17 + # Option 3/19 + # optionReset alts --reset + # type: Boolean min 0 max 1 + --reset) + # shellcheck disable=SC2034 + optionReset="1" + + if ((options_parse_optionParsedCountOptionReset >= 1 )); then + Log::displayError "Command ${SCRIPT_NAME} - Option ${options_parse_arg} - Maximum number of option occurrences reached(1)" + return 1 + fi + ((++options_parse_optionParsedCountOptionReset)) + ;; + + # Option 4/19 + # optionDumpKind alts --dump-kind|-k + # type: String min 0 max 1 + # authorizedValues: auto|sql|tar|tar.gz|gz|tar.individual.sql.gz + --dump-kind | -k) + shift + if (($# == 0)); then + Log::displayError "Command ${SCRIPT_NAME} - Option ${options_parse_arg} - a value needs to be specified" + return 1 + fi + if [[ ! "$1" =~ auto|sql|tar|tar.gz|gz|tar.individual.sql.gz ]]; then + Log::displayError "Command ${SCRIPT_NAME} - Option ${options_parse_arg} - value '$1' is not part of authorized values(auto, sql, tar, tar.gz, gz, tar.individual.sql.gz)" + return 1 + fi + + if ((options_parse_optionParsedCountOptionDumpKind >= 1 )); then + Log::displayError "Command ${SCRIPT_NAME} - Option ${options_parse_arg} - Maximum number of option occurrences reached(1)" + return 1 + fi + ((++options_parse_optionParsedCountOptionDumpKind)) + # shellcheck disable=SC2034 + optionDumpKind="$1" + ;; + + # Option 5/19 # optionTargetDsn alts --target-dsn|-t # type: String min 0 max 1 --target-dsn | -t) @@ -2097,7 +2162,7 @@ dbImportStreamCommandParse() { optionTargetDsn="$1" ;; - # Option 4/17 + # Option 6/19 # optionCharacterSet alts --character-set|-c # type: String min 0 max 1 --character-set | -c) @@ -2116,7 +2181,7 @@ dbImportStreamCommandParse() { optionCharacterSet="$1" ;; - # Option 5/17 + # Option 7/19 # optionHelp alts --help|-h # type: Boolean min 0 max 1 --help | -h) @@ -2132,7 +2197,7 @@ dbImportStreamCommandParse() { ;; - # Option 6/17 + # Option 8/19 # optionConfig alts --config # type: Boolean min 0 max 1 --config) @@ -2146,7 +2211,7 @@ dbImportStreamCommandParse() { ((++options_parse_optionParsedCountOptionConfig)) ;; - # Option 7/17 + # Option 9/19 # optionBashFrameworkConfig alts --bash-framework-config # type: String min 0 max 1 --bash-framework-config) @@ -2167,7 +2232,7 @@ dbImportStreamCommandParse() { ;; - # Option 8/17 + # Option 10/19 # optionInfoVerbose alts --verbose|-v # type: Boolean min 0 max 1 --verbose | -v) @@ -2185,7 +2250,7 @@ dbImportStreamCommandParse() { ;; - # Option 9/17 + # Option 11/19 # optionDebugVerbose alts -vv # type: Boolean min 0 max 1 -vv) @@ -2203,7 +2268,7 @@ dbImportStreamCommandParse() { ;; - # Option 10/17 + # Option 12/19 # optionTraceVerbose alts -vvv # type: Boolean min 0 max 1 -vvv) @@ -2221,7 +2286,7 @@ dbImportStreamCommandParse() { ;; - # Option 11/17 + # Option 13/19 # optionLogLevel alts --log-level # type: String min 0 max 1 # authorizedValues: OFF|ERR|ERROR|WARN|WARNING|INFO|DEBUG|TRACE @@ -2249,7 +2314,7 @@ dbImportStreamCommandParse() { ;; - # Option 12/17 + # Option 14/19 # optionLogFile alts --log-file # type: String min 0 max 1 --log-file) @@ -2272,7 +2337,7 @@ dbImportStreamCommandParse() { ;; - # Option 13/17 + # Option 15/19 # optionDisplayLevel alts --display-level # type: String min 0 max 1 # authorizedValues: OFF|ERR|ERROR|WARN|WARNING|INFO|DEBUG|TRACE @@ -2300,7 +2365,7 @@ dbImportStreamCommandParse() { ;; - # Option 14/17 + # Option 16/19 # optionNoColor alts --no-color # type: Boolean min 0 max 1 --no-color) @@ -2318,7 +2383,7 @@ dbImportStreamCommandParse() { ;; - # Option 15/17 + # Option 17/19 # optionTheme alts --theme # type: String min 0 max 1 # authorizedValues: default|default-force|noColor @@ -2346,7 +2411,7 @@ dbImportStreamCommandParse() { ;; - # Option 16/17 + # Option 18/19 # optionVersion alts --version # type: Boolean min 0 max 1 --version) @@ -2362,7 +2427,7 @@ dbImportStreamCommandParse() { ;; - # Option 17/17 + # Option 19/19 # optionQuiet alts --quiet|-q # type: Boolean min 0 max 1 --quiet | -q) @@ -2459,6 +2524,8 @@ dbImportStreamCommandParse() { + + if ((options_parse_argParsedCountArgDumpFile < 1 )); then Log::displayError "Command ${SCRIPT_NAME} - Argument 'argDumpFile' should be provided at least 1 time(s)" return 1 @@ -2492,7 +2559,7 @@ dbImportStreamCommandHelp() { # ------------------------------------------ # usage/options section # ------------------------------------------ - optionsAltList=("[--profile|-p ]" "[--tables ]" "[--target-dsn|-t ]" "[--character-set|-c ]" "[--help|-h]" "[--config]" "[--bash-framework-config ]" "[--verbose|-v]" "[-vv]" "[-vvv]" "[--log-level ]" "[--log-file ]" "[--display-level ]" "[--no-color]" "[--theme ]" "[--version]" "[--quiet|-q]" + optionsAltList=("[--profile|-p ]" "[--tables ]" "[--reset]" "[--dump-kind|-k ]" "[--target-dsn|-t ]" "[--character-set|-c ]" "[--help|-h]" "[--config]" "[--bash-framework-config ]" "[--verbose|-v]" "[-vv]" "[-vvv]" "[--log-level ]" "[--log-file ]" "[--display-level ]" "[--no-color]" "[--theme ]" "[--version]" "[--quiet|-q]" ) Array::wrap2 " " 80 2 "${__HELP_TITLE_COLOR}USAGE:${__RESET_COLOR}" \ "dbImportStream" "${optionsAltList[@]}" @@ -2528,6 +2595,37 @@ dbImportStreamCommandHelp() { Array::wrap2 ' ' 76 4 " " "Import only table specified in the list." "If aws mode, ignore profile option." "" echo + echo + echo -e "${__HELP_TITLE_COLOR}SOURCE OPTIONS:${__RESET_COLOR}" + echo -e " ${__HELP_OPTION_COLOR}--reset${__HELP_NORMAL} {single}" + Array::wrap2 ' ' 76 4 " " "Reset the table before importing data using TRUNCATE TABLE or DROP TABLE before CREATE TABLE." + echo + + + echo -e " ${__HELP_OPTION_COLOR}--dump-kind${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-k ${__HELP_NORMAL} {single}" + Array::wrap2 ' ' 76 4 " " "Kind of dump to import." "" + echo + + echo " Possible values: " + Array::wrap2 ' ' 76 8 " - ${__OPTION_COLOR}auto:${__RESET_COLOR} Deduce dump kind from filename: + ${__HELP_EXAMPLE}.sql${__HELP_NORMAL} -> sql + ${__HELP_EXAMPLE}.tar${__HELP_NORMAL} -> tar + ${__HELP_EXAMPLE}.tar.gz or .tgz${__HELP_NORMAL} -> tar.gz + ${__HELP_EXAMPLE}.gz${__HELP_NORMAL} -> gz" + echo + Array::wrap2 ' ' 76 8 " - ${__OPTION_COLOR}sql:${__RESET_COLOR} dump file is a .sql file." + echo + Array::wrap2 ' ' 76 8 " - ${__OPTION_COLOR}tar:${__RESET_COLOR} dump file is a .tar file containing multiple .sql files, one per table." + echo + Array::wrap2 ' ' 76 8 " - ${__OPTION_COLOR}tar.gz:${__RESET_COLOR} dump file is a .tar file gzipped containing multiple .sql files, one per table." + echo + Array::wrap2 ' ' 76 8 " - ${__OPTION_COLOR}gz:${__RESET_COLOR} dump file is a gz file containing a single .sql file." + echo + Array::wrap2 ' ' 76 8 " - ${__OPTION_COLOR}tar.individual.sql.gz:${__RESET_COLOR} dump file is a tar file containing multiple .sql.gz files, one per table." + echo + + Array::wrap2 ' ' 76 6 " Default value: " "auto" + echo echo echo -e "${__HELP_TITLE_COLOR}TARGET OPTIONS:${__RESET_COLOR}" echo -e " ${__HELP_OPTION_COLOR}--target-dsn${__HELP_NORMAL}, ${__HELP_OPTION_COLOR}-t ${__HELP_NORMAL} {single}" @@ -2654,7 +2752,7 @@ dbImportStreamCommandHelp() { # ------------------------------------------ echo echo -n -e "${__HELP_TITLE_COLOR}VERSION: ${__RESET_COLOR}" - echo "3.0" + echo "4.0" # ------------------------------------------ # author section # ------------------------------------------ @@ -2689,31 +2787,47 @@ main() { Linux::requireTarCommand Compiler::Embed::extractFileFromBase64 \ - "${PERSISTENT_TMPDIR:-/tmp}/1a25e8fe87d1af27f299ca9f5af14fe2fad803ddce20d659e5f6784c6056c9cc/dbImportStreamScript" \ - "QkVHSU57CiAgd3JpdGU9MQp9CnsKICBidWZmZXIgPSBzdWJzdHIoJDAsIDEsIDE1MCkKICBsaW5l \ -ID0gJDAKICBpZihtYXRjaChidWZmZXIsIC9eTE9DSyBUQUJMRVMgYChbXmBdKylgIFdSSVRFOyQv \ -LCBhcnIpICE9IDApIHsKICAgICMgY2hlY2sgaWYgaW5zZXJ0cyBhcmUgcGFydCBvZiB0aGUgcHJv \ -ZmlsZQogICAgdGFibGVOYW1lPWFyclsxXQogICAgaWYgKCEgKHRhYmxlTmFtZSBpbiBtYXApKSB7 \ -CiAgICAgIHByb2ZpbGVDbWQgPSAiZWNobyAnIiB0YWJsZU5hbWUgIicgfCAiIFBST0ZJTEVfQ09N \ -TUFORCAiIHwgZ3JlcCAtcSAiIHRhYmxlTmFtZQogICAgICBtYXBbdGFibGVOYW1lXSA9IChzeXN0 \ -ZW0ocHJvZmlsZUNtZCkgPT0gMCkKICAgIH0KICAgIGlmIChtYXBbdGFibGVOYW1lXSkgewogICAg \ -ICBwcmludCAiXDAzM1s0NG0iICJiZWdpbiBpbnNlcnQgIiB0YWJsZU5hbWUgIlwwMzNbMG0iICA+ \ -ICIvZGV2L3N0ZGVyciIKICAgICAgbGluZSA9IGxpbmUgIlxuVFJVTkNBVEUgVEFCTEUgYCIgdGFi \ -bGVOYW1lICJgOyIKICAgICAgd3JpdGU9MQogICAgfSBlbHNlIHsKICAgICAgcHJpbnQgImlnbm9y \ -ZSB0YWJsZSAiIHRhYmxlTmFtZSAgPiAiL2Rldi9zdGRlcnIiCiAgICAgIHdyaXRlPTAKICAgIH0K \ -ICB9IGVsc2UgaWYobWF0Y2goYnVmZmVyLCAvXmNvbW1pdDskLywgYXJyKSAhPSAwKSB7CiAgICB3 \ -cml0ZT0xCiAgfSBlbHNlIGlmKG1hdGNoKGJ1ZmZlciwgL1NFVCBOQU1FUyAoW14gXSspLywgYXJy \ -KSAhPSAwKSB7CiAgICBpZiAoQ0hBUkFDVEVSX1NFVCAhPSAiIikgewogICAgICBzdWIoL1NFVCBO \ -QU1FUyAoW14gXSspLywgIlNFVCBOQU1FUyAiIENIQVJBQ1RFUl9TRVQsIGxpbmUpCiAgICB9CiAg \ -ICB3cml0ZT0xCiAgfSBlbHNlIGlmKG1hdGNoKGJ1ZmZlciwgL1NFVCBjaGFyYWN0ZXJfc2V0X2Ns \ -aWVudCA9IChbXiBdKykvLCBhcnIpICE9IDAgJiYgc3Vic3RyKGFyclsxXSwgMCwgMSkgIT0gIkAi \ -KSB7CiAgICBpZiAoQ0hBUkFDVEVSX1NFVCAhPSAiIikgewogICAgICBzdWIoL1NFVCBjaGFyYWN0 \ -ZXJfc2V0X2NsaWVudCA9IChbXiBdKykvLCAiU0VUIGNoYXJhY3Rlcl9zZXRfY2xpZW50ID0gIiBD \ -SEFSQUNURVJfU0VULCBsaW5lKQogICAgfQogICAgd3JpdGU9MQogIH0KCgogIGlmICh3cml0ZSA9 \ -PSAxKSB7CiAgICBwcmludCBsaW5lCiAgfQp9Cg==" \ + "${PERSISTENT_TMPDIR:-/tmp}/f60daf3f7c7389022cfa34ad93f1f1d27253a0cbd47d93580fbf65ff7d20cca8/dbImportStreamScript" \ + "QkVHSU57CiAgd3JpdGU9MQp9CgpmdW5jdGlvbiBTaG91bGRJbmNsdWRlVGFibGUodGFibGVOYW1l \ +KSB7CiAgaWYgKCEgKHRhYmxlTmFtZSBpbiBtYXApKSB7CiAgICBwcm9maWxlQ21kID0gImVjaG8g \ +JyIgdGFibGVOYW1lICInIHwgIiBQUk9GSUxFX0NPTU1BTkQgIiB8IGdyZXAgLXEgIiB0YWJsZU5h \ +bWUKICAgIG1hcFt0YWJsZU5hbWVdID0gKHN5c3RlbShwcm9maWxlQ21kKSA9PSAwKQogICAgaWYg \ +KG1hcFt0YWJsZU5hbWVdKSB7CiAgICAgIHByaW50ICJcMDMzWzQ0bSIgImJlZ2luIGluc2VydCAi \ +IHRhYmxlTmFtZSAiXDAzM1swbSIgID4gIi9kZXYvc3RkZXJyIgogICAgfSBlbHNlIHsKICAgICAg \ +cHJpbnQgImV4Y2x1ZGUgdGFibGUgIiB0YWJsZU5hbWUgID4gIi9kZXYvc3RkZXJyIgogICAgfQog \ +IH0KICByZXR1cm4gbWFwW3RhYmxlTmFtZV0KfQoKewogIGJ1ZmZlciA9IHN1YnN0cigkMCwgMSwg \ +MTUwKQogIGxpbmUgPSAkMAogIGlmKG1hdGNoKGJ1ZmZlciwgL15MT0NLIFRBQkxFUyBgKFteYF0r \ +KWAgV1JJVEU7JC8sIGFycikgIT0gMCkgewogICAgIyBjaGVjayBpZiBpbnNlcnRzIGFyZSBwYXJ0 \ +IG9mIHRoZSBwcm9maWxlCiAgICB0YWJsZU5hbWU9YXJyWzFdCiAgICBpZiAoU2hvdWxkSW5jbHVk \ +ZVRhYmxlKHRhYmxlTmFtZSkpIHsKICAgICAgaWYgKFJFU0VUPT0xKSB7CiAgICAgICAgcHJpbnQg \ +IlwwMzNbNDRtIiAicmVzZXQgbW9kZSAtIHRydW5jYXRlIHRhYmxlICIgdGFibGVOYW1lICJcMDMz \ +WzBtIiAgPiAiL2Rldi9zdGRlcnIiCiAgICAgICAgbGluZSA9ICJcblRSVU5DQVRFIFRBQkxFIGAi \ +IHRhYmxlTmFtZSAiYCBJRiBFWElTVFM7XG4iIGxpbmUKICAgICAgfQogICAgICB3cml0ZT0xCiAg \ +ICB9IGVsc2UgewogICAgICB3cml0ZT0wCiAgICB9CiAgfSBlbHNlIGlmKG1hdGNoKGJ1ZmZlciwg \ +L15DUkVBVEUgVEFCTEUgYChbXmBdKylgIFwoJC8sIGFycikgIT0gMCkgewogICAgdGFibGVOYW1l \ +PWFyclsxXQogICAgaWYgKFNob3VsZEluY2x1ZGVUYWJsZSh0YWJsZU5hbWUpICYmIFJFU0VUPT0x \ +KSB7CiAgICAgIHByaW50ICJcMDMzWzQ0bSIgInJlc2V0IG1vZGUgLSBkcm9wIHRhYmxlICIgdGFi \ +bGVOYW1lICJcMDMzWzBtIiAgPiAiL2Rldi9zdGRlcnIiCiAgICAgIGxpbmUgPSAiXG5EUk9QIFRB \ +QkxFIElGIEVYSVNUUyBgIiB0YWJsZU5hbWUgImA7XG5DUkVBVEUgVEFCTEUgYCIgdGFibGVOYW1l \ +ICJgIChcbiIKICAgIH0gZWxzZSB7CiAgICAgIGxpbmUgPSAiQ1JFQVRFIFRBQkxFIElGIE5PVCBF \ +WElTVFMgYCIgdGFibGVOYW1lICJgIChcbiIKICAgIH0KICAgIHdyaXRlPTEKICB9IGVsc2UgaWYo \ +bWF0Y2goYnVmZmVyLCAvXklOU0VSVCBJTlRPIGAoW15gXSspYCBWQUxVRVMvLCBhcnIpICE9IDAp \ +IHsKICAgICMgY2hlY2sgaWYgaW5zZXJ0cyBhcmUgcGFydCBvZiB0aGUgcHJvZmlsZQogICAgdGFi \ +bGVOYW1lPWFyclsxXQogICAgd3JpdGU9U2hvdWxkSW5jbHVkZVRhYmxlKHRhYmxlTmFtZSkKICB9 \ +IGVsc2UgaWYobWF0Y2goYnVmZmVyLCAvXmNvbW1pdDskLywgYXJyKSAhPSAwKSB7CiAgICB3cml0 \ +ZT0xCiAgfSBlbHNlIGlmKG1hdGNoKGJ1ZmZlciwgL1NFVCBOQU1FUyAoW0EtWmEtejAtOV8tXSsp \ +LywgYXJyKSAhPSAwKSB7CiAgICBpZiAoQ0hBUkFDVEVSX1NFVCAhPSAiIikgewogICAgICBzdWIo \ +L1NFVCBOQU1FUyAoW0EtWmEtejAtOV8tXSspLywgIlNFVCBOQU1FUyAiIENIQVJBQ1RFUl9TRVQs \ +IGxpbmUpCiAgICB9CiAgICB3cml0ZT0xCiAgfSBlbHNlIGlmKG1hdGNoKGJ1ZmZlciwgL1NFVCBj \ +aGFyYWN0ZXJfc2V0X2NsaWVudCA9IChbQS1aYS16MC05Xy1dKykvLCBhcnIpICE9IDAgJiYgc3Vi \ +c3RyKGFyclsxXSwgMCwgMSkgIT0gIkAiKSB7CiAgICBpZiAoQ0hBUkFDVEVSX1NFVCAhPSAiIikg \ +ewogICAgICBzdWIoL1NFVCBjaGFyYWN0ZXJfc2V0X2NsaWVudCA9IChbQS1aYS16MC05Xy1dKykv \ +LCAiU0VUIGNoYXJhY3Rlcl9zZXRfY2xpZW50ID0gIiBDSEFSQUNURVJfU0VULCBsaW5lKQogICAg \ +fQogICAgd3JpdGU9MQogIH0KCiAgaWYgKHdyaXRlID09IDEpIHsKICAgIHByaW50IGxpbmUKICB9 \ +Cn0K" \ "644" -declare -gx embed_file_dbImportStreamScript="${PERSISTENT_TMPDIR:-/tmp}/1a25e8fe87d1af27f299ca9f5af14fe2fad803ddce20d659e5f6784c6056c9cc/dbImportStreamScript" +declare -gx embed_file_dbImportStreamScript="${PERSISTENT_TMPDIR:-/tmp}/f60daf3f7c7389022cfa34ad93f1f1d27253a0cbd47d93580fbf65ff7d20cca8/dbImportStreamScript" # create db instances @@ -2735,16 +2849,136 @@ declare status=0 # shellcheck disable=SC2034 declare -a pipeStatus=() +deduceDumpKindFromFilename() { + local dumpFile="$1" + Log::displayInfo "Deducing dump file format from filename ${dumpFile}" + if [[ "${dumpFile}" =~ \.sql$ ]]; then + echo "sql" + elif [[ "${dumpFile}" =~ \.tar$ ]]; then + echo "tar" + elif [[ "${dumpFile}" =~ (\.tar\.gz|\.tgz)$ ]]; then + echo "tar.gz" + elif [[ "${dumpFile}" =~ \.tar\.individual\.sql\.gz$ ]]; then + echo "tar.individual.sql.gz" + elif [[ "${dumpFile}" =~ \.gz$ ]]; then + echo "gz" + else + Log::displayError "Unable to deduce dump file format from filename ${dumpFile} - use --dump-kind option to specify the format" + return 1 + fi +} + +isAlpine() { + [[ "$(Linux::getDistributorId)" =~ alpine ]] +} + +loadTarDumpFile() { + local dumpFile="$1" + tar xf "${dumpFile}" +} + +loadGzipDumpFile() { + local dumpFile="$1" + zcat "${dumpFile}" +} + +# alpine tar does not support --wildcards, +# so we need to extract all files and cat them +alpineTarWildcardsAlternative() { + local dumpFile="$1" + local tarOptions="$2" + local tempDir + tempDir="$(mktemp -d)" + ( + trap 'rm -rf "${tempDir}"' EXIT + cd "${tempDir}" || exit 1 + tar "${tarOptions}" "${dumpFile}" + ls -al + cat ./*.sql + ) +} + +loadTgzIndividualSql() { + local dumpFile="$1" + if isAlpine; then + alpineTarWildcardsAlternative "${dumpFile}" "xzf" + else + tar xOzf "${dumpFile}" --wildcards --no-anchored '*.sql' + fi +} + +loadTarIndividualSql() { + local dumpFile="$1" + if isAlpine; then + alpineTarWildcardsAlternative "${dumpFile}" "xf" + else + tar xOf "${dumpFile}" --wildcards --no-anchored '*.sql' + fi +} + +loadTarIndividualSqlGz() { + local dumpFile="$1" + local sqlGzFile + local -a sqlGzFiles + mapfile -t sqlGzFiles < <(tar tf "${dumpFile}" | sort | grep -E '\.sql\.gz$') + for sqlGzFile in "${sqlGzFiles[@]}"; do + LOG_LAST_LOG_DATE_INIT=1 Log::computeDuration + tar xOf "${dumpFile}" "${sqlGzFile}" | zcat || { + if [[ $? = 141 ]]; then + return 0 + fi + # fail if any error during mysql or awk + return $? + } + DISPLAY_DURATION=1 Log::displayInfo "Processed ${sqlGzFile}" >&2 + done + return 0 +} + +loadDumpFile() { + local dumpFile="$1" + local dumpKind="$2" + if [[ "${dumpKind}" == "sql" ]]; then + cat "${dumpFile}" + elif [[ "${dumpKind}" == "tar" ]]; then + loadTarDumpFile "${dumpFile}" + elif [[ "${dumpKind}" == "gz" ]]; then + loadGzipDumpFile "${dumpFile}" + elif [[ "${dumpKind}" == "tar" ]]; then + loadTarIndividualSql "${dumpFile}" + elif [[ "${dumpKind}" == "tar.gz" ]]; then + loadTgzIndividualSql "${dumpFile}" + elif [[ "${dumpKind}" == "tar.individual.sql.gz" ]]; then + loadTarIndividualSqlGz "${dumpFile}" + else + Log::displayError "Unsupported dump file format for file ${dumpFile}" + exit 1 + fi + return 0 +} + +# shellcheck disable=SC2154 +if [[ "${optionDumpKind}" = "auto" ]]; then + # shellcheck disable=SC2154 + optionDumpKind="$(deduceDumpKindFromFilename "${argDumpFile}")" + Log::displayInfo "Deduced dump file format is ${optionDumpKind}" +fi +# try process substitution to avoid SIGPIPE +# https://oneuptime.com/blog/post/2026-01-24-bash-broken-pipe/view +Log::displayInfo "Loading dump file ${argDumpFile} with format ${optionDumpKind}" # shellcheck disable=2086,2154 ( - if [[ "${argDumpFile}" =~ \.tar.gz$ ]]; then - tar xOfz "${argDumpFile}" - elif [[ "${argDumpFile}" =~ \.gz$ ]]; then - zcat "${argDumpFile}" - fi + loadDumpFile "${argDumpFile}" "${optionDumpKind}" || { + if [[ $? = 141 ]]; then + return 0 + fi + # fail if any error during mysql or awk + return $? + } ) | ( awk \ -v PROFILE_COMMAND="${profileCommandFile}" \ + -v RESET="${optionReset}" \ -v CHARACTER_SET="${remoteCharacterSet}" \ -f "${embed_file_dbImportStreamScript}" \ - @@ -2753,11 +2987,10 @@ declare -a pipeStatus=() "--defaults-extra-file=${dbTargetInstance['AUTH_FILE']}" \ ${dbTargetInstance['DB_IMPORT_OPTIONS']} \ "${argTargetDbName}" -) || ( - # zcat will continue to write to stdout whereas awk has finished if table has been found +) || { # we detect this case because zcat will return code 141 because pipe closed Bash::handlePipelineFailure status pipeStatus -) +} } diff --git a/src/_binaries/Database/dbImport/dbImport-binary.yaml b/src/_binaries/Database/dbImport/dbImport-binary.yaml index e4565422..bf55d8a6 100644 --- a/src/_binaries/Database/dbImport/dbImport-binary.yaml +++ b/src/_binaries/Database/dbImport/dbImport-binary.yaml @@ -5,6 +5,7 @@ extends: - '${BASH_TOOLS_ROOT_DIR}/src/_binaries/commandDefinitions/optionsProfile.yaml' - '${BASH_TOOLS_ROOT_DIR}/src/_binaries/commandDefinitions/optionsTables.yaml' - '${BASH_TOOLS_ROOT_DIR}/src/_binaries/commandDefinitions/optionsMysqlSource.yaml' + - '${BASH_TOOLS_ROOT_DIR}/src/_binaries/commandDefinitions/optionsDumpKind.yaml' - '${BASH_TOOLS_ROOT_DIR}/src/_binaries/commandDefinitions/optionsDefault.yaml' - '${FRAMEWORK_ROOT_DIR}/src/_binaries/commandDefinitions/defaultCommand.yaml' - '${FRAMEWORK_ROOT_DIR}/src/_binaries/commandDefinitions/frameworkConfig.yaml' @@ -19,7 +20,7 @@ binData: commands: default: functionName: dbImportCommand - version: '3.0' + version: '4.0' copyrightBeginYear: 2020 commandName: dbImport beforeParseCallbacks: diff --git a/src/_binaries/Database/dbImport/dbImport-main.sh b/src/_binaries/Database/dbImport/dbImport-main.sh index c0ba6418..24f590ee 100755 --- a/src/_binaries/Database/dbImport/dbImport-main.sh +++ b/src/_binaries/Database/dbImport/dbImport-main.sh @@ -163,16 +163,24 @@ if [[ -z "${optionFromAws}" ]]; then fi fi Log::displayInfo "import remote to local from file ${remoteDbDumpTempFile}" +# shellcheck disable=SC2154 declare -a dbImportStreamOptions=( --profile "${optionProfile}" --target-dsn "${optionTargetDsn}" --character-set "${targetCharacterSet}" + --dump-kind "${optionDumpKind}" ) if [[ -n "${optionTables:-}" ]]; then dbImportStreamOptions+=( --tables "${optionTables}" ) fi +# shellcheck disable=SC2154 +if [[ "${optionReset}" = "1" ]]; then + dbImportStreamOptions+=( + --reset + ) +fi time ( "${CURRENT_DIR}/dbImportStream" \ "${dbImportStreamOptions[@]}" \ diff --git a/src/_binaries/Database/dbImport/dbImport-options.sh b/src/_binaries/Database/dbImport/dbImport-options.sh index 9ef914d0..decbd564 100755 --- a/src/_binaries/Database/dbImport/dbImport-options.sh +++ b/src/_binaries/Database/dbImport/dbImport-options.sh @@ -40,6 +40,12 @@ longDescriptionFunction() { echo echo -e " 2. import from S3" echo -e " ${__HELP_EXAMPLE}dbImport --from-aws awsFile.tar.gz --target-dsn localhost fromDb toDb${__HELP_NORMAL}" + echo + echo -e " ${__HELP_TITLE}Changelog${__HELP_NORMAL}" + echo -e " ${__HELP_EXAMPLE}4.0 (2026-04-14)${__HELP_NORMAL}" + echo -e " - add support for tar.gz and tar.individual.sql.gz dump files" + echo -e " ${__HELP_EXAMPLE}3.0${__HELP_NORMAL}" + echo -e " - initial version" Db::checkRequirements } diff --git a/src/_binaries/Database/dbImport/dbImport.bats b/src/_binaries/Database/dbImport/dbImport.bats index 8462a9f2..3a16b942 100755 --- a/src/_binaries/Database/dbImport/dbImport.bats +++ b/src/_binaries/Database/dbImport/dbImport.bats @@ -221,12 +221,10 @@ function Database::dbImport::remote_db_dump_already_present_from_db { #@test } function Database::dbImport::remote_db_fully_functional_from_aws { #@test - + (cd "${BATS_TEST_DIRNAME}/testsData" && tar cvzf "${HOME}/.bash-tools/dbImportDumps/fromDb.tar.gz.temp" 'dump.sql') stub aws \ 's3 ls --human-readable s3://s3server/exports/fromDb.tar.gz : exit 0' \ - "s3 cp s3://s3server/exports/fromDb.tar.gz '${HOME}/.bash-tools/dbImportDumps/fromDb.tar.gz' : touch '${HOME}/.bash-tools/dbImportDumps/fromDb.tar.gz'; exit 0" - stub tar \ - "xOfz '${HOME}/.bash-tools/dbImportDumps/fromDb.tar.gz' : cat '${BATS_TEST_DIRNAME}/testsData/dump.sql'" + "s3 cp s3://s3server/exports/fromDb.tar.gz '${HOME}/.bash-tools/dbImportDumps/fromDb.tar.gz' : mv '${HOME}/.bash-tools/dbImportDumps/fromDb.tar.gz.temp' '${HOME}/.bash-tools/dbImportDumps/fromDb.tar.gz'; exit 0" # call 5 (order 9): create target db # call 7 (order 11): import data dump into db @@ -247,7 +245,7 @@ function Database::dbImport::remote_db_fully_functional_from_aws { #@test function Database::dbImport::remote_db_dump_already_present_from_aws { #@test stub aws # create false dump 1 day in the past - tar cvzf "${HOME}/.bash-tools/dbImportDumps/fromDb.tar.gz" "${BATS_TEST_DIRNAME}/testsData/dump.sql" + (cd "${BATS_TEST_DIRNAME}/testsData" && tar cvzf "${HOME}/.bash-tools/dbImportDumps/fromDb.tar.gz" "dump.sql") touch -d@$(($(date +%s) + 86400)) "${HOME}/.bash-tools/dbImportDumps/fromDb.tar.gz" # call 5 (order 2): create target db (after dumps have been done) # call 7 (order 4): import data dump into db @@ -285,7 +283,7 @@ function Database::dbImport::import_local_dump_not_aws_with_tables_filter { #@te run "${binDir}/dbImport" --verbose -f default.local fromDb toDb --tables dataTable,otherTable 2>&1 assert_output --partial "db created" assert_output --partial "import structure dump" - assert_output --partial "ignore table emptyTable" + assert_output --partial "exclude table emptyTable" assert_output --partial "Import database duration : " assert_output --partial "begin insert otherTable" assert_output --partial "begin insert dataTable" @@ -311,7 +309,7 @@ function Database::dbImport::import_from_aws_with_tables_filter { #@test run "${binDir}/dbImport" --verbose --from-aws fromDb.tar.gz toDb --tables dataTable,otherTable 2>&1 assert_output --partial "Import database duration : " - assert_output --partial "ignore table emptyTable" + assert_output --partial "exclude table emptyTable" assert_output --partial "begin insert dataTable" assert_output --partial "begin insert otherTable" [[ -f "${HOME}/.bash-tools/dbImportDumps/fromDb.tar.gz" ]] diff --git a/src/_binaries/Database/dbImport/testsData/dbImport.help.txt b/src/_binaries/Database/dbImport/testsData/dbImport.help.txt index 8969ca82..97197dc6 100644 --- a/src/_binaries/Database/dbImport/testsData/dbImport.help.txt +++ b/src/_binaries/Database/dbImport/testsData/dbImport.help.txt @@ -5,9 +5,10 @@ USAGE: dbImport [--collation-name|-o ] [--target-dsn|-t ] [--character-set|-c ] [--profile|-p ] [--tables ] - [--skip-schema|-s] [--from-dsn|-f ] [--from-aws|-a ] [--help|-h] - [--config] [--bash-framework-config ] [--verbose|-v] - [-vv] [-vvv] [--log-level ] [--log-file ] + [--skip-schema|-s] [--from-dsn|-f ] [--from-aws|-a ] [--reset] + [--dump-kind|-k ] [--help|-h] [--config] + [--bash-framework-config ] [--verbose|-v] [-vv] [-vvv] + [--log-level ] [--log-file ] [--display-level ] [--no-color] [--theme ] [--version] [--quiet|-q] @@ -46,6 +47,27 @@ (Only .gz or tar.gz file are supported). This option is incompatible with -f|--from-dsn option. + --reset {single} + Reset the table before importing data using TRUNCATE TABLE or DROP TABLE + before CREATE TABLE. + --dump-kind, -k  {single} + Kind of dump to import. + + Possible values: + - auto: Deduce dump kind from filename: + .sql -> sql + .tar -> tar + .tar.gz or .tgz -> tar.gz + .gz -> gz + - sql: dump file is a .sql file. + - tar: dump file is a .tar file containing multiple .sql files, o + ne per table. + - tar.gz: dump file is a .tar file gzipped containing multiple .s + ql files, one per table. + - gz: dump file is a gz file containing a single .sql file. + - tar.individual.sql.gz: dump file is a tar file containing multi + ple .sql.gz files, one per table. + Default value: auto GLOBAL OPTIONS: --help, -h {single} @@ -140,7 +162,13 @@ 2. import from S3 dbImport --from-aws awsFile.tar.gz --target-dsn localhost fromDb toDb -VERSION: 3.0 + Changelog + 4.0 (2026-04-14) + - add support for tar.gz and tar.individual.sql.gz dump files + 3.0 + - initial version + +VERSION: 4.0 AUTHOR: [François Chastanet](https://github.com/fchastanet) diff --git a/src/_binaries/Database/dbImportStream/dbImportStream-binary.yaml b/src/_binaries/Database/dbImportStream/dbImportStream-binary.yaml index fbc209a7..7908fdbf 100644 --- a/src/_binaries/Database/dbImportStream/dbImportStream-binary.yaml +++ b/src/_binaries/Database/dbImportStream/dbImportStream-binary.yaml @@ -1,6 +1,7 @@ extends: - '${BASH_TOOLS_ROOT_DIR}/src/_binaries/commandDefinitions/optionsProfile.yaml' - '${BASH_TOOLS_ROOT_DIR}/src/_binaries/commandDefinitions/optionsTables.yaml' + - '${BASH_TOOLS_ROOT_DIR}/src/_binaries/commandDefinitions/optionsDumpKind.yaml' - '${BASH_TOOLS_ROOT_DIR}/src/_binaries/commandDefinitions/optionsVersion.yaml' - '${BASH_TOOLS_ROOT_DIR}/src/_binaries/commandDefinitions/optionsMysqlTarget.yaml' - '${BASH_TOOLS_ROOT_DIR}/src/_binaries/commandDefinitions/optionsDefault.yaml' @@ -17,7 +18,7 @@ binData: commands: default: functionName: dbImportStreamCommand - version: '3.0' + version: '4.0' copyrightBeginYear: 2020 commandName: dbImportStream beforeParseCallbacks: diff --git a/src/_binaries/Database/dbImportStream/dbImportStream-main.sh b/src/_binaries/Database/dbImportStream/dbImportStream-main.sh index 33f25e00..02cd03b0 100755 --- a/src/_binaries/Database/dbImportStream/dbImportStream-main.sh +++ b/src/_binaries/Database/dbImportStream/dbImportStream-main.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash + # @embed "${BASH_TOOLS_ROOT_DIR}/src/_binaries/Database/dbImportStream/dbImportStream.awk" AS dbImportStreamScript # create db instances @@ -20,16 +21,136 @@ declare status=0 # shellcheck disable=SC2034 declare -a pipeStatus=() +deduceDumpKindFromFilename() { + local dumpFile="$1" + Log::displayInfo "Deducing dump file format from filename ${dumpFile}" + if [[ "${dumpFile}" =~ \.sql$ ]]; then + echo "sql" + elif [[ "${dumpFile}" =~ \.tar$ ]]; then + echo "tar" + elif [[ "${dumpFile}" =~ (\.tar\.gz|\.tgz)$ ]]; then + echo "tar.gz" + elif [[ "${dumpFile}" =~ \.tar\.individual\.sql\.gz$ ]]; then + echo "tar.individual.sql.gz" + elif [[ "${dumpFile}" =~ \.gz$ ]]; then + echo "gz" + else + Log::displayError "Unable to deduce dump file format from filename ${dumpFile} - use --dump-kind option to specify the format" + return 1 + fi +} + +isAlpine() { + [[ "$(Linux::getDistributorId)" =~ alpine ]] +} + +loadTarDumpFile() { + local dumpFile="$1" + tar xf "${dumpFile}" +} + +loadGzipDumpFile() { + local dumpFile="$1" + zcat "${dumpFile}" +} + +# alpine tar does not support --wildcards, +# so we need to extract all files and cat them +alpineTarWildcardsAlternative() { + local dumpFile="$1" + local tarOptions="$2" + local tempDir + tempDir="$(mktemp -d)" + ( + trap 'rm -rf "${tempDir}"' EXIT + cd "${tempDir}" || exit 1 + tar "${tarOptions}" "${dumpFile}" + ls -al + cat ./*.sql + ) +} + +loadTgzIndividualSql() { + local dumpFile="$1" + if isAlpine; then + alpineTarWildcardsAlternative "${dumpFile}" "xzf" + else + tar xOzf "${dumpFile}" --wildcards --no-anchored '*.sql' + fi +} + +loadTarIndividualSql() { + local dumpFile="$1" + if isAlpine; then + alpineTarWildcardsAlternative "${dumpFile}" "xf" + else + tar xOf "${dumpFile}" --wildcards --no-anchored '*.sql' + fi +} + +loadTarIndividualSqlGz() { + local dumpFile="$1" + local sqlGzFile + local -a sqlGzFiles + mapfile -t sqlGzFiles < <(tar tf "${dumpFile}" | sort | grep -E '\.sql\.gz$') + for sqlGzFile in "${sqlGzFiles[@]}"; do + LOG_LAST_LOG_DATE_INIT=1 Log::computeDuration + tar xOf "${dumpFile}" "${sqlGzFile}" | zcat || { + if [[ $? = 141 ]]; then + return 0 + fi + # fail if any error during mysql or awk + return $? + } + DISPLAY_DURATION=1 Log::displayInfo "Processed ${sqlGzFile}" >&2 + done + return 0 +} + +loadDumpFile() { + local dumpFile="$1" + local dumpKind="$2" + if [[ "${dumpKind}" == "sql" ]]; then + cat "${dumpFile}" + elif [[ "${dumpKind}" == "tar" ]]; then + loadTarDumpFile "${dumpFile}" + elif [[ "${dumpKind}" == "gz" ]]; then + loadGzipDumpFile "${dumpFile}" + elif [[ "${dumpKind}" == "tar" ]]; then + loadTarIndividualSql "${dumpFile}" + elif [[ "${dumpKind}" == "tar.gz" ]]; then + loadTgzIndividualSql "${dumpFile}" + elif [[ "${dumpKind}" == "tar.individual.sql.gz" ]]; then + loadTarIndividualSqlGz "${dumpFile}" + else + Log::displayError "Unsupported dump file format for file ${dumpFile}" + exit 1 + fi + return 0 +} + +# shellcheck disable=SC2154 +if [[ "${optionDumpKind}" = "auto" ]]; then + # shellcheck disable=SC2154 + optionDumpKind="$(deduceDumpKindFromFilename "${argDumpFile}")" + Log::displayInfo "Deduced dump file format is ${optionDumpKind}" +fi +# try process substitution to avoid SIGPIPE +# https://oneuptime.com/blog/post/2026-01-24-bash-broken-pipe/view +Log::displayInfo "Loading dump file ${argDumpFile} with format ${optionDumpKind}" # shellcheck disable=2086,2154 ( - if [[ "${argDumpFile}" =~ \.tar.gz$ ]]; then - tar xOfz "${argDumpFile}" - elif [[ "${argDumpFile}" =~ \.gz$ ]]; then - zcat "${argDumpFile}" - fi + loadDumpFile "${argDumpFile}" "${optionDumpKind}" || { + if [[ $? = 141 ]]; then + return 0 + fi + # fail if any error during mysql or awk + return $? + } ) | ( awk \ -v PROFILE_COMMAND="${profileCommandFile}" \ + -v RESET="${optionReset}" \ -v CHARACTER_SET="${remoteCharacterSet}" \ -f "${embed_file_dbImportStreamScript}" \ - @@ -38,8 +159,7 @@ declare -a pipeStatus=() "--defaults-extra-file=${dbTargetInstance['AUTH_FILE']}" \ ${dbTargetInstance['DB_IMPORT_OPTIONS']} \ "${argTargetDbName}" -) || ( - # zcat will continue to write to stdout whereas awk has finished if table has been found +) || { # we detect this case because zcat will return code 141 because pipe closed Bash::handlePipelineFailure status pipeStatus -) +} diff --git a/src/_binaries/Database/dbImportStream/dbImportStream-options.sh b/src/_binaries/Database/dbImportStream/dbImportStream-options.sh index beb477a9..ea534028 100755 --- a/src/_binaries/Database/dbImportStream/dbImportStream-options.sh +++ b/src/_binaries/Database/dbImportStream/dbImportStream-options.sh @@ -28,6 +28,12 @@ longDescriptionFunction() { fromDsnOptionLongDescription echo profileOptionLongDescription + echo + echo -e " ${__HELP_TITLE}Changelog${__HELP_NORMAL}" + echo -e " ${__HELP_EXAMPLE}4.0 (2026-04-14)${__HELP_NORMAL}" + echo -e " - add support for tar.gz and tar.individual.sql.gz dump files" + echo -e " ${__HELP_EXAMPLE}3.0${__HELP_NORMAL}" + echo -e " - initial version" } dbImportStreamCommandCallback() { diff --git a/src/_binaries/Database/dbImportStream/dbImportStream.awk b/src/_binaries/Database/dbImportStream/dbImportStream.awk index 14ed7b59..e3b97890 100644 --- a/src/_binaries/Database/dbImportStream/dbImportStream.awk +++ b/src/_binaries/Database/dbImportStream/dbImportStream.awk @@ -1,39 +1,62 @@ BEGIN{ write=1 } + +function ShouldIncludeTable(tableName) { + if (! (tableName in map)) { + profileCmd = "echo '" tableName "' | " PROFILE_COMMAND " | grep -q " tableName + map[tableName] = (system(profileCmd) == 0) + if (map[tableName]) { + print "\033[44m" "begin insert " tableName "\033[0m" > "/dev/stderr" + } else { + print "exclude table " tableName > "/dev/stderr" + } + } + return map[tableName] +} + { buffer = substr($0, 1, 150) line = $0 if(match(buffer, /^LOCK TABLES `([^`]+)` WRITE;$/, arr) != 0) { # check if inserts are part of the profile tableName=arr[1] - if (! (tableName in map)) { - profileCmd = "echo '" tableName "' | " PROFILE_COMMAND " | grep -q " tableName - map[tableName] = (system(profileCmd) == 0) - } - if (map[tableName]) { - print "\033[44m" "begin insert " tableName "\033[0m" > "/dev/stderr" - line = line "\nTRUNCATE TABLE `" tableName "`;" + if (ShouldIncludeTable(tableName)) { + if (RESET==1) { + print "\033[44m" "reset mode - truncate table " tableName "\033[0m" > "/dev/stderr" + line = "\nTRUNCATE TABLE `" tableName "` IF EXISTS;\n" line + } write=1 } else { - print "ignore table " tableName > "/dev/stderr" write=0 } + } else if(match(buffer, /^CREATE TABLE `([^`]+)` \($/, arr) != 0) { + tableName=arr[1] + if (ShouldIncludeTable(tableName) && RESET==1) { + print "\033[44m" "reset mode - drop table " tableName "\033[0m" > "/dev/stderr" + line = "\nDROP TABLE IF EXISTS `" tableName "`;\nCREATE TABLE `" tableName "` (\n" + } else { + line = "CREATE TABLE IF NOT EXISTS `" tableName "` (\n" + } + write=1 + } else if(match(buffer, /^INSERT INTO `([^`]+)` VALUES/, arr) != 0) { + # check if inserts are part of the profile + tableName=arr[1] + write=ShouldIncludeTable(tableName) } else if(match(buffer, /^commit;$/, arr) != 0) { write=1 - } else if(match(buffer, /SET NAMES ([^ ]+)/, arr) != 0) { + } else if(match(buffer, /SET NAMES ([A-Za-z0-9_-]+)/, arr) != 0) { if (CHARACTER_SET != "") { - sub(/SET NAMES ([^ ]+)/, "SET NAMES " CHARACTER_SET, line) + sub(/SET NAMES ([A-Za-z0-9_-]+)/, "SET NAMES " CHARACTER_SET, line) } write=1 - } else if(match(buffer, /SET character_set_client = ([^ ]+)/, arr) != 0 && substr(arr[1], 0, 1) != "@") { + } else if(match(buffer, /SET character_set_client = ([A-Za-z0-9_-]+)/, arr) != 0 && substr(arr[1], 0, 1) != "@") { if (CHARACTER_SET != "") { - sub(/SET character_set_client = ([^ ]+)/, "SET character_set_client = " CHARACTER_SET, line) + sub(/SET character_set_client = ([A-Za-z0-9_-]+)/, "SET character_set_client = " CHARACTER_SET, line) } write=1 } - if (write == 1) { print line } diff --git a/src/_binaries/Database/dbImportStream/testsData/dbImportStream.help.txt b/src/_binaries/Database/dbImportStream/testsData/dbImportStream.help.txt index c60eef15..7b0933eb 100644 --- a/src/_binaries/Database/dbImportStream/testsData/dbImportStream.help.txt +++ b/src/_binaries/Database/dbImportStream/testsData/dbImportStream.help.txt @@ -3,10 +3,10 @@ USAGE: dbImportStream [OPTIONS] [ARGUMENTS] USAGE: dbImportStream [--profile|-p ] - [--tables ] [--target-dsn|-t ] - [--character-set|-c ] [--help|-h] [--config] - [--bash-framework-config ] [--verbose|-v] [-vv] [-vvv] - [--log-level ] [--log-file ] + [--tables ] [--reset] [--dump-kind|-k ] + [--target-dsn|-t ] [--character-set|-c ] [--help|-h] + [--config] [--bash-framework-config ] [--verbose|-v] + [-vv] [-vvv] [--log-level ] [--log-file ] [--display-level ] [--no-color] [--theme ] [--version] [--quiet|-q] @@ -23,6 +23,29 @@ If aws mode, ignore profile option. +SOURCE OPTIONS: + --reset {single} + Reset the table before importing data using TRUNCATE TABLE or DROP TABLE + before CREATE TABLE. + --dump-kind, -k  {single} + Kind of dump to import. + + Possible values: + - auto: Deduce dump kind from filename: + .sql -> sql + .tar -> tar + .tar.gz or .tgz -> tar.gz + .gz -> gz + - sql: dump file is a .sql file. + - tar: dump file is a .tar file containing multiple .sql files, o + ne per table. + - tar.gz: dump file is a .tar file gzipped containing multiple .s + ql files, one per table. + - gz: dump file is a gz file containing a single .sql file. + - tar.individual.sql.gz: dump file is a tar file containing multi + ple .sql.gz files, one per table. + Default value: auto + TARGET OPTIONS: --target-dsn, -t  {single} Dsn to use for target database. @@ -113,7 +136,13 @@ - default - none -VERSION: 3.0 + Changelog + 4.0 (2026-04-14) + - add support for tar.gz and tar.individual.sql.gz dump files + 3.0 + - initial version + +VERSION: 4.0 AUTHOR: [François Chastanet](https://github.com/fchastanet) diff --git a/src/_binaries/commandDefinitions/optionsDumpKind.yaml b/src/_binaries/commandDefinitions/optionsDumpKind.yaml new file mode 100644 index 00000000..f3f4ed0f --- /dev/null +++ b/src/_binaries/commandDefinitions/optionsDumpKind.yaml @@ -0,0 +1,52 @@ +--- +binData: + commands: + default: + optionGroups: + groupSourceDbOptions: + title: 'SOURCE OPTIONS:' + options: + - alts: + - '--reset' + group: groupSourceDbOptions + type: Boolean + help: >- + Reset the table before importing data + using TRUNCATE TABLE or DROP TABLE before CREATE TABLE. + variableName: optionReset + - alts: + - --dump-kind + - -k + group: groupSourceDbOptions + help: | + Kind of dump to import. + helpValueName: dumpKind + type: String + variableName: optionDumpKind + defaultValue: auto + authorizedValues: + - value: auto + help: >- + Deduce dump kind from filename: + ${__HELP_EXAMPLE}.sql${__HELP_NORMAL} -> sql + ${__HELP_EXAMPLE}.tar${__HELP_NORMAL} -> tar + ${__HELP_EXAMPLE}.tar.gz or .tgz${__HELP_NORMAL} -> tar.gz + ${__HELP_EXAMPLE}.gz${__HELP_NORMAL} -> gz + - value: sql + help: >- + dump file is a .sql file. + - value: tar + help: >- + dump file is a .tar file containing multiple .sql files, + one per table. + - value: tar.gz + help: >- + dump file is a .tar file gzipped containing multiple + .sql files, one per table. + - value: gz + help: >- + dump file is a gz file containing a single .sql file. + - value: tar.individual.sql.gz + help: >- + dump file is a tar file containing multiple + .sql.gz files, one per table.