From 3d5ff490a69887f1438956e5ec95b4363364dc15 Mon Sep 17 00:00:00 2001 From: Shahriyar Rzayev Date: Sun, 3 Oct 2021 19:07:37 +0400 Subject: [PATCH 1/4] feature: add new option for generate default config file in default path (#448) As discussed in #446 it can be valuable to add an commandline option to generate the default config file is this was somehow eliminated by setup. --generate-config-file flag option was added. Sample run will be something like this: autoxtrabackup --generate-config-file --verbose 2021-10-03 19:03:07 INFO [autoxtrabackup:278] Default config file is generated in /home/shako/.autoxtrabackup/autoxtrabackup.cnf 2021-10-03 19:03:07 INFO [autoxtrabackup:313] Xtrabackup command history: 2021-10-03 19:03:07 INFO [autoxtrabackup:315] ['command', 'xtrabackup_function', 'start time', 'end time', 'duration', 'exit code'] 2021-10-03 19:03:07 INFO [autoxtrabackup:316] Autoxtrabackup completed successfully! --- mysql_autoxtrabackup/autoxtrabackup.py | 22 ++++++++++++++----- .../general_conf/generalops.py | 18 +++++++-------- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/mysql_autoxtrabackup/autoxtrabackup.py b/mysql_autoxtrabackup/autoxtrabackup.py index 3da19e0..2ab19b7 100644 --- a/mysql_autoxtrabackup/autoxtrabackup.py +++ b/mysql_autoxtrabackup/autoxtrabackup.py @@ -17,6 +17,7 @@ from mysql_autoxtrabackup.backup_prepare.prepare import Prepare from mysql_autoxtrabackup.general_conf import path_config from mysql_autoxtrabackup.general_conf.generalops import GeneralClass +from mysql_autoxtrabackup.general_conf.generate_default_conf import GenerateDefaultConfig from mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner from mysql_autoxtrabackup.utils import version @@ -103,12 +104,11 @@ def validate_file(file: str) -> Optional[bool]: # filename extension should be .cnf pattern = re.compile(r".*\.cnf") - if pattern.match(file): - # Lastly the file should have all 5 required headers - if check_file_content(file): - return None - else: + if not pattern.match(file): raise ValueError("Invalid file extension. Expecting .cnf") + # Lastly the file should have all 5 required headers + if check_file_content(file): + return None return None @@ -133,6 +133,12 @@ def validate_file(file: str) -> Optional[bool]: show_default=True, help="Read options from the given file", ) +@click.option( + "--generate-config-file", + is_flag=True, + is_eager=True, + help="Create a config file template in default directory" +) @click.option("--tag", help="Pass the tag string for each backup") @click.option("--show-tags", is_flag=True, help="Show backup tags and exit") @click.option("-v", "--verbose", is_flag=True, help="Be verbose (print to console)") @@ -188,6 +194,7 @@ def all_procedure( log_file, log, defaults_file, + generate_config_file, dry_run, log_file_max_bytes, log_file_backup_count, @@ -256,6 +263,7 @@ def all_procedure( and dry_run is False and show_tags is False and run_server is False + and generate_config_file is False ): print_help(ctx, None, value=True) @@ -264,6 +272,10 @@ def all_procedure( elif show_tags and defaults_file: backup_ = Backup(config=defaults_file) backup_.show_tags(backup_dir=str(backup_options.get("backup_dir"))) + elif generate_config_file: + gen_ = GenerateDefaultConfig() + gen_.generate_config_file() + logger.info(f"Default config file is generated in {defaults_file}") elif prepare: prepare_ = Prepare(config=defaults_file, dry_run=dry_run_, tag=tag) prepare_.prepare_backup_and_copy_back() diff --git a/mysql_autoxtrabackup/general_conf/generalops.py b/mysql_autoxtrabackup/general_conf/generalops.py index e446348..66f063e 100644 --- a/mysql_autoxtrabackup/general_conf/generalops.py +++ b/mysql_autoxtrabackup/general_conf/generalops.py @@ -95,11 +95,10 @@ def backup_archive_options(self) -> Dict[str, Union[str, float]]: archive_max_size = self.con.get(section, "max_archive_size", fallback=None) if archive_max_size: archive_max_size = humanfriendly.parse_size(archive_max_size) - else: - if self.con.get(section, "archive_max_size", fallback=None): - archive_max_size = humanfriendly.parse_size( - self.con.get(section, "archive_max_size", fallback=None) - ) + elif self.con.get(section, "archive_max_size", fallback=None): + archive_max_size = humanfriendly.parse_size( + self.con.get(section, "archive_max_size", fallback=None) + ) # backward compatible with old config 'max_archive_duration' and newer 'archive_max_duration' archive_max_duration = self.con.get( @@ -107,11 +106,10 @@ def backup_archive_options(self) -> Dict[str, Union[str, float]]: ) if archive_max_duration: archive_max_duration = humanfriendly.parse_timespan(archive_max_duration) - else: - if self.con.get(section, "archive_max_size", fallback=None): - archive_max_duration = humanfriendly.parse_timespan( - self.con.get(section, "archive_max_size", fallback=None) - ) + elif self.con.get(section, "archive_max_size", fallback=None): + archive_max_duration = humanfriendly.parse_timespan( + self.con.get(section, "archive_max_size", fallback=None) + ) return { "archive_dir": self.con.get(section, "archive_dir", fallback=None), # type: ignore From 5737bb1524d231547dfda88d5e57d35cb501be28 Mon Sep 17 00:00:00 2001 From: Shahriyar Rzayev Date: Sun, 3 Oct 2021 19:53:53 +0400 Subject: [PATCH 2/4] bugfix: ValueError: could not convert string to float: 'None' (#449) The wrong config option was used and it fails when the tool tries to use backup archive feature. Now it will raise BackupArchiveNotConfigured error if options were not set and also it will check for archive_max_duration --- mysql_autoxtrabackup/autoxtrabackup.py | 6 ++++-- mysql_autoxtrabackup/backup_backup/backup_archive.py | 9 +++++++++ mysql_autoxtrabackup/general_conf/generalops.py | 4 ++-- mysql_autoxtrabackup/process_runner/errors.py | 11 +++++++++++ 4 files changed, 26 insertions(+), 4 deletions(-) diff --git a/mysql_autoxtrabackup/autoxtrabackup.py b/mysql_autoxtrabackup/autoxtrabackup.py index 2ab19b7..0cc8365 100644 --- a/mysql_autoxtrabackup/autoxtrabackup.py +++ b/mysql_autoxtrabackup/autoxtrabackup.py @@ -17,7 +17,9 @@ from mysql_autoxtrabackup.backup_prepare.prepare import Prepare from mysql_autoxtrabackup.general_conf import path_config from mysql_autoxtrabackup.general_conf.generalops import GeneralClass -from mysql_autoxtrabackup.general_conf.generate_default_conf import GenerateDefaultConfig +from mysql_autoxtrabackup.general_conf.generate_default_conf import ( + GenerateDefaultConfig, +) from mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner from mysql_autoxtrabackup.utils import version @@ -137,7 +139,7 @@ def validate_file(file: str) -> Optional[bool]: "--generate-config-file", is_flag=True, is_eager=True, - help="Create a config file template in default directory" + help="Create a config file template in default directory", ) @click.option("--tag", help="Pass the tag string for each backup") @click.option("--show-tags", is_flag=True, help="Show backup tags and exit") diff --git a/mysql_autoxtrabackup/backup_backup/backup_archive.py b/mysql_autoxtrabackup/backup_backup/backup_archive.py index 51ff78e..09fa9bf 100644 --- a/mysql_autoxtrabackup/backup_backup/backup_archive.py +++ b/mysql_autoxtrabackup/backup_backup/backup_archive.py @@ -7,6 +7,7 @@ from mysql_autoxtrabackup.backup_backup.backup_builder import BackupBuilderChecker from mysql_autoxtrabackup.general_conf import path_config from mysql_autoxtrabackup.general_conf.generalops import GeneralClass +from mysql_autoxtrabackup.process_runner.errors import BackupArchiveNotConfigured from mysql_autoxtrabackup.process_runner.process_runner import ProcessRunner from mysql_autoxtrabackup.utils import helpers @@ -126,6 +127,14 @@ def clean_old_archives(self) -> None: archive_dir = str(self.backup_archive_options.get("archive_dir")) # Finding if last full backup older than the interval or more from now! cleanup_msg = "Removing archive {}/{} due to {}" + if not self.backup_archive_options.get( + "archive_max_duration", None + ) and not self.backup_archive_options.get("archive_max_size", None): + raise BackupArchiveNotConfigured( + expression="BackupArchiveNotConfigured", + message="You need to both set archive_max_size and archive_max_duration in config file.", + ) + for archive in helpers.sorted_ls(archive_dir): if "_archive" in archive: archive_date = datetime.strptime(archive, "%Y-%m-%d_%H-%M-%S_archive") diff --git a/mysql_autoxtrabackup/general_conf/generalops.py b/mysql_autoxtrabackup/general_conf/generalops.py index 66f063e..f670ad2 100644 --- a/mysql_autoxtrabackup/general_conf/generalops.py +++ b/mysql_autoxtrabackup/general_conf/generalops.py @@ -106,9 +106,9 @@ def backup_archive_options(self) -> Dict[str, Union[str, float]]: ) if archive_max_duration: archive_max_duration = humanfriendly.parse_timespan(archive_max_duration) - elif self.con.get(section, "archive_max_size", fallback=None): + elif self.con.get(section, "archive_max_duration", fallback=None): archive_max_duration = humanfriendly.parse_timespan( - self.con.get(section, "archive_max_size", fallback=None) + self.con.get(section, "archive_max_duration", fallback=None) ) return { diff --git a/mysql_autoxtrabackup/process_runner/errors.py b/mysql_autoxtrabackup/process_runner/errors.py index c472318..e32d72d 100644 --- a/mysql_autoxtrabackup/process_runner/errors.py +++ b/mysql_autoxtrabackup/process_runner/errors.py @@ -55,3 +55,14 @@ def __init__(self, expression: str, message: str) -> None: self.expression = expression self.message = message log_error(self.expression, self.message) + + +class BackupArchiveNotConfigured(Error): + """ + Exception raised when archive_max_size and archive_max_duration configs are not set + """ + + def __init__(self, expression: str, message: str) -> None: + self.expression = expression + self.message = message + log_error(self.expression, self.message) From c6d6dee5685f4b803c3ccbae5213f2f7b598be15 Mon Sep 17 00:00:00 2001 From: Shahriyar Rzayev Date: Sun, 3 Oct 2021 19:59:47 +0400 Subject: [PATCH 3/4] Version bump --- mysql_autoxtrabackup/utils/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mysql_autoxtrabackup/utils/version.py b/mysql_autoxtrabackup/utils/version.py index 2fd781c..bd41eab 100644 --- a/mysql_autoxtrabackup/utils/version.py +++ b/mysql_autoxtrabackup/utils/version.py @@ -1,3 +1,3 @@ __all__ = "VERSION" -VERSION = "2.0.2" +VERSION = "2.0.3" From e433f4816d4073e6ab1545d6ec4960e92479b66a Mon Sep 17 00:00:00 2001 From: Shahriyar Rzayev Date: Sun, 3 Oct 2021 20:12:34 +0400 Subject: [PATCH 4/4] Updated the History.md --- HISTORY.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/HISTORY.md b/HISTORY.md index 53f8c22..cc21289 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,3 +1,8 @@ +## v2.0.3 (2021-10-03) + +* , #449 by @ShahriyarR +* , #448 by @ShahriyarR + ## v2.0.2 (2021-05-06) * Increased code coverage and did code base refactoring, #444 by @shahriyarr