| Path : /var/www/html/yedekledb24/DatabaseBackupMasterv8/ |
|
B-Con CMD Config cPanel C-Rdp D-Log Info Jump Mass Ransom Symlink vHost Zone-H |
| Current File : /var/www/html/yedekledb24/DatabaseBackupMasterv8/perform_incremental_mysql_backup.py |
import os
import subprocess
import tempfile
import logging
import shutil
import tarfile
import traceback
from models import BackupLog
from pathlib import Path
import storage_utils
logger = logging.getLogger(__name__)
def perform_incremental_mysql_backup(db_conn, temp_file, job, parent_log_id=None):
"""
Perform incremental MySQL/MariaDB backup using XtraBackup or MariaBackup.
Args:
db_conn (DatabaseConnection): Database connection object
temp_file (str): Path to temporary file for backup
job (BackupJob): Backup job configuration
parent_log_id (int, optional): ID of parent backup log entry
Returns:
bool: True if backup successful, False otherwise
"""
# Geçici dizinleri takip etmek için liste
temp_dirs = []
try:
# Detect database type and select the appropriate backup tool
db_type = db_conn.db_type.lower()
# Select the appropriate backup tool based on database type
if db_type == 'mariadb':
backup_tool = 'mariabackup'
logger.info("Using MariaBackup for MariaDB")
elif db_type == 'mysql':
backup_tool = 'xtrabackup'
logger.info("Using Percona XtraBackup for MySQL")
else:
logger.error(f"Unsupported database type for incremental backup: {db_type}")
return False
# Check if the selected backup tool is installed
try:
subprocess.run([backup_tool, '--version'], capture_output=True, check=True)
except (subprocess.SubprocessError, FileNotFoundError):
logger.error(f"{backup_tool} is not installed. Cannot perform incremental backup.")
return False
# Set MySQL/MariaDB connection parameters
mysql_args = {
'host': db_conn.host,
'port': db_conn.port,
'user': db_conn.username,
'password': db_conn.password,
'database': db_conn.database
}
# Create a temporary directory for the backup
backup_dir = tempfile.mkdtemp(prefix=f"backup_{mysql_args['database']}_")
temp_dirs.append(backup_dir)
# Determine backup type
backup_type = job.backup_method # 'full', 'incremental', 'differential'
# Optimum parametreler ile komut oluştur
cmd = [
backup_tool,
'--backup',
f'--target-dir={backup_dir}',
f'--host={mysql_args["host"]}',
f'--port={mysql_args["port"]}',
f'--user={mysql_args["user"]}',
f'--password={mysql_args["password"]}',
'--compress', # Yedekleri sıkıştır
'--compress-threads=4', # Sıkıştırma için 4 thread kullan
'--parallel=2' # Paralel işlem için 2 thread kullan
]
# For incremental/differential backup, find the base backup
parent_backup_dir = None
if backup_type in ['incremental', 'differential']:
if parent_log_id:
parent_backup = BackupLog.query.get(parent_log_id)
else:
# Find the last full backup for this job
parent_backup = BackupLog.query.filter_by(
job_id=job.id,
backup_method='full',
status='success'
).order_by(BackupLog.timestamp.desc()).first()
if not parent_backup:
logger.error(f"No parent backup found for {backup_type} backup. Performing full backup instead.")
backup_type = 'full'
else:
logger.info(f"Using parent backup ID {parent_backup.id} from {parent_backup.timestamp}")
# Create a temporary directory for the parent backup
parent_temp_dir = tempfile.mkdtemp(prefix=f"backup_parent_")
temp_dirs.append(parent_temp_dir)
parent_backup_file = os.path.join(parent_temp_dir, "parent_backup.tar.gz")
# Download the parent backup file
try:
logger.info(f"Downloading parent backup from storage: {parent_backup.backup_file}")
download_success = storage_utils.download_file_from_storage(
job.storage,
parent_backup.backup_file,
parent_backup_file
)
if not download_success:
logger.error("Failed to download parent backup from storage")
backup_type = 'full' # Fall back to full backup
elif not os.path.exists(parent_backup_file):
logger.error(f"Downloaded parent backup file does not exist: {parent_backup_file}")
backup_type = 'full' # Fall back to full backup
else:
# Extract the parent backup
parent_backup_dir = os.path.join(parent_temp_dir, "extracted")
os.makedirs(parent_backup_dir, exist_ok=True)
logger.info(f"Checking parent backup file format: {parent_backup_file}")
# Eğer dosya ismi .sql, .zip veya .gpg içeriyorsa, bu mysqldump ile oluşturulmuş bir dosyadır
if '.sql' in parent_backup.backup_file or '.zip' in parent_backup.backup_file or '.gpg' in parent_backup.backup_file:
logger.warning("Parent backup appears to be in SQL format, not compatible with incremental backups.")
logger.warning("To use incremental backups, you need at least one successful full backup using mariabackup.")
backup_type = 'full' # Tam yedeklemeye geri dön
else:
logger.info(f"Extracting parent backup file: {parent_backup_file}")
try:
# Dosya formatını kontrol et
if parent_backup_file.endswith('.gpg'):
logger.warning("Parent backup is encrypted and cannot be used for incremental backup.")
backup_type = 'full'
# Dosya formatını kontrol et - tar.gz mi değil mi?
try:
with tarfile.open(parent_backup_file, 'r:gz') as tar:
tar.extractall(path=parent_backup_dir)
# Find the extracted directory in parent_backup_dir
subdirs = [d for d in os.listdir(parent_backup_dir) if os.path.isdir(os.path.join(parent_backup_dir, d))]
if subdirs:
parent_backup_dir = os.path.join(parent_backup_dir, subdirs[0])
logger.info(f"Using extracted parent backup directory: {parent_backup_dir}")
else:
logger.error("No valid directory found in extracted parent backup")
backup_type = 'full' # Fall back to full backup
except tarfile.ReadError:
logger.warning("Parent backup is not in tar.gz format. Falling back to full backup.")
backup_type = 'full' # Tam yedeklemeye geri dön
except Exception as extract_err:
logger.error(f"Failed to extract parent backup: {str(extract_err)}")
backup_type = 'full' # Fall back to full backup
except Exception as download_err:
logger.error(f"Error accessing parent backup: {str(download_err)}")
backup_type = 'full' # Fall back to full backup
# Add incremental backup parameters if we have a valid parent backup dir
if backup_type != 'full' and parent_backup_dir and os.path.isdir(parent_backup_dir):
if backup_type == 'incremental':
logger.info(f"Setting up incremental backup with basedir: {parent_backup_dir}")
cmd.append('--incremental')
cmd.append(f'--incremental-basedir={parent_backup_dir}')
elif backup_type == 'differential':
# For differential backup, find the original full backup (base)
if parent_backup.backup_method == 'full':
logger.info(f"Setting up differential backup using full backup as base")
cmd.append('--incremental')
cmd.append(f'--incremental-basedir={parent_backup_dir}')
else:
logger.info(f"Setting up differential backup, searching for base full backup")
# Traverse up the backup chain until we find a full backup
base_backup = parent_backup
while base_backup and base_backup.backup_method != 'full':
base_backup = BackupLog.query.get(base_backup.parent_log_id)
if base_backup and base_backup.id != parent_backup.id:
# Download and extract the base backup
base_temp_dir = tempfile.mkdtemp(prefix=f"backup_base_")
temp_dirs.append(base_temp_dir)
base_backup_file = os.path.join(base_temp_dir, "base_backup.tar.gz")
logger.info(f"Downloading base full backup ID {base_backup.id} from storage")
download_success = storage_utils.download_file_from_storage(
job.storage,
base_backup.backup_file,
base_backup_file
)
if download_success and os.path.exists(base_backup_file):
base_backup_dir = os.path.join(base_temp_dir, "extracted")
os.makedirs(base_backup_dir, exist_ok=True)
try:
with tarfile.open(base_backup_file, 'r:gz') as tar:
tar.extractall(path=base_backup_dir)
subdirs = [d for d in os.listdir(base_backup_dir) if os.path.isdir(os.path.join(base_backup_dir, d))]
if subdirs:
base_backup_dir = os.path.join(base_backup_dir, subdirs[0])
logger.info(f"Using base backup directory: {base_backup_dir}")
cmd.append('--incremental')
cmd.append(f'--incremental-basedir={base_backup_dir}')
else:
logger.error("No valid directory found in extracted base backup")
backup_type = 'full' # Fall back to full backup
except Exception as extract_err:
logger.error(f"Failed to extract base backup: {str(extract_err)}")
backup_type = 'full' # Fall back to full backup
else:
logger.error("Failed to download base backup from storage")
backup_type = 'full' # Fall back to full backup
else:
logger.error("Could not find valid base full backup for differential backup")
backup_type = 'full' # Fall back to full backup
else:
logger.warning(f"Invalid parent backup directory or backup type is now 'full'")
# Add any extra options specified for XtraBackup/MariaBackup
if hasattr(job, 'xtrabackup_options') and job.xtrabackup_options:
extra_options = job.xtrabackup_options.split()
cmd.extend(extra_options)
logger.info(f"Added user-specified options: {job.xtrabackup_options}")
# Log final backup type and command
if backup_type == 'full':
logger.info("Executing FULL backup")
else:
logger.info(f"Executing {backup_type.upper()} backup")
logger.debug(f"Backup command: {' '.join(cmd)}")
# Execute backup command with timeout
result = subprocess.run(cmd, capture_output=True, text=True, timeout=3600) # 1 hour timeout
if result.returncode != 0:
logger.error(f"Backup error ({result.returncode}): {result.stderr}")
# InnoDB hatası varsa, otomatik olarak mysqldump'a geçiş yap
if "InnoDB" in result.stderr or "mariadbd got signal 6" in result.stderr:
logger.warning("Detected InnoDB or corruption error in MariaBackup/XtraBackup")
logger.warning("Falling back to standard mysqldump method")
# Standart mysqldump komutunu hazırla
mysqldump_cmd = [
'mysqldump',
f'--host={mysql_args["host"]}',
f'--port={mysql_args["port"]}',
f'--user={mysql_args["user"]}',
f'--password={mysql_args["password"]}',
'--single-transaction',
'--quick',
'--routines',
'--triggers',
'--add-drop-table',
'--compress',
mysql_args["database"]
]
# temp_file dosya uzantısını düzeltmemiz gerekebilir
# sql dosyası değilse, sql uzantılı hale getir
if not temp_file.endswith('.sql'):
temp_file = os.path.splitext(temp_file)[0] + '.sql'
logger.info(f"Executing fallback mysqldump to {temp_file}")
with open(temp_file, 'w') as f:
try:
dump_result = subprocess.run(mysqldump_cmd, stdout=f, stderr=subprocess.PIPE, text=True)
if dump_result.returncode != 0:
logger.error(f"Fallback mysqldump error: {dump_result.stderr}")
return False
else:
logger.info("Fallback mysqldump completed successfully")
return True
except Exception as dump_err:
logger.error(f"Fallback mysqldump exception: {str(dump_err)}")
return False
# Diğer hatalar için başarısız olarak işaretle
return False
# For full backups, prepare the backup to make it consistent
if backup_type == 'full':
# backup-my.cnf dosyasını kontrol et ve gerekirse oluştur
backup_my_cnf_path = os.path.join(backup_dir, 'backup-my.cnf')
if not os.path.exists(backup_my_cnf_path):
logger.info("Creating backup-my.cnf file for preparation step")
with open(backup_my_cnf_path, 'w') as f:
f.write("[mysqld]\n")
f.write("innodb_checksum_algorithm=crc32\n")
f.write("innodb_log_checksums=ON\n")
f.write("innodb_data_file_path=ibdata1:12M:autoextend\n")
f.write("innodb_log_files_in_group=2\n")
f.write("innodb_log_file_size=50331648\n")
f.write("innodb_page_size=16384\n")
f.write("innodb_undo_directory=./\n")
f.write("innodb_undo_tablespaces=0\n")
prepare_cmd = [
backup_tool,
'--prepare',
f'--target-dir={backup_dir}'
]
# MySQL için XtraBackup kullanıyorsak, ek parametre ekle
if 'xtrabackup' in backup_tool:
prepare_cmd.append('--skip-innodb-redo-log-restore') # Bu parametre sadece MySQL XtraBackup için geçerli
logger.info("Preparing full backup to ensure consistency")
prepare_result = subprocess.run(prepare_cmd, capture_output=True, text=True)
if prepare_result.returncode != 0:
logger.error(f"Backup prepare error: {prepare_result.stderr}")
# InnoDB hatası varsa, otomatik olarak mysqldump'a geçiş yap
if "InnoDB" in prepare_result.stderr or "mariadbd got signal" in prepare_result.stderr:
logger.warning("Detected InnoDB or corruption error in prepare step")
logger.warning("Falling back to standard mysqldump method")
# Standart mysqldump komutunu hazırla
mysqldump_cmd = [
'mysqldump',
f'--host={mysql_args["host"]}',
f'--port={mysql_args["port"]}',
f'--user={mysql_args["user"]}',
f'--password={mysql_args["password"]}',
'--single-transaction',
'--quick',
'--routines',
'--triggers',
'--add-drop-table',
'--compress',
mysql_args["database"]
]
# temp_file dosya uzantısını düzelt
if not temp_file.endswith('.sql'):
temp_file = os.path.splitext(temp_file)[0] + '.sql'
logger.info(f"Executing fallback mysqldump to {temp_file}")
with open(temp_file, 'w') as f:
try:
dump_result = subprocess.run(mysqldump_cmd, stdout=f, stderr=subprocess.PIPE, text=True)
if dump_result.returncode != 0:
logger.error(f"Fallback mysqldump error: {dump_result.stderr}")
logger.warning("Skipping backup preparation, proceeding with raw backup")
else:
logger.info("Fallback mysqldump completed successfully")
return True
except Exception as dump_err:
logger.error(f"Fallback mysqldump exception: {str(dump_err)}")
logger.warning("Skipping backup preparation, proceeding with raw backup")
else:
# Hata olsa bile devam et ve hazırlama adımını atla
logger.warning("Skipping backup preparation, proceeding with raw backup")
else:
logger.info("Backup preparation completed successfully")
# Create a tarball of the backup directory
target_dir = os.path.dirname(temp_file)
os.makedirs(target_dir, exist_ok=True)
logger.info(f"Creating compressed archive of backup: {temp_file}")
with tarfile.open(temp_file, "w:gz") as tar:
tar.add(backup_dir, arcname=os.path.basename(backup_dir))
# If encryption is enabled, encrypt the tarball
# Artırımlı yedekleme türü için şifreleme devre dışı bırakıldı
if backup_type == 'full' and hasattr(job, 'encrypt_backup') and job.encrypt_backup and hasattr(job, 'encryption_type') and job.encryption_type == 'symmetric' and hasattr(job, 'encryption_passphrase') and job.encryption_passphrase:
try:
logger.info("Encrypting backup file with GPG")
# Önce basitleştirilmiş versiyonu dene
try:
from encryption_utils_simplified import encrypt_file
except ImportError:
# Sonra normal versiyonu dene
from encryption_utils import encrypt_file
encrypted_file = encrypt_file(
temp_file,
recipients=None,
output_file=f"{temp_file}.gpg",
passphrase=job.encryption_passphrase
)
if encrypted_file:
# Remove the unencrypted file and update the temp_file path
os.unlink(temp_file)
temp_file = encrypted_file
logger.info(f"Backup file encrypted successfully: {encrypted_file}")
else:
logger.error("Failed to encrypt backup file")
except Exception as e:
logger.error(f"Encryption error: {str(e)}")
elif backup_type != 'full' and hasattr(job, 'encrypt_backup') and job.encrypt_backup:
logger.info("Encryption skipped for incremental/differential backup to maintain proper backup chain")
# Record final backup type in log message
logger.info(f"Successfully completed {backup_type.upper()} backup using {backup_tool}")
# Başarılı yedekleme durumunu bildirin
return True
except Exception as e:
logger.error(f"Backup error: {str(e)}")
logger.error(traceback.format_exc())
return False
finally:
# Cleanup temporary directories even if there's an error
for temp_dir in temp_dirs:
try:
if os.path.exists(temp_dir):
logger.debug(f"Cleaning up temporary directory: {temp_dir}")
shutil.rmtree(temp_dir)
except Exception as e:
logger.warning(f"Failed to clean up temporary directory {temp_dir}: {str(e)}")
# Continue with cleanup even if one directory fails