| Path : /var/www/html/yedekledb24/DatabaseBackupMasterv3/ |
|
B-Con CMD Config cPanel C-Rdp D-Log Info Jump Mass Ransom Symlink vHost Zone-H |
| Current File : /var/www/html/yedekledb24/DatabaseBackupMasterv3/backup_utils.py |
import os
import time
import subprocess
import tempfile
import logging
import shutil
import gzip
from datetime import datetime
from models import DatabaseConnection, StorageTarget, BackupJob, BackupLog
from storage_utils import upload_to_storage, get_storage_space
# Import Flask-related modules only after app is created to avoid circular imports
from app import db
logger = logging.getLogger(__name__)
# We now import format_size from utils.py
from utils import format_size
def test_database_connection(db_type, host, port, username, password, database):
"""
Test connection to a database server.
Args:
db_type (str): Database type (mysql, mariadb, postgresql)
host (str): Database host
port (int): Database port
username (str): Database username
password (str): Database password
database (str): Database name
Returns:
bool: True if connection successful, False otherwise
"""
try:
if db_type in ['mysql', 'mariadb']:
import pymysql
conn = pymysql.connect(
host=host,
port=port,
user=username,
password=password,
database=database,
connect_timeout=5
)
conn.close()
return True
elif db_type == 'postgresql':
import psycopg2
conn = psycopg2.connect(
host=host,
port=port,
user=username,
password=password,
dbname=database,
connect_timeout=5
)
conn.close()
return True
else:
logger.error(f"Unsupported database type: {db_type}")
return False
except Exception as e:
logger.error(f"Database connection error for {db_type} {host}:{port}/{database}: {str(e)}")
return False
def perform_mysql_backup(db_conn, temp_file):
"""
Perform MySQL/MariaDB backup using mysqldump with optimized parameters.
Args:
db_conn (DatabaseConnection): Database connection object
temp_file (str): Path to temporary file for backup
Returns:
bool: True if backup successful, False otherwise
"""
try:
# Create password file for mysqldump (safer than command line password)
pwd_file = tempfile.NamedTemporaryFile(delete=False)
pwd_file.write(db_conn.password.encode())
pwd_file.close()
# Build mysqldump command with optimized parameters
cmd = [
'mysqldump',
f'--host={db_conn.host}',
f'--port={db_conn.port}',
f'--user={db_conn.username}',
f'--password={db_conn.password}',
'--single-transaction', # Consistent backups without locking tables
'--quick', # Retrieve rows one by one (better for large tables)
'--routines', # Include stored procedures
'--triggers', # Include triggers
'--events', # Include events
'--add-drop-table', # Include DROP TABLE statements
'--add-drop-database', # Include DROP DATABASE statements
'--create-options', # Include all CREATE TABLE options
'--extended-insert', # Use multi-row INSERT syntax (more efficient)
'--compress', # Compress data between client and server if both support it
'--max-allowed-packet=1G', # Allow large table/blob handling
# Removed '--set-gtid-purged=OFF' for compatibility with older MySQL versions
'--databases', db_conn.database,
'-r', temp_file
]
# Create a compressed version if required
compressed_file = temp_file + '.gz'
# Execute command
result = subprocess.run(cmd, capture_output=True)
# Remove password file
os.unlink(pwd_file.name)
if result.returncode != 0:
logger.error(f"mysqldump error: {result.stderr.decode()}")
return False
# Compress the backup file
try:
with open(temp_file, 'rb') as f_in:
import gzip
with gzip.open(compressed_file, 'wb') as f_out:
shutil.copyfileobj(f_in, f_out)
# Replace original file with compressed version
os.remove(temp_file)
shutil.move(compressed_file, temp_file)
logger.info(f"Compressed MySQL backup file, reducing size significantly")
except Exception as e:
logger.warning(f"Could not compress backup file: {str(e)}")
# Continue with uncompressed file
return True
except Exception as e:
logger.error(f"MySQL backup error: {str(e)}")
return False
def perform_postgresql_backup(db_conn, temp_file):
"""
Perform PostgreSQL backup using pg_dump with optimized parameters.
Args:
db_conn (DatabaseConnection): Database connection object
temp_file (str): Path to temporary file for backup
Returns:
bool: True if backup successful, False otherwise
"""
try:
# Set environment variables for PostgreSQL
env = os.environ.copy()
env['PGHOST'] = db_conn.host
env['PGPORT'] = str(db_conn.port)
env['PGUSER'] = db_conn.username
env['PGPASSWORD'] = db_conn.password
env['PGDATABASE'] = db_conn.database
# Build pg_dump command with optimized parameters
cmd = [
'pg_dump',
'--format=custom', # Custom format for compression and flexibility
'--blobs', # Include large objects
'--create', # Include commands to create database
'--clean', # Include commands to clean (drop) objects before recreating
'--if-exists', # Use IF EXISTS when dropping objects
'--verbose', # Verbose mode
'--compress=9', # Maximum compression level
'--file=' + temp_file # Output file
]
# Execute command
result = subprocess.run(cmd, env=env, capture_output=True)
if result.returncode != 0:
logger.error(f"pg_dump error: {result.stderr.decode()}")
return False
# No need for additional compression as the custom format is already compressed
return True
except Exception as e:
logger.error(f"PostgreSQL backup error: {str(e)}")
return False
def run_backup(job, send_email=True):
"""
Run a backup job.
Args:
job (BackupJob): Backup job to run
send_email (bool): Whether to send email notification
Returns:
bool: True if backup successful, False otherwise
"""
start_time = time.time()
job_name = job.name
backup_log = BackupLog(job_id=job.id, status='failed')
try:
logger.info(f"Starting backup job: {job_name}")
# Get database connection
db_conn = DatabaseConnection.query.get(job.database_id)
if not db_conn:
msg = f"Database connection not found for job {job_name}"
logger.error(msg)
backup_log.message = msg
db.session.add(backup_log)
db.session.commit()
# Send email notification for failure
if send_email:
try:
from email_utils import send_backup_notification
send_backup_notification(backup_log.id)
except Exception as email_err:
logger.error(f"Failed to send email notification: {str(email_err)}")
return False
# Get storage target
storage = StorageTarget.query.get(job.storage_id)
if not storage:
msg = f"Storage target not found for job {job_name}"
logger.error(msg)
backup_log.message = msg
db.session.add(backup_log)
db.session.commit()
# Send email notification for failure
if send_email:
try:
from email_utils import send_backup_notification
send_backup_notification(backup_log.id)
except Exception as email_err:
logger.error(f"Failed to send email notification: {str(email_err)}")
return False
# Check storage space
storage_warning = None
try:
free_space, total_space = get_storage_space(storage)
percent_free = (free_space / total_space) * 100 if total_space > 0 else 0
if percent_free < 10:
storage_warning = f"Warning: Storage target {storage.name} has less than 10% free space ({percent_free:.1f}% remaining)"
logger.warning(storage_warning)
# We continue with the backup, just log a warning
except Exception as e:
storage_warning = f"Could not check storage space: {str(e)}"
logger.warning(storage_warning)
# Continue with backup attempt
# Create temporary file for backup
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
if db_conn.db_type in ['mysql', 'mariadb']:
backup_filename = f"{db_conn.database}_{timestamp}.sql"
if db_conn.db_type == 'mariadb':
backup_filename = f"mariadb_{db_conn.database}_{timestamp}.sql"
else:
backup_filename = f"{db_conn.database}_{timestamp}.dump"
temp_file = os.path.join(tempfile.gettempdir(), backup_filename)
# Perform backup based on database type
backup_success = False
if db_conn.db_type in ['mysql', 'mariadb']:
backup_success = perform_mysql_backup(db_conn, temp_file)
elif db_conn.db_type == 'postgresql':
backup_success = perform_postgresql_backup(db_conn, temp_file)
else:
msg = f"Unsupported database type: {db_conn.db_type}"
logger.error(msg)
backup_log.message = msg
db.session.add(backup_log)
db.session.commit()
# Send email notification for failure
if send_email:
try:
from email_utils import send_backup_notification
send_backup_notification(backup_log.id)
except Exception as email_err:
logger.error(f"Failed to send email notification: {str(email_err)}")
return False
if not backup_success:
msg = "Backup creation failed"
logger.error(msg)
backup_log.message = msg
db.session.add(backup_log)
db.session.commit()
# Send email notification for failure
if send_email:
try:
from email_utils import send_backup_notification
send_backup_notification(backup_log.id)
except Exception as email_err:
logger.error(f"Failed to send email notification: {str(email_err)}")
return False
# Get backup file size
backup_size = os.path.getsize(temp_file)
# Upload backup to storage
try:
remote_path = upload_to_storage(storage, temp_file, backup_filename)
# Update backup log with success
backup_log.status = 'success'
# Include storage warning in message if applicable
if storage_warning:
backup_log.message = f"Backup completed successfully. {storage_warning}"
else:
backup_log.message = "Backup completed successfully"
backup_log.backup_file = remote_path
backup_log.backup_size = backup_size
except Exception as e:
msg = f"Failed to upload backup: {str(e)}"
logger.error(msg)
backup_log.message = msg
db.session.add(backup_log)
db.session.commit()
# Send email notification for failure
if send_email:
try:
from email_utils import send_backup_notification
send_backup_notification(backup_log.id)
except Exception as email_err:
logger.error(f"Failed to send email notification: {str(email_err)}")
return False
finally:
# Clean up temporary file
try:
os.remove(temp_file)
except:
pass
# Update job's last run time
job.last_run = datetime.utcnow()
# Calculate duration
duration = time.time() - start_time
backup_log.duration = duration
db.session.add(backup_log)
db.session.commit()
logger.info(f"Backup job {job_name} completed successfully in {duration:.2f} seconds")
# Send email notification for success
if send_email:
try:
from email_utils import send_backup_notification
send_backup_notification(backup_log.id)
except Exception as email_err:
logger.error(f"Failed to send email notification: {str(email_err)}")
return True
except Exception as e:
# Log the error
msg = f"Backup job {job_name} failed: {str(e)}"
logger.error(msg)
# Update backup log with failure
backup_log.message = msg
db.session.add(backup_log)
db.session.commit()
# Send email notification for failure
if send_email:
try:
from email_utils import send_backup_notification
send_backup_notification(backup_log.id)
except Exception as email_err:
logger.error(f"Failed to send email notification: {str(email_err)}")
return False