| Path : /var/www/html/yedekledb24/DatabaseBackupMaster/ |
|
B-Con CMD Config cPanel C-Rdp D-Log Info Jump Mass Ransom Symlink vHost Zone-H |
| Current File : /var/www/html/yedekledb24/DatabaseBackupMaster/storage_utils.py |
import os
import logging
import tempfile
import shutil
from datetime import datetime
from ftplib import FTP
from smb.SMBConnection import SMBConnection
import tempfile
# Flag to check if NFS support is available
NFS_SUPPORT = False
try:
import nfs4_share as nfs4
NFS_SUPPORT = True
except ImportError:
logger = logging.getLogger(__name__)
logger.warning("NFS support not available - nfs4-share module not installed")
# Flag to check if SFTP support is available
SFTP_SUPPORT = False
try:
import paramiko
SFTP_SUPPORT = True
except ImportError:
logger = logging.getLogger(__name__)
logger.warning("SFTP support not available - paramiko module not installed")
logger = logging.getLogger(__name__)
def test_storage_connection(storage_type, host, port, username, password, path):
"""
Test connection to a storage target.
Args:
storage_type (str): Storage type (smb, nfs, ftp)
host (str): Storage host
port (int): Storage port
username (str): Username for storage
password (str): Password for storage
path (str): Storage path
Returns:
bool: True if connection successful, False otherwise
"""
try:
if storage_type == 'smb':
# Test SMB connection
conn = SMBConnection(
username,
password,
'backup-app', # Client name
host, # Server name
use_ntlm_v2=True
)
# If port is None, use default
if port is None:
port = 445
connected = conn.connect(host, port)
if not connected:
return False
# Try to list files to verify access
shares = conn.listShares()
share_names = [share.name for share in shares]
# Extract share name from path
share_name = path.split('/')[0]
if share_name not in share_names:
logger.error(f"Share {share_name} not found in available shares: {share_names}")
return False
conn.close()
return True
elif storage_type == 'nfs':
# Test NFS connection
if not NFS_SUPPORT:
logger.error("NFS support is not available - nfs4-share module not installed")
return False
try:
# Create a temporary directory for mounting
mount_dir = tempfile.mkdtemp()
# Use nfs4mount from nfs4 package
client = nfs4.NFS4Client(host, proto='tcp')
export = client.get_export(path)
mount = export.mount(mount_dir)
# Test if we can list the directory
os.listdir(mount_dir)
# Clean up
mount.umount()
os.rmdir(mount_dir)
return True
except Exception as e:
logger.error(f"NFS connection error: {str(e)}")
return False
elif storage_type == 'ftp':
# Test FTP connection
if port is None:
port = 21
ftp = FTP()
ftp.connect(host, port)
ftp.login(username, password)
# Try to change to the specified directory
if path:
ftp.cwd(path)
ftp.quit()
return True
elif storage_type == 'sftp':
# Test SFTP connection
if not SFTP_SUPPORT:
logger.error("SFTP support is not available - paramiko module not installed")
return False
# Use default port if not specified
if port is None:
port = 22
try:
# Create SSH client
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(
hostname=host,
port=port,
username=username,
password=password
)
# Create SFTP client
sftp = ssh.open_sftp()
# Try to change to the specified directory
if path:
sftp.chdir(path)
# List files to verify access
sftp.listdir()
# Clean up
sftp.close()
ssh.close()
return True
except Exception as e:
logger.error(f"SFTP connection error: {str(e)}")
return False
else:
logger.error(f"Unsupported storage type: {storage_type}")
return False
except Exception as e:
logger.error(f"Storage connection error for {storage_type} {host}: {str(e)}")
return False
def get_storage_space(storage):
"""
Get available and total space for a storage target.
Args:
storage (StorageTarget): Storage target object
Returns:
tuple: (free_space, total_space) in bytes
"""
try:
if storage.storage_type == 'smb':
# Get space info for SMB share
conn = SMBConnection(
storage.username,
storage.password,
'backup-app', # Client name
storage.host, # Server name
use_ntlm_v2=True
)
# If port is None, use default
port = storage.port if storage.port is not None else 445
connected = conn.connect(storage.host, port)
if not connected:
raise Exception("Could not connect to SMB share")
# Extract share name from path
share_name = storage.path.split('/')[0]
# Get disk information
disk_info = conn.getStorageInfo(share_name)
conn.close()
# Calculate free and total space
free_space = disk_info.free_units * disk_info.block_size
total_space = disk_info.total_units * disk_info.block_size
return free_space, total_space
elif storage.storage_type == 'nfs':
# Get space info for NFS share
if not NFS_SUPPORT:
logger.error("NFS support is not available - nfs4-share module not installed")
# Return default values if NFS support is not available
return 1024 * 1024 * 1024, 10 * 1024 * 1024 * 1024 # 1GB free of 10GB
try:
# Create a temporary directory for mounting
mount_dir = tempfile.mkdtemp()
# Use nfs4mount from nfs4 package
client = nfs4.NFS4Client(storage.host, proto='tcp')
export = client.get_export(storage.path)
mount = export.mount(mount_dir)
# Get filesystem stats
stats = os.statvfs(mount_dir)
# Clean up
mount.umount()
os.rmdir(mount_dir)
# Calculate free and total space
free_space = stats.f_bfree * stats.f_frsize
total_space = stats.f_blocks * stats.f_frsize
return free_space, total_space
except Exception as e:
logger.error(f"Error getting NFS space: {str(e)}")
# Return some default values if we can't get the actual space
return 0, 1
elif storage.storage_type == 'ftp':
# FTP doesn't have a standard way to get space info
# Most FTP servers support the AVBL command, but it's not universal
try:
port = storage.port if storage.port is not None else 21
ftp = FTP()
ftp.connect(storage.host, port)
ftp.login(storage.username, storage.password)
# Try to get available space using AVBL command
try:
free_space = ftp.voidcmd('AVBL /')
# Extract the number from the response
free_space = int(free_space.split()[1])
# We don't have a way to get total space, so assume free space is 50%
total_space = free_space * 2
except:
# If AVBL is not supported, return default values
free_space = 1000 * 1024 * 1024 # 1 GB free
total_space = 10000 * 1024 * 1024 # 10 GB total
ftp.quit()
return free_space, total_space
except Exception as e:
logger.error(f"Error getting FTP space: {str(e)}")
# Return some default values if we can't get the actual space
return 0, 1
elif storage.storage_type == 'sftp':
# SFTP doesn't have a standard way to get space info like df command
# We need to execute commands over SSH
if not SFTP_SUPPORT:
logger.error("SFTP support is not available - paramiko module not installed")
# Return default values if SFTP support is not available
return 1024 * 1024 * 1024, 10 * 1024 * 1024 * 1024 # 1GB free of 10GB
try:
# Use default port if not specified
port = storage.port if storage.port is not None else 22
# Create SSH client
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(
hostname=storage.host,
port=port,
username=storage.username,
password=storage.password
)
# Execute 'df' command to get disk usage information
# -P: POSIX output format, -k: 1k blocks
cmd = f"df -P -k {storage.path} | tail -1"
stdin, stdout, stderr = ssh.exec_command(cmd)
# Parse output
output = stdout.read().decode().strip()
# Check if we got valid output
if output:
# Parse the output which is in format:
# Filesystem 1024-blocks Used Available Capacity Mounted on
# /dev/sda1 61267136 40896464 17254904 71% /
parts = output.split()
if len(parts) >= 4:
total_space = int(parts[1]) * 1024 # Convert from 1K blocks to bytes
free_space = int(parts[3]) * 1024 # Convert from 1K blocks to bytes
ssh.close()
return free_space, total_space
# If we couldn't parse the output or got no output, fallback to default values
ssh.close()
# Default values
free_space = 1000 * 1024 * 1024 # 1 GB free
total_space = 10000 * 1024 * 1024 # 10 GB total
return free_space, total_space
except Exception as e:
logger.error(f"Error getting SFTP space: {str(e)}")
# Return default values if we can't get the actual space
return 1000 * 1024 * 1024, 10000 * 1024 * 1024 # 1GB free of 10GB
else:
logger.error(f"Unsupported storage type: {storage.storage_type}")
# Return some default values
return 0, 1
except Exception as e:
logger.error(f"Error getting storage space for {storage.name}: {str(e)}")
# Return some default values
return 0, 1
def download_file_from_storage(storage, remote_file, local_file):
"""
Download a file from the storage target.
Args:
storage (StorageTarget): Storage target object
remote_file (str): Path to the file on remote storage
local_file (str): Path where the file should be saved locally
Returns:
bool: True if download was successful, False otherwise
"""
try:
if storage.storage_type == 'smb':
# Download from SMB share
conn = SMBConnection(
storage.username,
storage.password,
'backup-app', # Client name
storage.host, # Server name
use_ntlm_v2=True
)
# If port is None, use default
port = storage.port if storage.port is not None else 445
connected = conn.connect(storage.host, port)
if not connected:
raise Exception("Could not connect to SMB share")
# Extract share name and path from storage path
parts = storage.path.split('/')
share_name = parts[0]
# If remote_file is a full path, extract just the filename for SMB
remote_filename = os.path.basename(remote_file)
# Build the full remote path relative to share
remote_dir = '/'.join(parts[1:]) if len(parts) > 1 else ''
if remote_dir:
remote_path = f"{remote_dir}/{remote_filename}"
else:
remote_path = remote_filename
# Download the file
with open(local_file, 'wb') as file_obj:
conn.retrieveFile(share_name, remote_path, file_obj)
conn.close()
return True
elif storage.storage_type == 'nfs':
# Download from NFS share
if not NFS_SUPPORT:
logger.error("NFS support is not available - nfs4-share module not installed")
return False
try:
# Create a temporary directory for mounting
mount_dir = tempfile.mkdtemp()
# Use nfs4mount from nfs4 package
client = nfs4.NFS4Client(storage.host, proto='tcp')
export = client.get_export(storage.path)
mount = export.mount(mount_dir)
# Extract the filename from the remote path
remote_filename = os.path.basename(remote_file)
# Copy the file from the mounted directory
source_path = os.path.join(mount_dir, remote_filename)
if os.path.exists(source_path):
shutil.copy2(source_path, local_file)
# Clean up
mount.umount()
os.rmdir(mount_dir)
return True
else:
logger.error(f"File not found on NFS share: {source_path}")
# Clean up
mount.umount()
os.rmdir(mount_dir)
return False
except Exception as e:
logger.error(f"Error downloading from NFS: {str(e)}")
return False
elif storage.storage_type == 'ftp':
# Download from FTP server
port = storage.port if storage.port is not None else 21
ftp = FTP()
ftp.connect(storage.host, port)
ftp.login(storage.username, storage.password)
# Change to the specified directory
if storage.path:
try:
ftp.cwd(storage.path)
except:
logger.error(f"Directory not found on FTP server: {storage.path}")
ftp.quit()
return False
# Extract the filename from the remote path
remote_filename = os.path.basename(remote_file)
# Download the file
try:
with open(local_file, 'wb') as file_obj:
ftp.retrbinary(f'RETR {remote_filename}', file_obj.write)
ftp.quit()
return True
except Exception as e:
logger.error(f"Error downloading file from FTP: {str(e)}")
ftp.quit()
return False
elif storage.storage_type == 'sftp':
# Download from SFTP server
if not SFTP_SUPPORT:
logger.error("SFTP support is not available - paramiko module not installed")
return False
try:
# Use default port if not specified
port = storage.port if storage.port is not None else 22
# Create SSH client
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(
hostname=storage.host,
port=port,
username=storage.username,
password=storage.password
)
# Create SFTP client
sftp = ssh.open_sftp()
# Change to the specified directory
if storage.path:
try:
sftp.chdir(storage.path)
except:
logger.error(f"Directory not found on SFTP server: {storage.path}")
sftp.close()
ssh.close()
return False
# Extract the filename from the remote path
remote_filename = os.path.basename(remote_file)
# Build the remote path
remote_path = f"{storage.path}/{remote_filename}"
# Download the file
try:
sftp.get(remote_path, local_file)
sftp.close()
ssh.close()
return True
except Exception as e:
logger.error(f"Error downloading file from SFTP: {str(e)}")
sftp.close()
ssh.close()
return False
except Exception as e:
logger.error(f"Error connecting to SFTP server: {str(e)}")
return False
else:
logger.error(f"Unsupported storage type: {storage.storage_type}")
return False
except Exception as e:
logger.error(f"Error downloading file from storage: {str(e)}")
return False
def upload_to_storage(storage, local_file, remote_filename):
"""
Upload a file to the storage target.
Args:
storage (StorageTarget): Storage target object
local_file (str): Path to local file
remote_filename (str): Filename to use on the remote storage
Returns:
str: Path to the uploaded file on the remote storage
"""
try:
if storage.storage_type == 'smb':
# Upload to SMB share
conn = SMBConnection(
storage.username,
storage.password,
'backup-app', # Client name
storage.host, # Server name
use_ntlm_v2=True
)
# If port is None, use default
port = storage.port if storage.port is not None else 445
connected = conn.connect(storage.host, port)
if not connected:
raise Exception("Could not connect to SMB share")
# Extract share name and path from storage path
parts = storage.path.split('/')
share_name = parts[0]
remote_dir = '/'.join(parts[1:]) if len(parts) > 1 else ''
# Build remote path
if remote_dir:
remote_path = f"{remote_dir}/{remote_filename}"
else:
remote_path = remote_filename
# Upload the file
with open(local_file, 'rb') as file_obj:
conn.storeFile(share_name, remote_path, file_obj)
conn.close()
return f"{storage.path}/{remote_filename}"
elif storage.storage_type == 'nfs':
# Upload to NFS share
if not NFS_SUPPORT:
logger.error("NFS support is not available - nfs4-share module not installed")
raise Exception("NFS support is not available - nfs4-share module not installed")
try:
# Create a temporary directory for mounting
mount_dir = tempfile.mkdtemp()
# Use nfs4mount from nfs4 package
client = nfs4.NFS4Client(storage.host, proto='tcp')
export = client.get_export(storage.path)
mount = export.mount(mount_dir)
# Copy the file to the mounted directory
dest_path = os.path.join(mount_dir, remote_filename)
shutil.copy2(local_file, dest_path)
# Clean up
mount.umount()
os.rmdir(mount_dir)
return f"{storage.path}/{remote_filename}"
except Exception as e:
logger.error(f"Error uploading to NFS: {str(e)}")
raise
elif storage.storage_type == 'ftp':
# Upload to FTP server
port = storage.port if storage.port is not None else 21
ftp = FTP()
ftp.connect(storage.host, port)
ftp.login(storage.username, storage.password)
# Change to the specified directory
if storage.path:
# Create directory if it doesn't exist
try:
ftp.cwd(storage.path)
except:
# Try to create directory structure
path_parts = storage.path.strip('/').split('/')
current_path = ''
for part in path_parts:
current_path += '/' + part
try:
ftp.cwd(current_path)
except:
ftp.mkd(current_path)
ftp.cwd(current_path)
# Upload the file
# FTP zaman aşımı ayarı - soket varsa ayarlayalım
if hasattr(ftp, 'sock') and ftp.sock:
ftp.sock.settimeout(1800) # 30 dakika zaman aşımı - büyük dosyalar için yeterli olmalı
# Chunk tabanlı yükleme ekleyelim - büyük dosyalar için daha güvenilir
file_size = os.path.getsize(local_file)
# Dosyayı yükle
with open(local_file, 'rb') as file_obj:
ftp.storbinary(f"STOR {remote_filename}", file_obj, blocksize=8192)
logger.info(f"FTP upload completed successfully. File size: {file_size/1024/1024:.2f} MB")
# Bağlantıyı güvenli kapatma denemeleri
if hasattr(ftp, 'sock') and ftp.sock:
try:
# Zaman aşımı ayarla (3 saniye)
ftp.sock.settimeout(3)
except:
pass
try:
ftp.quit()
except Exception as e:
logger.warning(f"FTP bağlantısını kapatma hatası (yedek yüklendi, sadece bağlantı kapatılamadı): {str(e)}")
try:
# Sessizce bağlantıyı kapat
ftp.close()
except:
pass
return f"{storage.path}/{remote_filename}"
elif storage.storage_type == 'sftp':
# Upload to SFTP server
if not SFTP_SUPPORT:
logger.error("SFTP support is not available - paramiko module not installed")
raise Exception("SFTP support is not available - paramiko module not installed")
# Use default port if not specified
port = storage.port if storage.port is not None else 22
try:
# Create SSH client
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(
hostname=storage.host,
port=port,
username=storage.username,
password=storage.password
)
# Create SFTP client
sftp = ssh.open_sftp()
# Check if the directory exists and create it if not
if storage.path:
try:
sftp.chdir(storage.path)
except IOError:
# Directory doesn't exist, create it
# Split the path and create each directory
path_parts = storage.path.strip('/').split('/')
current_path = ''
for part in path_parts:
if not part:
continue
current_path += '/' + part
try:
sftp.chdir(current_path)
except IOError:
sftp.mkdir(current_path)
sftp.chdir(current_path)
# Build remote path
remote_path = storage.path.rstrip('/') + '/' + remote_filename
# Upload the file
sftp.put(local_file, remote_filename)
# Clean up
sftp.close()
ssh.close()
return remote_path
except Exception as e:
logger.error(f"Error uploading to SFTP: {str(e)}")
raise
else:
raise Exception(f"Unsupported storage type: {storage.storage_type}")
except Exception as e:
logger.error(f"Error uploading to {storage.name}: {str(e)}")
raise