世界上只有一种英雄主义,就是看清生活的真相之后依然热爱生活!

Python 脚本备份 MySQL 数据库和系统目录到阿里云 OSS

2019-06-05
141次查阅
2019/7/4

阿里云对象存储服务(Object Storage Service,简称 OSS)为您提供基于网络的数据存取服务。使用 OSS,您可以通过网络随时存储和调用包括文本、图片、音频和视频等在内的各种非结构化数据文件。

该脚本主要功能是备份系统目录和MySQL数据库,并上传到阿里云OSS,使用之前先安装 OSS Python SDK:https://help.aliyun.com/document_detail/85288.html

脚本内容(Python3):

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
# sk(at)1987.name 2019
# 备份 文件目录数据 和 网站数据库 到 阿里云OSS

import os
import sys
import time
import oss2
import socket
import subprocess
from itertools import islice

# 备份数据库名
BACKUP_DATABASES = ['dbanme_1', 'dbname_2', 'dbname_3']

# 需要备份的目录
BACKUP_DIRECTORY = {'dir_name_1': '/path/to/dir1',
                    'dir_name_2': '/path/to/dir2',
                    'dir_name_3': '/path/to/dir3'}

# 因为备份MySQL数据使用的是mysqldump导出sql,所以需要定义MySQL路径
MYSQL_BASE_DIR = '/usr/local/mysql'

# 定义一个临时存储数据的目录
ARCHIVE_DIR = '/data/backup'

# MySQL数据库用户密码
DB_USER = 'root'
DB_PWD = 'xxxxxxxxxxxxxxxxxx'
DB_POST = '3306'
DB_HOST = '127.0.0.1'

# OSS 认证配置
OSS_ACCESS_KEY_ID = 'LTxxxxxxxxxxxx5Zi'
OSS_ACCESS_KEY_SECRET = 'Ibxxxxxxxxxxxxxxxxxxxxc4y'
OSS_ENDPOINT = 'http://oss-cn-hongkong-internal.aliyuncs.com'
OSS_BUCKET = 'server-data-backup'

#################################
# 以下不需要修改
#################################

HOST_NAME = socket.gethostname()
DATE = time.strftime("%Y-%m-%d", time.localtime())

# oss login
def oss_login():
    global use_bucket
    auth = oss2.Auth(OSS_ACCESS_KEY_ID, OSS_ACCESS_KEY_SECRET)
    use_bucket = oss2.Bucket(auth, OSS_ENDPOINT, OSS_BUCKET)

# delete files on oss
def oss_delete(obj):
    if isinstance(obj, list):
        result = use_bucket.batch_delete_objects(obj)
        print('\n'.join(result.deleted_keys))
    else:
        use_bucket.delete_object(obj)

# upload files to oss
def oss_upload(obj):
    object_name = '{}_{}'.format(HOST_NAME, os.path.basename(obj))
    # 这里判断 OSS 上是否存在同名的文件,如果存在就删除后再上传
    exist = use_bucket.object_exists(object_name)
    if exist:
        oss_delete(object_name)
    # 上传
    use_bucket.put_object_from_file(object_name, obj)

# show files list
def oss_show(num=10):
    for i in islice(oss2.ObjectIterator(use_bucket), num):
        print(i.key)

# run shell command
def run_shell(command, work_dir=ARCHIVE_DIR):
    result = subprocess.run(command,
                            shell=True,
                            cwd=work_dir,
                            stdout=subprocess.DEVNULL,
                            stderr=subprocess.DEVNULL)
    return True if result.returncode == 0 else False

# 目录打包
def backup_dir_pack(dirs):
    '''
    dirs 是目录字典,格式为:
    dirs = {'package_name_1': '/path/to/dir_1',
            'package_name_2': '/path/to/dir_2',
            'package_name_3': '/path/to/dir_3'}
    '''
    packages = []
    for package_name, path in dirs.items():
        # 打包目录的父路径
        parent_path = os.path.dirname(path)
        # 打包目录名
        dir_name = os.path.basename(path)
        # 压缩后的文件名(绝对路径)
        compress_to = '{}/{}.tar.gz'.format(ARCHIVE_DIR, package_name)
        if os.path.isdir(path):
            cmd = 'tar czf {} -C {} {}'.format(compress_to,
                                               parent_path,
                                               dir_name)
            if run_shell(cmd):
                print('{} compression successful'.format(path))
                packages.append(compress_to)
            else:
                print('{} compression faile'.format(path))
                sys.exit(1)
        else:
            print('{} not found'.format(path))
            sys.exit(1)
    return packages

# 文件打包
def backup_file_pack(files, package_name, whether_to_delete=False):
    '''
    files         是文件列表,格式为:files = ['/aa/a.log', '/bb/b.log']
    package_name  是打包后的文件名
    whether_to_delete = True      打包后删除源文件
                        False     打包后不删
    '''
    # 检测文件是否存在
    for i in files:
        if not os.path.isfile(i):
            print('{} not found'.format(i))
            sys.exit(1)

    # 压缩文件列表
    seq = (x for x in files)
    f = ' '.join(seq)
    # 压缩后的文件名(绝对路径)
    compress_to = '{}/{}.tar.gz'.format(ARCHIVE_DIR, package_name)
    cmd = 'tar czPf {} {}'.format(compress_to, f)
    msg = 'compression successful' if run_shell(cmd) else 'compression failed'
    print('{}.tar.gz {}'.format(package_name, msg))
    if whether_to_delete:
        run_shell('rm -f {}'.format(f))
    return compress_to

# 执行mysqldump命令
def mysqldump(dbname, user=DB_USER, pwd=DB_PWD, post=DB_POST, host=DB_HOST):
    # 当前时间
    now = time.strftime("%Y-%m-%d-%H-%M-%S", time.localtime())
    # mysqldump 命令路径
    mysqldump = '{}/bin/mysqldump'.format(MYSQL_BASE_DIR)
    # 附加参数:
    # add_par = '--no-defaults --set-gtid-purged=OFF --hex-blob --single-transaction'
    # 上面参数一般用于阿里云RDS备份,非RDS数据库可以不用
    add_par = ''
    # 导出的 sql 文件(绝对路径)
    export_to = '{ap}/{db}_{time}.sql'.format(db=dbname,
                                              ap=ARCHIVE_DIR,
                                              time=now)
    cmd_format = '{cmd} -h{h} -u{u} -p{p} {db} > {f}'
    cmd = cmd_format.format(cmd=mysqldump,
                            h=host,
                            u=user,
                            p=pwd,
                            db=dbname,
                            f=export_to)
    try:
        if run_shell(cmd):
            # 返回导出的sql文件路径用于打包
            return export_to
        else:
            print('{} export failed'.format(dbname))
            sys.exit(1)
    except Exception as e:
        print(e)
        sys.exit(1)

# 备份数据库
def backup_mysql_db(db):
    # db 为数据库名列表,格式为:db = ['db1', 'db2', 'db3']
    if isinstance(db, list):
        '''
        sql_files = []
         for i in db:
            sql_file = mysqldump(i)
            sql_files.append(sql_file)
        '''
        package = 'mysql_data'
        sql_files = list(map(mysqldump, db))
        return backup_file_pack(sql_files, package, True)
    else:
        print('db list format error')
        sys.exit(1)

def main():
    oss_login()
    # 备份目录文件并上传至 OSS
    backup_dir_packages = backup_dir_pack(BACKUP_DIRECTORY)
    for i in backup_dir_packages:
        oss_upload(i)
        os.remove(i)

    # 备份数据库并上传至 OSS
    databases_package = backup_mysql_db(BACKUP_DATABASES)
    oss_upload(databases_package)
    os.remove(databases_package)

# 执行
if __name__ == '__main__':
    main()

评论

想说点什么?