MINIO服务器基于AWS S3 SDK 文件分片上传及下载（C++实现）

服务器 0

MINIO服务器基于AWS S3 SDK 文件分片上传及下载（C++实现）

  • 基于aws-s3安装
    • 安装
  • 初始化S3连接
  • 上传文件
    • 一般文件上传
    • 分块上传
  • 下载文件
  • 参考资料
  • 下载地址

基于aws-s3安装

安装

安装环境依赖：

Debian/Ubuntu-based systems : sudo apt-get install libcurl4-openssl-devlibssl-dev uuid-dev zlib1g-dev libpulse-dev cmakeRedhat/Fedora-based systems : sudo dnf install libcurl-devel openssl-devellibuuid-devel pulseaudio-devel cmake

获取SDK源码并安装：

git clone --recurse-submodules https://github.com/aws/aws-sdk-cpp.gitcd aws-sdk-cppmkdir buildcd buildcmake ..make -j8sudo make install

项目中CMakeLists.txt配置：

cmake_minimum_required(VERSION 3.19)project(project_name)set(CMAKE_CXX_STANDARD 11) # Find the AWS SDK for C++ package.find_package(AWSSDK CONFIG COMPONENTS core s3 REQUIRED) include_directories(${AWSSDK_INCLUDE_DIRS})add_executable(project_name mian.cpp)target_link_libraries(project_name PRIVATE  aws-cpp-sdk-s3)

初始化S3连接

S3Client InitS3Client(const Aws::String IPPort, const Aws::String& accessKeyId, const Aws::String& secretKey, std::string securityToken){    // 初始化 S3 Client    Aws::Client::ClientConfiguration cfg;    cfg.endpointOverride = IPPort;    cfg.scheme = Aws::Http::Scheme::HTTP;    cfg.verifySSL = false;    Aws::Auth::AWSCredentials cred(accessKeyId, secretKey, securityToken);    S3Client client(cred, cfg,                    Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, false);    return client;}

上传文件

一般文件上传

/// 判断待上传的文件是否存在inline bool file_exists(const string& name){    struct stat buffer;    return (stat(name.c_str(), &buffer) == 0);}/// 上传文件bool doPutObject(const Aws::String &objectKey, const Aws::String &fromBucket, S3Client client, const string& file_name, const Aws::String &region){    // 判断文件是否存在    if (!file_exists(file_name)) {        cout << "ERROR: 找不到这个文件，这个文件不存在"             << endl;        return false;    }    /// 初始化上传请求    Aws::S3::Model::PutObjectRequest object_request;    object_request.SetBucket(fromBucket);    object_request.SetKey(objectKey);    /// 建立文件输入串流，可为任何串流    const shared_ptr<Aws::IOStream> input_data =            Aws::MakeShared<Aws::FStream>("SampleAllocationTag",                                          file_name.c_str(),                                          ios_base::in | ios_base::binary);    object_request.SetBody(input_data);    /// 开始上传至S3    Aws::S3::Model::PutObjectOutcome put_object_outcome = client.PutObject(object_request);    if (!put_object_outcome.IsSuccess()) {        auto error = put_object_outcome.GetError();        cout << "ERROR: " << error.GetExceptionName() << ": "             << error.GetMessage() << endl;        return false;    }else {        cout << "success" << endl;        return true;    }}/// 启动上传程序bool StartUpLoad(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_names, std::string object_names, std::string imagename, std::string securityToken){    // 初始化    Aws::SDKOptions options;    options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;    Aws::InitAPI(options);    /// 设置ip、acccessKey、secretKey    Aws::String IP = Ips;    Aws::String accessKeyId = acccessKey;    Aws::String secretKey = secretKeyS;    S3Client client;    client = InitS3Client(IP, accessKeyId, secretKey, securityToken);    Aws::InitAPI(options);    {        /// 设置上传的目标桶和文件路径        const Aws::String bucket_name = bucket_names;        const Aws::String object_name = object_names;        if (!doPutObject(object_name, bucket_name, client, imagename, " ")) {            std::string errorInfo =  object_name + " 上传失败";            writeLog(L_ERROR, errorInfo.c_str());            return false;        }    }    writeLog(L_INFO, "====> 文件上传成功");    return true;}

分块上传

/// MINIO 切片上传bool mainCutUpload(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_names, std::string object_names, std::string imagename, std::string securityToken){    Aws::SDKOptions options;    options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;    Aws::InitAPI(options);    Aws::String IP = Ips;    Aws::String accessKeyId = acccessKey;    Aws::String secretKey = secretKeyS;    /// 初始化S3服务    S3Client client = InitS3Client(IP, accessKeyId, secretKey, securityToken);    Aws::InitAPI(options);    {        /// 设置上传的目标桶和文件路径        const Aws::String bucket_name = bucket_names;        const Aws::String object_name = object_names;        std::string bucket = bucket_name;        std::string key = object_name;        /// 初始化分块上传任务        Aws::S3::Model::CreateMultipartUploadRequest create_request;        create_request.SetBucket(bucket.c_str());        create_request.SetKey(key.c_str());        Aws::S3::Model::CreateMultipartUploadOutcome outcome = client.CreateMultipartUpload(create_request);        /// 获取上传ID        std::string upload_id = outcome.GetResult().GetUploadId();        std::cout << "upload id is:" << upload_id << std::endl;        /// 创建分段上传输出        if (!outcome.IsSuccess()) {            auto err = outcome.GetError();            std::cout << "Error: CreateMultipartUpload: " <<                      err.GetExceptionName() << ": " << err.GetMessage() << std::endl;            return false;        }        else            std::cout << "Success: CreateMultipartUpload: " << bucket << std::endl;        std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetBucket() << std::endl;        std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetKey() << std::endl;        std::cout << "Success: CreateMultipartUpload: " << outcome.GetResult().GetUploadId() << std::endl;        Aws::String UploadId = upload_id;        /// 启动分块上传        Aws::S3::Model::UploadPartRequest request1;        request1.SetBucket(bucket);        request1.SetKey(key);        /// 每个分片大小        long partSize = 5 * 1024 * 1024;        /// 读取文件        std::fstream file(imagename.c_str(),std::ios::in | std::ios::binary);        file.seekg(0,std::ios::end);        long fileSize = file.tellg();   /// 获取文件大小        file.seekg(0, std::ios::beg);        char* buffer = new char[partSize];  // 定义缓存        /// 初始化上传变量        long filePosition = 0;        Aws::Vector<Aws::S3::Model::CompletedPart> completeParts;        int partNumber = 1;        /// 开始上传        while(filePosition < fileSize){            partSize = std::min(partSize,(fileSize - filePosition));            file.read(buffer,partSize);            Aws::S3::Model::UploadPartRequest uploadPartRequest;            uploadPartRequest.WithBucket(bucket).WithKey(key).WithUploadId(UploadId).WithPartNumber(partNumber).WithContentLength(partSize);            Aws::String str(buffer,partSize);            auto input_data = Aws::MakeShared<Aws::StringStream>("UploadPartStream",str);            uploadPartRequest.SetBody(input_data);            filePosition += partSize;            auto uploadPartResult = client.UploadPart(uploadPartRequest);            completeParts.push_back(Aws::S3::Model::CompletedPart().WithETag(uploadPartResult.GetResult().GetETag()).WithPartNumber(partNumber));            memset(buffer, 0, partSize);            ++partNumber;        }        // 完成分块上传        Aws::S3::Model::CompleteMultipartUploadRequest request2;        request2.SetBucket(bucket);        request2.SetKey(key);        request2.SetUploadId(UploadId);        Aws::S3::Model::CompletedMultipartUpload completed_multipart_upload;        completed_multipart_upload.SetParts(completeParts);        request2.SetMultipartUpload(completed_multipart_upload);        Aws::S3::Model::CompleteMultipartUploadOutcome outcome3 = client.CompleteMultipartUpload(request2);        if (outcome3.IsSuccess()) {            std::cout << "Success: CompleteMultipartUpload " << std::endl;            return true;        }        else        {            std::cout << "Error: CompleteMultipartUpload: " <<                      outcome.GetError().GetMessage() << std::endl;            return false;        }    }}

下载文件

/// 从MINIO中下载文件或数据bool doGetObject(const Aws::String &objectKey, const Aws::String &fromBucket, S3Client client, const Aws::String savePath,const Aws::String &region){    /// 初始化下载请求    Aws::S3::Model::GetObjectRequest object_request;    object_request.SetBucket(fromBucket);    object_request.SetKey(objectKey);    /// 从S3服务器中下载数据    Aws::S3::Model::GetObjectOutcome get_object_outcome = client.GetObject(object_request);    if (get_object_outcome.IsSuccess()) {        /// 保存文件        Aws::OFStream local_file;        local_file.open(savePath, std::ios::out | std::ios::binary);        local_file << get_object_outcome.GetResultWithOwnership().GetBody().rdbuf();        std::cout << "Done!" << std::endl;        return true;    } else {        auto err = get_object_outcome.GetError();        std::cout << "Error: GetObject: " << err.GetExceptionName() << ": " << err.GetMessage() << std::endl;        return false;    }}/// 启动下载程序bool StartDownLoad(std::string Ips, std::string acccessKey, std::string secretKeyS, std::string bucket_name, std::string downPath, std::string imagePath, std::string securityToken,    bool &flags){    // 初始化Aws API    Aws::SDKOptions options;    options.loggingOptions.logLevel = Aws::Utils::Logging::LogLevel::Debug;    Aws::InitAPI(options);    /// 设置ip、acccessKey、secretKey    Aws::String IP = Ips;    Aws::String accessKeyId = acccessKey;    Aws::String secretKey = secretKeyS;    /// 初始化S3服务    S3Client client = InitS3Client(IP, accessKeyId, secretKey, securityToken);    /// 根据get接口的桶名称设置文件所在的桶    Aws::S3::Model::Bucket temps{};    std::string stringK = bucket_name;    temps.SetName(stringK);    /// 查找MINIO相关数据    Aws::S3::Model::ListObjectsRequest objects_request;    objects_request.WithBucket(temps.GetName());   /// 设置桶路径    objects_request.SetPrefix(downPath);   /// 设置文件夹路径    auto list_objects_outcome = client.ListObjects(objects_request);  /// 判断存储桶及相关连接是否有效    if (flags){        writeLog(L_INFO, "====> 文件下载终止");        return false;    }    if (list_objects_outcome.IsSuccess())    {        /// 获取该目录下的前一千个文件        Aws::Vector<Aws::S3::Model::Object> object_list =                list_objects_outcome.GetResult().GetContents();        if (object_list.size() == 1000){            /// 当下载数量大于一千时 循环下载文件            while(object_list.size() == 1000){                auto finally_object = object_list[0];                for (auto const &s3_object : object_list)                {                    if (flags){                        writeLog(L_INFO, "====> 文件下载终止");                        return false;                    }                    /// 设置桶名称                    const Aws::String bucket_name = stringK;                    /// 设置待下载的文件名称                    const Aws::String object_name = s3_object.GetKey();                    /// 设置保存文件路径及文件名称                    std::string NewPath;                    size_t indexNew = object_name.find_last_of('/');                    NewPath = object_name.substr(indexNew + 1, object_name.length());                    const Aws::String SavePath = imagePath + NewPath;                    size_t indexK = downPath.find_last_of('/');                    std::string newPathEs = downPath.substr(0, indexK);                    std::string strK;                    strK = imagePath + "/" + NewPath;                    const Aws::String SavePaths = strK;                    //筛选数据                    std::string fileName;                    size_t index = object_name.find_last_of('.');                    fileName = object_name.substr(index + 1, object_name.length());                    if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){                        /// 启动下载                        if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) {                            std::string errorInfo =  object_name + " 下载失败";                            writeLog(L_ERROR, errorInfo.c_str());                            continue;                        }                    }                }                finally_object = object_list[999];                /// 重新配置相关路径                objects_request.WithBucket(temps.GetName());   /// 设置桶路径                objects_request.SetPrefix(finally_object.GetKey());   /// 设置新文件夹路径                list_objects_outcome = client.ListObjects(objects_request);  /// 判断存储桶及相关连接是否有效                if (list_objects_outcome.IsSuccess()){                    object_list = list_objects_outcome.GetResult().GetContents();                }else{                    writeLog(L_ERROR, "====> 再次连接失败");                }            }            for (auto const &s3_object : object_list)            {                if (flags){                    writeLog(L_INFO, "====> 文件下载终止");                    return false;                }                /// 设置桶名称                const Aws::String bucket_name = stringK;                /// 设置待下载的文件名称                const Aws::String object_name = s3_object.GetKey();                /// 设置保存文件路径及文件名称                std::string NewPath;                size_t indexNew = object_name.find_last_of('/');                NewPath = object_name.substr(indexNew + 1, object_name.length());                const Aws::String SavePath = imagePath + NewPath;                size_t indexK = downPath.find_last_of('/');                std::string newPathEs = downPath.substr(0, indexK);                std::string strK;                strK = imagePath + "/" + NewPath;                const Aws::String SavePaths = strK;                //筛选数据                std::string fileName;                size_t index = object_name.find_last_of('.');                fileName = object_name.substr(index + 1, object_name.length());                if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){                    /// 启动下载                    if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) {                        std::string errorInfo =  object_name + " 下载失败";                        writeLog(L_ERROR, errorInfo.c_str());                        continue;                    }                }            }        }else{            for (auto const &s3_object : object_list)            {                if (flags){                    writeLog(L_INFO, "====> 文件下载终止");                    return false;                }                /// 设置桶名称                const Aws::String bucket_name = stringK;                /// 设置待下载的文件名称                const Aws::String object_name = s3_object.GetKey();                /// 设置保存文件路径及文件名称                std::string NewPath;                size_t indexNew = object_name.find_last_of('/');                NewPath = object_name.substr(indexNew + 1, object_name.length());                const Aws::String SavePath = imagePath +  NewPath;                size_t indexK = downPath.find_last_of('/');                std::string newPathEs = downPath.substr(0, indexK);                std::string strK;                strK = imagePath + "/" + NewPath;                const Aws::String SavePaths = strK;                //筛选数据                std::string fileName;                size_t index = object_name.find_last_of('.');                fileName = object_name.substr(index + 1, object_name.length());                if (fileName == "JPG" || fileName == "jpeg" || fileName == "JPEG" || fileName == "jpg"){                    /// 启动下载                    if (!doGetObject(object_name, bucket_name, client,SavePaths,"")) {                        std::string errorInfo =  object_name + " 下载失败";                        writeLog(L_ERROR, errorInfo.c_str());                        continue;                    }                }            }        }    }    writeLog(L_INFO, "====> 文件下载成功");    return true;}

参考资料

SDK文档资料
C++_SDK.pdf

下载地址

实现下载的整体类代码下载
具体内容如下
在这里插入图片描述

也许您对下面的内容还感兴趣：