Amazon S3 或 Amazon Simple Storage Service 是 Amazon Web Services (AWS) 提供的一项服务,它通过 Web 服务接口提供对象存储。
Amazon Simple Storage Service(广泛称为 Amazon S3)是一种高度可扩展、快速且持久的解决方案,适用于任何数据类型的对象级存储。与我们都习惯的操作系统不同,Amazon S3 不会将文件存储在文件系统中,而是将文件存储为对象。对象存储允许用户上传文件、视频和文档,就像您将文件、视频和文档上传到流行的云存储产品(如 Dropbox 和 Google Drive)一样。这使得 Amazon S3 非常灵活且与平台无关。
<?xml version="1.0" encoding="UTF-8"?> <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion> <artifactId>aws</artifactId> <name>aws</name> <version>1.0</version> <groupId>chenwc</groupId> <packaging>jar</packaging> <properties> <java.version>1.8</java.version> <aws.java.sdk.version>2.18.16</aws.java.sdk.version> <slf4j.version>1.7.36</slf4j.version> <log4j.version>1.2.17</log4j.version> <maven.compiler.plugin.version>3.6.1</maven.compiler.plugin.version> </properties> <dependencyManagement> <dependencies> <dependency> <groupId>software.amazon.awssdk</groupId> <artifactId>bom</artifactId> <version>${aws.java.sdk.version}</version> <type>pom</type> <scope>import</scope> </dependency> </dependencies> </dependencyManagement> <dependencies> <dependency> <groupId>software.amazon.awssdk</groupId> <artifactId>s3</artifactId> <exclusions> <exclusion> <groupId>software.amazon.awssdk</groupId> <artifactId>netty-nio-client</artifactId> </exclusion> <exclusion> <groupId>software.amazon.awssdk</groupId> <artifactId>apache-client</artifactId> </exclusion> <exclusion> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> </exclusion> </exclusions> </dependency> <dependency> <groupId>software.amazon.awssdk</groupId> <artifactId>apache-client</artifactId> </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-log4j12</artifactId> <version>${slf4j.version}</version> </dependency> <dependency> <groupId>org.slf4j</groupId> <artifactId>slf4j-api</artifactId> <version>${slf4j.version}</version> </dependency> <dependency> <groupId>log4j</groupId> <artifactId>log4j</artifactId> <version>${log4j.version}</version> </dependency> </dependencies> <build> <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-compiler-plugin</artifactId> <version>${maven.compiler.plugin.version}</version> </plugin> </plugins> </build> </project>
package com.chenwc.util; import software.amazon.awssdk.regions.Region; import java.io.InputStream; import java.util.Properties; /** * AWS配置 * @author chenwc */ public class AwsConfig { private String appKey; private String secretKey; private String bucketName; private String endpoint; private Region region; public AwsConfig() { Properties properties = new Properties(); //对于maven项目,AwsConfig 类的包目录是 com/chenwc/util,要返回三级目录才能读取到 src/main/resource 目录下的文件 InputStream is = AwsConfig.class.getResourceAsStream("../../../aws.properties"); try { properties.load(is); this.appKey = properties.getProperty("appKey"); this.secretKey = properties.getProperty("secretKey"); this.bucketName = properties.getProperty("bucketName"); this.endpoint = properties.getProperty("endpoint"); this.region = Region.of(properties.getProperty("region")); } catch (Exception e) { // TODO: handle exception e.printStackTrace(); } finally { try { if (null != is) { is.close(); } } catch (Exception e2) { // TODO: handle exception e2.printStackTrace(); } } } public String getAppKey() { return appKey; } public void setAppKey(String appKey) { this.appKey = appKey; } public String getSecretKey() { return secretKey; } public void setSecretKey(String secretKey) { this.secretKey = secretKey; } public String getBucketName() { return bucketName; } public void setBucketName(String bucketName) { this.bucketName = bucketName; } public String getEndpoint() { return endpoint; } public void setEndpoint(String endpoint) { this.endpoint = endpoint; } public Region getRegion() { return region; } public void setRegion(Region region) { this.region = region; } }
appKey=e7Ip4Oq4WZlxyt31 secretKey=BMKcPToCeLlwiBOKC3NKeXbKAXjTdh79 bucketName=ueditor endpoint=http://192.168.100.115:9000 region=us-east-2
package com.chenwc.util; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.core.ResponseBytes; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.core.waiters.WaiterResponse; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.*; import software.amazon.awssdk.services.s3.waiters.S3Waiter; import java.io.*; import java.net.URI; import java.text.DecimalFormat; import java.util.ArrayList; import java.util.List; /** * AWS 操作工具类 * * @author chenwc */ public class AwsUtil { private static final Logger log = LoggerFactory.getLogger(AwsUtil.class); /** * 块文件大小,10M */ private static final Integer chunkFileSize = 10 * 1024 * 1024; /** * 线程安全的 DecimalFormat */ private static final ThreadLocal<DecimalFormat> threadLocalDecimalFormat = new ThreadLocal<>(); /** * 获取 DecimalFormat * * @return DecimalFormat */ public static DecimalFormat getDecimalFormat() { DecimalFormat df = threadLocalDecimalFormat.get(); if (df == null) { df = new DecimalFormat("#.00"); threadLocalDecimalFormat.set(df); } return df; } /** * 创建s3客户端 * * @param awsConfig AwsConfig * @return s3客户端 */ public static S3Client getS3Client(AwsConfig awsConfig) { AwsBasicCredentials awsCreds = AwsBasicCredentials.create( awsConfig.getAppKey(), awsConfig.getSecretKey()); return S3Client.builder() .region(awsConfig.getRegion()) .credentialsProvider(StaticCredentialsProvider.create(awsCreds)) .endpointOverride(URI.create(awsConfig.getEndpoint())) //强制使用地址类型风格的url,如果设置为false会使用域名类型 .forcePathStyle(true) .build(); } /** * 使用S3Waiter对象创建存储桶 * * @param s3Client S3Client * @param bucketName 桶名 */ public static void createBucket(S3Client s3Client, String bucketName) { try { if (listBucket(s3Client, bucketName)) { log.info("桶名:{} 已存在,不需要再创建!", bucketName); return; } //创建 S3Waiter 对象 S3Waiter s3Waiter = s3Client.waiter(); CreateBucketRequest bucketRequest = CreateBucketRequest.builder() .bucket(bucketName) .build(); //创建请求 s3Client.createBucket(bucketRequest); HeadBucketRequest bucketRequestWait = HeadBucketRequest.builder() .bucket(bucketName) .build(); // 等待创建存储桶并打印出响应。 WaiterResponse<HeadBucketResponse> waiterResponse = s3Waiter.waitUntilBucketExists(bucketRequestWait); boolean isExists = waiterResponse.matched().response().isPresent(); log.info("存储桶已存在: {}", isExists); log.info(bucketName + " 存储桶创建完成"); } catch (S3Exception e) { e.printStackTrace(); log.error(e.awsErrorDetails().errorMessage()); } } /** * 列出服务器上所有存储桶 * * @param s3Client S3Client */ public static void listBucket(S3Client s3Client) { // List buckets ListBucketsRequest listBucketsRequest = ListBucketsRequest.builder().build(); ListBucketsResponse listBucketsResponse = s3Client.listBuckets(listBucketsRequest); listBucketsResponse.buckets().forEach(x -> log.info("桶名: " + x.name())); } /** * 判断桶名是否已存在 * * @param s3Client S3Client * @param bucketName 桶名 * @return 已存在返回true,不存在返回false */ public static boolean listBucket(S3Client s3Client, String bucketName) { // List buckets ListBucketsRequest listBucketsRequest = ListBucketsRequest.builder().build(); ListBucketsResponse listBucketsResponse = s3Client.listBuckets(listBucketsRequest); List<Bucket> list = listBucketsResponse.buckets(); for (Bucket b : list) { if (b.name().equals(bucketName)) { return true; } } return false; } /** * 清空存储桶内所有对象并删除存储桶 * * @param s3 S3Client * @param bucketName 桶名 */ public static void deleteObjectsInBucket(S3Client s3, String bucketName) { try { // 要删除存储桶,必须首先删除存储桶中的所有对象。 ListObjectsV2Request listObjectsV2Request = ListObjectsV2Request.builder() .bucket(bucketName) .build(); ListObjectsV2Response listObjectsV2Response; //删除存储桶中的所有对象。 do { listObjectsV2Response = s3.listObjectsV2(listObjectsV2Request); for (S3Object s3Object : listObjectsV2Response.contents()) { DeleteObjectRequest request = DeleteObjectRequest.builder() .bucket(bucketName) .key(s3Object.key()) .build(); s3.deleteObject(request); } } while (listObjectsV2Response.isTruncated()); //删除空存储桶 DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder() .bucket(bucketName) .build(); s3.deleteBucket(deleteBucketRequest); } catch (S3Exception e) { e.printStackTrace(); log.error(e.awsErrorDetails().errorMessage()); } } /** * 上传对象 * * @param s3Client S3Client * @param bucketName 桶名 * @param key 对象在桶内的存储路径+文件名 * @param file 上传文件对象 * @return 是否上传成功,是返回true,否返回false */ public static boolean uploadObject(S3Client s3Client, String bucketName, String key, File file) { log.info("--------------------------开始上传对象: {} ---------------------------------", key); if (listBucketObjects(s3Client, bucketName, key)) { log.info("key 为:{} 的文件在桶:{} 已存在,不需要重复上传", key, bucketName); return false; } log.info("在存储桶上传对象: {}", bucketName); log.info("上传对象 Key : {}", key); printObjectSize(file.length()); PutObjectRequest objectRequest = PutObjectRequest.builder() .bucket(bucketName) .key(key) .build(); PutObjectResponse putObjectResponse = s3Client.putObject(objectRequest, RequestBody.fromFile(file)); boolean isUploadSuccessful = putObjectResponse.sdkHttpResponse().isSuccessful(); log.info("isUploadSuccessful: {}", isUploadSuccessful); log.info("uploadObject statusCode: {}", putObjectResponse.sdkHttpResponse().statusCode()); log.info("uploadObject statusText: {}", putObjectResponse.sdkHttpResponse().statusText()); log.info("--------------------------上传对象: {} 完成---------------------------------", key); return isUploadSuccessful; } /** * 分片上传对象 * * @param s3Client S3Client * @param bucketName 桶名 * @param key 对象在桶内的存储路径+文件名 * @param file 上传文件对象 * @return 是否上传成功,是返回true,否返回false */ public static boolean multipartUploadObject(S3Client s3Client, String bucketName, String key, File file) { log.info("--------------------------分片上传对象开始: {} ---------------------------------", key); if (listBucketObjects(s3Client, bucketName, key)) { log.info("key 为:{} 的文件在桶:{} 已存在,不需要重复上传", key, bucketName); return false; } // 首先创建多部分上传并获取上传id log.info("在存储桶分片上传对象: {}", bucketName); log.info("分片上传对象 Key: {}", key); printObjectSize(file.length()); CreateMultipartUploadRequest createMultipartUploadRequest = CreateMultipartUploadRequest.builder() .bucket(bucketName) .key(key) .build(); CreateMultipartUploadResponse response = s3Client.createMultipartUpload(createMultipartUploadRequest); String uploadId = response.uploadId(); log.info("uploadId: " + uploadId); //文件分块数量 int fileNum = (int) Math.ceil(file.length() * 1.0 / chunkFileSize); List<CompletedPart> completedPartList = new ArrayList<>(); try { FileInputStream fis = new FileInputStream(file); int available = fis.available(); log.info("available: {}", available); if (available <= 0) { log.info("待读取文件为空文件,读取文件失败"); return false; } byte[] result = new byte[available]; //读取总的字节数 int readBytesNum = fis.read(result); for (int i = 1; i <= fileNum; i++) { log.info("正在上传第:{} 个文件分片....", i); byte[] upload; if (i < fileNum) { upload = new byte[chunkFileSize]; //src 源数组,srcPos 从源数组哪个位置开始拷贝,dest 目标数组,destPos 从目标数组哪个位置开始复制,length 复制多少长度 int startIndex = (i - 1) * chunkFileSize; log.info("startIndex: {}", startIndex); log.info("endIndex: {}", startIndex + chunkFileSize); System.arraycopy(result, startIndex, upload, 0, chunkFileSize); } //最后一块 else { upload = new byte[(int) (file.length() - chunkFileSize * (i - 1))]; int startIndex = (i - 1) * chunkFileSize; int length = (int) (file.length() - chunkFileSize * (i - 1)); log.info("startIndex: {}", startIndex); log.info("endIndex: {}", startIndex + length); System.arraycopy(result, startIndex, upload, 0, length); } //上载对象的所有不同部分 UploadPartRequest uploadPartRequest = UploadPartRequest.builder() .bucket(bucketName) .key(key) .uploadId(uploadId) .partNumber(i).build(); String etag = s3Client.uploadPart(uploadPartRequest, RequestBody.fromBytes(upload)).eTag(); CompletedPart part = CompletedPart.builder().partNumber(i).eTag(etag).build(); completedPartList.add(part); log.info("第:{} 个文件分片上传完成....", i); } } catch (Exception e) { e.printStackTrace(); return false; } // 最后调用completeMultipartUpload操作,告诉S3合并所有上传的部分并完成多部分操作。 CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder() .parts(completedPartList) .build(); CompleteMultipartUploadRequest completeMultipartUploadRequest = CompleteMultipartUploadRequest.builder() .bucket(bucketName) .key(key) .uploadId(uploadId) .multipartUpload(completedMultipartUpload) .build(); CompleteMultipartUploadResponse completeMultipartUploadResponse = s3Client.completeMultipartUpload(completeMultipartUploadRequest); boolean isUploadSuccessful = completeMultipartUploadResponse.sdkHttpResponse().isSuccessful(); log.info("isMultipartUploadSuccessful: {}", isUploadSuccessful); log.info("MultipartUploadObject statusCode: {}", completeMultipartUploadResponse.sdkHttpResponse().statusCode()); log.info("MultipartUploadObject statusText: {}", completeMultipartUploadResponse.sdkHttpResponse().statusText()); log.info("--------------------------上传对象: {} 完成---------------------------------", key); log.info("--------------------------分片上传对象结束: {} ---------------------------------", key); return isUploadSuccessful; } /** * 下载对象 * * @param s3Client S3Client * @param bucketName 桶名 * @param key 对象在桶内的存储路径+文件名 * @param sourcePath 待写入本地文件全路径+文件名 * @return 是否下载成功,是返回true,否返回false */ public static boolean downloadObject(S3Client s3Client, String bucketName, String key, String sourcePath) { log.info("--------------------------开始下载对象: {} ---------------------------------", key); GetObjectRequest getObjectRequest = GetObjectRequest.builder() .bucket(bucketName) .key(key) .build(); ResponseBytes<GetObjectResponse> responseBytes = s3Client.getObjectAsBytes(getObjectRequest); InputStream inputStream = responseBytes.asInputStream(); convertInputStreamToLocalFile(inputStream, sourcePath); GetObjectResponse getObjectResponse = responseBytes.response(); boolean isDownloadSuccessful = getObjectResponse.sdkHttpResponse().isSuccessful(); log.info("isDownloadSuccessful: {}", isDownloadSuccessful); log.info("downloadObject statusCode: {}", getObjectResponse.sdkHttpResponse().statusCode()); log.info("downloadObject statusText: {}", getObjectResponse.sdkHttpResponse().statusText()); log.info("--------------------------下载对象: {} 完成---------------------------------", key); return isDownloadSuccessful; } /** * 删除对象 * * @param s3Client S3Client * @param bucketName 桶名 * @param key 对象在桶内的存储路径+文件名 * @return 是否删除成功,是返回true,否返回false */ public static boolean deleteObject(S3Client s3Client, String bucketName, String key) { log.info("--------------------------开始删除对象: {} ---------------------------------", key); if (!listBucketObjects(s3Client, bucketName, key)) { log.info("key 为: {} 文件不存在,请勿重复删除!", key); return true; } else { log.info("key 为: {} 文件存在,可以进行删除操作!", key); } DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder() .bucket(bucketName) .key(key) .build(); DeleteObjectResponse response = s3Client.deleteObject(deleteObjectRequest); boolean isDeleteSuccessful = response.sdkHttpResponse().isSuccessful(); log.info("isDeleteSuccessful: {}", isDeleteSuccessful); log.info("deleteObject statusCode: {}", response.sdkHttpResponse().statusCode()); log.info("deleteObject statusText: {}", response.sdkHttpResponse().statusText()); log.info("--------------------------删除对象: {} 完成---------------------------------", key); if (isDeleteSuccessful) { log.info("存储桶: {} 中 key: {} 的文件删除成功!", bucketName, key); return true; } else { log.info("存储桶: {} 中 key: {} 的文件删除失败!", bucketName, key); return false; } } /** * 列出存储桶内所有对象 * * @param s3 S3Client * @param bucketName 桶名 */ public static void listBucketObjects(S3Client s3, String bucketName) { try { ListObjectsRequest listObjects = ListObjectsRequest .builder() .bucket(bucketName) .build(); ListObjectsResponse res = s3.listObjects(listObjects); List<S3Object> objects = res.contents(); for (S3Object myValue : objects) { log.info("对象 Key: " + myValue.key()); printObjectSize(myValue.size()); log.info("对象拥有者: " + myValue.owner()); } } catch (S3Exception e) { log.error(e.awsErrorDetails().errorMessage()); } } /** * 查找在同一个桶内有没有相同key的文件 * * @param s3 S3Client * @param bucketName 桶名 * @param key 对象在桶内的存储路径+文件名 * @return 存在返回true,不存在返回false */ public static boolean listBucketObjects(S3Client s3, String bucketName, String key) { try { ListObjectsRequest listObjects = ListObjectsRequest .builder() .bucket(bucketName) .build(); ListObjectsResponse res = s3.listObjects(listObjects); List<S3Object> objects = res.contents(); for (S3Object myValue : objects) { if (myValue.key().equals(key)) { return true; } } } catch (S3Exception e) { log.error(e.awsErrorDetails().errorMessage()); } return false; } /** * 复制对象 * * @param s3 S3Client * @param fromBucket 源桶名 * @param objectKey 待复制对象 * @param toBucket 目标桶名 */ public static void copyBucketObject(S3Client s3, String fromBucket, String objectKey, String toBucket) { CopyObjectRequest copyReq = CopyObjectRequest.builder() .sourceBucket(fromBucket) .sourceKey(objectKey) .destinationBucket(toBucket) .destinationKey(objectKey) .build(); try { CopyObjectResponse copyRes = s3.copyObject(copyReq); log.info("对象复制结果:{}", copyRes.copyObjectResult().toString()); } catch (S3Exception e) { log.error(e.awsErrorDetails().errorMessage()); } } /** * 把 InputStream 流写入本地文件 * * @param inputStream InputStream 流 * @param filePath 待写入本地文件全路径+文件名 */ public static File convertInputStreamToLocalFile(InputStream inputStream, String filePath) { File file = new File(filePath); FileOutputStream fos = null; try { if (!file.exists()) { if (!file.createNewFile()) { log.info("创建文件: {} 失败!", filePath); return null; } } fos = new FileOutputStream(file); byte[] b = new byte[1024]; while ((inputStream.read(b)) != -1) { // 写入数据 fos.write(b); } } catch (IOException e) { e.printStackTrace(); } finally { try { if (null != inputStream) { inputStream.close(); } if (null != fos) { //保存数据 fos.close(); } } catch (IOException e) { e.printStackTrace(); } } return file; } /** * 打印对象大小 * * @param myValueSize 对象大小 */ private static void printObjectSize(Long myValueSize) { if (myValueSize < 1024) { log.info("对象大小: " + myValueSize + " B"); } else if (calKb(myValueSize) < 1024) { log.info("对象大小: " + getDecimalFormat().format(calKb(myValueSize)) + " KB"); } else if (calKb(calKb(myValueSize)) < 1024) { log.info("对象大小: " + getDecimalFormat().format(calKb(calKb(myValueSize))) + " MB"); } else { log.info("对象大小: " + getDecimalFormat().format(calKb(calKb(calKb(myValueSize)))) + " GB"); } } /** * bytes 转成 kb * * @param val bytes * @return kb */ private static double calKb(double val) { return val / 1024.00; } }
package com.chenwc; import com.chenwc.util.AwsConfig; import com.chenwc.util.AwsUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import software.amazon.awssdk.services.s3.S3Client; import java.io.File; /** * 测试 * @author chenwc */ public class Main { private static final Logger log = LoggerFactory.getLogger(Main.class); public static void main(String[] args){ AwsConfig awsConfig = new AwsConfig(); S3Client s3Client = AwsUtil.getS3Client(awsConfig); //创建存储桶 AwsUtil.createBucket(s3Client, awsConfig.getBucketName()); //列出所有存储桶 AwsUtil.listBucket(s3Client); //删除对象 AwsUtil.deleteObject(s3Client, awsConfig.getBucketName(),"apollo-adminservice-2.0.1-github.zip"); //删除存储桶内所有对象并删除存储桶 //AwsUtil.deleteObjectsInBucket(s3Client, awsConfig.getBucketName()); //AwsUtil.listBucket(s3Client); //上传对象 File file = new File("D:\\Downloads\\mysql-connector-java-5.1.49.jar"); AwsUtil.uploadObject(s3Client, awsConfig.getBucketName(), file.getName(), file); //上传对象 file = new File("D:\\Downloads\\elasticsearch-7.17.7-linux-x86_64.tar.gz"); AwsUtil.uploadObject(s3Client, awsConfig.getBucketName(), file.getName(), file); //上传对象 file = new File("D:\\Downloads\\微信图片_20221126230842.jpg"); AwsUtil.uploadObject(s3Client, awsConfig.getBucketName(), file.getName(), file); //分片上传对象 File file2 = new File("D:\\Downloads\\apollo-adminservice-2.0.1-github.zip"); AwsUtil.multipartUploadObject(s3Client, awsConfig.getBucketName(), file2.getName(), file2); //下载对象 AwsUtil.downloadObject(s3Client, awsConfig.getBucketName(), "微信图片_20221126230842.jpg", "D:\\Downloads\\微信图片_20221126230842-" + System.currentTimeMillis() + ".jpg"); //列出存储桶所有对象 AwsUtil.listBucketObjects(s3Client, awsConfig.getBucketName()); //复制对象 AwsUtil.copyBucketObject(s3Client, awsConfig.getBucketName(), file.getName(), "exporter1"); //断开连接 s3Client.close(); System.exit(0); } }