0%

AWS S3 对象存储服务

虽然亚马逊云非常牛逼,虽然亚马逊云财大气粗,虽然亚马逊用的人也非常多,可是这个文档我简直无法接受,特别是客服,令人发指的回复速度,瞬间让人无语,可是毕竟牛逼.忍了,躺一次坑而已

1.图片上传

1.1

S3 Java SDK 分两个版本,1.0和2.0 , 1.0 的S3对象是AmazonS3 ,2.0的忘记了,下面的示例代码,是1.0版本的

1.2导包

1
<dependencyManagement>
2
        <dependencies>
3
            <dependency>
4
                <groupId>com.amazonaws</groupId>
5
                <artifactId>aws-java-sdk-bom</artifactId>
6
                <version>1.11.558</version>
7
                <type>pom</type>
8
                <scope>import</scope>
9
            </dependency>
10
        </dependencies>
11
    </dependencyManagement>
12
13
    <dependencies>
14
        <dependency>
15
            <groupId>com.amazonaws</groupId>
16
            <artifactId>aws-java-sdk-s3</artifactId>
17
        </dependency>
18
        <dependency>
19
            <groupId>com.amazonaws</groupId>
20
            <artifactId>aws-java-sdk-s3control</artifactId>
21
        </dependency>
22
    </dependencies>

1.3代码,已经可以运行了,但是还需要配置Id和秘钥

1
public static void main(String[] args) throws Exception {
2
        String clientRegion = "ap-east-1";
3
        String bucketName = "hotupdates";
4
        String keyName = "abc/demo.txt";
5
        String filePath = "C:\\Users\\Owner\\Desktop\\demo.txt";
6
7
        try {
8
            AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
9
                    .withRegion(clientRegion)
10
                    .withCredentials(new ProfileCredentialsProvider())
11
                    .build();
12
            TransferManager tm = TransferManagerBuilder.standard()
13
                    .withS3Client(s3Client)
14
                    .build();
15
            Upload upload = tm.upload(bucketName, keyName, new File(filePath));
16
            System.out.println("Object upload started");
17
            upload.waitForCompletion();
18
            tm.shutdownNow();
19
            System.out.println("Object upload complete");
20
        }
21
        catch(AmazonServiceException e) {
22
            e.printStackTrace();
23
        }
24
        catch(SdkClientException e) {
25
            e.printStackTrace();
26
        }
27
    }

1.4 配置ID和秘钥

Windows 在C:\Users\xxx\.aws这个文件下面创建两个文件,没有后缀的哟
文件1:config

1
[default]
2
region = ap-east-1

文件2:credentials

1
[default]
2
aws_access_key_id=xxxxxxxxxxxxxxxxxxx
3
aws_secret_access_key=xxxxxxxxxxxxxxxxxxx

1.5 到这就可以运行了

1.6 想要在上传的时候打印进度条

1
public class HighLevelMultipartUpload2 {
2
3
    public static void main(String[] args) throws Exception {
4
        String clientRegion = "ap-east-1";
5
        String bucketName = "hotupdates";
6
        String keyName = "abc/demo.txt";
7
        String filePath = "C:\\Users\\Owner\\Desktop\\demo.txt";
8
9
        try {
10
            AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
11
                    .withRegion(clientRegion)
12
                    .withCredentials(new ProfileCredentialsProvider())
13
                    .build();
14
            TransferManager tm = TransferManagerBuilder.standard()
15
                    .withS3Client(s3Client)
16
                    .build();
17
18
            // TransferManager processes all transfers asynchronously,
19
            // so this call returns immediately.
20
            Upload upload = tm.upload(bucketName, keyName, new File(filePath));
21
            System.out.println("Object upload started");
22
23
            // Optionally, wait for the upload to finish before continuing.
24
            upload.waitForCompletion();
25
26
27
            tm.shutdownNow();
28
//            XferMgrProgress.showTransferProgress(upload);
29
//            XferMgrProgress.waitForCompletion(upload);
30
31
32
            System.out.println("Object upload complete");
33
        }
34
        catch(AmazonServiceException e) {
35
            e.printStackTrace();
36
        }
37
        catch(SdkClientException e) {
38
            e.printStackTrace();
39
        }
40
    }
41
}
1
public class XferMgrProgress
2
{
3
    // waits for the transfer to complete, catching any exceptions that occur.
4
    public static void waitForCompletion(Transfer xfer)
5
    {
6
        try {
7
            xfer.waitForCompletion();
8
        } catch (AmazonServiceException e) {
9
            System.err.println("Amazon service error: " + e.getMessage());
10
            System.exit(1);
11
        } catch (AmazonClientException e) {
12
            System.err.println("Amazon client error: " + e.getMessage());
13
            System.exit(1);
14
        } catch (InterruptedException e) {
15
            System.err.println("Transfer interrupted: " + e.getMessage());
16
            System.exit(1);
17
        }
18
    }
19
20
    // Prints progress while waiting for the transfer to finish.
21
    public static void showTransferProgress(Transfer xfer)
22
    {
23
        // print the transfer's human-readable description
24
        System.out.println(xfer.getDescription());
25
        // print an empty progress bar...
26
        printProgressBar(0.0);
27
        // update the progress bar while the xfer is ongoing.
28
        do {
29
            try {
30
                Thread.sleep(100);
31
            } catch (InterruptedException e) {
32
                return;
33
            }
34
            // Note: so_far and total aren't used, they're just for
35
            // documentation purposes.
36
            TransferProgress progress = xfer.getProgress();
37
            long so_far = progress.getBytesTransferred();
38
            long total = progress.getTotalBytesToTransfer();
39
            double pct = progress.getPercentTransferred();
40
            eraseProgressBar();
41
            printProgressBar(pct);
42
        } while (xfer.isDone() == false);
43
        // print the final state of the transfer.
44
        Transfer.TransferState xfer_state = xfer.getState();
45
        System.out.println(": " + xfer_state);
46
    }
47
48
    // Prints progress of a multiple file upload while waiting for it to finish.
49
    public static void showMultiUploadProgress(MultipleFileUpload multi_upload)
50
    {
51
        // print the upload's human-readable description
52
        System.out.println(multi_upload.getDescription());
53
54
        Collection<? extends Upload> sub_xfers = new ArrayList<Upload>();
55
        sub_xfers = multi_upload.getSubTransfers();
56
57
        do {
58
            System.out.println("\nSubtransfer progress:\n");
59
            for (Upload u : sub_xfers) {
60
                System.out.println("  " + u.getDescription());
61
                if (u.isDone()) {
62
                    Transfer.TransferState xfer_state = u.getState();
63
                    System.out.println("  " + xfer_state);
64
                } else {
65
                    TransferProgress progress = u.getProgress();
66
                    double pct = progress.getPercentTransferred();
67
                    printProgressBar(pct);
68
                    System.out.println();
69
                }
70
            }
71
72
            // wait a bit before the next update.
73
            try {
74
                Thread.sleep(200);
75
            } catch (InterruptedException e) {
76
                return;
77
            }
78
        } while (multi_upload.isDone() == false);
79
        // print the final state of the transfer.
80
        Transfer.TransferState xfer_state = multi_upload.getState();
81
        System.out.println("\nMultipleFileUpload " + xfer_state);
82
    }
83
84
    // prints a simple text progressbar: [#####     ]
85
    public static void printProgressBar(double pct)
86
    {
87
        // if bar_size changes, then change erase_bar (in eraseProgressBar) to
88
        // match.
89
        final int bar_size = 40;
90
        final String empty_bar = "                                        ";
91
        final String filled_bar = "########################################";
92
        int amt_full = (int)(bar_size * (pct / 100.0));
93
        System.out.format("  [%s%s]", filled_bar.substring(0, amt_full),
94
                empty_bar.substring(0, bar_size - amt_full));
95
    }
96
97
    // erases the progress bar.
98
    public static void eraseProgressBar()
99
    {
100
        // erase_bar is bar_size (from printProgressBar) + 4 chars.
101
        final String erase_bar = "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b";
102
        System.out.format(erase_bar);
103
    }
104
105
    public static void uploadFileWithListener(String file_path,
106
                                              String bucket_name, String key_prefix, boolean pause)
107
    {
108
        System.out.println("file: " + file_path +
109
                (pause ? " (pause)" : ""));
110
111
        String key_name = null;
112
        if (key_prefix != null) {
113
            key_name = key_prefix + '/' + file_path;
114
        } else {
115
            key_name = file_path;
116
        }
117
118
        File f = new File(file_path);
119
        TransferManager xfer_mgr = TransferManagerBuilder.standard().build();
120
        try {
121
            Upload u = xfer_mgr.upload(bucket_name, key_name, f);
122
            // print an empty progress bar...
123
            printProgressBar(0.0);
124
            u.addProgressListener(new ProgressListener() {
125
                public void progressChanged(ProgressEvent e) {
126
                    double pct = e.getBytesTransferred() * 100.0 / e.getBytes();
127
                    eraseProgressBar();
128
                    printProgressBar(pct);
129
                }
130
            });
131
            // block with Transfer.waitForCompletion()
132
            XferMgrProgress.waitForCompletion(u);
133
            // print the final state of the transfer.
134
            Transfer.TransferState xfer_state = u.getState();
135
            System.out.println(": " + xfer_state);
136
        } catch (AmazonServiceException e) {
137
            System.err.println(e.getErrorMessage());
138
            System.exit(1);
139
        }
140
        xfer_mgr.shutdownNow();
141
    }
142
143
    public static void uploadDirWithSubprogress(String dir_path,
144
                                                String bucket_name, String key_prefix, boolean recursive,
145
                                                boolean pause)
146
    {
147
        System.out.println("directory: " + dir_path + (recursive ?
148
                " (recursive)" : "") + (pause ? " (pause)" : ""));
149
150
        TransferManager xfer_mgr = new TransferManager();
151
        try {
152
            MultipleFileUpload multi_upload = xfer_mgr.uploadDirectory(
153
                    bucket_name, key_prefix, new File(dir_path), recursive);
154
            // loop with Transfer.isDone()
155
            XferMgrProgress.showMultiUploadProgress(multi_upload);
156
            // or block with Transfer.waitForCompletion()
157
            XferMgrProgress.waitForCompletion(multi_upload);
158
        } catch (AmazonServiceException e) {
159
            System.err.println(e.getErrorMessage());
160
            System.exit(1);
161
        }
162
        xfer_mgr.shutdownNow();
163
    }
164
165
    public static void main(String[] args)
166
    {
167
        final String USAGE = "\n" +
168
                "Usage:\n" +
169
                "    XferMgrProgress [--recursive] [--pause] <s3_path> <local_path>\n\n" +
170
                "Where:\n" +
171
                "    --recursive - Only applied if local_path is a directory.\n" +
172
                "                  Copies the contents of the directory recursively.\n\n" +
173
                "    --pause     - Attempt to pause+resume the upload. This may not work for\n" +
174
                "                  small files.\n\n" +
175
                "    s3_path     - The S3 destination (bucket/path) to upload the file(s) to.\n\n" +
176
                "    local_path  - The path to a local file or directory path to upload to S3.\n\n" +
177
                "Examples:\n" +
178
                "    XferMgrProgress public_photos/cat_happy.png my_photos/funny_cat.png\n" +
179
                "    XferMgrProgress public_photos my_photos/cat_sad.png\n" +
180
                "    XferMgrProgress public_photos my_photos\n\n";
181
182
        if (args.length < 2) {
183
            System.out.println(USAGE);
184
            System.exit(1);
185
        }
186
187
        int cur_arg = 0;
188
        boolean recursive = false;
189
        boolean pause = false;
190
191
        // first, parse any switches
192
        while (args[cur_arg].startsWith("--")) {
193
            if (args[cur_arg].equals("--recursive")) {
194
                recursive = true;
195
            } else if (args[cur_arg].equals("--pause")) {
196
                pause = true;
197
            } else {
198
                System.out.println("Unknown argument: " + args[cur_arg]);
199
                System.out.println(USAGE);
200
                System.exit(1);
201
            }
202
            cur_arg += 1;
203
        }
204
205
        // only the first '/' character is of interest to get the bucket name.
206
        // Subsequent ones are part of the key name.
207
        String s3_path[] = args[cur_arg].split("/", 2);
208
        cur_arg += 1;
209
210
        String bucket_name = s3_path[0];
211
        String key_prefix = null;
212
        if (s3_path.length > 1) {
213
            key_prefix = s3_path[1];
214
        }
215
216
        String local_path = args[cur_arg];
217
218
        // check to see if local path is a directory or file...
219
        File f = new File(args[cur_arg]);
220
        if (f.exists() == false) {
221
            System.out.println("Input path doesn't exist: " + args[cur_arg]);
222
            System.exit(1);
223
        }
224
        else if (f.isDirectory()) {
225
            uploadDirWithSubprogress(local_path, bucket_name, key_prefix,
226
                    recursive, pause);
227
        } else {
228
            uploadFileWithListener(local_path, bucket_name, key_prefix, pause);
229
        }
230
    }
231
}

参考官网:https://docs.aws.amazon.com/zh_cn/sdk-for-java/v1/developer-guide/credentials.html

2.其余的一些配置

2.1跨域

1
<CORSConfiguration>
2
 <CORSRule>
3
   <AllowedOrigin>*</AllowedOrigin>
4
   <AllowedMethod>PUT</AllowedMethod>
5
   <AllowedMethod>POST</AllowedMethod>
6
   <AllowedMethod>DELETE</AllowedMethod>
7
   <AllowedMethod>GET</AllowedMethod>
8
   <AllowedMethod>HEAD</AllowedMethod>
9
   <AllowedHeader>*</AllowedHeader>
10
  <MaxAgeSeconds>3000</MaxAgeSeconds>
11
 </CORSRule>
12
</CORSConfiguration>

2.2 存储桶策略

1
{
2
    "Version": "2012-10-17",
3
    "Statement": [
4
        {
5
            "Sid": "Stmt1546506260886",
6
            "Effect": "Allow",
7
            "Principal": {
8
                "AWS": "*"
9
            },
10
            "Action": "s3:GetObject",
11
            "Resource": "arn:aws:s3:::apk-online/*"
12
        }
13
    ]
14
}

apk-online 是桶名