Web 直接上传 S3 技术方案验证 | Golang 实现
服务端接收用户上传文件时的一般方案是：“客户端 → 服务器 → S3”
这样带来一些问题，上传文件会占用服务器的 CPU、内存、带宽，大文件会占用磁盘存储空间，如果文件不需要进一步处理，一种优雅的方式是由客户端（e.g. Web 网页）直接上传文件到 S3，即： “客户端 → S3”
方式一：Pre-Signed URL 预签名
方案简介
服务端用已定义的 Key 向 S3 申请预签名 URL，这个 URL 带有失效时间，把它提供给客户端（e.g. Web 网页），可直接通过 HTTP PUT 请求，将数据直接以二进制流的方式上传。
适合上传单个大文件的上传
安全保证
已授权的临时 URL 暴露给外部上传，安全性保证有以下几点：
服务端生成 Key 天然限定了资源 URI，客户端只能上传文件到指定位置
服务端应该设置合理的 Timeout 超时时间，到达超时时间，S3 将中断上传，同时不会对文件进行保存
服务端可以设置 Content-Type 和 Content-Length，这些信息会被签名到 Signed URL 中，客户端上传的内容不匹配时 S3 会跑出 403 错误
代码示例
以下代码模拟了获取 Pre-Signed URL、通过 Put 方式上传文件到 S3、上传成功后获取下载 Sign URL 的过程
package main
import (
"bytes"
"fmt"
"net/http"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
)
// GenerateUploadPresignedURL creates a presigned URL for uploading.
func GenerateUploadPresignedURL(svc *s3.S3, bucketName, objectKey string, expiration time.Duration) (string, error) {
req, _ := svc.PutObjectRequest(&s3.PutObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
// ContentType: aws.String("text/plain"), // "application/json"
// ContentLength: aws.Int64(19),
})
// Generate presigned URL
signedURL, err := req.Presign(expiration)
if err != nil {
return "", fmt.Errorf("error generating upload presigned URL: %w", err)
}
return signedURL, nil
}
// UploadContentToS3 uploads content to S3 using the provided presigned URL.
func UploadContentToS3(signedURL, content, contentType string) error {
req, err := http.NewRequest("PUT", signedURL, bytes.NewBuffer([]byte(content)))
if err != nil {
return fmt.Errorf("error creating PUT request: %w", err)
}
req.Header.Set("Content-Type", contentType)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("error executing PUT request: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("upload failed with status code %d", resp.StatusCode)
}
fmt.Println("Content uploaded successfully")
return nil
}
// GenerateDownloadPresignedURL creates a presigned URL for downloading.
func GenerateDownloadPresignedURL(svc *s3.S3, bucketName, objectKey string, expiration time.Duration) (string, error) {
req, _ := svc.GetObjectRequest(&s3.GetObjectInput{
Bucket: aws.String(bucketName),
Key: aws.String(objectKey),
})
// Generate presigned URL
signedURL, err := req.Presign(expiration)
if err != nil {
return "", fmt.Errorf("error generating download presigned URL: %w", err)
}
return signedURL, nil
}
func GetContent() string {
CNTimeLocation, err := time.LoadLocation("Asia/Shanghai")
if err != nil {
panic(err)
}
return time.Now().In(CNTimeLocation).Format("2006-01-02 15:04:05")
}
func main() {
const (
region = "cn-northwest-1"
bucketName = "<your-bucket-name>"
objectKey = "<s3-object-target-key>"
expiration = 15 * time.Minute // URL expiration time
accessKey = "<your-access-key>"
secretKey = "<your-secret-key>"
)
// Initialize session with credentials
sess, err := session.NewSession(&aws.Config{
Region: aws.String(region),
Credentials: credentials.NewStaticCredentials(
accessKey, secretKey, "",
),
})
if err != nil {
fmt.Printf("Error creating AWS session: %v\n", err)
return
}
svc := s3.New(sess)
// Generate and print upload presigned URL
uploadURL, err := GenerateUploadPresignedURL(svc, bucketName, objectKey, expiration)
if err != nil {
fmt.Printf("Error generating upload presigned URL: %v\n", err)
return
}
fmt.Printf("Upload URL: %s\n", uploadURL)
// Upload content to S3
content := GetContent()
contentType := "text/plain"
if err := UploadContentToS3(uploadURL, content, contentType); err != nil {
fmt.Printf("Error uploading content: %v\n", err)
return
}
// Generate and print download presigned URL
downloadURL, err := GenerateDownloadPresignedURL(svc, bucketName, objectKey, expiration)
if err != nil {
fmt.Printf("Error generating download presigned URL: %v\n", err)
return
}
fmt.Printf("Download URL: %s\n", downloadURL)
}
输出
值得注意的是 Put 方法申请的 URL 仅能用于上传，不能用于下载和查看，代码中申请的 GET 方法 Signed URL 可供查看和下载
交互流程
用户选择文件 → 前端计算 Content-Type、Content-Length、Content-MD5 等信息提交到服务器 → 服务器生成 Pre-Signed URL 返回给客户端 → 客户端上传文件到 S3 → 上传成功后，将上传成功信息通知到服务器
方式二：S3 URL + Signing Fields（Form Post）
方案简介
这是基于 AWS S3 独有特性的实现方式，优点是客户端使用更加灵活（可以给客户端更多自定义的权限）
由服务端生成 Form 字段键值（例如：X-Amz-Date、X-Amz-Signature），将这些信息返回给客户端，客户端提交 Post Form 表单到 S3 的时候携带
相较于 Pre-Signed URL Put 方式，Form 表单申请到的签名信息可以在有效期内复用，更适合多文件上传
补充
S3 不支持批量上传，如需多文件批量上传需由客户端实现，循环提交表单，这很适合批量上传小文件
相应的，批量上传场景在方式一中则比较繁琐，因为每提交一次文件都需要向服务器申请一次 Pre-Signed URL，客户端没办法自定义 Key 路径进行上传
Form Post 方式灵活的同时，因为使用了 AWS 的策略，其它 S3 兼容的存储无法使用这种方式，不够通用
安全保证
服务端可以通过策略限制上传文件的大小、类型为一个范围，例如：上传的文件需在 1 ～ 50MB 大小、文件类型需要是 image/* 类型等，同时，也需要设置合理的过期时间。
策略示例
这个策略是为 Amazon S3 使用的预签名 POST 上传协议编写的，它描述了允许上传操作必须满足的一系列条件，这种策略确保上传到 S3 存储桶的请求符合策略中指定的条件
{ "expiration": "2015-12-30T12:00:00.000Z",
"conditions": [
{"bucket": "sigv4examplebucket"},
["starts-with", "$key", "user/user1/"],
{"acl": "public-read"},
{"success_action_redirect": "http://sigv4examplebucket.s3.amazonaws.com/successful_upload.html"},
["starts-with", "$Content-Type", "image/"],
{"x-amz-meta-uuid": "14365123651274"},
{"x-amz-server-side-encryption": "AES256"},
["starts-with", "$x-amz-meta-tag", ""],
{"x-amz-credential": "AKIAIOSFODNN7EXAMPLE/20151229/us-east-1/s3/aws4_request"},
{"x-amz-algorithm": "AWS4-HMAC-SHA256"},
{"x-amz-date": "20151229T000000Z" }
]
}
参考文档：https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-post-example.html
代码示例
需要使用 aws-sdk-go-v2 版本的 AWS SDK，Post 方式由 Form 参数指定传到那里、传什么文件
package main
import (
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"encoding/hex"
"encoding/json"
"fmt"
"log"
"path/filepath"
"time"
"github.com/aws/aws-sdk-go-v2/aws"
"github.com/aws/aws-sdk-go-v2/config"
"github.com/aws/aws-sdk-go-v2/credentials"
)
// AWS configuration constants
const (
Region = "cn-northwest-1"
BucketName = "<your-bucket-name>"
AccessKey = "<your-access-key>"
SecretKey = "<your-secret-key>"
)
func main() {
// Load AWS configuration
cfg, err := loadAWSConfig(context.TODO())
if err != nil {
log.Fatalf("unable to load SDK config: %v", err)
}
// Define the object key
keyPrefix := "s3-web-upload"
// Set time values
timeStamp := time.Now().UTC()
shortDate, amzDate := formatDates(timeStamp)
// Create the AWS credential string
credential := fmt.Sprintf("%s/%s/%s/s3/aws4_request", AccessKey, shortDate, Region)
// Create the policy
expiration := timeStamp.Add(15 * time.Minute)
policy := createPolicy(BucketName, keyPrefix, credential, amzDate, expiration)
// Retrieve signing credentials
creds, err := cfg.Credentials.Retrieve(context.TODO())
if err != nil {
log.Fatalf("unable to retrieve credentials: %v", err)
}
// Sign the policy
signature, err := generateSignature(policy, creds, Region, shortDate)
if err != nil {
log.Fatalf("unable to sign policy: %v", err)
}
// Print the necessary fields for POST request
printPostFields(BucketName, policy, creds, signature, credential, amzDate)
// Generate and print the CURL command, upload txt
objectKey := "s3-web-upload/test03-post-form/hello.txt"
curlCommand := generateCURLCommand(BucketName, objectKey, "text/plain", policy, credential, amzDate, signature)
fmt.Println("\nCURL Command:")
fmt.Println(curlCommand)
// Generate and print the CURL command, upload jpg
objectKey = "s3-web-upload/test03-post-form/hello.jpg"
curlCommand = generateCURLCommand(BucketName, objectKey, "image/jpeg", policy, credential, amzDate, signature)
fmt.Println("\nCURL Command:")
fmt.Println(curlCommand)
}
// loadAWSConfig loads AWS configuration using provided credentials and region
func loadAWSConfig(ctx context.Context) (aws.Config, error) {
return config.LoadDefaultConfig(ctx,
config.WithRegion(Region),
config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(AccessKey, SecretKey, "")),
)
}
// formatDates generates formatted date strings for use in AWS policies and signatures
func formatDates(timeStamp time.Time) (shortDate, amzDate string) {
shortDate = timeStamp.Format("20060102")
amzDate = timeStamp.Format("20060102T150405Z")
return
}
type s3PostPolicy struct {
Expiration string `json:"expiration"`
Conditions []interface{} `json:"conditions"`
}
// createPolicy generates a base64 encoded JSON policy document with given conditions
func createPolicy(bucket, keyPrefix, credential, amzDate string, expiration time.Time) string {
conditions := []interface{}{
map[string]string{"bucket": bucket},
map[string]string{"x-amz-algorithm": "AWS4-HMAC-SHA256"},
map[string]string{"x-amz-credential": credential},
map[string]string{"x-amz-date": amzDate},
[]interface{}{"content-length-range", 0, 10485760}, // 10MB
[]interface{}{"starts-with", "$key", keyPrefix},
[]interface{}{"starts-with", "$Content-Type", ""}, // e.g. image/
}
policy := s3PostPolicy{
Expiration: expiration.UTC().Format(time.RFC3339),
Conditions: conditions,
}
policyBytes, _ := json.Marshal(policy)
return base64.StdEncoding.EncodeToString(policyBytes)
}
// generateSignature creates an AWS v4 signature for the provided policy
func generateSignature(policy string, creds aws.Credentials, region, date string) (string, error) {
signingKey := deriveSigningKey(creds.SecretAccessKey, date, region, "s3")
signature := hmacSHA256(signingKey, policy)
return hex.EncodeToString(signature), nil
}
// deriveSigningKey derives the signing key used for AWS signature v4
func deriveSigningKey(secret, date, region, service string) []byte {
kDate := hmacSHA256([]byte("AWS4"+secret), date)
kRegion := hmacSHA256(kDate, region)
kService := hmacSHA256(kRegion, service)
return hmacSHA256(kService, "aws4_request")
}
// hmacSHA256 performs HMAC-SHA256 hashing algorithm with given key and data
func hmacSHA256(key []byte, data string) []byte {
h := hmac.New(sha256.New, key)
h.Write([]byte(data))
return h.Sum(nil)
}
// printPostFields displays the constructed fields needed for S3 POST requests
func printPostFields(bucketName, policy string, creds aws.Credentials, signature, credential, amzDate string) {
fmt.Printf("URL: https://%s.s3.%s.amazonaws.com.cn\n", bucketName, Region)
fmt.Println("Fields:")
//fmt.Printf("Key: %s\n", key)
fmt.Printf("AWSAccessKeyId: %s\n", creds.AccessKeyID)
fmt.Printf("Policy: %s\n", policy)
fmt.Printf("x-amz-signature: %s\n", signature)
fmt.Printf("x-amz-credential: %s\n", credential)
fmt.Printf("x-amz-algorithm: AWS4-HMAC-SHA256\n")
fmt.Printf("x-amz-date: %s\n", amzDate)
}
// generateCURLCommand generates a CURL command for uploading a file to S3
func generateCURLCommand(bucketName, key, contentType, policy, credential, amzDate, signature string) string {
filename := filepath.Base(key)
curlTemplate := `curl -X POST \
-F "key=%s" \
-F "Content-Type=%s" \
-F "X-Amz-Credential=%s" \
-F "X-Amz-Algorithm=AWS4-HMAC-SHA256" \
-F "X-Amz-Date=%s" \
-F "Policy=%s" \
-F "X-Amz-Signature=%s" \
-F "file=@%s" \
https://%s.s3.%s.amazonaws.com.cn/`
return fmt.Sprintf(curlTemplate, key, contentType, credential, amzDate, policy, signature, filename, bucketName, Region)
}
输出
运行这两个 CURL 示例命令，即可提交当前文件夹下的 hello.txt 和 hello.jpg 文件（先准备好）
未报错则说明上传成功，否则 S3 会抛出错误原因。
交互流程
用户选择文件 → 从服务器获取 URL + Form Fields → 客户端上传文件到 S3（循环上传多个文件） → 上传成功后，将上传成功信息提交给服务器
参考
Example: Browser-Based Upload using HTTP POST (Using AWS Signature Version 4)
Browser-Based Uploads Using POST (AWS Signature Version 4)
GPT-4o & Claude 3.5 Sonnet 提供部分示例代码