Go实战--golang资源管理七牛云对象存储(bucketManager)

生命不止，继续 go go go !!!

之前学习了七牛与的golang SDK，我们主要介绍了如何通过golang上传文件到七牛bucket：
Go实战–golang上传文件到七牛云对象存储(github.com/qiniu/api.v7)

今天，与大家一起学习bucket中资源管理。

bucket.go源码

关于资源管理的方法都是位于bucket.go，大概五百多行的代码，不算很长，这里贴过来：

package storage

import (
    "context"
    "encoding/base64"
    "errors"
    "fmt"
    "net/url"
    "strconv"
    "strings"

    "github.com/qiniu/api.v7/auth/qbox"
    "github.com/qiniu/x/rpc.v7"
)

// 资源管理相关的默认域名
const (
    DefaultRsHost  = "rs.qiniu.com"
    DefaultRsfHost = "rsf.qiniu.com"
    DefaultAPIHost = "api.qiniu.com"
    DefaultPubHost = "pu.qbox.me:10200"
)

// FileInfo 文件基本信息
type FileInfo struct {
    Hash     string `json:"hash"`
    Fsize    int64  `json:"fsize"`
    PutTime  int64  `json:"putTime"`
    MimeType string `json:"mimeType"`
    Type     int    `json:"type"`
}

func (f *FileInfo) String() string {
    str := ""
    str += fmt.Sprintf("Hash: %sn",f.Hash)
    str += fmt.Sprintf("Fsize: %dn",f.Fsize)
    str += fmt.Sprintf("PutTime: %dn",f.PutTime)
    str += fmt.Sprintf("MimeType: %sn",f.MimeType)
    str += fmt.Sprintf("Type: %dn",f.Type)
    return str
}

// FetchRet 资源抓取的返回值
type FetchRet struct {
    Hash     string `json:"hash"`
    Fsize    int64  `json:"fsize"`
    MimeType string `json:"mimeType"`
    Key      string `json:"key"`
}

func (r *FetchRet) String() string {
    str := ""
    str += fmt.Sprintf("Key: %sn",r.Key)
    str += fmt.Sprintf("Hash: %sn",r.Hash)
    str += fmt.Sprintf("Fsize: %dn",r.Fsize)
    str += fmt.Sprintf("MimeType: %sn",r.MimeType)
    return str
}

// ListItem 为文件列举的返回值
type ListItem struct {
    Key      string `json:"key"`
    Hash     string `json:"hash"`
    Fsize    int64  `json:"fsize"`
    PutTime  int64  `json:"putTime"`
    MimeType string `json:"mimeType"`
    Type     int    `json:"type"`
    EndUser  string `json:"endUser"`
}

func (l *ListItem) String() string {
    str := ""
    str += fmt.Sprintf("Hash: %sn",l.Hash)
    str += fmt.Sprintf("Fsize: %dn",l.Fsize)
    str += fmt.Sprintf("PutTime: %dn",l.PutTime)
    str += fmt.Sprintf("MimeType: %sn",l.MimeType)
    str += fmt.Sprintf("Type: %dn",l.Type)
    str += fmt.Sprintf("EndUser: %sn",l.EndUser)
    return str
}

// BatchOpRet 为批量执行操作的返回值
// 批量操作支持 stat，copy，delete，move，chgm，chtype，deleteAfterDays几个操作
// 其中 stat 为获取文件的基本信息，如果文件存在则返回基本信息，如果文件不存在返回 error 。
// 其他的操作，如果成功，则返回 code，不成功会同时返回 error 信息，可以根据 error 信息来判断问题所在。
type BatchOpRet struct {
    Code int `json:"code,omitempty"`
    Data struct {
        Hash     string `json:"hash"`
        Fsize    int64  `json:"fsize"`
        PutTime  int64  `json:"putTime"`
        MimeType string `json:"mimeType"`
        Type     int    `json:"type"`
        Error    string `json:"error"`
    } `json:"data,omitempty"`
}

// BucketManager 提供了对资源进行管理的操作
type BucketManager struct {
    client *rpc.Client
    mac    *qbox.Mac
    cfg    *Config
}

// NewBucketManager 用来构建一个新的资源管理对象
func NewBucketManager(mac *qbox.Mac,cfg *Config) *BucketManager {
    if cfg == nil {
        cfg = &Config{}
    }

    return &BucketManager{
        client: NewClient(mac,nil),mac:    mac,cfg:    cfg,}
}

// NewBucketManagerEx 用来构建一个新的资源管理对象
func NewBucketManagerEx(mac *qbox.Mac,cfg *Config,client *rpc.Client) *BucketManager {
    if cfg == nil {
        cfg = &Config{}
    }

    if client == nil {
        client = NewClient(mac,nil)
    }

    return &BucketManager{
        client: client,}
}

// Buckets 用来获取空间列表，如果指定了 shared 参数为 true，那么一同列表被授权访问的空间
func (m *BucketManager) Buckets(shared bool) (buckets []string,err error) {
    ctx := context.TODO()
    var reqHost string

    scheme := "http://"
    if m.cfg.UseHTTPS {
        scheme = "https://"
    }

    reqHost = fmt.Sprintf("%s%s",scheme,DefaultRsHost)
    reqURL := fmt.Sprintf("%s/buckets?shared=%v",reqHost,shared)
    err = m.client.Call(ctx,&buckets,"POST",reqURL)
    return
}

// Stat 用来获取一个文件的基本信息
func (m *BucketManager) Stat(bucket,key string) (info FileInfo,err error) {
    ctx := context.TODO()
    reqHost,reqErr := m.rsHost(bucket)
    if reqErr != nil {
        err = reqErr
        return
    }

    reqURL := fmt.Sprintf("%s%s",URIStat(bucket,key))
    err = m.client.Call(ctx,&info,reqURL)
    return
}

// Delete 用来删除空间中的一个文件
func (m *BucketManager) Delete(bucket,key string) (err error) {
    ctx := context.TODO()
    reqHost,reqErr := m.rsHost(bucket)
    if reqErr != nil {
        err = reqErr
        return
    }
    reqURL := fmt.Sprintf("%s%s",URIDelete(bucket,nil,reqURL)
    return
}

// Copy 用来创建已有空间中的文件的一个新的副本
func (m *BucketManager) Copy(srcBucket,srcKey,destBucket,destKey string,force bool) (err error) {
    ctx := context.TODO()
    reqHost,reqErr := m.rsHost(srcBucket)
    if reqErr != nil {
        err = reqErr
        return
    }

    reqURL := fmt.Sprintf("%s%s",URICopy(srcBucket,destKey,force))
    err = m.client.Call(ctx,reqURL)
    return
}

// Move 用来将空间中的一个文件移动到新的空间或者重命名
func (m *BucketManager) Move(srcBucket,URIMove(srcBucket,reqURL)
    return
}

// ChangeMime 用来更新文件的MimeType
func (m *BucketManager) ChangeMime(bucket,key,newMime string) (err error) {
    ctx := context.TODO()
    reqHost,URIChangeMime(bucket,newMime))
    err = m.client.Call(ctx,reqURL)
    return
}

// ChangeType 用来更新文件的存储类型，0表示普通存储，1表示低频存储
func (m *BucketManager) ChangeType(bucket,key string,fileType int) (err error) {
    ctx := context.TODO()
    reqHost,URIChangeType(bucket,fileType))
    err = m.client.Call(ctx,reqURL)
    return
}

// DeleteAfterDays 用来更新文件生命周期，如果 days 设置为0，则表示取消文件的定期删除功能，永久存储
func (m *BucketManager) DeleteAfterDays(bucket,days int) (err error) {
    ctx := context.TODO()
    reqHost,URIDeleteAfterDays(bucket,days))
    err = m.client.Call(ctx,reqURL)
    return
}

// Batch 接口提供了资源管理的批量操作，支持 stat，copy，move，delete，chgm，chtype，deleteAfterDays几个接口
func (m *BucketManager) Batch(operations []string) (batchOpRet []BatchOpRet,err error) {
    if len(operations) > 1000 {
        err = errors.New("batch operation count exceeds the limit of 1000")
        return
    }
    ctx := context.TODO()
    scheme := "http://"
    if m.cfg.UseHTTPS {
        scheme = "https://"
    }
    reqURL := fmt.Sprintf("%s%s/batch",DefaultRsHost)
    params := map[string][]string{
        "op": operations,}
    err = m.client.CallWithForm(ctx,&batchOpRet,reqURL,params)
    return
}

// Fetch 根据提供的远程资源链接来抓取一个文件到空间并已指定文件名保存
func (m *BucketManager) Fetch(resURL,bucket,key string) (fetchRet FetchRet,reqErr := m.iovipHost(bucket)
    if reqErr != nil {
        err = reqErr
        return
    }
    reqURL := fmt.Sprintf("%s%s",uriFetch(resURL,&fetchRet,reqURL)
    return
}

// FetchWithoutKey 根据提供的远程资源链接来抓取一个文件到空间并以文件的内容hash作为文件名
func (m *BucketManager) FetchWithoutKey(resURL,bucket string) (fetchRet FetchRet,uriFetchWithoutKey(resURL,bucket))
    err = m.client.Call(ctx,reqURL)
    return
}

// Prefetch 用来同步镜像空间的资源和镜像源资源内容
func (m *BucketManager) Prefetch(bucket,uriPrefetch(bucket,reqURL)
    return
}

// SetImage 用来设置空间镜像源
func (m *BucketManager) SetImage(siteURL,bucket string) (err error) {
    ctx := context.TODO()
    reqURL := fmt.Sprintf("http://%s%s",DefaultPubHost,uriSetImage(siteURL,reqURL)
    return
}

// SetImageWithHost 用来设置空间镜像源，额外添加回源Host头部
func (m *BucketManager) SetImageWithHost(siteURL,host string) (err error) {
    ctx := context.TODO()
    reqURL := fmt.Sprintf("http://%s%s",uriSetImageWithHost(siteURL,host))
    err = m.client.Call(ctx,reqURL)
    return
}

// UnsetImage 用来取消空间镜像源设置
func (m *BucketManager) UnsetImage(bucket string) (err error) {
    ctx := context.TODO()
    reqURL := fmt.Sprintf("http://%s%s",uriUnsetImage(bucket))
    err = m.client.Call(ctx,reqURL)
    return err
}

type listFilesRet struct {
    Marker         string     `json:"marker"`
    Items          []ListItem `json:"items"`
    CommonPrefixes []string   `json:"commonPrefixes"`
}

// ListFiles 用来获取空间文件列表，可以根据需要指定文件的前缀 prefix，文件的目录 delimiter，循环列举的时候下次
// 列举的位置 marker，以及每次返回的文件的最大数量limit，其中limit最大为1000。
func (m *BucketManager) ListFiles(bucket,prefix,delimiter,marker string,limit int) (entries []ListItem,commonPrefixes []string,nextMarker string,hasNext bool,err error) {
    if limit <= 0 || limit > 1000 {
        err = errors.New("invalid list limit,only allow [1,1000]")
        return
    }

    ctx := context.TODO()
    reqHost,reqErr := m.rsfHost(bucket)
    if reqErr != nil {
        err = reqErr
        return
    }

    ret := listFilesRet{}
    reqURL := fmt.Sprintf("%s%s",uriListFiles(bucket,marker,limit))
    err = m.client.Call(ctx,&ret,reqURL)
    if err != nil {
        return
    }

    commonPrefixes = ret.CommonPrefixes
    nextMarker = ret.Marker
    entries = ret.Items
    if ret.Marker != "" {
        hasNext = true
    }

    return
}

func (m *BucketManager) rsHost(bucket string) (rsHost string,err error) {
    var zone *Zone
    if m.cfg.Zone != nil {
        zone = m.cfg.Zone
    } else {
        if v,zoneErr := GetZone(m.mac.AccessKey,bucket); zoneErr != nil {
            err = zoneErr
            return
        } else {
            zone = v
        }
    }

    scheme := "http://"
    if m.cfg.UseHTTPS {
        scheme = "https://"
    }

    rsHost = fmt.Sprintf("%s%s",zone.RsHost)
    return
}

func (m *BucketManager) rsfHost(bucket string) (rsfHost string,bucket); zoneErr != nil {
            err = zoneErr
            return
        } else {
            zone = v
        }
    }

    scheme := "http://"
    if m.cfg.UseHTTPS {
        scheme = "https://"
    }

    rsfHost = fmt.Sprintf("%s%s",zone.RsfHost)
    return
}

func (m *BucketManager) iovipHost(bucket string) (iovipHost string,bucket); zoneErr != nil {
            err = zoneErr
            return
        } else {
            zone = v
        }
    }

    scheme := "http://"
    if m.cfg.UseHTTPS {
        scheme = "https://"
    }

    iovipHost = fmt.Sprintf("%s%s",zone.IovipHost)
    return
}

// 构建op的方法，导出的方法支持在Batch操作中使用

// URIStat 构建 stat 接口的请求命令
func URIStat(bucket,key string) string {
    return fmt.Sprintf("/stat/%s",EncodedEntry(bucket,key))
}

// URIDelete 构建 delete 接口的请求命令
func URIDelete(bucket,key string) string {
    return fmt.Sprintf("/delete/%s",key))
}

// URICopy 构建 copy 接口的请求命令
func URICopy(srcBucket,force bool) string {
    return fmt.Sprintf("/copy/%s/%s/force/%v",EncodedEntry(srcBucket,srcKey),EncodedEntry(destBucket,destKey),force)
}

// URIMove 构建 move 接口的请求命令
func URIMove(srcBucket,force bool) string {
    return fmt.Sprintf("/move/%s/%s/force/%v",force)
}

// URIDeleteAfterDays 构建 deleteAfterDays 接口的请求命令
func URIDeleteAfterDays(bucket,days int) string {
    return fmt.Sprintf("/deleteAfterDays/%s/%d",key),days)
}

// URIChangeMime 构建 chgm 接口的请求命令
func URIChangeMime(bucket,newMime string) string {
    return fmt.Sprintf("/chgm/%s/mime/%s",base64.URLEncoding.EncodeToString([]byte(newMime)))
}

// URIChangeType 构建 chtype 接口的请求命令
func URIChangeType(bucket,fileType int) string {
    return fmt.Sprintf("/chtype/%s/type/%d",fileType)
}

// 构建op的方法，非导出的方法无法用在Batch操作中
func uriFetch(resURL,key string) string {
    return fmt.Sprintf("/fetch/%s/to/%s",base64.URLEncoding.EncodeToString([]byte(resURL)),key))
}

func uriFetchWithoutKey(resURL,bucket string) string {
    return fmt.Sprintf("/fetch/%s/to/%s",EncodedEntryWithoutKey(bucket))
}

func uriPrefetch(bucket,key string) string {
    return fmt.Sprintf("/prefetch/%s",key))
}

func uriSetImage(siteURL,bucket string) string {
    return fmt.Sprintf("/image/%s/from/%s",base64.URLEncoding.EncodeToString([]byte(siteURL)))
}

func uriSetImageWithHost(siteURL,host string) string {
    return fmt.Sprintf("/image/%s/from/%s/host/%s",base64.URLEncoding.EncodeToString([]byte(siteURL)),base64.URLEncoding.EncodeToString([]byte(host)))
}

func uriUnsetImage(bucket string) string {
    return fmt.Sprintf("/unimage/%s",bucket)
}

func uriListFiles(bucket,limit int) string {
    query := make(url.Values)
    query.Add("bucket",bucket)
    if prefix != "" {
        query.Add("prefix",prefix)
    }
    if delimiter != "" {
        query.Add("delimiter",delimiter)
    }
    if marker != "" {
        query.Add("marker",marker)
    }
    if limit > 0 {
        query.Add("limit",strconv.FormatInt(int64(limit), 10))
    }
    return fmt.Sprintf("/list?%s",query.Encode())
}

// EncodedEntry 生成URL Safe Base64编码的 Entry
func EncodedEntry(bucket,key string) string {
    entry := fmt.Sprintf("%s:%s",key)
    return base64.URLEncoding.EncodeToString([]byte(entry))
}

// EncodedEntryWithoutKey 生成 key 为null的情况下 URL Safe Base64编码的Entry
func EncodedEntryWithoutKey(bucket string) string {
    return base64.URLEncoding.EncodeToString([]byte(bucket))
}

// MakePublicURL 用来生成公开空间资源下载链接
func MakePublicURL(domain,key string) (finalUrl string) {
    srcUrl := fmt.Sprintf("%s/%s",domain,key)
    srcUri,_ := url.Parse(srcUrl)
    finalUrl = srcUri.String()
    return
}

// MakePrivateURL 用来生成私有空间资源下载链接
func MakePrivateURL(mac *qbox.Mac,deadline int64) (privateURL string) {
    publicURL := MakePublicURL(domain,key)
    urlToSign := publicURL
    if strings.Contains(publicURL,"?") {
        urlToSign = fmt.Sprintf("%s&e=%d",urlToSign,deadline)
    } else {
        urlToSign = fmt.Sprintf("%s?e=%d",deadline)
    }
    token := mac.Sign([]byte(urlToSign))
    privateURL = fmt.Sprintf("%s&token=%s",token)
    return
}

资源管理–实战

获取文件类型
更改文件类型
更改文件mime
重命名文件
创建文件副本
删除文件

package main

import (
    "fmt"

    "github.com/qiniu/api.v7/auth/qbox"
    "github.com/qiniu/api.v7/storage"
)

func main() {
    accessKey := "TgVGKnpCMLDI6hSS4rSWE3g-FZjMPf6ZbcX0Kd7c"
    secretKey := "zqZvH3fNVaggw00oc9wCrcWKgeeiV7WITFTFds7H"
    bucket := "wangshubotest"
    key := "2.log"

    mac := qbox.NewMac(accessKey,secretKey)

    cfg := storage.Config{}
    cfg.Zone = &storage.ZoneHuadong
    cfg.UseHTTPS = false
    cfg.UseCdnDomains = false

    bucketManager := storage.NewBucketManager(mac,&cfg)

    // Get file info
    fmt.Println("------Get file info------")
    fileInfo,sErr := bucketManager.Stat(bucket,key)
    if sErr != nil {
        fmt.Println(sErr)
        return
    }
    fmt.Println(fileInfo.String())
    fmt.Println(storage.ParsePutTime(fileInfo.PutTime))

    // Change the mime of the file
    fmt.Println("------Change the mime of the file------")
    newMime := "image/x-png"
    err := bucketManager.ChangeMime(bucket,newMime)
    if err != nil {
        fmt.Println(err)
        return
    }
    fileInfo,sErr = bucketManager.Stat(bucket,key)
    if sErr != nil {
        fmt.Println(sErr)
        return
    }
    fmt.Println(fileInfo.String())
    fmt.Println(storage.ParsePutTime(fileInfo.PutTime))

    // Change filetype
    fmt.Println("------Change filetype------")
    fileType := 1
    err = bucketManager.ChangeType(bucket,fileType)
    if err != nil {
        fmt.Println(err)
        return
    }
    fileInfo,key)
    if sErr != nil {
        fmt.Println(sErr)
        return
    }
    fmt.Println(fileInfo.String())
    fmt.Println(storage.ParsePutTime(fileInfo.PutTime))

    //Copy file
    fmt.Println("------Copy file------")
    destBucket := bucket
    destKey := "2_copy.log"
    force := false
    err = bucketManager.Copy(bucket,force)
    if err != nil {
        fmt.Println(err)
        return
    }
    fileInfo,destKey)
    if sErr != nil {
        fmt.Println(sErr)
        return
    }
    fmt.Println(fileInfo.String())
    fmt.Println(storage.ParsePutTime(fileInfo.PutTime))

    //Rename file
    fmt.Println("------Rename file------")
    destKey = "2_new.log"
    force = false
    err = bucketManager.Move(bucket,destKey)
    if sErr != nil {
        fmt.Println(sErr)
        return
    }
    fmt.Println(fileInfo.String())
    fmt.Println(storage.ParsePutTime(fileInfo.PutTime))

    // Delete file
    fmt.Println("------Delete file------")
    err = bucketManager.Delete(bucket,key)
    if err != nil {
        fmt.Println(err)
        return
    }
    fileInfo,key)
    if sErr != nil {
        fmt.Println(sErr)
        return
    }
    fmt.Println(fileInfo.String())
    fmt.Println(storage.ParsePutTime(fileInfo.PutTime))
}

输出：

------Get file info------
Hash:     Fh3tNUEoiaj-qkNcP915nRuiAm4-
Fsize:    20
PutTime:  15083847282270721
MimeType: text/plain
Type:     0

2017-10-19 11:45:28.2270721 +0800 CST
------Change the mime of the file------
Hash:     Fh3tNUEoiaj-qkNcP915nRuiAm4-
Fsize:    20
PutTime:  15083847282270721
MimeType: image/x-png
Type:     0

2017-10-19 11:45:28.2270721 +0800 CST
------Change filetype------
Hash:     Fh3tNUEoiaj-qkNcP915nRuiAm4-
Fsize:    20
PutTime:  15083847282270721
MimeType: image/x-png
Type:     1

2017-10-19 11:45:28.2270721 +0800 CST
------Copy file------
Hash:     Fh3tNUEoiaj-qkNcP915nRuiAm4-
Fsize:    20
PutTime:  15083847373416725
MimeType: image/x-png
Type:     1

2017-10-19 11:45:37.3416725 +0800 CST
------Rename file------
Hash:     Fh3tNUEoiaj-qkNcP915nRuiAm4-
Fsize:    20
PutTime:  15083847376868294
MimeType: image/x-png
Type:     1

2017-10-19 11:45:37.6868294 +0800 CST
------Delete file------
no such file or directory