service/vendor/github.com/aliyun/aliyun-oss-go-sdk/oss/utils.go

610 lines
15 KiB
Go

package oss
import (
"bytes"
"errors"
"fmt"
"hash/crc32"
"hash/crc64"
"io"
"net/http"
"os"
"runtime"
"strconv"
"strings"
"time"
"unicode/utf8"
)
var sys_name string
var sys_release string
var sys_machine string
var (
escQuot = []byte(""") // shorter than """
escApos = []byte("'") // shorter than "'"
escAmp = []byte("&")
escLT = []byte("<")
escGT = []byte(">")
escTab = []byte("	")
escNL = []byte("
")
escCR = []byte("
")
escFFFD = []byte("\uFFFD") // Unicode replacement character
)
func init() {
sys_name = runtime.GOOS
sys_release = "-"
sys_machine = runtime.GOARCH
}
// userAgent gets user agent
// It has the SDK version information, OS information and GO version
func userAgent() string {
sys := getSysInfo()
return fmt.Sprintf("aliyun-sdk-go/%s (%s/%s/%s;%s)", Version, sys.name,
sys.release, sys.machine, runtime.Version())
}
type sysInfo struct {
name string // OS name such as windows/Linux
release string // OS version 2.6.32-220.23.2.ali1089.el5.x86_64 etc
machine string // CPU type amd64/x86_64
}
// getSysInfo gets system info
// gets the OS information and CPU type
func getSysInfo() sysInfo {
return sysInfo{name: sys_name, release: sys_release, machine: sys_machine}
}
// GetRangeConfig gets the download range from the options.
func GetRangeConfig(options []Option) (*UnpackedRange, error) {
rangeOpt, err := FindOption(options, HTTPHeaderRange, nil)
if err != nil || rangeOpt == nil {
return nil, err
}
return ParseRange(rangeOpt.(string))
}
// UnpackedRange
type UnpackedRange struct {
HasStart bool // Flag indicates if the start point is specified
HasEnd bool // Flag indicates if the end point is specified
Start int64 // Start point
End int64 // End point
}
// InvalidRangeError returns invalid range error
func InvalidRangeError(r string) error {
return fmt.Errorf("InvalidRange %s", r)
}
func GetRangeString(unpackRange UnpackedRange) string {
var strRange string
if unpackRange.HasStart && unpackRange.HasEnd {
strRange = fmt.Sprintf("%d-%d", unpackRange.Start, unpackRange.End)
} else if unpackRange.HasStart {
strRange = fmt.Sprintf("%d-", unpackRange.Start)
} else if unpackRange.HasEnd {
strRange = fmt.Sprintf("-%d", unpackRange.End)
}
return strRange
}
// ParseRange parse various styles of range such as bytes=M-N
func ParseRange(normalizedRange string) (*UnpackedRange, error) {
var err error
hasStart := false
hasEnd := false
var start int64
var end int64
// Bytes==M-N or ranges=M-N
nrSlice := strings.Split(normalizedRange, "=")
if len(nrSlice) != 2 || nrSlice[0] != "bytes" {
return nil, InvalidRangeError(normalizedRange)
}
// Bytes=M-N,X-Y
rSlice := strings.Split(nrSlice[1], ",")
rStr := rSlice[0]
if strings.HasSuffix(rStr, "-") { // M-
startStr := rStr[:len(rStr)-1]
start, err = strconv.ParseInt(startStr, 10, 64)
if err != nil {
return nil, InvalidRangeError(normalizedRange)
}
hasStart = true
} else if strings.HasPrefix(rStr, "-") { // -N
len := rStr[1:]
end, err = strconv.ParseInt(len, 10, 64)
if err != nil {
return nil, InvalidRangeError(normalizedRange)
}
if end == 0 { // -0
return nil, InvalidRangeError(normalizedRange)
}
hasEnd = true
} else { // M-N
valSlice := strings.Split(rStr, "-")
if len(valSlice) != 2 {
return nil, InvalidRangeError(normalizedRange)
}
start, err = strconv.ParseInt(valSlice[0], 10, 64)
if err != nil {
return nil, InvalidRangeError(normalizedRange)
}
hasStart = true
end, err = strconv.ParseInt(valSlice[1], 10, 64)
if err != nil {
return nil, InvalidRangeError(normalizedRange)
}
hasEnd = true
}
return &UnpackedRange{hasStart, hasEnd, start, end}, nil
}
// AdjustRange returns adjusted range, adjust the range according to the length of the file
func AdjustRange(ur *UnpackedRange, size int64) (start, end int64) {
if ur == nil {
return 0, size
}
if ur.HasStart && ur.HasEnd {
start = ur.Start
end = ur.End + 1
if ur.Start < 0 || ur.Start >= size || ur.End > size || ur.Start > ur.End {
start = 0
end = size
}
} else if ur.HasStart {
start = ur.Start
end = size
if ur.Start < 0 || ur.Start >= size {
start = 0
}
} else if ur.HasEnd {
start = size - ur.End
end = size
if ur.End < 0 || ur.End > size {
start = 0
end = size
}
}
return
}
// GetNowSec returns Unix time, the number of seconds elapsed since January 1, 1970 UTC.
// gets the current time in Unix time, in seconds.
func GetNowSec() int64 {
return time.Now().Unix()
}
// GetNowNanoSec returns t as a Unix time, the number of nanoseconds elapsed
// since January 1, 1970 UTC. The result is undefined if the Unix time
// in nanoseconds cannot be represented by an int64. Note that this
// means the result of calling UnixNano on the zero Time is undefined.
// gets the current time in Unix time, in nanoseconds.
func GetNowNanoSec() int64 {
return time.Now().UnixNano()
}
// GetNowGMT gets the current time in GMT format.
func GetNowGMT() string {
return time.Now().UTC().Format(http.TimeFormat)
}
// FileChunk is the file chunk definition
type FileChunk struct {
Number int // Chunk number
Offset int64 // Chunk offset
Size int64 // Chunk size.
}
// SplitFileByPartNum splits big file into parts by the num of parts.
// Split the file with specified parts count, returns the split result when error is nil.
func SplitFileByPartNum(fileName string, chunkNum int) ([]FileChunk, error) {
if chunkNum <= 0 || chunkNum > 10000 {
return nil, errors.New("chunkNum invalid")
}
file, err := os.Open(fileName)
if err != nil {
return nil, err
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return nil, err
}
if int64(chunkNum) > stat.Size() {
return nil, errors.New("oss: chunkNum invalid")
}
var chunks []FileChunk
var chunk = FileChunk{}
var chunkN = (int64)(chunkNum)
for i := int64(0); i < chunkN; i++ {
chunk.Number = int(i + 1)
chunk.Offset = i * (stat.Size() / chunkN)
if i == chunkN-1 {
chunk.Size = stat.Size()/chunkN + stat.Size()%chunkN
} else {
chunk.Size = stat.Size() / chunkN
}
chunks = append(chunks, chunk)
}
return chunks, nil
}
// SplitFileByPartSize splits big file into parts by the size of parts.
// Splits the file by the part size. Returns the FileChunk when error is nil.
func SplitFileByPartSize(fileName string, chunkSize int64) ([]FileChunk, error) {
if chunkSize <= 0 {
return nil, errors.New("chunkSize invalid")
}
file, err := os.Open(fileName)
if err != nil {
return nil, err
}
defer file.Close()
stat, err := file.Stat()
if err != nil {
return nil, err
}
var chunkN = stat.Size() / chunkSize
if chunkN >= 10000 {
return nil, errors.New("Too many parts, please increase part size")
}
var chunks []FileChunk
var chunk = FileChunk{}
for i := int64(0); i < chunkN; i++ {
chunk.Number = int(i + 1)
chunk.Offset = i * chunkSize
chunk.Size = chunkSize
chunks = append(chunks, chunk)
}
if stat.Size()%chunkSize > 0 {
chunk.Number = len(chunks) + 1
chunk.Offset = int64(len(chunks)) * chunkSize
chunk.Size = stat.Size() % chunkSize
chunks = append(chunks, chunk)
}
return chunks, nil
}
// GetPartEnd calculates the end position
func GetPartEnd(begin int64, total int64, per int64) int64 {
if begin+per > total {
return total - 1
}
return begin + per - 1
}
// CrcTable returns the table constructed from the specified polynomial
var CrcTable = func() *crc64.Table {
return crc64.MakeTable(crc64.ECMA)
}
// CrcTable returns the table constructed from the specified polynomial
var crc32Table = func() *crc32.Table {
return crc32.MakeTable(crc32.IEEE)
}
// choiceTransferPartOption choices valid option supported by Uploadpart or DownloadPart
func ChoiceTransferPartOption(options []Option) []Option {
var outOption []Option
listener, _ := FindOption(options, progressListener, nil)
if listener != nil {
outOption = append(outOption, Progress(listener.(ProgressListener)))
}
payer, _ := FindOption(options, HTTPHeaderOssRequester, nil)
if payer != nil {
outOption = append(outOption, RequestPayer(PayerType(payer.(string))))
}
versionId, _ := FindOption(options, "versionId", nil)
if versionId != nil {
outOption = append(outOption, VersionId(versionId.(string)))
}
trafficLimit, _ := FindOption(options, HTTPHeaderOssTrafficLimit, nil)
if trafficLimit != nil {
speed, _ := strconv.ParseInt(trafficLimit.(string), 10, 64)
outOption = append(outOption, TrafficLimitHeader(speed))
}
respHeader, _ := FindOption(options, responseHeader, nil)
if respHeader != nil {
outOption = append(outOption, GetResponseHeader(respHeader.(*http.Header)))
}
return outOption
}
// ChoiceCompletePartOption choices valid option supported by CompleteMulitiPart
func ChoiceCompletePartOption(options []Option) []Option {
var outOption []Option
listener, _ := FindOption(options, progressListener, nil)
if listener != nil {
outOption = append(outOption, Progress(listener.(ProgressListener)))
}
payer, _ := FindOption(options, HTTPHeaderOssRequester, nil)
if payer != nil {
outOption = append(outOption, RequestPayer(PayerType(payer.(string))))
}
acl, _ := FindOption(options, HTTPHeaderOssObjectACL, nil)
if acl != nil {
outOption = append(outOption, ObjectACL(ACLType(acl.(string))))
}
callback, _ := FindOption(options, HTTPHeaderOssCallback, nil)
if callback != nil {
outOption = append(outOption, Callback(callback.(string)))
}
callbackVar, _ := FindOption(options, HTTPHeaderOssCallbackVar, nil)
if callbackVar != nil {
outOption = append(outOption, CallbackVar(callbackVar.(string)))
}
respHeader, _ := FindOption(options, responseHeader, nil)
if respHeader != nil {
outOption = append(outOption, GetResponseHeader(respHeader.(*http.Header)))
}
forbidOverWrite, _ := FindOption(options, HTTPHeaderOssForbidOverWrite, nil)
if forbidOverWrite != nil {
if forbidOverWrite.(string) == "true" {
outOption = append(outOption, ForbidOverWrite(true))
} else {
outOption = append(outOption, ForbidOverWrite(false))
}
}
notification, _ := FindOption(options, HttpHeaderOssNotification, nil)
if notification != nil {
outOption = append(outOption, SetHeader(HttpHeaderOssNotification, notification))
}
return outOption
}
// ChoiceAbortPartOption choices valid option supported by AbortMultipartUpload
func ChoiceAbortPartOption(options []Option) []Option {
var outOption []Option
payer, _ := FindOption(options, HTTPHeaderOssRequester, nil)
if payer != nil {
outOption = append(outOption, RequestPayer(PayerType(payer.(string))))
}
respHeader, _ := FindOption(options, responseHeader, nil)
if respHeader != nil {
outOption = append(outOption, GetResponseHeader(respHeader.(*http.Header)))
}
return outOption
}
// ChoiceHeadObjectOption choices valid option supported by HeadObject
func ChoiceHeadObjectOption(options []Option) []Option {
var outOption []Option
// not select HTTPHeaderRange to get whole object length
payer, _ := FindOption(options, HTTPHeaderOssRequester, nil)
if payer != nil {
outOption = append(outOption, RequestPayer(PayerType(payer.(string))))
}
versionId, _ := FindOption(options, "versionId", nil)
if versionId != nil {
outOption = append(outOption, VersionId(versionId.(string)))
}
respHeader, _ := FindOption(options, responseHeader, nil)
if respHeader != nil {
outOption = append(outOption, GetResponseHeader(respHeader.(*http.Header)))
}
return outOption
}
func CheckBucketName(bucketName string) error {
nameLen := len(bucketName)
if nameLen < 3 || nameLen > 63 {
return fmt.Errorf("bucket name %s len is between [3-63],now is %d", bucketName, nameLen)
}
for _, v := range bucketName {
if !(('a' <= v && v <= 'z') || ('0' <= v && v <= '9') || v == '-') {
return fmt.Errorf("bucket name %s can only include lowercase letters, numbers, and -", bucketName)
}
}
if bucketName[0] == '-' || bucketName[nameLen-1] == '-' {
return fmt.Errorf("bucket name %s must start and end with a lowercase letter or number", bucketName)
}
return nil
}
func CheckObjectName(objectName string) error {
if len(objectName) == 0 {
return fmt.Errorf("object name is empty")
}
return nil
}
func GetReaderLen(reader io.Reader) (int64, error) {
var contentLength int64
var err error
switch v := reader.(type) {
case *bytes.Buffer:
contentLength = int64(v.Len())
case *bytes.Reader:
contentLength = int64(v.Len())
case *strings.Reader:
contentLength = int64(v.Len())
case *os.File:
fInfo, fError := v.Stat()
if fError != nil {
err = fmt.Errorf("can't get reader content length,%s", fError.Error())
} else {
contentLength = fInfo.Size()
}
case *io.LimitedReader:
contentLength = int64(v.N)
case *LimitedReadCloser:
contentLength = int64(v.N)
default:
err = fmt.Errorf("can't get reader content length,unkown reader type")
}
return contentLength, err
}
func LimitReadCloser(r io.Reader, n int64) io.Reader {
var lc LimitedReadCloser
lc.R = r
lc.N = n
return &lc
}
// LimitedRC support Close()
type LimitedReadCloser struct {
io.LimitedReader
}
func (lc *LimitedReadCloser) Close() error {
if closer, ok := lc.R.(io.ReadCloser); ok {
return closer.Close()
}
return nil
}
type DiscardReadCloser struct {
RC io.ReadCloser
Discard int
}
func (drc *DiscardReadCloser) Read(b []byte) (int, error) {
n, err := drc.RC.Read(b)
if drc.Discard == 0 || n <= 0 {
return n, err
}
if n <= drc.Discard {
drc.Discard -= n
return 0, err
}
realLen := n - drc.Discard
copy(b[0:realLen], b[drc.Discard:n])
drc.Discard = 0
return realLen, err
}
func (drc *DiscardReadCloser) Close() error {
closer, ok := drc.RC.(io.ReadCloser)
if ok {
return closer.Close()
}
return nil
}
func ConvertEmptyValueToNil(params map[string]interface{}, keys []string) {
for _, key := range keys {
value, ok := params[key]
if ok && value == "" {
// convert "" to nil
params[key] = nil
}
}
}
func EscapeLFString(str string) string {
var log bytes.Buffer
for i := 0; i < len(str); i++ {
if str[i] != '\n' {
log.WriteByte(str[i])
} else {
log.WriteString("\\n")
}
}
return log.String()
}
// EscapeString writes to p the properly escaped XML equivalent
// of the plain text data s.
func EscapeXml(s string) string {
var p strings.Builder
var esc []byte
hextable := "0123456789ABCDEF"
escPattern := []byte("&#x00;")
last := 0
for i := 0; i < len(s); {
r, width := utf8.DecodeRuneInString(s[i:])
i += width
switch r {
case '"':
esc = escQuot
case '\'':
esc = escApos
case '&':
esc = escAmp
case '<':
esc = escLT
case '>':
esc = escGT
case '\t':
esc = escTab
case '\n':
esc = escNL
case '\r':
esc = escCR
default:
if !isInCharacterRange(r) || (r == 0xFFFD && width == 1) {
if r >= 0x00 && r < 0x20 {
escPattern[3] = hextable[r>>4]
escPattern[4] = hextable[r&0x0f]
esc = escPattern
} else {
esc = escFFFD
}
break
}
continue
}
p.WriteString(s[last : i-width])
p.Write(esc)
last = i
}
p.WriteString(s[last:])
return p.String()
}
// Decide whether the given rune is in the XML Character Range, per
// the Char production of https://www.xml.com/axml/testaxml.htm,
// Section 2.2 Characters.
func isInCharacterRange(r rune) (inrange bool) {
return r == 0x09 ||
r == 0x0A ||
r == 0x0D ||
r >= 0x20 && r <= 0xD7FF ||
r >= 0xE000 && r <= 0xFFFD ||
r >= 0x10000 && r <= 0x10FFFF
}