new: Automatically pull Markdown files from Tencent Cloud COS

internal/storage/cos.go

@@ -0,0 +1,91 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"sync"
+
+	"MollyBlog/config"
+
+	"github.com/tencentyun/cos-go-sdk-v5"
+)
+
+func cosDownload(wg *sync.WaitGroup, c *cos.Client, keysCh <-chan []string) {
+	defer wg.Done()
+	for keys := range keysCh {
+		key := keys[0]
+		filename := keys[1]
+		_, err := c.Object.GetToFile(context.Background(), key, filename, nil)
+		if err != nil {
+			log.Println(err)
+		}
+
+		log.Println("download success: ", filename)
+	}
+}
+
+func CosLoadMarkdowns(config config.MConfig, localDir string) error {
+	u, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", config.Storage.COS.Bucket, config.Storage.COS.Region))
+	b := &cos.BaseURL{BucketURL: u}
+	c := cos.NewClient(b, &http.Client{
+		Transport: &cos.AuthorizationTransport{
+			SecretID:  config.Storage.COS.SecretId,
+			SecretKey: config.Storage.COS.SecretKey,
+		},
+	})
+
+	var wg sync.WaitGroup
+
+	keysCh := make(chan []string, 3)
+	threadPoolNum := 3
+	for i := 0; i < threadPoolNum; i++ {
+		wg.Add(1)
+		go cosDownload(&wg, c, keysCh)
+	}
+
+	marker := ""
+	prefix := config.Storage.COS.SavePath
+	encodingType := "url"
+
+	log.Println("start load markdown files")
+	isTruncated := true
+	for isTruncated {
+		opt := &cos.BucketGetOptions{
+			Prefix:       prefix,
+			Marker:       marker,
+			EncodingType: encodingType,
+		}
+
+		v, _, err := c.Bucket.Get(context.Background(), opt)
+		if err != nil {
+			return err
+		}
+
+		for _, c := range v.Contents {
+			key, _ := cos.DecodeURIComponent(c.Key)
+			localFile := path.Join(localDir, key)
+			if _, err := os.Stat(path.Dir(localFile)); err != nil && os.IsNotExist(err) {
+				_ = os.MkdirAll(path.Dir(localFile), os.ModePerm)
+			}
+			
+			if strings.HasSuffix(localFile, "/") {
+				continue
+			}
+
+			keysCh <- []string{key, localFile}
+		}
+		marker, _ = cos.DecodeURIComponent(v.NextMarker)
+		isTruncated = v.IsTruncated
+	}
+
+	close(keysCh)
+	wg.Wait()
+
+	return nil
+}