new: Automatically pull Markdown files from Tencent Cloud COS

config.yaml

@@ -1,6 +1,17 @@
 # app config #
 host: 0.0.0.0
 port: 8080
+storage:
+  src: _post/src
+  dst: _post/dst
+  type: COS
+  cos:
+    appid:
+    bucket:
+    region:
+    secret_id:
+    secret_key:
+    save_path:
 template: templates/default
 
 # site config #

config/mConfig.go

@@ -9,9 +9,10 @@ import (
 )
 
 type MConfig struct {
-	Host     string `yaml:"host"`
+	Host     string   `yaml:"host"`
-	Port     int    `yaml:"port"`
+	Port     int      `yaml:"port"`
-	Template string `yaml:"template"`
+	Storage  mStorage `yaml:"storage"`
+	Template string   `yaml:"template"`
 
 	MSite mSite `yaml:"site"`
 }

config/mStorage.go

@@ -0,0 +1,17 @@
+package config
+
+type mStorage struct {
+	SRC  string    `yaml:"src"`
+	DST  string    `yaml:"dst"`
+	Type string    `yaml:"type"`
+	COS  cosConfig `yaml:"cos"`
+}
+
+type cosConfig struct {
+	Appid     string `yaml:"appid"`
+	Bucket    string `yaml:"bucket"`
+	Region    string `yaml:"region"`
+	SecretId  string `yaml:"secret_id"`
+	SecretKey string `yaml:"secret_key"`
+	SavePath  string `yaml:"save_path"`
+}

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/88250/lute v1.7.6
 	github.com/gin-gonic/gin v1.10.0
 	github.com/huichen/wukong v0.0.0-20210824074240-ecbd39fa0b56
+	github.com/tencentyun/cos-go-sdk-v5 v0.7.59
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -15,6 +16,7 @@ require (
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bytedance/sonic v1.11.6 // indirect
 	github.com/bytedance/sonic/loader v0.1.1 // indirect
+	github.com/clbanning/mxj v1.8.4 // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
 	github.com/cloudwego/iasm v0.2.0 // indirect
 	github.com/cznic/fileutil v0.0.0-20181122101858-4d67cfea8c87 // indirect
@@ -32,6 +34,7 @@ require (
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.20.0 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/gopherjs/gopherjs v1.17.2 // indirect
 	github.com/huichen/murmur v0.0.0-20130808212358-e0489551cf51 // indirect
 	github.com/huichen/sego v0.0.0-20210824061530-c87651ea5c76 // indirect
@@ -39,8 +42,10 @@ require (
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/mozillazg/go-httpheader v0.2.1 // indirect
 	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect

go.sum

@@ -1,5 +1,6 @@
 github.com/88250/lute v1.7.6 h1:8F4i53POAR+oEkyClsdHYPU1A7Z8rdxfRfjarD+FL9w=
 github.com/88250/lute v1.7.6/go.mod h1:+wUqx/1kdFDbWtxn9LYJlaCOAeol2pjSO6w+WJTVQsg=
+github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
 github.com/adamzy/cedar-go v0.0.0-20170805034717-80a9c64b256d h1:ir/IFJU5xbja5UaBEQLjcvn7aAU01nqU/NUyOBEU+ew=
 github.com/adamzy/cedar-go v0.0.0-20170805034717-80a9c64b256d/go.mod h1:PRWNwWq0yifz6XDPZu48aSld8BWwBfr2JKB2bGWiEd4=
 github.com/adamzy/sego v0.0.0-20151004184924-5eab9a44f8e8/go.mod h1:KQxo+Xesl2wLJ3yJcX443KaoWzXpbPzU1GNRyE8kNEY=
@@ -11,6 +12,8 @@ github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc
 github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4=
 github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM=
 github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU=
+github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I=
+github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
 github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y=
 github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w=
 github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg=
@@ -53,11 +56,16 @@ github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBEx
 github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
 github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
+github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
 github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g=
 github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k=
 github.com/huichen/murmur v0.0.0-20130808212358-e0489551cf51 h1:86ZSBmeBzG7dWW2rx9imn5pVKgqe7YjWzZ9qhn4Z+9A=
@@ -78,11 +86,15 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs=
+github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ=
+github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
 github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
 github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -101,6 +113,10 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y=
+github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563/go.mod h1:uom4Nvi9W+Qkom0exYiJ9VWJjXwyxtPYTkKkaLMlfE0=
+github.com/tencentyun/cos-go-sdk-v5 v0.7.59 h1:bzB/0fj+gKCUvEfe+c4CNGkqTnrenxzTsLM2rMn3mHE=
+github.com/tencentyun/cos-go-sdk-v5 v0.7.59/go.mod h1:8+hG+mQMuRP/OIS9d83syAvXvrMj9HhkND6Q1fLghw0=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE=

internal/mApp/mApp.go

@@ -34,9 +34,9 @@ type MApp struct {
 	SrcFiles []model.MFileInfo
 }
 
-const (
+var (
-	SRC = "_post/src" // source markdown files
+	SRC = config.MConfigInstance.Storage.SRC // source markdown files
-	DST = "_post/dst" // destination html files
+	DST = config.MConfigInstance.Storage.DST // destination html files
 )
 
 func init() {

internal/mApp/mMarkdown.go

@@ -1,6 +1,8 @@
 package mApp
 
 import (
+	"MollyBlog/internal/storage"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -15,9 +17,23 @@ import (
 // loadMarkdownFiles load markdown source files
 func (ma *MApp) loadMarkdownFiles() error {
 	var markdownList []model.MFileInfo
+	var err error
 
-	markdownPath := SRC
+	markdownPath := ma.Config.Storage.SRC
-	err := filepath.Walk(markdownPath, func(path string, info os.FileInfo, err error) error {
+
+	switch ma.Config.Storage.Type {
+	case "COS":
+		err = storage.CosLoadMarkdowns(*ma.Config, markdownPath)
+		break
+	default:
+		return errors.New("Unsupported storage type: " + ma.Config.Storage.Type)
+	}
+
+	if err != nil {
+		return err
+	}
+
+	err = filepath.Walk(markdownPath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -47,7 +63,7 @@ func (ma *MApp) loadMarkdownFiles() error {
 
 // parseMarkdowns parse markdown files to html
 func (ma *MApp) parseMarkdowns() error {
-	htmlPath := DST
+	htmlPath := ma.Config.Storage.DST
 
 	for index, file := range ma.SrcFiles {
 		// read markdown file

internal/storage/cos.go

@@ -0,0 +1,91 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"strings"
+	"sync"
+
+	"MollyBlog/config"
+
+	"github.com/tencentyun/cos-go-sdk-v5"
+)
+
+func cosDownload(wg *sync.WaitGroup, c *cos.Client, keysCh <-chan []string) {
+	defer wg.Done()
+	for keys := range keysCh {
+		key := keys[0]
+		filename := keys[1]
+		_, err := c.Object.GetToFile(context.Background(), key, filename, nil)
+		if err != nil {
+			log.Println(err)
+		}
+
+		log.Println("download success: ", filename)
+	}
+}
+
+func CosLoadMarkdowns(config config.MConfig, localDir string) error {
+	u, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", config.Storage.COS.Bucket, config.Storage.COS.Region))
+	b := &cos.BaseURL{BucketURL: u}
+	c := cos.NewClient(b, &http.Client{
+		Transport: &cos.AuthorizationTransport{
+			SecretID:  config.Storage.COS.SecretId,
+			SecretKey: config.Storage.COS.SecretKey,
+		},
+	})
+
+	var wg sync.WaitGroup
+
+	keysCh := make(chan []string, 3)
+	threadPoolNum := 3
+	for i := 0; i < threadPoolNum; i++ {
+		wg.Add(1)
+		go cosDownload(&wg, c, keysCh)
+	}
+
+	marker := ""
+	prefix := config.Storage.COS.SavePath
+	encodingType := "url"
+
+	log.Println("start load markdown files")
+	isTruncated := true
+	for isTruncated {
+		opt := &cos.BucketGetOptions{
+			Prefix:       prefix,
+			Marker:       marker,
+			EncodingType: encodingType,
+		}
+
+		v, _, err := c.Bucket.Get(context.Background(), opt)
+		if err != nil {
+			return err
+		}
+
+		for _, c := range v.Contents {
+			key, _ := cos.DecodeURIComponent(c.Key)
+			localFile := path.Join(localDir, key)
+			if _, err := os.Stat(path.Dir(localFile)); err != nil && os.IsNotExist(err) {
+				_ = os.MkdirAll(path.Dir(localFile), os.ModePerm)
+			}
+			
+			if strings.HasSuffix(localFile, "/") {
+				continue
+			}
+
+			keysCh <- []string{key, localFile}
+		}
+		marker, _ = cos.DecodeURIComponent(v.NextMarker)
+		isTruncated = v.IsTruncated
+	}
+
+	close(keysCh)
+	wg.Wait()
+
+	return nil
+}

templates/default/html/archive.html

@@ -82,7 +82,7 @@
             </div>
 
             <!-- special info -->
-            <!-- <div class="col-lg-3 d-none d-lg-grid carousel slide special-info-area"></div>-->
+             <div class="col-lg-3 d-none d-lg-grid carousel slide special-info-area"></div>
 
         </div>
 

templates/default/html/index.html

@@ -81,7 +81,7 @@
             </div>
 
             <!-- special info -->
-            <!-- <div class="col-lg-3 d-none d-lg-grid carousel slide special-info-area"></div>-->
+             <div class="col-lg-3 d-none d-lg-grid carousel slide special-info-area"></div>
 
         </div>
     </div>