From 95f4496ab6cf81dc7cc6a2055c1694ce8b5e26fa Mon Sep 17 00:00:00 2001 From: yvling Date: Tue, 17 Dec 2024 22:12:12 +0800 Subject: [PATCH] new: Automatically pull Markdown files from Tencent Cloud COS --- config.yaml | 11 ++++ config/mConfig.go | 7 ++- config/mStorage.go | 17 ++++++ go.mod | 5 ++ go.sum | 16 +++++ internal/mApp/mApp.go | 6 +- internal/mApp/mMarkdown.go | 22 ++++++- internal/storage/cos.go | 91 +++++++++++++++++++++++++++++ templates/default/html/archive.html | 2 +- templates/default/html/index.html | 2 +- 10 files changed, 168 insertions(+), 11 deletions(-) create mode 100644 config/mStorage.go create mode 100644 internal/storage/cos.go diff --git a/config.yaml b/config.yaml index 8bb09a9..3016a75 100644 --- a/config.yaml +++ b/config.yaml @@ -1,6 +1,17 @@ # app config # host: 0.0.0.0 port: 8080 +storage: + src: _post/src + dst: _post/dst + type: COS + cos: + appid: + bucket: + region: + secret_id: + secret_key: + save_path: template: templates/default # site config # diff --git a/config/mConfig.go b/config/mConfig.go index 63c5953..bea98ea 100644 --- a/config/mConfig.go +++ b/config/mConfig.go @@ -9,9 +9,10 @@ import ( ) type MConfig struct { - Host string `yaml:"host"` - Port int `yaml:"port"` - Template string `yaml:"template"` + Host string `yaml:"host"` + Port int `yaml:"port"` + Storage mStorage `yaml:"storage"` + Template string `yaml:"template"` MSite mSite `yaml:"site"` } diff --git a/config/mStorage.go b/config/mStorage.go new file mode 100644 index 0000000..85d1f1b --- /dev/null +++ b/config/mStorage.go @@ -0,0 +1,17 @@ +package config + +type mStorage struct { + SRC string `yaml:"src"` + DST string `yaml:"dst"` + Type string `yaml:"type"` + COS cosConfig `yaml:"cos"` +} + +type cosConfig struct { + Appid string `yaml:"appid"` + Bucket string `yaml:"bucket"` + Region string `yaml:"region"` + SecretId string `yaml:"secret_id"` + SecretKey string `yaml:"secret_key"` + SavePath string `yaml:"save_path"` +} diff --git a/go.mod b/go.mod index c32d356..0647b68 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/88250/lute v1.7.6 github.com/gin-gonic/gin v1.10.0 github.com/huichen/wukong v0.0.0-20210824074240-ecbd39fa0b56 + github.com/tencentyun/cos-go-sdk-v5 v0.7.59 gopkg.in/yaml.v3 v3.0.1 ) @@ -15,6 +16,7 @@ require ( github.com/boltdb/bolt v1.3.1 // indirect github.com/bytedance/sonic v1.11.6 // indirect github.com/bytedance/sonic/loader v0.1.1 // indirect + github.com/clbanning/mxj v1.8.4 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/cznic/fileutil v0.0.0-20181122101858-4d67cfea8c87 // indirect @@ -32,6 +34,7 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.20.0 // indirect github.com/goccy/go-json v0.10.2 // indirect + github.com/google/go-querystring v1.0.0 // indirect github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/huichen/murmur v0.0.0-20130808212358-e0489551cf51 // indirect github.com/huichen/sego v0.0.0-20210824061530-c87651ea5c76 // indirect @@ -39,8 +42,10 @@ require ( github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/mapstructure v1.4.3 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/mozillazg/go-httpheader v0.2.1 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/twitchyliquid64/golang-asm v0.15.1 // indirect diff --git a/go.sum b/go.sum index 9d375a0..b405cf1 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,6 @@ github.com/88250/lute v1.7.6 h1:8F4i53POAR+oEkyClsdHYPU1A7Z8rdxfRfjarD+FL9w= github.com/88250/lute v1.7.6/go.mod h1:+wUqx/1kdFDbWtxn9LYJlaCOAeol2pjSO6w+WJTVQsg= +github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/adamzy/cedar-go v0.0.0-20170805034717-80a9c64b256d h1:ir/IFJU5xbja5UaBEQLjcvn7aAU01nqU/NUyOBEU+ew= github.com/adamzy/cedar-go v0.0.0-20170805034717-80a9c64b256d/go.mod h1:PRWNwWq0yifz6XDPZu48aSld8BWwBfr2JKB2bGWiEd4= github.com/adamzy/sego v0.0.0-20151004184924-5eab9a44f8e8/go.mod h1:KQxo+Xesl2wLJ3yJcX443KaoWzXpbPzU1GNRyE8kNEY= @@ -11,6 +12,8 @@ github.com/bytedance/sonic v1.11.6 h1:oUp34TzMlL+OY1OUWxHqsdkgC/Zfc85zGqw9siXjrc github.com/bytedance/sonic v1.11.6/go.mod h1:LysEHSvpvDySVdC2f87zGWf6CIKJcAvqab1ZaiQtds4= github.com/bytedance/sonic/loader v0.1.1 h1:c+e5Pt1k/cy5wMveRDyk2X4B9hF4g7an8N3zCYjJFNM= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= +github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= @@ -53,11 +56,16 @@ github.com/go-playground/validator/v10 v10.20.0 h1:K9ISHbSaI0lyB2eWMPJo+kOS/FBEx github.com/go-playground/validator/v10 v10.20.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/huichen/murmur v0.0.0-20130808212358-e0489551cf51 h1:86ZSBmeBzG7dWW2rx9imn5pVKgqe7YjWzZ9qhn4Z+9A= @@ -78,11 +86,15 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/mapstructure v1.4.3 h1:OVowDSCllw/YjdLkam3/sm7wEtOy59d8ndGgCcyj8cs= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= +github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -101,6 +113,10 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563/go.mod h1:uom4Nvi9W+Qkom0exYiJ9VWJjXwyxtPYTkKkaLMlfE0= +github.com/tencentyun/cos-go-sdk-v5 v0.7.59 h1:bzB/0fj+gKCUvEfe+c4CNGkqTnrenxzTsLM2rMn3mHE= +github.com/tencentyun/cos-go-sdk-v5 v0.7.59/go.mod h1:8+hG+mQMuRP/OIS9d83syAvXvrMj9HhkND6Q1fLghw0= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= github.com/ugorji/go/codec v1.2.12 h1:9LC83zGrHhuUA9l16C9AHXAqEV/2wBQ4nkvumAE65EE= diff --git a/internal/mApp/mApp.go b/internal/mApp/mApp.go index d6456e6..bb7d98f 100644 --- a/internal/mApp/mApp.go +++ b/internal/mApp/mApp.go @@ -34,9 +34,9 @@ type MApp struct { SrcFiles []model.MFileInfo } -const ( - SRC = "_post/src" // source markdown files - DST = "_post/dst" // destination html files +var ( + SRC = config.MConfigInstance.Storage.SRC // source markdown files + DST = config.MConfigInstance.Storage.DST // destination html files ) func init() { diff --git a/internal/mApp/mMarkdown.go b/internal/mApp/mMarkdown.go index c725984..fa82413 100644 --- a/internal/mApp/mMarkdown.go +++ b/internal/mApp/mMarkdown.go @@ -1,6 +1,8 @@ package mApp import ( + "MollyBlog/internal/storage" + "errors" "fmt" "io" "os" @@ -15,9 +17,23 @@ import ( // loadMarkdownFiles load markdown source files func (ma *MApp) loadMarkdownFiles() error { var markdownList []model.MFileInfo + var err error - markdownPath := SRC - err := filepath.Walk(markdownPath, func(path string, info os.FileInfo, err error) error { + markdownPath := ma.Config.Storage.SRC + + switch ma.Config.Storage.Type { + case "COS": + err = storage.CosLoadMarkdowns(*ma.Config, markdownPath) + break + default: + return errors.New("Unsupported storage type: " + ma.Config.Storage.Type) + } + + if err != nil { + return err + } + + err = filepath.Walk(markdownPath, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -47,7 +63,7 @@ func (ma *MApp) loadMarkdownFiles() error { // parseMarkdowns parse markdown files to html func (ma *MApp) parseMarkdowns() error { - htmlPath := DST + htmlPath := ma.Config.Storage.DST for index, file := range ma.SrcFiles { // read markdown file diff --git a/internal/storage/cos.go b/internal/storage/cos.go new file mode 100644 index 0000000..8157b11 --- /dev/null +++ b/internal/storage/cos.go @@ -0,0 +1,91 @@ +package storage + +import ( + "context" + "fmt" + "log" + "net/http" + "net/url" + "os" + "path" + "strings" + "sync" + + "MollyBlog/config" + + "github.com/tencentyun/cos-go-sdk-v5" +) + +func cosDownload(wg *sync.WaitGroup, c *cos.Client, keysCh <-chan []string) { + defer wg.Done() + for keys := range keysCh { + key := keys[0] + filename := keys[1] + _, err := c.Object.GetToFile(context.Background(), key, filename, nil) + if err != nil { + log.Println(err) + } + + log.Println("download success: ", filename) + } +} + +func CosLoadMarkdowns(config config.MConfig, localDir string) error { + u, _ := url.Parse(fmt.Sprintf("https://%s.cos.%s.myqcloud.com", config.Storage.COS.Bucket, config.Storage.COS.Region)) + b := &cos.BaseURL{BucketURL: u} + c := cos.NewClient(b, &http.Client{ + Transport: &cos.AuthorizationTransport{ + SecretID: config.Storage.COS.SecretId, + SecretKey: config.Storage.COS.SecretKey, + }, + }) + + var wg sync.WaitGroup + + keysCh := make(chan []string, 3) + threadPoolNum := 3 + for i := 0; i < threadPoolNum; i++ { + wg.Add(1) + go cosDownload(&wg, c, keysCh) + } + + marker := "" + prefix := config.Storage.COS.SavePath + encodingType := "url" + + log.Println("start load markdown files") + isTruncated := true + for isTruncated { + opt := &cos.BucketGetOptions{ + Prefix: prefix, + Marker: marker, + EncodingType: encodingType, + } + + v, _, err := c.Bucket.Get(context.Background(), opt) + if err != nil { + return err + } + + for _, c := range v.Contents { + key, _ := cos.DecodeURIComponent(c.Key) + localFile := path.Join(localDir, key) + if _, err := os.Stat(path.Dir(localFile)); err != nil && os.IsNotExist(err) { + _ = os.MkdirAll(path.Dir(localFile), os.ModePerm) + } + + if strings.HasSuffix(localFile, "/") { + continue + } + + keysCh <- []string{key, localFile} + } + marker, _ = cos.DecodeURIComponent(v.NextMarker) + isTruncated = v.IsTruncated + } + + close(keysCh) + wg.Wait() + + return nil +} diff --git a/templates/default/html/archive.html b/templates/default/html/archive.html index b9b989c..a4c2179 100644 --- a/templates/default/html/archive.html +++ b/templates/default/html/archive.html @@ -82,7 +82,7 @@ - +
diff --git a/templates/default/html/index.html b/templates/default/html/index.html index a88803c..6fd1d3b 100644 --- a/templates/default/html/index.html +++ b/templates/default/html/index.html @@ -81,7 +81,7 @@ - +