new: Improve the automatic update module, add authentication, and fix some bugs

config.yaml

@@ -1,6 +1,13 @@
 # app config #
 host: 0.0.0.0
 port: 8080
+cache_path: tmp
+
+# use this secret to update your posts
+update_endpoint: /update
+update_secret: 123456
+
+# configure object storage to automatically obtain article updates
 storage:
   src: _post/src
   dst: _post/dst
@@ -12,6 +19,8 @@ storage:
     secret_id:
     secret_key:
     save_path:
+
+# website template directory
 template: templates/default
 
 # site config #

config/mConfig.go

@@ -13,6 +13,9 @@ type MConfig struct {
 	Port           int      `yaml:"port"`
 	Storage        mStorage `yaml:"storage"`
 	Template       string   `yaml:"template"`
+	UpdateEndpoint string   `yaml:"update_endpoint"`
+	UpdateSecret   string   `yaml:"update_secret"`
+	CachePath      string   `yaml:"cache_path"`
 
 	MSite mSite `yaml:"site"`
 }

internal/mApp/mApp.go

@@ -1,11 +1,13 @@
 package mApp
 
 import (
-	"MollyBlog/config"
-	"MollyBlog/internal/model"
 	"fmt"
 	"html/template"
 	"log"
+	"os"
+
+	"MollyBlog/config"
+	"MollyBlog/internal/model"
 
 	"github.com/88250/lute"
 	"github.com/gin-gonic/gin"
@@ -88,7 +90,7 @@ func NewMApp(cfg *config.MConfig) *MApp {
 }
 
 // resetStorage before each update, delete the cache
-func (ma *MApp) resetStorage() {
+func (ma *MApp) resetStorage() error {
 	ma.Posts = nil
 	ma.Tags = nil
 	ma.Categories = nil
@@ -103,4 +105,42 @@ func (ma *MApp) resetStorage() {
 	ma.CategoriesCount = make(map[string]int)
 	ma.TaggedPosts = make(map[string][]*model.MPost)
 	ma.CategorizedPosts = make(map[string][]*model.MPost)
+
+	if ma.searcher != nil {
+		ma.searcher.Close()
+	}
+
+	var err error
+	err = os.RemoveAll(ma.Config.Storage.SRC)
+	if err != nil {
+		return err
+	}
+
+	_, err = os.Stat(ma.Config.Storage.SRC)
+	if os.IsNotExist(err) {
+		err = os.MkdirAll(ma.Config.Storage.SRC, os.ModePerm)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = os.RemoveAll(ma.Config.Storage.DST)
+	if err != nil {
+		return err
+	}
+
+	_, err = os.Stat(ma.Config.Storage.DST)
+	if os.IsNotExist(err) {
+		err = os.MkdirAll(ma.Config.Storage.DST, os.ModePerm)
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	err = os.RemoveAll(ma.Config.CachePath)
+	if err != nil {
+		return err
+	}
+
+	return nil
 }

internal/mApp/mHandler.go

@@ -404,7 +404,12 @@ func (ma *MApp) SearchHandler(ctx *gin.Context) {
 
 func (ma *MApp) UpdateBlogHandler(ctx *gin.Context) {
 	var err error
-	ma.resetStorage()
+	err = ma.resetStorage()
+	if err != nil {
+		log.Printf("reset storage error: %v\n", err)
+		_ = ctx.Error(err)
+		return
+	}
 
 	err = ma.loadMarkdownFiles()
 	if err != nil {

internal/mApp/mMarkdown.go

@@ -1,7 +1,6 @@
 package mApp
 
 import (
-	"MollyBlog/internal/storage"
 	"errors"
 	"fmt"
 	"io"
@@ -9,6 +8,7 @@ import (
 	"path/filepath"
 
 	"MollyBlog/internal/model"
+	"MollyBlog/internal/storage"
 	"MollyBlog/utils"
 
 	"gopkg.in/yaml.v3"

internal/mApp/mMiddleware.go

@@ -0,0 +1,21 @@
+package mApp
+
+import (
+	"net/http"
+
+	"MollyBlog/config"
+	
+	"github.com/gin-gonic/gin"
+)
+
+func (ma *MApp) AuthMiddleware(ctx *gin.Context) {
+	if ctx.Request.URL.Path == config.MConfigInstance.UpdateEndpoint {
+		secret := ctx.GetHeader("molly-secret")
+		if secret != config.MConfigInstance.UpdateSecret {
+			ctx.AbortWithStatus(http.StatusForbidden)
+			return
+		}
+	}
+
+	ctx.Next()
+}

internal/mApp/mRouter.go

@@ -1,6 +1,8 @@
 package mApp
 
 func (ma *MApp) loadRoutes() {
+	ma.engine.Use(ma.AuthMiddleware)
+
 	ma.engine.GET("/", ma.IndexHandler)
 	ma.engine.GET("/search", ma.SearchHandler)
 	ma.engine.GET("/archive", ma.ArchiveHandler)
@@ -8,5 +10,6 @@ func (ma *MApp) loadRoutes() {
 	ma.engine.GET("/tag/:hash", ma.TagHandler)
 	ma.engine.GET("/category/:hash", ma.CategoryHandler)
 
-	ma.engine.PUT("/update", ma.UpdateBlogHandler)
+	// update's endpoint
+	ma.engine.POST(ma.Config.UpdateEndpoint, ma.UpdateBlogHandler)
 }

internal/mApp/mSearch.go

@@ -1,6 +1,7 @@
 package mApp
 
 import (
+	"fmt"
 	"io"
 	"os"
 
@@ -9,13 +10,15 @@ import (
 )
 
 func (ma *MApp) loadPostIndex() {
+	cachePath := fmt.Sprintf("%s/persistent", ma.Config.CachePath)
+	ma.searcher = nil
 	ma.searcher = &engine.Engine{}
 
 	ma.searcher.Init(types.EngineInitOptions{
 		UsePersistentStorage:    true,
-		PersistentStorageFolder: "tmp",
+		PersistentStorageFolder: cachePath,
-		StopTokenFile:           "data/stop_tokens.txt",
+		StopTokenFile:           "data/search/stop_tokens.txt",
-		SegmenterDictionaries:   "data/dictionary.txt",
+		SegmenterDictionaries:   "data/search/dictionary.txt",
 		IndexerInitOptions: &types.IndexerInitOptions{
 			IndexType: types.LocationsIndex,
 		},
@@ -26,6 +29,7 @@ func (ma *MApp) loadPostIndex() {
 		postData, _ := io.ReadAll(postFile)
 
 		ma.searcher.IndexDocument(post.Index, types.DocumentIndexData{Content: string(postData)}, false)
+		_ = postFile.Close()
 	}
 
 	ma.searcher.FlushIndex()

internal/storage/cos.go

@@ -53,7 +53,6 @@ func CosLoadMarkdowns(config config.MConfig, localDir string) error {
 	prefix := config.Storage.COS.SavePath
 	encodingType := "url"
 	
-	log.Println("start load markdown files")
 	isTruncated := true
 	for isTruncated {
 		opt := &cos.BucketGetOptions{

main.go

@@ -9,17 +9,20 @@ import (
 )
 
 func init() {
-	_, err := os.Stat(mApp.SRC)
+	src := config.MConfigInstance.Storage.SRC
+	dst := config.MConfigInstance.Storage.DST
+
+	_, err := os.Stat(src)
 	if os.IsNotExist(err) {
-		err = os.MkdirAll(mApp.SRC, os.ModePerm)
+		err = os.MkdirAll(src, os.ModePerm)
 		if err != nil {
 			log.Fatal(err)
 		}
 	}
 
-	_, err = os.Stat(mApp.DST)
+	_, err = os.Stat(dst)
 	if os.IsNotExist(err) {
-		err = os.MkdirAll(mApp.DST, os.ModePerm)
+		err = os.MkdirAll(dst, os.ModePerm)
 		if err != nil {
 			log.Fatal(err)
 		}