Storage configuration support [storage] (#13314)

* Fix minio bug

* Add tests for storage configuration

* Change the Seek flag to keep compitable minio?

* Fix test when first-byte-pos of all ranges is greater than the resource length

Co-authored-by: techknowlogick <techknowlogick@gitea.io>

Author: lunny

integrations/integration_test.go

@@ -11,7 +11,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"log"
 	"net/http"
 	"net/http/cookiejar"
 	"net/http/httptest"
@@ -27,8 +26,10 @@ import (
 	"code.gitea.io/gitea/models"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/graceful"
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/queue"
 	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/storage"
 	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/routers"
 	"code.gitea.io/gitea/routers/routes"
@@ -59,6 +60,8 @@ func NewNilResponseRecorder() *NilResponseRecorder {
 }
 
 func TestMain(m *testing.M) {
+	defer log.Close()
+
 	managerCtx, cancel := context.WithCancel(context.Background())
 	graceful.InitManager(managerCtx)
 	defer cancel()
@@ -142,34 +145,38 @@ func initIntegrationTest() {
 	util.RemoveAll(models.LocalCopyPath())
 	setting.CheckLFSVersion()
 	setting.InitDBConfig()
+	if err := storage.Init(); err != nil {
+		fmt.Printf("Init storage failed: %v", err)
+		os.Exit(1)
+	}
 
 	switch {
 	case setting.Database.UseMySQL:
 		db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s)/",
 			setting.Database.User, setting.Database.Passwd, setting.Database.Host))
 		defer db.Close()
 		if err != nil {
-			log.Fatalf("sql.Open: %v", err)
+			log.Fatal("sql.Open: %v", err)
 		}
 		if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s", setting.Database.Name)); err != nil {
-			log.Fatalf("db.Exec: %v", err)
+			log.Fatal("db.Exec: %v", err)
 		}
 	case setting.Database.UsePostgreSQL:
 		db, err := sql.Open("postgres", fmt.Sprintf("postgres://%s:%s@%s/?sslmode=%s",
 			setting.Database.User, setting.Database.Passwd, setting.Database.Host, setting.Database.SSLMode))
 		defer db.Close()
 		if err != nil {
-			log.Fatalf("sql.Open: %v", err)
+			log.Fatal("sql.Open: %v", err)
 		}
 		dbrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM pg_database WHERE datname = '%s'", setting.Database.Name))
 		if err != nil {
-			log.Fatalf("db.Query: %v", err)
+			log.Fatal("db.Query: %v", err)
 		}
 		defer dbrows.Close()
 
 		if !dbrows.Next() {
 			if _, err = db.Exec(fmt.Sprintf("CREATE DATABASE %s", setting.Database.Name)); err != nil {
-				log.Fatalf("db.Exec: CREATE DATABASE: %v", err)
+				log.Fatal("db.Exec: CREATE DATABASE: %v", err)
 			}
 		}
 		// Check if we need to setup a specific schema
@@ -183,18 +190,18 @@ func initIntegrationTest() {
 		// This is a different db object; requires a different Close()
 		defer db.Close()
 		if err != nil {
-			log.Fatalf("sql.Open: %v", err)
+			log.Fatal("sql.Open: %v", err)
 		}
 		schrows, err := db.Query(fmt.Sprintf("SELECT 1 FROM information_schema.schemata WHERE schema_name = '%s'", setting.Database.Schema))
 		if err != nil {
-			log.Fatalf("db.Query: %v", err)
+			log.Fatal("db.Query: %v", err)
 		}
 		defer schrows.Close()
 
 		if !schrows.Next() {
 			// Create and setup a DB schema
 			if _, err = db.Exec(fmt.Sprintf("CREATE SCHEMA %s", setting.Database.Schema)); err != nil {
-				log.Fatalf("db.Exec: CREATE SCHEMA: %v", err)
+				log.Fatal("db.Exec: CREATE SCHEMA: %v", err)
 			}
 		}
 
@@ -203,10 +210,10 @@ func initIntegrationTest() {
 		db, err := sql.Open("mssql", fmt.Sprintf("server=%s; port=%s; database=%s; user id=%s; password=%s;",
 			host, port, "master", setting.Database.User, setting.Database.Passwd))
 		if err != nil {
-			log.Fatalf("sql.Open: %v", err)
+			log.Fatal("sql.Open: %v", err)
 		}
 		if _, err := db.Exec(fmt.Sprintf("If(db_id(N'%s') IS NULL) BEGIN CREATE DATABASE %s; END;", setting.Database.Name, setting.Database.Name)); err != nil {
-			log.Fatalf("db.Exec: %v", err)
+			log.Fatal("db.Exec: %v", err)
 		}
 		defer db.Close()
 	}

integrations/lfs_getobject_test.go

@@ -78,6 +78,7 @@ func storeAndGetLfs(t *testing.T, content *[]byte, extraHeader *http.Header, exp
 			}
 		}
 	}
+
 	resp := session.MakeRequest(t, req, expectedStatus)
 
 	return resp
@@ -210,7 +211,7 @@ func TestGetLFSRange(t *testing.T) {
 		{"bytes=0-10", "123456789\n", http.StatusPartialContent},
 		// end-range bigger than length-1 is ignored
 		{"bytes=0-11", "123456789\n", http.StatusPartialContent},
-		{"bytes=11-", "", http.StatusPartialContent},
+		{"bytes=11-", "Requested Range Not Satisfiable", http.StatusRequestedRangeNotSatisfiable},
 		// incorrect header value cause whole header to be ignored
 		{"bytes=-", "123456789\n", http.StatusOK},
 		{"foobar", "123456789\n", http.StatusOK},

integrations/mysql.ini.tmpl

@@ -45,27 +45,28 @@ START_SSH_SERVER = true
 OFFLINE_MODE     = false
 
 LFS_START_SERVER = true
-LFS_CONTENT_PATH = integrations/gitea-integration-mysql/datalfs-mysql
 LFS_JWT_SECRET   = Tv_MjmZuHqpIY6GFl12ebgkRAMt4RlWt0v4EHKSXO0w
-LFS_STORE_TYPE = minio
-LFS_SERVE_DIRECT = false
-LFS_MINIO_ENDPOINT = minio:9000
-LFS_MINIO_ACCESS_KEY_ID = 123456
-LFS_MINIO_SECRET_ACCESS_KEY = 12345678
-LFS_MINIO_BUCKET = gitea
-LFS_MINIO_LOCATION = us-east-1
-LFS_MINIO_BASE_PATH = lfs/
-LFS_MINIO_USE_SSL = false
+
+[lfs]
+MINIO_BASE_PATH = lfs/
 
 [attachment]
+MINIO_BASE_PATH = attachments/
+
+[avatars]
+MINIO_BASE_PATH = avatars/
+
+[repo-avatars]
+MINIO_BASE_PATH = repo-avatars/
+
+[storage]
 STORAGE_TYPE = minio
 SERVE_DIRECT = false
 MINIO_ENDPOINT = minio:9000
 MINIO_ACCESS_KEY_ID = 123456
 MINIO_SECRET_ACCESS_KEY = 12345678
 MINIO_BUCKET = gitea
 MINIO_LOCATION = us-east-1
-MINIO_BASE_PATH = attachments/
 MINIO_USE_SSL = false
 
 [mailer]
@@ -88,9 +89,6 @@ ENABLE_NOTIFY_MAIL                = true
 DISABLE_GRAVATAR              = false
 ENABLE_FEDERATED_AVATAR       = false
 
-AVATAR_UPLOAD_PATH            = integrations/gitea-integration-mysql/data/avatars
-REPOSITORY_AVATAR_UPLOAD_PATH = integrations/gitea-integration-mysql/data/repo-avatars
-
 [session]
 PROVIDER        = file
 PROVIDER_CONFIG = integrations/gitea-integration-mysql/data/sessions

modules/lfs/content_store.go

@@ -8,6 +8,7 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"io"
 	"os"
 
@@ -21,6 +22,21 @@ var (
 	errSizeMismatch = errors.New("Content size does not match")
 )
 
+// ErrRangeNotSatisfiable represents an error which request range is not satisfiable.
+type ErrRangeNotSatisfiable struct {
+	FromByte int64
+}
+
+func (err ErrRangeNotSatisfiable) Error() string {
+	return fmt.Sprintf("Requested range %d is not satisfiable", err.FromByte)
+}
+
+// IsErrRangeNotSatisfiable returns true if the error is an ErrRangeNotSatisfiable
+func IsErrRangeNotSatisfiable(err error) bool {
+	_, ok := err.(ErrRangeNotSatisfiable)
+	return ok
+}
+
 // ContentStore provides a simple file system based storage.
 type ContentStore struct {
 	storage.ObjectStorage
@@ -35,7 +51,12 @@ func (s *ContentStore) Get(meta *models.LFSMetaObject, fromByte int64) (io.ReadC
 		return nil, err
 	}
 	if fromByte > 0 {
-		_, err = f.Seek(fromByte, os.SEEK_CUR)
+		if fromByte >= meta.Size {
+			return nil, ErrRangeNotSatisfiable{
+				FromByte: fromByte,
+			}
+		}
+		_, err = f.Seek(fromByte, io.SeekStart)
 		if err != nil {
 			log.Error("Whilst trying to read LFS OID[%s]: Unable to seek to %d Error: %v", meta.Oid, fromByte, err)
 		}

modules/lfs/server.go

@@ -191,8 +191,12 @@ func getContentHandler(ctx *context.Context) {
 	contentStore := &ContentStore{ObjectStorage: storage.LFS}
 	content, err := contentStore.Get(meta, fromByte)
 	if err != nil {
-		// Errors are logged in contentStore.Get
-		writeStatus(ctx, 404)
+		if IsErrRangeNotSatisfiable(err) {
+			writeStatus(ctx, http.StatusRequestedRangeNotSatisfiable)
+		} else {
+			// Errors are logged in contentStore.Get
+			writeStatus(ctx, 404)
+		}
 		return
 	}
 	defer content.Close()

modules/setting/storage.go

@@ -32,14 +32,12 @@ func (s *Storage) MapTo(v interface{}) error {
 }
 
 func getStorage(name, typ string, overrides ...*ini.Section) Storage {
-	sectionName := "storage"
-	if len(name) > 0 {
-		sectionName = sectionName + "." + typ
-	}
+	const sectionName = "storage"
 	sec := Cfg.Section(sectionName)
 
 	if len(overrides) == 0 {
 		overrides = []*ini.Section{
+			Cfg.Section(sectionName + "." + typ),
 			Cfg.Section(sectionName + "." + name),
 		}
 	}