Postee as api 2

Makefile

@@ -23,6 +23,9 @@ fmt :
 test :
 	go test -race -coverprofile=coverage.txt -covermode=atomic ./router ./msgservice ./dbservice ./formatting ./data ./regoservice ./routes
 
+lint :
+	golangci-lint run
+
 cover :
 	go test ./msgservice ./dbservice ./router ./formatting ./data ./regoservice ./routes -v -coverprofile=cover.out
 	go tool cover -html=cover.out

README.md

@@ -448,15 +448,18 @@ See [Postee UI](PosteeUI.md) for details how to setup the Postee UI.
 ## Misc
 
 ### Data Persistency
-The Postee container uses BoltDB to store information about previously scanned images.
+The Postee container uses BoltDB or PostgreSQL to store information about previously scanned images.
 This is used to prevent resending messages that were already sent before.
-The size of the database can grow over time. Every image that is saved in the database uses 20K of storage.  
-
-Postee supports ‘PATH_TO_DB’ environment variable to change the database directory. To use, set the ‘PATH_TO_DB’ environment variable to point to the database file, for example: PATH_TO_DB="./database/webhook.db". By default, the directory for the database file is “/server/database/webhook.db”.
+The size of the database can grow over time. Every image that is saved in the Bolt database uses 20K of storage.  
+The default Postee Database is BoltDb. 
+
+Postee supports ‘PATH_TO_DB’ environment variable to change the bolt database directory. To use, set the ‘PATH_TO_DB’ environment variable to point to the bolt database file, for example: PATH_TO_DB="./database/webhook.db". By default, the directory for the bolt database file is “/server/database/webhook.db”.
 
 If you would like to persist the database file between restarts of the Postee container, then you should
 use a persistent storage option to mount the "/server/database" directory of the container.
 The "deploy/kubernetes" directory in this project contains an example deployment that includes a basic Host Persistency.
+
+To use PostgreSQL set the 'POSTGRES_URL' environment variable to your [connection URI](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING). If you would like to connect 2 or more Postee to 1 PostgreSQL to use unique tenant name in postee config file.
 
 ### Using environment variables in Postee Configuration File   
 Postee supports use of environment variables for *Output* fields: **User**, **Password** and **Token**. Add preffix `$` to the environment variable name in the configuration file, for example:

router/integrations.go → data/integrations.go

@@ -1,4 +1,4 @@
-package router
+package data
 
 type OutputSettings struct {
 	Name            string            `json:"name,omitempty"`

router/template.go → data/template.go

@@ -1,4 +1,4 @@
-package router
+package data
 
 type Template struct {
 	Name               string `json:"name"`

router/tenants.go → data/tenants.go

@@ -1,10 +1,11 @@
-package router
+package data
 
 import (
 	"github.com/aquasecurity/postee/routes"
 )
 
 type TenantSettings struct {
+	Name            string              `json:"name,omitempty"`
 	AquaServer      string              `json:"aqua-server,omitempty"`
 	DBMaxSize       int                 `json:"max-db-size,omitempty"`
 	DBRemoveOldData int                 `json:"delete-old-data,omitempty"`

data/utils.go

@@ -8,3 +8,9 @@ func ClearField(source string) string {
 	re := regexp.MustCompile(`[[:cntrl:]]|[\x{FFFD}]`)
 	return re.ReplaceAllString(source, "")
 }
+
+func CopyStringArray(src []string) []string {
+	dst := make([]string, len(src))
+	copy(dst, src)
+	return dst
+}

data/utils_test.go

@@ -2,6 +2,8 @@ package data
 
 import (
 	"testing"
+
+	"github.com/stretchr/testify/assert"
 )
 
 func TestClearField(t *testing.T) {
@@ -19,3 +21,11 @@ func TestClearField(t *testing.T) {
 		}
 	}
 }
+
+func TestCopyStringArray(t *testing.T) {
+	src := []string{"a", "b", "c"}
+	dst := CopyStringArray(src)
+	dst[0] = "x"
+	assert.Equal(t, "a", src[0], "TestCopyStringArray")
+	assert.Equal(t, "x", dst[0], "TestCopyStringArray")
+}

dbservice/boltdb/actions.go

@@ -0,0 +1,49 @@
+package boltdb
+
+import (
+	"time"
+
+	"github.com/aquasecurity/postee/dbservice/dbparam"
+	bolt "go.etcd.io/bbolt"
+)
+
+func (boltDb *BoltDb) MayBeStoreMessage(message []byte, messageKey string, expired *time.Time) (wasStored bool, err error) {
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	db, err := bolt.Open(boltDb.DbPath, 0666, nil)
+	if err != nil {
+		return false, err
+	}
+	defer db.Close()
+
+	if err = Init(db, dbparam.DbBucketName); err != nil {
+		return false, err
+	}
+	if err = Init(db, dbparam.DbBucketExpiryDates); err != nil {
+		return false, err
+	}
+
+	currentValue, err := dbSelect(db, dbparam.DbBucketName, messageKey)
+	if err != nil {
+		return false, err
+	}
+
+	if currentValue != nil {
+		return false, nil
+	} else {
+		bMessageKey := []byte(messageKey)
+		err = dbInsert(db, dbparam.DbBucketName, bMessageKey, message)
+		if err != nil {
+			return false, err
+		}
+		if expired != nil {
+			err = dbInsert(db, dbparam.DbBucketExpiryDates, []byte(expired.Format(dbparam.DateFmt)), bMessageKey)
+			if err != nil {
+				return false, err
+			}
+		}
+		return true, nil
+	}
+
+}

dbservice/boltdb/boltdb.go

@@ -0,0 +1,49 @@
+package boltdb
+
+import (
+	"os"
+	"path/filepath"
+	"sync"
+)
+
+var (
+	mutex sync.Mutex
+)
+
+type BoltDb struct {
+	DbPath string
+}
+
+func NewBoltDb() *BoltDb {
+	return &BoltDb{
+		DbPath: "/server/database/webhooks.db",
+	}
+}
+
+func (boltDb *BoltDb) ChangeDbPath(newPath string) {
+	mutex.Lock()
+	boltDb.DbPath = newPath
+	mutex.Unlock()
+}
+
+func (boltDb *BoltDb) SetNewDbPath(newPath string) error {
+	if newPath != "" {
+		if _, err := os.Stat(newPath); err != nil {
+			if os.IsNotExist(err) {
+				err = os.MkdirAll(filepath.Dir(newPath), os.ModePerm)
+				if err != nil {
+					return err
+				}
+			} else {
+				return err
+			}
+		}
+		boltDb.ChangeDbPath(newPath)
+	}
+	return nil
+}
+
+// unimplemented
+func (boltDb *BoltDb) Close() error {
+	return nil
+}

dbservice/boltdb/changedbpath_test.go

@@ -0,0 +1,18 @@
+package boltdb
+
+import (
+	"testing"
+)
+
+func TestChangeDbPath(t *testing.T) {
+	boltDb := NewBoltDb()
+	testPath := "/tmp/test.db"
+	storedPath := boltDb.DbPath
+	boltDb.ChangeDbPath(testPath)
+	defer func() {
+		boltDb.ChangeDbPath(storedPath)
+	}()
+	if boltDb.DbPath != testPath {
+		t.Errorf("path is not configured correctly, expected: %s, got %s", testPath, boltDb.DbPath)
+	}
+}

dbservice/boltdb/checker.go

@@ -0,0 +1,94 @@
+package boltdb
+
+import (
+	"bytes"
+	"time"
+
+	"github.com/aquasecurity/postee/dbservice/dbparam"
+	"github.com/aquasecurity/postee/log"
+	bolt "go.etcd.io/bbolt"
+)
+
+func (boltDb *BoltDb) CheckSizeLimit() {
+	if dbparam.DbSizeLimit == 0 {
+		return
+	}
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	db, err := bolt.Open(boltDb.DbPath, 0666, nil)
+	if err != nil {
+		log.Logger.Errorf("CheckSizeLimit: Can't open db: %s", boltDb.DbPath)
+		return
+	}
+	defer db.Close()
+
+	if err := db.Update(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(dbparam.DbBucketName))
+		if b == nil {
+			return nil
+		}
+		c := b.Cursor()
+		size := 0
+		for k, v := c.First(); k != nil; k, v = c.Next() {
+			size += len(v)
+		}
+		if size > dbparam.DbSizeLimit {
+			return tx.DeleteBucket([]byte(dbparam.DbBucketName))
+		}
+		return nil
+	}); err != nil {
+		log.Logger.Errorf("Unable to delete bucket: %v", err)
+		return
+	}
+}
+
+func (boltDb *BoltDb) CheckExpiredData() {
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	db, err := bolt.Open(boltDb.DbPath, 0666, nil)
+	if err != nil {
+		log.Logger.Errorf("CheckExpiredData: Can't open db: %s", boltDb.DbPath)
+		return
+	}
+	defer db.Close()
+
+	expired, err := boltDb.getExpired(db)
+	if err != nil {
+		log.Logger.Errorf("Can't select expired data: %v", err)
+		return
+	}
+
+	if err := dbDelete(db, dbparam.DbBucketName, expired); err != nil {
+		log.Logger.Errorf("Can't remove expired data: %v", err)
+	}
+}
+
+func (boltDb *BoltDb) getExpired(db *bolt.DB) (keys [][]byte, err error) {
+	keys = [][]byte{}
+	ttlKeys := [][]byte{}
+
+	if err = db.View(func(tx *bolt.Tx) error {
+		b := tx.Bucket([]byte(dbparam.DbBucketExpiryDates))
+		if b == nil {
+			return nil
+		}
+		c := b.Cursor()
+
+		max := []byte(time.Now().UTC().Format(dbparam.DateFmt)) //remove expired records
+		for k, v := c.First(); k != nil && bytes.Compare(k, max) <= 0; k, v = c.Next() {
+			keys = append(keys, v)
+			ttlKeys = append(ttlKeys, k)
+		}
+		return nil
+	}); err != nil {
+		return nil, err
+	}
+
+	if err = dbDelete(db, dbparam.DbBucketExpiryDates, ttlKeys); err != nil {
+		return nil, err
+	}
+
+	return
+}