fix(etc): move etc files into data

Signed-off-by: Jianxiang Ran <rxan_embedded@163.com>

internal/plugin/native/manager.go

@@ -57,8 +57,8 @@ type Manager struct {
 	// loaded symbols in current runtime
 	runtime map[string]*plugin.Plugin
 	// dirs
-	pluginDir string
-	etcDir    string
+	pluginDir     string
+	pluginConfDir string
 	// the access to db
 	db kv.KeyValue
 }
@@ -69,7 +69,7 @@ func InitManager() (*Manager, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot find plugins folder: %s", err)
 	}
-	etcDir, err := conf.GetConfLoc()
+	dataDir, err := conf.GetDataLoc()
 	if err != nil {
 		return nil, fmt.Errorf("cannot find etc folder: %s", err)
 	}
@@ -77,7 +77,7 @@ func InitManager() (*Manager, error) {
 	if err != nil {
 		return nil, fmt.Errorf("error when opening db: %v", err)
 	}
-	registry := &Manager{symbols: make(map[string]string), db: db, pluginDir: pluginDir, etcDir: etcDir, runtime: make(map[string]*plugin.Plugin)}
+	registry := &Manager{symbols: make(map[string]string), db: db, pluginDir: pluginDir, pluginConfDir: dataDir, runtime: make(map[string]*plugin.Plugin)}
 	manager = registry
 	plugins := make([]map[string]string, 3)
 	for i := range plugin2.PluginTypes {
@@ -282,15 +282,15 @@ func (rr *Manager) Register(t plugin2.PluginType, j plugin2.Plugin) error {
 
 	switch t {
 	case plugin2.SINK:
-		if err := meta.ReadSinkMetaFile(path.Join(rr.etcDir, plugin2.PluginTypes[t], name+`.json`), true); nil != err {
+		if err := meta.ReadSinkMetaFile(path.Join(rr.pluginConfDir, plugin2.PluginTypes[t], name+`.json`), true); nil != err {
 			conf.Log.Errorf("readSinkFile:%v", err)
 		}
 	case plugin2.SOURCE:
-		if err := meta.ReadSourceMetaFile(path.Join(rr.etcDir, plugin2.PluginTypes[t], name+`.json`), true); nil != err {
+		if err := meta.ReadSourceMetaFile(path.Join(rr.pluginConfDir, plugin2.PluginTypes[t], name+`.json`), true); nil != err {
 			conf.Log.Errorf("readSourceFile:%v", err)
 		}
 	case plugin2.FUNCTION:
-		if err := meta.ReadFuncMetaFile(path.Join(rr.etcDir, plugin2.PluginTypes[t], name+`.json`), true); nil != err {
+		if err := meta.ReadFuncMetaFile(path.Join(rr.pluginConfDir, plugin2.PluginTypes[t], name+`.json`), true); nil != err {
 			conf.Log.Errorf("readFuncFile:%v", err)
 		}
 	}
@@ -331,7 +331,7 @@ func (rr *Manager) Delete(t plugin2.PluginType, name string, stop bool) error {
 		soPath,
 	}
 	// Find etc folder
-	etcPath := path.Join(rr.etcDir, plugin2.PluginTypes[t], name)
+	etcPath := path.Join(rr.pluginConfDir, plugin2.PluginTypes[t], name)
 	if fi, err := os.Stat(etcPath); err == nil {
 		if fi.Mode().IsDir() {
 			paths = append(paths, etcPath)
@@ -339,16 +339,16 @@ func (rr *Manager) Delete(t plugin2.PluginType, name string, stop bool) error {
 	}
 	switch t {
 	case plugin2.SOURCE:
-		paths = append(paths, path.Join(rr.etcDir, plugin2.PluginTypes[t], name+".yaml"))
-		srcJsonPath := path.Join(rr.etcDir, plugin2.PluginTypes[plugin2.SOURCE], name+".json")
+		paths = append(paths, path.Join(rr.pluginConfDir, plugin2.PluginTypes[t], name+".yaml"))
+		srcJsonPath := path.Join(rr.pluginConfDir, plugin2.PluginTypes[plugin2.SOURCE], name+".json")
 		_ = os.RemoveAll(srcJsonPath)
 		meta.UninstallSource(name)
 	case plugin2.SINK:
-		sinkJsonPaths := path.Join(rr.etcDir, plugin2.PluginTypes[plugin2.SINK], name+".json")
+		sinkJsonPaths := path.Join(rr.pluginConfDir, plugin2.PluginTypes[plugin2.SINK], name+".json")
 		_ = os.RemoveAll(sinkJsonPaths)
 		meta.UninstallSink(name)
 	case plugin2.FUNCTION:
-		funcJsonPath := path.Join(rr.etcDir, plugin2.PluginTypes[plugin2.FUNCTION], name+".json")
+		funcJsonPath := path.Join(rr.pluginConfDir, plugin2.PluginTypes[plugin2.FUNCTION], name+".json")
 		_ = os.RemoveAll(funcJsonPath)
 		old := make([]string, 0)
 		if ok, err := rr.db.Get(name, &old); err != nil {
@@ -439,7 +439,7 @@ func (rr *Manager) install(t plugin2.PluginType, name, src string, shellParas []
 	expFiles := 1
 	if t == plugin2.SOURCE {
 		yamlFile = name + ".yaml"
-		yamlPath = path.Join(rr.etcDir, plugin2.PluginTypes[t], yamlFile)
+		yamlPath = path.Join(rr.pluginConfDir, plugin2.PluginTypes[t], yamlFile)
 		expFiles = 2
 	}
 	var revokeFiles []string
@@ -460,7 +460,7 @@ func (rr *Manager) install(t plugin2.PluginType, name, src string, shellParas []
 			revokeFiles = append(revokeFiles, yamlPath)
 			filenames = append(filenames, yamlPath)
 		} else if fileName == name+".json" {
-			jsonPath := path.Join(rr.etcDir, plugin2.PluginTypes[t], fileName)
+			jsonPath := path.Join(rr.pluginConfDir, plugin2.PluginTypes[t], fileName)
 			if err := filex.UnzipTo(file, jsonPath); nil != err {
 				conf.Log.Errorf("Failed to decompress the metadata %s file", fileName)
 			} else {
@@ -476,7 +476,7 @@ func (rr *Manager) install(t plugin2.PluginType, name, src string, shellParas []
 			revokeFiles = append(revokeFiles, soPath)
 			soName, version = parseName(fileName)
 		} else if strings.HasPrefix(fileName, "etc/") {
-			err = filex.UnzipTo(file, path.Join(rr.etcDir, plugin2.PluginTypes[t], strings.Replace(fileName, "etc", name, 1)))
+			err = filex.UnzipTo(file, path.Join(rr.pluginConfDir, plugin2.PluginTypes[t], strings.Replace(fileName, "etc", name, 1)))
 			if err != nil {
 				return version, err
 			}

internal/plugin/native/manager_test.go

@@ -141,7 +141,7 @@ func TestManager_Register(t *testing.T) {
 		if !reflect.DeepEqual(tt.err, err) {
 			t.Errorf("%d: error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.err, err)
 		} else if tt.err == nil {
-			err := checkFile(manager.pluginDir, manager.etcDir, tt.t, tt.n, tt.v, tt.lowerSo)
+			err := checkFile(manager.pluginDir, manager.pluginConfDir, tt.t, tt.n, tt.v, tt.lowerSo)
 			if err != nil {
 				t.Errorf("%d: error : %s\n\n", i, err)
 			}

internal/plugin/portable/manager.go

@@ -38,9 +38,9 @@ import (
 var manager *Manager
 
 type Manager struct {
-	pluginDir string
-	etcDir    string
-	reg       *registry // can be replaced with kv
+	pluginDir     string
+	pluginConfDir string
+	reg           *registry // can be replaced with kv
 }
 
 // InitManager must only be called once
@@ -49,7 +49,7 @@ func InitManager() (*Manager, error) {
 	if err != nil {
 		return nil, fmt.Errorf("cannot find plugins folder: %s", err)
 	}
-	etcDir, err := conf.GetConfLoc()
+	dataDir, err := conf.GetDataLoc()
 	if err != nil {
 		return nil, fmt.Errorf("cannot find etc folder: %s", err)
 	}
@@ -63,9 +63,9 @@ func InitManager() (*Manager, error) {
 	// Read plugin info from file system
 	pluginDir = filepath.Join(pluginDir, "portable")
 	m := &Manager{
-		pluginDir: pluginDir,
-		etcDir:    etcDir,
-		reg:       registry,
+		pluginDir:     pluginDir,
+		pluginConfDir: dataDir,
+		reg:           registry,
 	}
 	err = m.syncRegistry()
 	if err != nil {
@@ -133,17 +133,17 @@ func (m *Manager) doRegister(name string, pi *PluginInfo, isInit bool) error {
 
 	if !isInit {
 		for _, s := range pi.Sources {
-			if err := meta.ReadSourceMetaFile(path.Join(m.etcDir, plugin.PluginTypes[plugin.SOURCE], s+`.json`), true); nil != err {
+			if err := meta.ReadSourceMetaFile(path.Join(m.pluginConfDir, plugin.PluginTypes[plugin.SOURCE], s+`.json`), true); nil != err {
 				conf.Log.Errorf("read source json file:%v", err)
 			}
 		}
 		for _, s := range pi.Sinks {
-			if err := meta.ReadSinkMetaFile(path.Join(m.etcDir, plugin.PluginTypes[plugin.SINK], s+`.json`), true); nil != err {
+			if err := meta.ReadSinkMetaFile(path.Join(m.pluginConfDir, plugin.PluginTypes[plugin.SINK], s+`.json`), true); nil != err {
 				conf.Log.Errorf("read sink json file:%v", err)
 			}
 		}
 		for _, s := range pi.Functions {
-			if err := meta.ReadFuncMetaFile(path.Join(m.etcDir, plugin.PluginTypes[plugin.FUNCTION], s+`.json`), true); nil != err {
+			if err := meta.ReadFuncMetaFile(path.Join(m.pluginConfDir, plugin.PluginTypes[plugin.FUNCTION], s+`.json`), true); nil != err {
 				conf.Log.Errorf("read function json file:%v", err)
 			}
 		}
@@ -270,7 +270,7 @@ func (m *Manager) install(name, src string, shellParas []string) (resultErr erro
 	for _, file := range r.File {
 		fileName := file.Name
 		if strings.HasPrefix(fileName, "sources/") || strings.HasPrefix(fileName, "sinks/") || strings.HasPrefix(fileName, "functions/") {
-			target = path.Join(m.etcDir, fileName)
+			target = path.Join(m.pluginConfDir, fileName)
 		} else {
 			target = path.Join(pluginTarget, fileName)
 			if fileName == "install.sh" {
@@ -354,19 +354,19 @@ func (m *Manager) Delete(name string) error {
 	m.reg.Delete(name)
 	// delete files and uninstall metas
 	for _, s := range pinfo.Sources {
-		p := path.Join(m.etcDir, plugin.PluginTypes[plugin.SOURCE], s+".yaml")
+		p := path.Join(m.pluginConfDir, plugin.PluginTypes[plugin.SOURCE], s+".yaml")
 		os.Remove(p)
-		p = path.Join(m.etcDir, plugin.PluginTypes[plugin.SOURCE], s+".json")
+		p = path.Join(m.pluginConfDir, plugin.PluginTypes[plugin.SOURCE], s+".json")
 		os.Remove(p)
 		meta.UninstallSource(s)
 	}
 	for _, s := range pinfo.Sinks {
-		p := path.Join(m.etcDir, plugin.PluginTypes[plugin.SINK], s+".json")
+		p := path.Join(m.pluginConfDir, plugin.PluginTypes[plugin.SINK], s+".json")
 		os.Remove(p)
 		meta.UninstallSink(s)
 	}
 	for _, s := range pinfo.Functions {
-		p := path.Join(m.etcDir, plugin.PluginTypes[plugin.FUNCTION], s+".json")
+		p := path.Join(m.pluginConfDir, plugin.PluginTypes[plugin.FUNCTION], s+".json")
 		os.Remove(p)
 		meta.UninstallFunc(s)
 	}

internal/plugin/portable/manager_test.go

@@ -83,7 +83,7 @@ func TestManager_Install(t *testing.T) {
 		if !reflect.DeepEqual(tt.err, err) {
 			t.Errorf("%d: error mismatch:\n  exp=%s\n  got=%s\n\n", i, tt.err, err)
 		} else if tt.err == nil {
-			err := checkFileForMirror(manager.pluginDir, manager.etcDir, true)
+			err := checkFileForMirror(manager.pluginDir, manager.pluginConfDir, true)
 			if err != nil {
 				t.Errorf("%d: error : %s\n\n", i, err)
 			}
@@ -164,7 +164,7 @@ func TestDelete(t *testing.T) {
 	if err != nil {
 		t.Errorf("delete plugin error: %v", err)
 	}
-	err = checkFileForMirror(manager.pluginDir, manager.etcDir, false)
+	err = checkFileForMirror(manager.pluginDir, manager.pluginConfDir, false)
 	if err != nil {
 		t.Errorf("error : %s\n\n", err)
 	}

internal/server/rest.go

@@ -96,17 +96,26 @@ func jsonResponse(i interface{}, w http.ResponseWriter, logger api.Logger) {
 	}
 }
 
-func createRestServer(ip string, port int, needToken bool) *http.Server {
-	// Create upload path for upload api
-	etcDir, err := conf.GetConfLoc()
+func createPaths() {
+	dataDir, err := conf.GetDataLoc()
 	if err != nil {
 		panic(err)
 	}
-	uploadDir = filepath.Join(etcDir, "uploads")
-	err = os.MkdirAll(uploadDir, os.ModePerm)
-	if err != nil {
-		panic(err)
+	dirs := []string{"sources", "sinks", "functions", "services", "services/schemas"}
+
+	for _, v := range dirs {
+		// Create dir if not exist
+		realDir := filepath.Join(dataDir, v)
+		if _, err := os.Stat(realDir); os.IsNotExist(err) {
+			if err := os.MkdirAll(realDir, os.ModePerm); err != nil {
+				panic(err)
+			}
+		}
 	}
+}
+
+func createRestServer(ip string, port int, needToken bool) *http.Server {
+	createPaths()
 
 	r := mux.NewRouter()
 	r.HandleFunc("/", rootHandler).Methods(http.MethodGet, http.MethodPost)

internal/service/manager.go

@@ -52,7 +52,7 @@ func InitManager() (*Manager, error) {
 	mutex.Lock()
 	defer mutex.Unlock()
 	if singleton == nil {
-		dir := "etc/services"
+		dir := "data/services"
 		if kconf.IsTesting {
 			dir = "service/test"
 		}

internal/service/schema.go

@@ -69,7 +69,7 @@ var ( //Do not call these directly, use the get methods
 
 func ProtoParser() *protoparse.Parser {
 	once.Do(func() {
-		dir := "etc/services/schemas/"
+		dir := "data/services/schemas/"
 		if kconf.IsTesting {
 			dir = "service/test/schemas/"
 		}