fix(edgex): edgex yaml optional config can not update to json declare's field

Signed-off-by: Jianxiang Ran <rxan_embedded@163.com>

internal/conf/load.go

@@ -69,6 +69,26 @@ func LoadConfigFromPath(p string, c interface{}) error {
 	return mapstructure.Decode(configs, c)
 }
 
+func CorrectsConfigKeysByJson(configs map[string]interface{}, jsonFilePath string) error {
+	dir, err := GetConfLoc()
+	if err != nil {
+		return err
+	}
+	path := path.Join(dir, jsonFilePath)
+	m, err := loadJsonForYaml(path)
+	if err != nil {
+		return err
+	}
+	names, err := extractNamesFromProperties(m)
+	if err != nil {
+		return err
+	}
+
+	applyKeys(configs, names)
+
+	return nil
+}
+
 func getPrefix(p string) string {
 	_, file := path.Split(p)
 	return strings.ToUpper(strings.TrimSuffix(file, filepath.Ext(file)))
@@ -230,19 +250,30 @@ func extractNamesFromProperties(jsonMap map[string]interface{}) ([]string, error
 		return nil, fmt.Errorf("json map does not have properties value")
 	}
 	if propertiesAsMap, success := properties.(map[string]interface{}); success {
-		list := propertiesAsMap["default"]
-		if interfaceList, isList := list.([]interface{}); isList {
-			for _, element := range interfaceList {
-				if m, isMap := element.(map[string]interface{}); isMap {
-					n := m["name"]
-					if s, isString := n.(string); isString {
-						result = append(result, s)
-					}
-				}
-			}
-		}
+		re := extractNamesFromElement(propertiesAsMap)
+		result = append(result, re...)
 	} else {
 		return nil, fmt.Errorf("failed to cast to list of properties")
 	}
 	return result, nil
 }
+
+func extractNamesFromElement(jsonMap map[string]interface{}) []string {
+	result := make([]string, 0)
+	list := jsonMap["default"]
+	if interfaceList, isList := list.([]interface{}); isList {
+		for _, element := range interfaceList {
+			if m, isMap := element.(map[string]interface{}); isMap {
+				re := extractNamesFromElement(m)
+				result = append(result, re...)
+			}
+		}
+	} else {
+		n := jsonMap["name"]
+		if s, isString := n.(string); isString {
+			result = append(result, s)
+		}
+	}
+
+	return result
+}

internal/conf/load_test.go

@@ -66,6 +66,35 @@ func TestJsonCamelCase(t *testing.T) {
 	}
 }
 
+func TestNestedFields(t *testing.T) {
+	key := "EDGEX__DEFAULT__OPTIONAL__PASSWORD"
+	value := "password"
+
+	err := os.Setenv(key, value)
+	if err != nil {
+		t.Error(err)
+	}
+
+	const ConfigName = "sources/edgex.yaml"
+	c := make(map[string]interface{})
+	err = LoadConfigByName(ConfigName, &c)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if casted, success := c["default"].(map[string]interface{}); success {
+		if optional, ok := casted["optional"].(map[string]interface{}); ok {
+			if optional["Password"] != "password" {
+				t.Errorf("Password variable should set it to password")
+			}
+		} else {
+			t.Errorf("returned value does not contains map under 'optional' key")
+		}
+	} else {
+		t.Errorf("returned value does not contains map under 'Basic' key")
+	}
+}
+
 func TestKeysReplacement(t *testing.T) {
 	input := createRandomConfigMap()
 	expected := createExpectedRandomConfigMap()

internal/topo/connection/client_edgex.go

@@ -46,12 +46,18 @@ type EdgexConf struct {
 
 func (es *EdgexClient) CfgValidate(props map[string]interface{}) error {
 
-	c := &EdgexConf{}
-	err := cast.MapToStructStrict(props, c)
+	edgexJsonPath := "sources/edgex.json"
+	err := conf.CorrectsConfigKeysByJson(props, edgexJsonPath)
 	if err != nil {
 		return fmt.Errorf("read properties %v fail for connection selector %s with error: %v", props, es.selector.ConnSelectorCfg, err)
 	}
 
+	c := &EdgexConf{}
+	err = cast.MapToStructStrict(props, c)
+	if err != nil {
+		return fmt.Errorf("map config map to struct %v fail for connection selector %s with error: %v", props, es.selector.ConnSelectorCfg, err)
+	}
+
 	if c.Server == "" {
 		return fmt.Errorf("missing server property for connection selector %s", es.selector.ConnSelectorCfg)
 	}

internal/topo/connection/client_edgex_test.go

@@ -18,6 +18,7 @@ package connection
 
 import (
 	"github.com/edgexfoundry/go-mod-messaging/v2/pkg/types"
+	"reflect"
 	"testing"
 )
 
@@ -127,3 +128,39 @@ func TestEdgex_CfgValidate(t *testing.T) {
 		})
 	}
 }
+
+func TestEdgex_CorrectsConfigKey(t *testing.T) {
+	var props = map[string]interface{}{
+		"protocol": "tcp",
+		"server":   "127.0.0.1",
+		"port":     int64(1883),
+		"type":     "mqtt",
+		"optional": map[string]interface{}{
+			"clientid": "client1",
+			"username": "user1",
+			"password": "password",
+		},
+	}
+
+	es := &EdgexClient{
+		selector: &ConSelector{
+			ConnSelectorCfg: "testSelector",
+		},
+	}
+
+	err := es.CfgValidate(props)
+	if err != nil {
+		t.Errorf("Error %v", err)
+	}
+
+	expectOps := map[string]string{
+		"ClientId": "client1",
+		"Username": "user1",
+		"Password": "password",
+	}
+
+	if !reflect.DeepEqual(es.mbconf.Optional, expectOps) {
+		t.Errorf("CfgValidate() expect = %+v, actual %+v", expectOps, es.mbconf.Optional)
+	}
+
+}

internal/topo/node/node.go

@@ -170,6 +170,22 @@ func (o *defaultSinkNode) preprocess(data interface{}) (interface{}, bool) {
 	return data, false
 }
 
+func printable(m map[string]interface{}) map[string]interface{} {
+	printableMap := make(map[string]interface{})
+	for k, v := range m {
+		if strings.EqualFold(k, "password") {
+			printableMap[k] = "*"
+		} else {
+			if vm, ok := v.(map[string]interface{}); ok {
+				printableMap[k] = printable(vm)
+			} else {
+				printableMap[k] = v
+			}
+		}
+	}
+	return printableMap
+}
+
 func getSourceConf(ctx api.StreamContext, sourceType string, options *ast.Options) map[string]interface{} {
 	confkey := options.CONF_KEY
 	logger := ctx.GetLogger()
@@ -205,6 +221,6 @@ func getSourceConf(ctx api.StreamContext, sourceType string, options *ast.Option
 		f = "json"
 	}
 	props["format"] = strings.ToLower(f)
-	logger.Debugf("get conf for %s with conf key %s: %v", sourceType, confkey, props)
+	logger.Debugf("get conf for %s with conf key %s: %v", sourceType, confkey, printable(props))
 	return props
 }