ekuiper-new/extensions/sinks/sql/sql.go

// Copyright 2022-2023 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"encoding/json"
	"fmt"
	"reflect"
	"strings"

	"github.com/lf-edge/ekuiper/extensions/sqldatabase"
	"github.com/lf-edge/ekuiper/extensions/sqldatabase/driver"
	"github.com/lf-edge/ekuiper/extensions/util"
	"github.com/lf-edge/ekuiper/internal/topo/transform"
	"github.com/lf-edge/ekuiper/pkg/api"
	"github.com/lf-edge/ekuiper/pkg/ast"
	"github.com/lf-edge/ekuiper/pkg/cast"
	"github.com/lf-edge/ekuiper/pkg/errorx"
)

type sqlConfig struct {
	Url            string   `json:"url"`
	Table          string   `json:"table"`
	Fields         []string `json:"fields"`
	DataTemplate   string   `json:"dataTemplate"`
	TableDataField string   `json:"tableDataField"`
	DataField      string   `json:"dataField"`
	RowkindField   string   `json:"rowkindField"`
	KeyField       string   `json:"keyField"`
}

func (t *sqlConfig) buildInsertSql(ctx api.StreamContext, mapData map[string]interface{}) ([]string, string, error) {
	keys, vals, err := t.getKeyValues(ctx, mapData)
	if err != nil {
		return keys, "", err
	}
	if strings.HasPrefix(strings.ToLower(t.Url), "dm://") {
		for i, key := range keys {
			keys[i] = fmt.Sprintf(`"%v"`, key)
		}
	}
	sqlStr := "(" + strings.Join(vals, ",") + ")"
	return keys, sqlStr, nil
}

func (t *sqlConfig) getKeyValues(ctx api.StreamContext, mapData map[string]interface{}) ([]string, []string, error) {
	if 0 == len(mapData) {
		return nil, nil, fmt.Errorf("data is empty.")
	}
	logger := ctx.GetLogger()
	var keys, vals []string

	if len(t.Fields) != 0 {
		for _, k := range t.Fields {
			keys = append(keys, k)
			if v, ok := mapData[k]; ok && v != nil {
				if reflect.String == reflect.TypeOf(v).Kind() {
					vals = append(vals, fmt.Sprintf("'%v'", v))
				} else {
					vals = append(vals, fmt.Sprintf(`%v`, v))
				}
			} else {
				logger.Warnln("not found field:", k)
				vals = append(vals, fmt.Sprintf(`NULL`))
			}
		}
	} else {
		for k, v := range mapData {
			keys = append(keys, k)
			if reflect.String == reflect.TypeOf(v).Kind() {
				vals = append(vals, fmt.Sprintf("'%v'", v))
			} else {
				vals = append(vals, fmt.Sprintf(`%v`, v))
			}
		}
	}
	return keys, vals, nil
}

type sqlSink struct {
	conf *sqlConfig
	// The db connection instance
	db sqldatabase.DB
}

func (m *sqlSink) Configure(props map[string]interface{}) error {
	cfg := &sqlConfig{}
	err := cast.MapToStruct(props, cfg)
	if err != nil {
		return fmt.Errorf("read properties %v fail with error: %v", props, err)
	}
	if cfg.DataField == "" {
		cfg.DataField = cfg.TableDataField
	}
	if cfg.Url == "" {
		return fmt.Errorf("property Url is required")
	}
	if cfg.Table == "" {
		return fmt.Errorf("property Table is required")
	}
	if cfg.RowkindField != "" && cfg.KeyField == "" {
		return fmt.Errorf("keyField is required when rowkindField is set")
	}
	m.conf = cfg
	return nil
}

func (m *sqlSink) Open(ctx api.StreamContext) (err error) {
	logger := ctx.GetLogger()
	logger.Debugf("Opening sql sink")
	db, err := util.FetchDBToOneNode(util.GlobalPool, m.conf.Url)
	if err != nil {
		logger.Errorf("support build tags are %v", driver.KnownBuildTags())
		return err
	}
	m.db = db
	return
}

func (m *sqlSink) writeToDB(ctx api.StreamContext, sqlStr *string) error {
	ctx.GetLogger().Debugf(*sqlStr)
	r, err := m.db.Exec(*sqlStr)
	if err != nil {
		return fmt.Errorf("%s: %s", errorx.IOErr, err.Error())
	}
	d, err := r.RowsAffected()
	if err != nil {
		ctx.GetLogger().Errorf("get rows affected error: %s", err.Error())
	}
	ctx.GetLogger().Debugf("Rows affected: %d", d)
	return nil
}

func (m *sqlSink) Collect(ctx api.StreamContext, item interface{}) error {
	ctx.GetLogger().Debugf("sql sink receive %s", item)
	if m.conf.DataTemplate != "" {
		jsonBytes, _, err := ctx.TransformOutput(item)
		if err != nil {
			return err
		}
		tm := make(map[string]interface{})
		err = json.Unmarshal(jsonBytes, &tm)
		if err != nil {
			return fmt.Errorf("fail to decode data %s after applying dataTemplate for error %v", string(jsonBytes), err)
		}
		item = tm
	} else {
		tm, _, err := transform.TransItem(item, m.conf.DataField, m.conf.Fields)
		if err != nil {
			return fmt.Errorf("fail to 1 transform data %v for error %v!!!!!", item, err)
		}
		item = tm
	}

	var (
		table string
		err   error
	)
	switch v := item.(type) {
	case map[string]interface{}:
		table, err = ctx.ParseTemplate(m.conf.Table, v)
		if err != nil {
			ctx.GetLogger().Errorf("parse template for table %s error: %v", m.conf.Table, err)
			return err
		}
	case []map[string]interface{}:
		if len(v) == 0 {
			ctx.GetLogger().Warnf("empty data array")
			return nil
		}
		table, err = ctx.ParseTemplate(m.conf.Table, v[0])
		if err != nil {
			ctx.GetLogger().Errorf("parse template for table %s error: %v", m.conf.Table, err)
			return err
		}
	}

	var keys []string = nil
	var values []string = nil
	var vars string

	if m.conf.RowkindField == "" {
		switch v := item.(type) {
		case []map[string]interface{}:
			for _, mapData := range v {
				keys, vars, err = m.conf.buildInsertSql(ctx, mapData)
				if err != nil {
					return err
				}
				values = append(values, vars)
			}
			if keys != nil {
				sqlStr := fmt.Sprintf("INSERT INTO %s (%s) values ", table, strings.Join(keys, ",")) + strings.Join(values, ",") + ";"
				return m.writeToDB(ctx, &sqlStr)
			}
			return nil
		case map[string]interface{}:
			keys, vars, err = m.conf.buildInsertSql(ctx, v)
			if err != nil {
				return err
			}
			values = append(values, vars)
			if keys != nil {
				sqlStr := fmt.Sprintf("INSERT INTO %s (%s) values ", table, strings.Join(keys, ",")) + strings.Join(values, ",") + ";"
				return m.writeToDB(ctx, &sqlStr)
			}
			return nil
		case []interface{}:
			for _, data := range v {
				mapData, ok := data.(map[string]interface{})
				if !ok {
					ctx.GetLogger().Errorf("unsupported type: %T", data)
					return fmt.Errorf("unsupported type: %T", data)
				}

				keys, vars, err = m.conf.buildInsertSql(ctx, mapData)
				if err != nil {
					ctx.GetLogger().Errorf("sql sink build sql error %v for data", err, mapData)
					return err
				}
				values = append(values, vars)
			}

			if keys != nil {
				sqlStr := fmt.Sprintf("INSERT INTO %s (%s) values ", table, strings.Join(keys, ",")) + strings.Join(values, ",") + ";"
				return m.writeToDB(ctx, &sqlStr)
			}
			return nil
		default: // never happen
			return fmt.Errorf("unsupported type: %T", item)
		}
	} else {
		switch d := item.(type) {
		case []map[string]interface{}:
			for _, el := range d {
				err := m.save(ctx, table, el)
				if err != nil {
					ctx.GetLogger().Error(err)
				}
			}
		case map[string]interface{}:
			err := m.save(ctx, table, d)
			if err != nil {
				return err
			}
		case []interface{}:
			for _, vv := range d {
				el, ok := vv.(map[string]interface{})
				if !ok {
					ctx.GetLogger().Errorf("unsupported type: %T", vv)
					return fmt.Errorf("unsupported type: %T", vv)
				}
				err := m.save(ctx, table, el)
				if err != nil {
					ctx.GetLogger().Error(err)
				}
			}
		default:
			return fmt.Errorf("unrecognized format of %s", item)
		}
		return nil
	}
}

func (m *sqlSink) Close(_ api.StreamContext) error {
	if m.db != nil {
		return util.ReturnDBFromOneNode(util.GlobalPool, m.conf.Url)
	}
	return nil
}

// save save updatable data only to db
func (m *sqlSink) save(ctx api.StreamContext, table string, data map[string]interface{}) error {
	rowkind := ast.RowkindInsert
	c, ok := data[m.conf.RowkindField]
	if ok {
		rowkind, ok = c.(string)
		if !ok {
			return fmt.Errorf("rowkind field %s is not a string in data %v", m.conf.RowkindField, data)
		}
		if rowkind != ast.RowkindInsert && rowkind != ast.RowkindUpdate && rowkind != ast.RowkindDelete {
			return fmt.Errorf("invalid rowkind %s", rowkind)
		}
	}
	var sqlStr string
	switch rowkind {
	case ast.RowkindInsert:
		keys, vars, err := m.conf.buildInsertSql(ctx, data)
		if err != nil {
			return err
		}
		values := []string{vars}
		if keys != nil {
			sqlStr = fmt.Sprintf("INSERT INTO %s (%s) values ", table, strings.Join(keys, ",")) + strings.Join(values, ",") + ";"
		}
	case ast.RowkindUpdate:
		keyval, ok := data[m.conf.KeyField]
		if !ok {
			return fmt.Errorf("field %s does not exist in data %v", m.conf.KeyField, data)
		}
		keys, vals, err := m.conf.getKeyValues(ctx, data)
		if err != nil {
			return err
		}
		sqlStr = fmt.Sprintf("UPDATE %s SET ", table)
		for i, key := range keys {
			if i != 0 {
				sqlStr += ","
			}
			sqlStr += fmt.Sprintf("%s=%s", key, vals[i])
		}
		if _, ok := keyval.(string); ok {
			sqlStr += fmt.Sprintf(" WHERE %s = '%s';", m.conf.KeyField, keyval)
		} else {
			sqlStr += fmt.Sprintf(" WHERE %s = %v;", m.conf.KeyField, keyval)
		}
	case ast.RowkindDelete:
		keyval, ok := data[m.conf.KeyField]
		if !ok {
			return fmt.Errorf("field %s does not exist in data %v", m.conf.KeyField, data)
		}
		if _, ok := keyval.(string); ok {
			sqlStr = fmt.Sprintf("DELETE FROM %s WHERE %s = '%s';", table, m.conf.KeyField, keyval)
		} else {
			sqlStr = fmt.Sprintf("DELETE FROM %s WHERE %s = %v;", table, m.conf.KeyField, keyval)
		}
	default:
		return fmt.Errorf("invalid rowkind %s", rowkind)
	}
	return m.writeToDB(ctx, &sqlStr)
}

func Sql() api.Sink {
	return &sqlSink{}
}