ekuiper-new/internal/topo/operator/preprocessor.go

// Copyright 2021 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package operator

import (
	"fmt"
	"github.com/lf-edge/ekuiper/internal/xsql"
	"github.com/lf-edge/ekuiper/pkg/api"
	"github.com/lf-edge/ekuiper/pkg/cast"
)

type Preprocessor struct {
	//Pruned stream fields. Could be streamField(with data type info) or string
	defaultFieldProcessor
	allMeta        bool
	metaFields     []string //only needed if not allMeta
	isEventTime    bool
	isSchemaless   bool
	timestampField string
}

func NewPreprocessor(isSchemaless bool, fields []interface{}, allMeta bool, metaFields []string, iet bool, timestampField string, timestampFormat string, isBinary bool, strictValidation bool) (*Preprocessor, error) {
	p := &Preprocessor{
		allMeta: allMeta, metaFields: metaFields, isEventTime: iet,
		isSchemaless: isSchemaless, timestampField: timestampField}
	p.defaultFieldProcessor = defaultFieldProcessor{
		streamFields: fields, isBinary: isBinary, timestampFormat: timestampFormat, strictValidation: strictValidation,
	}
	return p, nil
}

/*
 *	input: *xsql.Tuple
 *	output: *xsql.Tuple
 */
func (p *Preprocessor) Apply(ctx api.StreamContext, data interface{}, _ *xsql.FunctionValuer, _ *xsql.AggregateFunctionValuer) interface{} {
	log := ctx.GetLogger()
	tuple, ok := data.(*xsql.Tuple)
	if !ok {
		return fmt.Errorf("expect tuple data type")
	}

	log.Debugf("preprocessor receive %s", tuple.Message)
	var (
		result map[string]interface{}
		err    error
	)
	if !p.isSchemaless && p.streamFields != nil {
		result, err = p.processField(tuple, nil)
		if err != nil {
			return fmt.Errorf("error in preprocessor: %s", err)
		}
		tuple.Message = result
	} else {
		result = tuple.Message
	}
	if p.isEventTime {
		if t, ok := result[p.timestampField]; ok {
			if ts, err := cast.InterfaceToUnixMilli(t, p.timestampFormat); err != nil {
				return fmt.Errorf("cannot convert timestamp field %s to timestamp with error %v", p.timestampField, err)
			} else {
				tuple.Timestamp = ts
				log.Debugf("preprocessor calculate timstamp %d", tuple.Timestamp)
			}
		} else {
			return fmt.Errorf("cannot find timestamp field %s in tuple %v", p.timestampField, result)
		}
	}
	// No need to reconstruct meta as the memory has been allocated earlier
	//if !p.allMeta && p.metaFields != nil && len(p.metaFields) > 0 {
	//	newMeta := make(xsql.Metadata)
	//	for _, f := range p.metaFields {
	//		if m, ok := tuple.Metadata.Value(f, ""); ok {
	//			newMeta[f] = m
	//		}
	//	}
	//	tuple.Metadata = newMeta
	//}
	return tuple
}