feat(function): allow resize function to output in raw format

Signed-off-by: Jiyong Huang <huangjy@emqx.io>

docs/en_US/sqls/custom_functions.md

@@ -48,10 +48,10 @@ accumulateWordCount(avg,sep) example
 
 Image processing currently only supports the formats of `png` and `jpeg` 
 
-| Function  | Example                            | Description                                                                                                                                |
-|-----------|------------------------------------|--------------------------------------------------------------------------------------------------------------------------------------------|
-| resize    | resize(avg,width, height)          | Create a scaled image with new dimensions (width, height). If width or height is set to 0, it is set to the reserved value of aspect ratio |
-| thumbnail | thumbnail(avg,maxWidth, maxHeight) | Reduce the image that retains the aspect ratio to the maximum size (maxWidth, maxHeight).                                                  |
+| Function  | Example                             | Description                                                                                                                                                                                                                                                                     |
+|-----------|-------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| resize    | resize(avg, width, height, [isRaw]) | Create a scaled image with new dimensions (width, height). If width or height is set to 0, it is set to the reserved value of aspect ratio. isRaw is optional, specifies whether to output raw data instead of encoded format like jpeg which is commonly used in AI inference. |
+| thumbnail | thumbnail(avg,maxWidth, maxHeight)  | Reduce the image that retains the aspect ratio to the maximum size (maxWidth, maxHeight).                                                                                                                                                                                       |
 
 resize(avg,width, height) example
 

docs/zh_CN/sqls/custom_functions.md

@@ -48,10 +48,10 @@ accumulateWordCount(avg,sep) 示例
 
 图像处理目前暂时只支持`png`和`jpeg`格式
 
-| 函数        | 示例                                 | 说明                                                      |
-|-----------|------------------------------------|---------------------------------------------------------|
-| resize    | resize(avg,width, height)          | 创建具有新尺寸（宽度，高度）的缩放图像。如果 width 或 height 设置为0，则将其设置为长宽比保留值 |
-| thumbnail | thumbnail(avg,maxWidth, maxHeight) | 将保留宽高比的图像缩小到最大尺寸( maxWidth，maxHeight)。                  |
+| 函数        | 示例                                  | 说明                                                                                                |
+|-----------|-------------------------------------|---------------------------------------------------------------------------------------------------|
+| resize    | resize(avg, width, height, [isRaw]) | 创建具有新尺寸（宽度，高度）的缩放图像。如果 width 或 height 设置为0，则将其设置为长宽比保留值。isRaw 为可选参数，用于指定是否输出原始未编码数据，常用于 AI 模型推理中。 |
+| thumbnail | thumbnail(avg,maxWidth, maxHeight)  | 将保留宽高比的图像缩小到最大尺寸( maxWidth，maxHeight)。                                                            |
 
 resize(avg,width, height)示例
 

extensions/functions/image/image.json

@@ -20,7 +20,7 @@
 	"name": "image",
 	"functions": [{
 		"name": "resize",
-		"example": "resize(image,width, height)",
+		"example": "resize(image,width, height, isRaw)",
 		"hint": {
 			"en_US": "Creates a scaled image with new dimensions (width, height) .If either width or height is set to 0, it will be set to an aspect ratio preserving value.",
 			"zh_CN": "创建具有新尺寸（宽度，高度）的缩放图像。如果width或height设置为0，则将其设置为长宽比保留值。"
@@ -68,6 +68,20 @@
 					"en_US": "Input height",
 					"zh_CN": "输入高度"
 				}
+			},{
+				"name": "isRaw",
+				"hidden": true,
+				"optional": true,
+				"control": "radio",
+				"type": "bool",
+				"hint": {
+					"en_US": "Whether to output raw data, set to true when using by most AI inference",
+					"zh_CN": "输出未编码原始数据，通常 AI 推断需要原始数据"
+				},
+				"label": {
+					"en_US": "Output Raw Data",
+					"zh_CN": "输出原始数据"
+				}
 			}
 		],
 		"outputs": [

extensions/functions/image/resize.go

@@ -1,4 +1,4 @@
-// Copyright 2021 EMQ Technologies Co., Ltd.
+// Copyright 2021-2022 EMQ Technologies Co., Ltd.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,6 +20,7 @@ import (
 	"github.com/lf-edge/ekuiper/pkg/api"
 	"github.com/nfnt/resize"
 	"image"
+	"image/gif"
 	"image/jpeg"
 	"image/png"
 )
@@ -28,13 +29,17 @@ type imageResize struct {
 }
 
 func (f *imageResize) Validate(args []interface{}) error {
-	if len(args) != 3 {
-		return fmt.Errorf("The resize function supports 3 parameters, but got %d", len(args))
+	if len(args) < 3 {
+		return fmt.Errorf("The resize function must have at least 3 parameters, but got %d", len(args))
 	}
 	return nil
 }
 
-func (f *imageResize) Exec(args []interface{}, _ api.FunctionContext) (interface{}, bool) {
+func (f *imageResize) IsAggregate() bool {
+	return false
+}
+
+func (f *imageResize) Exec(args []interface{}, ctx api.FunctionContext) (interface{}, bool) {
 	arg, ok := args[0].([]byte)
 	if !ok {
 		return fmt.Errorf("arg[0] is not a bytea, got %v", args[0]), false
@@ -47,28 +52,53 @@ func (f *imageResize) Exec(args []interface{}, _ api.FunctionContext) (interface
 	if !ok || 0 > height {
 		return fmt.Errorf("arg[2] is not a bigint, got %v", args[2]), false
 	}
+	isRaw := false
+	if len(args) > 3 {
+		isRaw, ok = args[3].(bool)
+		if !ok {
+			return fmt.Errorf("arg[3] is not a bool, got %v", args[3]), false
+		}
+	}
+	ctx.GetLogger().Debugf("resize: %d %d, output raw %v", width, height, isRaw)
+
 	img, format, err := image.Decode(bytes.NewReader(arg))
 	if nil != err {
 		return fmt.Errorf("image decode error:%v", err), false
 	}
-	img = resize.Resize(uint(width), uint(height), img, resize.Bilinear)
 
-	var b []byte
-	buf := bytes.NewBuffer(b)
-	switch format {
-	case "png":
-		err = png.Encode(buf, img)
-	case "jpeg":
-		err = jpeg.Encode(buf, img, nil)
-	default:
-		return fmt.Errorf("%s image type is not currently supported", format), false
-	}
-	if nil != err {
-		return fmt.Errorf("image encode error:%v", err), false
+	img = resize.Resize(uint(width), uint(height), img, resize.Bilinear)
+	if isRaw {
+		bounds := img.Bounds()
+		dx, dy := bounds.Dx(), bounds.Dy()
+		bb := make([]byte, width*height*3)
+		for y := 0; y < dy; y++ {
+			for x := 0; x < dx; x++ {
+				col := img.At(x, y)
+				r, g, b, _ := col.RGBA()
+				bb[(y*dx+x)*3+0] = byte(float64(r) / 255.0)
+				bb[(y*dx+x)*3+1] = byte(float64(g) / 255.0)
+				bb[(y*dx+x)*3+2] = byte(float64(b) / 255.0)
+			}
+		}
+		return bb, true
+	} else {
+		var b []byte
+		buf := bytes.NewBuffer(b)
+		switch format {
+		case "png":
+			err = png.Encode(buf, img)
+		case "jpeg":
+			err = jpeg.Encode(buf, img, nil)
+		case "gif":
+			err = gif.Encode(buf, img, nil)
+		default:
+			return fmt.Errorf("%s image type is not currently supported", format), false
+		}
+		if nil != err {
+			return fmt.Errorf("image encode error:%v", err), false
+		}
+		return buf.Bytes(), true
 	}
-	return buf.Bytes(), true
 }
 
-func (f *imageResize) IsAggregate() bool {
-	return false
-}
+var ResizeWithChan imageResize

extensions/functions/image/resize_test.go

@@ -0,0 +1,59 @@
+// Copyright 2022 EMQ Technologies Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"github.com/lf-edge/ekuiper/internal/conf"
+	kctx "github.com/lf-edge/ekuiper/internal/topo/context"
+	"github.com/lf-edge/ekuiper/internal/topo/state"
+	"github.com/lf-edge/ekuiper/pkg/api"
+	"io/ioutil"
+	"reflect"
+	"testing"
+)
+
+func TestResize(t *testing.T) {
+	contextLogger := conf.Log.WithField("rule", "testExec")
+	ctx := kctx.WithValue(kctx.Background(), kctx.LoggerKey, contextLogger)
+	tempStore, _ := state.CreateStore("mockRule0", api.AtMostOnce)
+	fctx := kctx.NewDefaultFuncContext(ctx.WithMeta("mockRule0", "test", tempStore), 2)
+	var tests = []struct {
+		image  string
+		result string
+	}{
+		{
+			image:  "img.png",
+			result: "resized.png",
+		},
+	}
+	for i, tt := range tests {
+		payload, err := ioutil.ReadFile(tt.image)
+		if err != nil {
+			t.Errorf("Failed to read image file %s", tt.image)
+			continue
+		}
+		resized, err := ioutil.ReadFile(tt.result)
+		if err != nil {
+			t.Errorf("Failed to read result image file %s", tt.result)
+			continue
+		}
+		result, _ := ResizeWithChan.Exec([]interface{}{
+			payload, 100, 100,
+		}, fctx)
+		if !reflect.DeepEqual(result, resized) {
+			t.Errorf("%d result mismatch", i)
+		}
+	}
+}