fix(upload): support url for file upload (#23) (#2097)

Signed-off-by: Jianxiang Ran <jianxiang.ran@emqx.io>
Signed-off-by: Jianxiang Ran <rxan_embedded@163.com>

docs/en_US/api/restapi/uploads.md

@@ -2,7 +2,7 @@ The eKuiper REST api for configuration file uploads allows you to upload configu
 
 ## Upload a configuration file
 
-The API supports to upload a local file or provide the text content of file. The upload request will save the file into your `${dataPath}/uploads`. It will override the existed file of the same name. The response is the absolute path of the uploaded file which you can refer in other configurations.
+The API supports to upload a local file, provide the text content of file or upload a http file link. The upload request will save the file into your `${dataPath}/uploads`. It will override the existed file of the same name. The response is the absolute path of the uploaded file which you can refer in other configurations.
 
 ### Upload by a file
 
@@ -43,6 +43,19 @@ POST http://localhost:9081/config/uploads
 }
 ```
 
+### Upload by HTTP file link
+
+Should put the file in HTTP Server in advance
+
+```shell
+POST http://localhost:9081/config/uploads
+
+{
+  "name": "my.json",
+  "file": "http://127.0.0.1:80/my.json"
+}
+```
+
 ## Show uploaded file list
 
 The API is used for displaying all files in the `${dataPath}/uploads` path.

docs/zh_CN/api/restapi/uploads.md

@@ -2,7 +2,7 @@ eKuiper REST api允许您上传配置文件并列出所有上传的文件。
 
 ## 上传配置文件
 
-支持两种方式上传配置文件：上传文件或者提供文件名和文本内容。上传请求将把文件保存到你的 `${dataPath}/uploads` 。它将覆盖现有的同名文件。返回的响应是上传文件的绝对路径，从而可以在其他配置中使用。
+支持以下方式上传配置文件：上传文件，提供文件名和文本内容或者提供文件下载链接。上传请求将把文件保存到你的 `${dataPath}/uploads` 。它将覆盖现有的同名文件。返回的响应是上传文件的绝对路径，从而可以在其他配置中使用。
 
 ### 上传文件
 
@@ -43,6 +43,19 @@ POST http://localhost:9081/config/uploads
 }
 ```
 
+### 通过文件链接创建文件
+
+文件需提前放到 HTTP 服务器，并提供对外下载链接
+
+```shell
+POST http://localhost:9081/config/uploads
+
+{
+  "name": "my.json",
+  "file": "http://127.0.0.1:80/my.json"
+}
+```
+
 ## 获取上传文件的列表
 
 该API用于显示 `${dataPath}/uploads` 路径中的所有文件。

internal/server/rest.go

@@ -170,8 +170,50 @@ func createRestServer(ip string, port int, needToken bool) *http.Server {
 }
 
 type fileContent struct {
-	Name    string `json:"name"`
-	Content string `json:"content"`
+	Name     string `json:"name"`
+	Content  string `json:"content"`
+	FilePath string `json:"file"`
+}
+
+func (f *fileContent) Validate() error {
+	if f.Content == "" && f.FilePath == "" {
+		return fmt.Errorf("invalid body: content or FilePath is required")
+	}
+	if f.Name == "" {
+		return fmt.Errorf("invalid body: name is required")
+	}
+	return nil
+}
+
+func upload(file *fileContent) error {
+	err := getFile(file)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func getFile(file *fileContent) error {
+	filePath := filepath.Join(uploadDir, file.Name)
+	dst, err := os.Create(filePath)
+	if err != nil {
+		return err
+	}
+	defer dst.Close()
+
+	if file.FilePath != "" {
+		err := httpx.DownloadFile(filePath, file.FilePath)
+		if err != nil {
+			return err
+		}
+	} else {
+		_, err := dst.Write([]byte(file.Content))
+		if err != nil {
+			return err
+		}
+	}
+	return nil
 }
 
 func fileUploadHandler(w http.ResponseWriter, r *http.Request) {
@@ -187,20 +229,16 @@ func fileUploadHandler(w http.ResponseWriter, r *http.Request) {
 				handleError(w, err, "Invalid body: Error decoding file json", logger)
 				return
 			}
-			if fc.Content == "" || fc.Name == "" {
-				handleError(w, nil, "Invalid body: name and content are required", logger)
-				return
-			}
-			filePath := filepath.Join(uploadDir, fc.Name)
-			dst, err := os.Create(filePath)
-			defer dst.Close()
+			err = fc.Validate()
 			if err != nil {
-				handleError(w, err, "Error creating the file", logger)
+				handleError(w, err, "Invalid body: missing necessary field", logger)
 				return
 			}
-			_, err = dst.Write([]byte(fc.Content))
+
+			filePath := filepath.Join(uploadDir, fc.Name)
+			err = upload(fc)
 			if err != nil {
-				handleError(w, err, "Error writing the file", logger)
+				handleError(w, err, "Upload error: getFile has error", logger)
 				return
 			}
 			w.WriteHeader(http.StatusCreated)

internal/server/rest_test.go

@@ -379,6 +379,17 @@ func (suite *RestTestSuite) Test_fileUpload() {
 	suite.r.ServeHTTP(w, req)
 	assert.Equal(suite.T(), http.StatusCreated, w.Code)
 
+	postContent := map[string]string{}
+	postContent["Name"] = "test1.txt"
+	postContent["file"] = "file://" + uploadDir + "/test.txt"
+
+	bdy, _ := json.Marshal(postContent)
+	req, _ = http.NewRequest(http.MethodPost, "http://localhost:8080/config/uploads", bytes.NewBuffer(bdy))
+	req.Header["Content-Type"] = []string{"application/json"}
+	w = httptest.NewRecorder()
+	suite.r.ServeHTTP(w, req)
+	assert.Equal(suite.T(), http.StatusCreated, w.Code)
+
 	req, _ = http.NewRequest(http.MethodGet, "http://localhost:8080/config/uploads", bytes.NewBufferString("any"))
 	w = httptest.NewRecorder()
 	suite.r.ServeHTTP(w, req)
@@ -388,6 +399,50 @@ func (suite *RestTestSuite) Test_fileUpload() {
 	w = httptest.NewRecorder()
 	suite.r.ServeHTTP(w, req)
 	assert.Equal(suite.T(), http.StatusOK, w.Code)
+
+	req, _ = http.NewRequest(http.MethodDelete, "http://localhost:8080/config/uploads/test1.txt", bytes.NewBufferString("any"))
+	w = httptest.NewRecorder()
+	suite.r.ServeHTTP(w, req)
+	assert.Equal(suite.T(), http.StatusOK, w.Code)
+	os.Remove(uploadDir)
+}
+
+func (suite *RestTestSuite) Test_fileUploadValidate() {
+	fileJson := `{"Name": "test.txt", "Content": test}`
+	req, _ := http.NewRequest(http.MethodPost, "http://localhost:8080/config/uploads", bytes.NewBufferString(fileJson))
+	req.Header["Content-Type"] = []string{"application/json"}
+	os.Mkdir(uploadDir, 0o777)
+	w := httptest.NewRecorder()
+	suite.r.ServeHTTP(w, req)
+	assert.Equal(suite.T(), http.StatusBadRequest, w.Code)
+
+	fileJson = `{"Name": "test.txt", "Contents": "test"}`
+	req, _ = http.NewRequest(http.MethodPost, "http://localhost:8080/config/uploads", bytes.NewBufferString(fileJson))
+	req.Header["Content-Type"] = []string{"application/json"}
+	os.Mkdir(uploadDir, 0o777)
+	w = httptest.NewRecorder()
+	suite.r.ServeHTTP(w, req)
+	assert.Equal(suite.T(), http.StatusBadRequest, w.Code)
+
+	fileJson = `{"Content": "test"}`
+	req, _ = http.NewRequest(http.MethodPost, "http://localhost:8080/config/uploads", bytes.NewBufferString(fileJson))
+	req.Header["Content-Type"] = []string{"application/json"}
+	os.Mkdir(uploadDir, 0o777)
+	w = httptest.NewRecorder()
+	suite.r.ServeHTTP(w, req)
+	assert.Equal(suite.T(), http.StatusBadRequest, w.Code)
+
+	postContent := map[string]string{}
+	postContent["Name"] = "test1.txt"
+	postContent["file"] = "file://" + uploadDir + "/test.txt"
+
+	bdy, _ := json.Marshal(postContent)
+	req, _ = http.NewRequest(http.MethodPost, "http://localhost:8080/config/uploads", bytes.NewBuffer(bdy))
+	req.Header["Content-Type"] = []string{"application/json"}
+	w = httptest.NewRecorder()
+	suite.r.ServeHTTP(w, req)
+	assert.Equal(suite.T(), http.StatusBadRequest, w.Code)
+
 	os.Remove(uploadDir)
 }