rest.go 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845
  1. // Copyright 2021-2022 EMQ Technologies Co., Ltd.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package server
  15. import (
  16. "bytes"
  17. "encoding/json"
  18. "fmt"
  19. "github.com/lf-edge/ekuiper/internal/conf"
  20. "github.com/lf-edge/ekuiper/internal/meta"
  21. "github.com/lf-edge/ekuiper/internal/pkg/httpx"
  22. "github.com/lf-edge/ekuiper/internal/processor"
  23. "io"
  24. "net/http"
  25. "os"
  26. "path/filepath"
  27. "runtime"
  28. "strconv"
  29. "strings"
  30. "time"
  31. "github.com/gorilla/handlers"
  32. "github.com/gorilla/mux"
  33. "github.com/lf-edge/ekuiper/internal/server/middleware"
  34. "github.com/lf-edge/ekuiper/pkg/api"
  35. "github.com/lf-edge/ekuiper/pkg/ast"
  36. "github.com/lf-edge/ekuiper/pkg/errorx"
  37. "github.com/lf-edge/ekuiper/pkg/infra"
  38. )
  39. const (
  40. ContentType = "Content-Type"
  41. ContentTypeJSON = "application/json"
  42. )
  43. var uploadDir string
  44. type statementDescriptor struct {
  45. Sql string `json:"sql,omitempty"`
  46. }
  47. func decodeStatementDescriptor(reader io.ReadCloser) (statementDescriptor, error) {
  48. sd := statementDescriptor{}
  49. err := json.NewDecoder(reader).Decode(&sd)
  50. // Problems decoding
  51. if err != nil {
  52. return sd, fmt.Errorf("Error decoding the statement descriptor: %v", err)
  53. }
  54. return sd, nil
  55. }
  56. // Handle applies the specified error and error concept to the HTTP response writer
  57. func handleError(w http.ResponseWriter, err error, prefix string, logger api.Logger) {
  58. message := prefix
  59. if message != "" {
  60. message += ": "
  61. }
  62. message += err.Error()
  63. logger.Error(message)
  64. var ec int
  65. switch e := err.(type) {
  66. case *errorx.Error:
  67. switch e.Code() {
  68. case errorx.NOT_FOUND:
  69. ec = http.StatusNotFound
  70. default:
  71. ec = http.StatusBadRequest
  72. }
  73. default:
  74. ec = http.StatusBadRequest
  75. }
  76. http.Error(w, message, ec)
  77. }
  78. func jsonResponse(i interface{}, w http.ResponseWriter, logger api.Logger) {
  79. w.Header().Add(ContentType, ContentTypeJSON)
  80. jsonByte, err := json.Marshal(i)
  81. if err != nil {
  82. handleError(w, err, "", logger)
  83. }
  84. w.Header().Add("Content-Length", strconv.Itoa(len(jsonByte)))
  85. _, err = w.Write(jsonByte)
  86. // Problems encoding
  87. if err != nil {
  88. handleError(w, err, "", logger)
  89. }
  90. }
  91. func jsonByteResponse(buffer bytes.Buffer, w http.ResponseWriter, logger api.Logger) {
  92. w.Header().Add(ContentType, ContentTypeJSON)
  93. w.Header().Add("Content-Length", strconv.Itoa(buffer.Len()))
  94. _, err := w.Write(buffer.Bytes())
  95. // Problems encoding
  96. if err != nil {
  97. handleError(w, err, "", logger)
  98. }
  99. }
  100. func createRestServer(ip string, port int, needToken bool) *http.Server {
  101. dataDir, err := conf.GetDataLoc()
  102. if err != nil {
  103. panic(err)
  104. }
  105. uploadDir = filepath.Join(dataDir, "uploads")
  106. r := mux.NewRouter()
  107. r.HandleFunc("/", rootHandler).Methods(http.MethodGet, http.MethodPost)
  108. r.HandleFunc("/ping", pingHandler).Methods(http.MethodGet)
  109. r.HandleFunc("/streams", streamsHandler).Methods(http.MethodGet, http.MethodPost)
  110. r.HandleFunc("/streams/{name}", streamHandler).Methods(http.MethodGet, http.MethodDelete, http.MethodPut)
  111. r.HandleFunc("/streams/{name}/schema", streamSchemaHandler).Methods(http.MethodGet)
  112. r.HandleFunc("/tables", tablesHandler).Methods(http.MethodGet, http.MethodPost)
  113. r.HandleFunc("/tables/{name}", tableHandler).Methods(http.MethodGet, http.MethodDelete, http.MethodPut)
  114. r.HandleFunc("/tables/{name}/schema", tableSchemaHandler).Methods(http.MethodGet)
  115. r.HandleFunc("/rules", rulesHandler).Methods(http.MethodGet, http.MethodPost)
  116. r.HandleFunc("/rules/{name}", ruleHandler).Methods(http.MethodDelete, http.MethodGet, http.MethodPut)
  117. r.HandleFunc("/rules/{name}/status", getStatusRuleHandler).Methods(http.MethodGet)
  118. r.HandleFunc("/rules/{name}/start", startRuleHandler).Methods(http.MethodPost)
  119. r.HandleFunc("/rules/{name}/stop", stopRuleHandler).Methods(http.MethodPost)
  120. r.HandleFunc("/rules/{name}/restart", restartRuleHandler).Methods(http.MethodPost)
  121. r.HandleFunc("/rules/{name}/topo", getTopoRuleHandler).Methods(http.MethodGet)
  122. r.HandleFunc("/ruleset/export", exportHandler).Methods(http.MethodPost)
  123. r.HandleFunc("/ruleset/import", importHandler).Methods(http.MethodPost)
  124. r.HandleFunc("/config/uploads", fileUploadHandler).Methods(http.MethodPost, http.MethodGet)
  125. r.HandleFunc("/config/uploads/{name}", fileDeleteHandler).Methods(http.MethodDelete)
  126. r.HandleFunc("/data/export", configurationExportHandler).Methods(http.MethodGet)
  127. r.HandleFunc("/data/import", configurationImportHandler).Methods(http.MethodPost)
  128. r.HandleFunc("/data/import/status", configurationStatusHandler).Methods(http.MethodGet)
  129. // Register extended routes
  130. for k, v := range components {
  131. logger.Infof("register rest endpoint for component %s", k)
  132. v.rest(r)
  133. }
  134. if needToken {
  135. r.Use(middleware.Auth)
  136. }
  137. server := &http.Server{
  138. Addr: fmt.Sprintf("%s:%d", ip, port),
  139. // Good practice to set timeouts to avoid Slowloris attacks.
  140. WriteTimeout: time.Second * 60 * 5,
  141. ReadTimeout: time.Second * 60 * 5,
  142. IdleTimeout: time.Second * 60,
  143. Handler: handlers.CORS(handlers.AllowedHeaders([]string{"Accept", "Accept-Language", "Content-Type", "Content-Language", "Origin", "Authorization"}), handlers.AllowedMethods([]string{"POST", "GET", "PUT", "DELETE", "HEAD"}))(r),
  144. }
  145. server.SetKeepAlivesEnabled(false)
  146. return server
  147. }
  148. type fileContent struct {
  149. Name string `json:"name"`
  150. Content string `json:"content"`
  151. }
  152. func fileUploadHandler(w http.ResponseWriter, r *http.Request) {
  153. switch r.Method {
  154. // Upload or overwrite a file
  155. case http.MethodPost:
  156. switch r.Header.Get("Content-Type") {
  157. case "application/json":
  158. fc := &fileContent{}
  159. defer r.Body.Close()
  160. err := json.NewDecoder(r.Body).Decode(fc)
  161. if err != nil {
  162. handleError(w, err, "Invalid body: Error decoding file json", logger)
  163. return
  164. }
  165. if fc.Content == "" || fc.Name == "" {
  166. handleError(w, nil, "Invalid body: name and content are required", logger)
  167. return
  168. }
  169. filePath := filepath.Join(uploadDir, fc.Name)
  170. dst, err := os.Create(filePath)
  171. defer dst.Close()
  172. if err != nil {
  173. handleError(w, err, "Error creating the file", logger)
  174. return
  175. }
  176. _, err = dst.Write([]byte(fc.Content))
  177. if err != nil {
  178. handleError(w, err, "Error writing the file", logger)
  179. return
  180. }
  181. w.WriteHeader(http.StatusCreated)
  182. w.Write([]byte(filePath))
  183. default:
  184. // Maximum upload of 1 GB files
  185. err := r.ParseMultipartForm(1024 << 20)
  186. if err != nil {
  187. handleError(w, err, "Error parse the multi part form", logger)
  188. return
  189. }
  190. // Get handler for filename, size and headers
  191. file, handler, err := r.FormFile("uploadFile")
  192. if err != nil {
  193. handleError(w, err, "Error Retrieving the File", logger)
  194. return
  195. }
  196. defer file.Close()
  197. // Create file
  198. filePath := filepath.Join(uploadDir, handler.Filename)
  199. dst, err := os.Create(filePath)
  200. defer dst.Close()
  201. if err != nil {
  202. handleError(w, err, "Error creating the file", logger)
  203. return
  204. }
  205. // Copy the uploaded file to the created file on the filesystem
  206. if _, err := io.Copy(dst, file); err != nil {
  207. handleError(w, err, "Error writing the file", logger)
  208. return
  209. }
  210. w.WriteHeader(http.StatusCreated)
  211. w.Write([]byte(filePath))
  212. }
  213. case http.MethodGet:
  214. // Get the list of files in the upload directory
  215. files, err := os.ReadDir(uploadDir)
  216. if err != nil {
  217. handleError(w, err, "Error reading the file upload dir", logger)
  218. return
  219. }
  220. fileNames := make([]string, len(files))
  221. for i, f := range files {
  222. fileNames[i] = filepath.Join(uploadDir, f.Name())
  223. }
  224. jsonResponse(fileNames, w, logger)
  225. }
  226. }
  227. func fileDeleteHandler(w http.ResponseWriter, r *http.Request) {
  228. vars := mux.Vars(r)
  229. name := vars["name"]
  230. filePath := filepath.Join(uploadDir, name)
  231. e := os.Remove(filePath)
  232. if e != nil {
  233. handleError(w, e, "Error deleting the file", logger)
  234. return
  235. }
  236. w.WriteHeader(http.StatusOK)
  237. w.Write([]byte("ok"))
  238. }
  239. type information struct {
  240. Version string `json:"version"`
  241. Os string `json:"os"`
  242. Arch string `json:"arch"`
  243. UpTimeSeconds int64 `json:"upTimeSeconds"`
  244. }
  245. // The handler for root
  246. func rootHandler(w http.ResponseWriter, r *http.Request) {
  247. defer r.Body.Close()
  248. switch r.Method {
  249. case http.MethodGet, http.MethodPost:
  250. w.WriteHeader(http.StatusOK)
  251. info := new(information)
  252. info.Version = version
  253. info.UpTimeSeconds = time.Now().Unix() - startTimeStamp
  254. info.Os = runtime.GOOS
  255. info.Arch = runtime.GOARCH
  256. byteInfo, _ := json.Marshal(info)
  257. w.Write(byteInfo)
  258. }
  259. }
  260. func pingHandler(w http.ResponseWriter, _ *http.Request) {
  261. w.WriteHeader(http.StatusOK)
  262. }
  263. func sourcesManageHandler(w http.ResponseWriter, r *http.Request, st ast.StreamType) {
  264. defer r.Body.Close()
  265. switch r.Method {
  266. case http.MethodGet:
  267. var (
  268. content []string
  269. err error
  270. kind string
  271. )
  272. if st == ast.TypeTable {
  273. kind = r.URL.Query().Get("kind")
  274. if kind == "scan" {
  275. kind = ast.StreamKindScan
  276. } else if kind == "lookup" {
  277. kind = ast.StreamKindLookup
  278. } else {
  279. kind = ""
  280. }
  281. }
  282. if kind != "" {
  283. content, err = streamProcessor.ShowTable(kind)
  284. } else {
  285. content, err = streamProcessor.ShowStream(st)
  286. }
  287. if err != nil {
  288. handleError(w, err, fmt.Sprintf("%s command error", strings.Title(ast.StreamTypeMap[st])), logger)
  289. return
  290. }
  291. jsonResponse(content, w, logger)
  292. case http.MethodPost:
  293. v, err := decodeStatementDescriptor(r.Body)
  294. if err != nil {
  295. handleError(w, err, "Invalid body", logger)
  296. return
  297. }
  298. content, err := streamProcessor.ExecStreamSql(v.Sql)
  299. if err != nil {
  300. handleError(w, err, fmt.Sprintf("%s command error", strings.Title(ast.StreamTypeMap[st])), logger)
  301. return
  302. }
  303. w.WriteHeader(http.StatusCreated)
  304. w.Write([]byte(content))
  305. }
  306. }
  307. func sourceManageHandler(w http.ResponseWriter, r *http.Request, st ast.StreamType) {
  308. defer r.Body.Close()
  309. vars := mux.Vars(r)
  310. name := vars["name"]
  311. switch r.Method {
  312. case http.MethodGet:
  313. content, err := streamProcessor.DescStream(name, st)
  314. if err != nil {
  315. handleError(w, err, fmt.Sprintf("describe %s error", ast.StreamTypeMap[st]), logger)
  316. return
  317. }
  318. jsonResponse(content, w, logger)
  319. case http.MethodDelete:
  320. content, err := streamProcessor.DropStream(name, st)
  321. if err != nil {
  322. handleError(w, err, fmt.Sprintf("delete %s error", ast.StreamTypeMap[st]), logger)
  323. return
  324. }
  325. w.WriteHeader(http.StatusOK)
  326. w.Write([]byte(content))
  327. case http.MethodPut:
  328. v, err := decodeStatementDescriptor(r.Body)
  329. if err != nil {
  330. handleError(w, err, "Invalid body", logger)
  331. return
  332. }
  333. content, err := streamProcessor.ExecReplaceStream(name, v.Sql, st)
  334. if err != nil {
  335. handleError(w, err, fmt.Sprintf("%s command error", strings.Title(ast.StreamTypeMap[st])), logger)
  336. return
  337. }
  338. w.WriteHeader(http.StatusOK)
  339. w.Write([]byte(content))
  340. }
  341. }
  342. // list or create streams
  343. func streamsHandler(w http.ResponseWriter, r *http.Request) {
  344. sourcesManageHandler(w, r, ast.TypeStream)
  345. }
  346. // describe or delete a stream
  347. func streamHandler(w http.ResponseWriter, r *http.Request) {
  348. sourceManageHandler(w, r, ast.TypeStream)
  349. }
  350. // list or create tables
  351. func tablesHandler(w http.ResponseWriter, r *http.Request) {
  352. sourcesManageHandler(w, r, ast.TypeTable)
  353. }
  354. func tableHandler(w http.ResponseWriter, r *http.Request) {
  355. sourceManageHandler(w, r, ast.TypeTable)
  356. }
  357. func streamSchemaHandler(w http.ResponseWriter, r *http.Request) {
  358. sourceSchemaHandler(w, r, ast.TypeStream)
  359. }
  360. func tableSchemaHandler(w http.ResponseWriter, r *http.Request) {
  361. sourceSchemaHandler(w, r, ast.TypeTable)
  362. }
  363. func sourceSchemaHandler(w http.ResponseWriter, r *http.Request, st ast.StreamType) {
  364. vars := mux.Vars(r)
  365. name := vars["name"]
  366. content, err := streamProcessor.GetInferredJsonSchema(name, st)
  367. if err != nil {
  368. handleError(w, err, fmt.Sprintf("get schema of %s error", ast.StreamTypeMap[st]), logger)
  369. return
  370. }
  371. jsonResponse(content, w, logger)
  372. }
  373. // list or create rules
  374. func rulesHandler(w http.ResponseWriter, r *http.Request) {
  375. defer r.Body.Close()
  376. switch r.Method {
  377. case http.MethodPost:
  378. body, err := io.ReadAll(r.Body)
  379. if err != nil {
  380. handleError(w, err, "Invalid body", logger)
  381. return
  382. }
  383. id, err := createRule("", string(body))
  384. if err != nil {
  385. handleError(w, err, "", logger)
  386. return
  387. }
  388. result := fmt.Sprintf("Rule %s was created successfully.", id)
  389. w.WriteHeader(http.StatusCreated)
  390. w.Write([]byte(result))
  391. case http.MethodGet:
  392. content, err := getAllRulesWithStatus()
  393. if err != nil {
  394. handleError(w, err, "Show rules error", logger)
  395. return
  396. }
  397. jsonResponse(content, w, logger)
  398. }
  399. }
  400. // describe or delete a rule
  401. func ruleHandler(w http.ResponseWriter, r *http.Request) {
  402. defer r.Body.Close()
  403. vars := mux.Vars(r)
  404. name := vars["name"]
  405. switch r.Method {
  406. case http.MethodGet:
  407. rule, err := ruleProcessor.GetRuleJson(name)
  408. if err != nil {
  409. handleError(w, err, "describe rule error", logger)
  410. return
  411. }
  412. w.Header().Add(ContentType, ContentTypeJSON)
  413. w.Write([]byte(rule))
  414. case http.MethodDelete:
  415. deleteRule(name)
  416. content, err := ruleProcessor.ExecDrop(name)
  417. if err != nil {
  418. handleError(w, err, "delete rule error", logger)
  419. return
  420. }
  421. w.WriteHeader(http.StatusOK)
  422. w.Write([]byte(content))
  423. case http.MethodPut:
  424. _, err := ruleProcessor.GetRuleById(name)
  425. if err != nil {
  426. handleError(w, err, "not found this rule", logger)
  427. return
  428. }
  429. body, err := io.ReadAll(r.Body)
  430. if err != nil {
  431. handleError(w, err, "Invalid body", logger)
  432. return
  433. }
  434. err = updateRule(name, string(body))
  435. if err != nil {
  436. handleError(w, err, "restart rule error", logger)
  437. return
  438. }
  439. // Update to db after validation
  440. _, err = ruleProcessor.ExecUpdate(name, string(body))
  441. var result string
  442. if err != nil {
  443. handleError(w, err, "Update rule error, suggest to delete it and recreate", logger)
  444. return
  445. } else {
  446. result = fmt.Sprintf("Rule %s was updated successfully.", name)
  447. }
  448. w.WriteHeader(http.StatusOK)
  449. w.Write([]byte(result))
  450. }
  451. }
  452. // get status of a rule
  453. func getStatusRuleHandler(w http.ResponseWriter, r *http.Request) {
  454. defer r.Body.Close()
  455. vars := mux.Vars(r)
  456. name := vars["name"]
  457. content, err := getRuleStatus(name)
  458. if err != nil {
  459. handleError(w, err, "get rule status error", logger)
  460. return
  461. }
  462. w.Header().Set(ContentType, ContentTypeJSON)
  463. w.WriteHeader(http.StatusOK)
  464. w.Write([]byte(content))
  465. }
  466. // start a rule
  467. func startRuleHandler(w http.ResponseWriter, r *http.Request) {
  468. defer r.Body.Close()
  469. vars := mux.Vars(r)
  470. name := vars["name"]
  471. err := startRule(name)
  472. if err != nil {
  473. handleError(w, err, "start rule error", logger)
  474. return
  475. }
  476. w.WriteHeader(http.StatusOK)
  477. w.Write([]byte(fmt.Sprintf("Rule %s was started", name)))
  478. }
  479. // stop a rule
  480. func stopRuleHandler(w http.ResponseWriter, r *http.Request) {
  481. defer r.Body.Close()
  482. vars := mux.Vars(r)
  483. name := vars["name"]
  484. result := stopRule(name)
  485. w.WriteHeader(http.StatusOK)
  486. w.Write([]byte(result))
  487. }
  488. // restart a rule
  489. func restartRuleHandler(w http.ResponseWriter, r *http.Request) {
  490. defer r.Body.Close()
  491. vars := mux.Vars(r)
  492. name := vars["name"]
  493. err := restartRule(name)
  494. if err != nil {
  495. handleError(w, err, "restart rule error", logger)
  496. return
  497. }
  498. w.WriteHeader(http.StatusOK)
  499. w.Write([]byte(fmt.Sprintf("Rule %s was restarted", name)))
  500. }
  501. // get topo of a rule
  502. func getTopoRuleHandler(w http.ResponseWriter, r *http.Request) {
  503. defer r.Body.Close()
  504. vars := mux.Vars(r)
  505. name := vars["name"]
  506. content, err := getRuleTopo(name)
  507. if err != nil {
  508. handleError(w, err, "get rule topo error", logger)
  509. return
  510. }
  511. w.Header().Set(ContentType, ContentTypeJSON)
  512. w.Write([]byte(content))
  513. }
  514. type rulesetInfo struct {
  515. Content string `json:"content"`
  516. FilePath string `json:"file"`
  517. }
  518. func importHandler(w http.ResponseWriter, r *http.Request) {
  519. rsi := &rulesetInfo{}
  520. err := json.NewDecoder(r.Body).Decode(rsi)
  521. if err != nil {
  522. handleError(w, err, "Invalid body: Error decoding json", logger)
  523. return
  524. }
  525. if rsi.Content != "" && rsi.FilePath != "" {
  526. handleError(w, nil, "Invalid body: Cannot specify both content and file", logger)
  527. return
  528. } else if rsi.Content == "" && rsi.FilePath == "" {
  529. handleError(w, nil, "Invalid body: must specify content or file", logger)
  530. return
  531. }
  532. content := []byte(rsi.Content)
  533. if rsi.FilePath != "" {
  534. reader, err := httpx.ReadFile(rsi.FilePath)
  535. if err != nil {
  536. handleError(w, nil, "Fail to read file", logger)
  537. return
  538. }
  539. defer reader.Close()
  540. buf := new(bytes.Buffer)
  541. _, err = io.Copy(buf, reader)
  542. if err != nil {
  543. handleError(w, err, "fail to convert file", logger)
  544. return
  545. }
  546. content = buf.Bytes()
  547. }
  548. rules, counts, err := rulesetProcessor.Import(content)
  549. if err != nil {
  550. handleError(w, nil, "Import ruleset error", logger)
  551. return
  552. }
  553. infra.SafeRun(func() error {
  554. for _, name := range rules {
  555. rul, ee := ruleProcessor.GetRuleById(name)
  556. if ee != nil {
  557. logger.Error(ee)
  558. continue
  559. }
  560. reply := recoverRule(rul)
  561. if reply != "" {
  562. logger.Error(reply)
  563. }
  564. }
  565. return nil
  566. })
  567. w.Write([]byte(fmt.Sprintf("imported %d streams, %d tables and %d rules", counts[0], counts[1], counts[2])))
  568. }
  569. func exportHandler(w http.ResponseWriter, r *http.Request) {
  570. const name = "ekuiper_export.json"
  571. exported, _, err := rulesetProcessor.Export()
  572. if err != nil {
  573. handleError(w, err, "export error", logger)
  574. return
  575. }
  576. w.Header().Set("Content-Type", "application/octet-stream")
  577. w.Header().Add("Content-Disposition", "Attachment")
  578. http.ServeContent(w, r, name, time.Now(), exported)
  579. }
  580. type Configuration struct {
  581. Streams map[string]string `json:"streams"`
  582. Tables map[string]string `json:"tables"`
  583. Rules map[string]string `json:"rules"`
  584. NativePlugins map[string]string `json:"nativePlugins"`
  585. PortablePlugins map[string]string `json:"portablePlugins"`
  586. SourceConfig map[string]string `json:"sourceConfig"`
  587. SinkConfig map[string]string `json:"sinkConfig"`
  588. ConnectionConfig map[string]string `json:"connectionConfig"`
  589. Service map[string]string `json:"Service"`
  590. Schema map[string]string `json:"Schema"`
  591. }
  592. func configurationExport() ([]byte, error) {
  593. conf := &Configuration{
  594. Streams: make(map[string]string),
  595. Tables: make(map[string]string),
  596. Rules: make(map[string]string),
  597. NativePlugins: make(map[string]string),
  598. PortablePlugins: make(map[string]string),
  599. SourceConfig: make(map[string]string),
  600. SinkConfig: make(map[string]string),
  601. ConnectionConfig: make(map[string]string),
  602. Service: make(map[string]string),
  603. Schema: make(map[string]string),
  604. }
  605. ruleSet := rulesetProcessor.ExportRuleSet()
  606. if ruleSet != nil {
  607. conf.Streams = ruleSet.Streams
  608. conf.Tables = ruleSet.Tables
  609. conf.Rules = ruleSet.Rules
  610. }
  611. conf.NativePlugins = pluginExport()
  612. conf.PortablePlugins = portablePluginExport()
  613. conf.Service = serviceExport()
  614. conf.Schema = schemaExport()
  615. yamlCfg := meta.GetConfigurations()
  616. conf.SourceConfig = yamlCfg.Sources
  617. conf.SinkConfig = yamlCfg.Sinks
  618. conf.ConnectionConfig = yamlCfg.Connections
  619. return json.Marshal(conf)
  620. }
  621. func configurationExportHandler(w http.ResponseWriter, r *http.Request) {
  622. const name = "ekuiper_export.json"
  623. jsonBytes, _ := configurationExport()
  624. w.Header().Set("Content-Type", "application/octet-stream")
  625. w.Header().Add("Content-Disposition", "Attachment")
  626. http.ServeContent(w, r, name, time.Now(), bytes.NewReader(jsonBytes))
  627. }
  628. func configurationReset() {
  629. _ = resetAllRules()
  630. _ = resetAllStreams()
  631. pluginReset()
  632. portablePluginsReset()
  633. serviceReset()
  634. schemaReset()
  635. meta.ResetConfigs()
  636. }
  637. func configurationImport(data []byte, reboot bool) error {
  638. conf := &Configuration{
  639. Streams: make(map[string]string),
  640. Tables: make(map[string]string),
  641. Rules: make(map[string]string),
  642. NativePlugins: make(map[string]string),
  643. PortablePlugins: make(map[string]string),
  644. SourceConfig: make(map[string]string),
  645. SinkConfig: make(map[string]string),
  646. ConnectionConfig: make(map[string]string),
  647. Service: make(map[string]string),
  648. Schema: make(map[string]string),
  649. }
  650. err := json.Unmarshal(data, conf)
  651. if err != nil {
  652. return fmt.Errorf("configuration unmarshal with error %v", err)
  653. }
  654. if reboot {
  655. err = pluginImport(conf.NativePlugins)
  656. if err != nil {
  657. return fmt.Errorf("pluginImportHandler with error %v", err)
  658. }
  659. err = schemaImport(conf.Schema)
  660. if err != nil {
  661. return fmt.Errorf("schemaImport with error %v", err)
  662. }
  663. }
  664. portablePluginImport(conf.PortablePlugins)
  665. serviceImport(conf.Service)
  666. yamlCfgSet := meta.YamlConfigurationSet{
  667. Sources: conf.SourceConfig,
  668. Sinks: conf.SinkConfig,
  669. Connections: conf.ConnectionConfig,
  670. }
  671. meta.LoadConfigurations(yamlCfgSet)
  672. ruleSet := processor.Ruleset{
  673. Streams: conf.Streams,
  674. Tables: conf.Tables,
  675. Rules: conf.Rules,
  676. }
  677. rulesetProcessor.ImportRuleSet(ruleSet)
  678. return nil
  679. }
  680. type configurationInfo struct {
  681. Content string `json:"content"`
  682. FilePath string `json:"file"`
  683. }
  684. func configurationImportHandler(w http.ResponseWriter, r *http.Request) {
  685. cb := r.URL.Query().Get("stop")
  686. stop := cb == "1"
  687. rsi := &configurationInfo{}
  688. err := json.NewDecoder(r.Body).Decode(rsi)
  689. if err != nil {
  690. handleError(w, err, "Invalid body: Error decoding json", logger)
  691. return
  692. }
  693. if rsi.Content != "" && rsi.FilePath != "" {
  694. handleError(w, nil, "Invalid body: Cannot specify both content and file", logger)
  695. return
  696. } else if rsi.Content == "" && rsi.FilePath == "" {
  697. handleError(w, nil, "Invalid body: must specify content or file", logger)
  698. return
  699. }
  700. content := []byte(rsi.Content)
  701. if rsi.FilePath != "" {
  702. reader, err := httpx.ReadFile(rsi.FilePath)
  703. if err != nil {
  704. handleError(w, nil, "Fail to read file", logger)
  705. return
  706. }
  707. defer reader.Close()
  708. buf := new(bytes.Buffer)
  709. _, err = io.Copy(buf, reader)
  710. if err != nil {
  711. handleError(w, err, "fail to convert file", logger)
  712. return
  713. }
  714. content = buf.Bytes()
  715. }
  716. configurationReset()
  717. err = configurationImport(content, stop)
  718. if err != nil {
  719. handleError(w, err, "Import configuration error", logger)
  720. return
  721. }
  722. if stop {
  723. go func() {
  724. time.Sleep(1 * time.Second)
  725. os.Exit(100)
  726. }()
  727. }
  728. w.WriteHeader(http.StatusOK)
  729. }
  730. func configurationStatusExport() Configuration {
  731. conf := Configuration{
  732. Streams: make(map[string]string),
  733. Tables: make(map[string]string),
  734. Rules: make(map[string]string),
  735. NativePlugins: make(map[string]string),
  736. PortablePlugins: make(map[string]string),
  737. SourceConfig: make(map[string]string),
  738. SinkConfig: make(map[string]string),
  739. ConnectionConfig: make(map[string]string),
  740. Service: make(map[string]string),
  741. Schema: make(map[string]string),
  742. }
  743. ruleSet := rulesetProcessor.ExportRuleSetStatus()
  744. if ruleSet != nil {
  745. conf.Streams = ruleSet.Streams
  746. conf.Tables = ruleSet.Tables
  747. conf.Rules = ruleSet.Rules
  748. }
  749. conf.NativePlugins = pluginStatusExport()
  750. conf.PortablePlugins = portablePluginStatusExport()
  751. conf.Service = serviceStatusExport()
  752. conf.Schema = schemaStatusExport()
  753. yamlCfgStatus := meta.GetConfigurationStatus()
  754. conf.SourceConfig = yamlCfgStatus.Sources
  755. conf.SinkConfig = yamlCfgStatus.Sinks
  756. conf.ConnectionConfig = yamlCfgStatus.Connections
  757. return conf
  758. }
  759. func configurationStatusHandler(w http.ResponseWriter, r *http.Request) {
  760. defer r.Body.Close()
  761. content := configurationStatusExport()
  762. jsonResponse(content, w, logger)
  763. }