window_rule_test.go 52 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544
  1. package topotest
  2. import (
  3. "github.com/emqx/kuiper/xstream"
  4. "github.com/emqx/kuiper/xstream/api"
  5. "testing"
  6. )
  7. func TestWindow(t *testing.T) {
  8. //Reset
  9. streamList := []string{"demo", "demoError", "demo1", "sessionDemo", "table1"}
  10. HandleStream(false, streamList, t)
  11. var tests = []RuleTest{
  12. {
  13. Name: `TestWindowRule1`,
  14. Sql: `SELECT * FROM demo GROUP BY HOPPINGWINDOW(ss, 2, 1)`,
  15. R: [][]map[string]interface{}{
  16. {{
  17. "color": "red",
  18. "size": float64(3),
  19. "ts": float64(1541152486013),
  20. }, {
  21. "color": "blue",
  22. "size": float64(6),
  23. "ts": float64(1541152486822),
  24. }},
  25. {{
  26. "color": "red",
  27. "size": float64(3),
  28. "ts": float64(1541152486013),
  29. }, {
  30. "color": "blue",
  31. "size": float64(6),
  32. "ts": float64(1541152486822),
  33. }, {
  34. "color": "blue",
  35. "size": float64(2),
  36. "ts": float64(1541152487632),
  37. }},
  38. {{
  39. "color": "blue",
  40. "size": float64(2),
  41. "ts": float64(1541152487632),
  42. }, {
  43. "color": "yellow",
  44. "size": float64(4),
  45. "ts": float64(1541152488442),
  46. }},
  47. {{
  48. "color": "yellow",
  49. "size": float64(4),
  50. "ts": float64(1541152488442),
  51. }, {
  52. "color": "red",
  53. "size": float64(1),
  54. "ts": float64(1541152489252),
  55. }},
  56. },
  57. M: map[string]interface{}{
  58. "op_1_preprocessor_demo_0_exceptions_total": int64(0),
  59. "op_1_preprocessor_demo_0_process_latency_us": int64(0),
  60. "op_1_preprocessor_demo_0_records_in_total": int64(5),
  61. "op_1_preprocessor_demo_0_records_out_total": int64(5),
  62. "op_3_project_0_exceptions_total": int64(0),
  63. "op_3_project_0_process_latency_us": int64(0),
  64. "op_3_project_0_records_in_total": int64(4),
  65. "op_3_project_0_records_out_total": int64(4),
  66. "sink_mockSink_0_exceptions_total": int64(0),
  67. "sink_mockSink_0_records_in_total": int64(4),
  68. "sink_mockSink_0_records_out_total": int64(4),
  69. "source_demo_0_exceptions_total": int64(0),
  70. "source_demo_0_records_in_total": int64(5),
  71. "source_demo_0_records_out_total": int64(5),
  72. "op_2_window_0_exceptions_total": int64(0),
  73. "op_2_window_0_process_latency_us": int64(0),
  74. "op_2_window_0_records_in_total": int64(5),
  75. "op_2_window_0_records_out_total": int64(4),
  76. },
  77. }, {
  78. Name: `TestWindowRule2`,
  79. Sql: `SELECT color, ts FROM demo where size > 2 GROUP BY tumblingwindow(ss, 1)`,
  80. R: [][]map[string]interface{}{
  81. {{
  82. "color": "red",
  83. "ts": float64(1541152486013),
  84. }, {
  85. "color": "blue",
  86. "ts": float64(1541152486822),
  87. }},
  88. {{
  89. "color": "yellow",
  90. "ts": float64(1541152488442),
  91. }},
  92. },
  93. M: map[string]interface{}{
  94. "op_1_preprocessor_demo_0_exceptions_total": int64(0),
  95. "op_1_preprocessor_demo_0_process_latency_us": int64(0),
  96. "op_1_preprocessor_demo_0_records_in_total": int64(5),
  97. "op_1_preprocessor_demo_0_records_out_total": int64(5),
  98. "op_4_project_0_exceptions_total": int64(0),
  99. "op_4_project_0_process_latency_us": int64(0),
  100. "op_4_project_0_records_in_total": int64(2),
  101. "op_4_project_0_records_out_total": int64(2),
  102. "sink_mockSink_0_exceptions_total": int64(0),
  103. "sink_mockSink_0_records_in_total": int64(2),
  104. "sink_mockSink_0_records_out_total": int64(2),
  105. "source_demo_0_exceptions_total": int64(0),
  106. "source_demo_0_records_in_total": int64(5),
  107. "source_demo_0_records_out_total": int64(5),
  108. "op_3_window_0_exceptions_total": int64(0),
  109. "op_3_window_0_process_latency_us": int64(0),
  110. "op_3_window_0_records_in_total": int64(3),
  111. "op_3_window_0_records_out_total": int64(2),
  112. "op_2_filter_0_exceptions_total": int64(0),
  113. "op_2_filter_0_process_latency_us": int64(0),
  114. "op_2_filter_0_records_in_total": int64(5),
  115. "op_2_filter_0_records_out_total": int64(3),
  116. },
  117. }, {
  118. Name: `TestWindowRule3`,
  119. Sql: `SELECT color, temp, demo.ts as ts1, demo1.ts as ts2 FROM demo INNER JOIN demo1 ON ts1 = ts2 GROUP BY SlidingWindow(ss, 1)`,
  120. R: [][]map[string]interface{}{
  121. {{
  122. "color": "red",
  123. "temp": 25.5,
  124. "ts1": float64(1541152486013),
  125. "ts2": float64(1541152486013),
  126. }}, {{
  127. "color": "red",
  128. "temp": 25.5,
  129. "ts1": float64(1541152486013),
  130. "ts2": float64(1541152486013),
  131. }}, {{
  132. "color": "red",
  133. "temp": 25.5,
  134. "ts1": float64(1541152486013),
  135. "ts2": float64(1541152486013),
  136. }}, {{
  137. "color": "blue",
  138. "temp": 28.1,
  139. "ts1": float64(1541152487632),
  140. "ts2": float64(1541152487632),
  141. }}, {{
  142. "color": "blue",
  143. "temp": 28.1,
  144. "ts1": float64(1541152487632),
  145. "ts2": float64(1541152487632),
  146. }}, {{
  147. "color": "blue",
  148. "temp": 28.1,
  149. "ts1": float64(1541152487632),
  150. "ts2": float64(1541152487632),
  151. }, {
  152. "color": "yellow",
  153. "temp": 27.4,
  154. "ts1": float64(1541152488442),
  155. "ts2": float64(1541152488442),
  156. }}, {{
  157. "color": "yellow",
  158. "temp": 27.4,
  159. "ts1": float64(1541152488442),
  160. "ts2": float64(1541152488442),
  161. }}, {{
  162. "color": "yellow",
  163. "temp": 27.4,
  164. "ts1": float64(1541152488442),
  165. "ts2": float64(1541152488442),
  166. }, {
  167. "color": "red",
  168. "temp": 25.5,
  169. "ts1": float64(1541152489252),
  170. "ts2": float64(1541152489252),
  171. }},
  172. },
  173. M: map[string]interface{}{
  174. "op_1_preprocessor_demo_0_exceptions_total": int64(0),
  175. "op_1_preprocessor_demo_0_process_latency_us": int64(0),
  176. "op_1_preprocessor_demo_0_records_in_total": int64(5),
  177. "op_1_preprocessor_demo_0_records_out_total": int64(5),
  178. "op_2_preprocessor_demo1_0_exceptions_total": int64(0),
  179. "op_2_preprocessor_demo1_0_process_latency_us": int64(0),
  180. "op_2_preprocessor_demo1_0_records_in_total": int64(5),
  181. "op_2_preprocessor_demo1_0_records_out_total": int64(5),
  182. "op_5_project_0_exceptions_total": int64(0),
  183. "op_5_project_0_process_latency_us": int64(0),
  184. "op_5_project_0_records_in_total": int64(8),
  185. "op_5_project_0_records_out_total": int64(8),
  186. "sink_mockSink_0_exceptions_total": int64(0),
  187. "sink_mockSink_0_records_in_total": int64(8),
  188. "sink_mockSink_0_records_out_total": int64(8),
  189. "source_demo_0_exceptions_total": int64(0),
  190. "source_demo_0_records_in_total": int64(5),
  191. "source_demo_0_records_out_total": int64(5),
  192. "source_demo1_0_exceptions_total": int64(0),
  193. "source_demo1_0_records_in_total": int64(5),
  194. "source_demo1_0_records_out_total": int64(5),
  195. "op_3_window_0_exceptions_total": int64(0),
  196. "op_3_window_0_process_latency_us": int64(0),
  197. "op_3_window_0_records_in_total": int64(10),
  198. "op_3_window_0_records_out_total": int64(10),
  199. "op_4_join_0_exceptions_total": int64(0),
  200. "op_4_join_0_process_latency_us": int64(0),
  201. "op_4_join_0_records_in_total": int64(10),
  202. "op_4_join_0_records_out_total": int64(8),
  203. },
  204. T: &xstream.PrintableTopo{
  205. Sources: []string{"source_demo", "source_demo1"},
  206. Edges: map[string][]string{
  207. "source_demo": {"op_1_preprocessor_demo"},
  208. "source_demo1": {"op_2_preprocessor_demo1"},
  209. "op_1_preprocessor_demo": {"op_3_window"},
  210. "op_2_preprocessor_demo1": {"op_3_window"},
  211. "op_3_window": {"op_4_join"},
  212. "op_4_join": {"op_5_project"},
  213. "op_5_project": {"sink_mockSink"},
  214. },
  215. },
  216. }, {
  217. Name: `TestWindowRule4`,
  218. Sql: `SELECT color, count(*) as c FROM demo GROUP BY SlidingWindow(ss, 2), color ORDER BY color`,
  219. R: [][]map[string]interface{}{
  220. {{
  221. "color": "red",
  222. "c": float64(1),
  223. }}, {{
  224. "color": "blue",
  225. "c": float64(1),
  226. }, {
  227. "color": "red",
  228. "c": float64(1),
  229. }}, {{
  230. "color": "blue",
  231. "c": float64(2),
  232. }, {
  233. "color": "red",
  234. "c": float64(1),
  235. }}, {{
  236. "color": "blue",
  237. "c": float64(2),
  238. }, {
  239. "color": "yellow",
  240. "c": float64(1),
  241. }}, {{
  242. "color": "blue",
  243. "c": float64(1),
  244. }, {
  245. "color": "red",
  246. "c": float64(1),
  247. }, {
  248. "color": "yellow",
  249. "c": float64(1),
  250. }},
  251. },
  252. M: map[string]interface{}{
  253. "op_1_preprocessor_demo_0_exceptions_total": int64(0),
  254. "op_1_preprocessor_demo_0_process_latency_us": int64(0),
  255. "op_1_preprocessor_demo_0_records_in_total": int64(5),
  256. "op_1_preprocessor_demo_0_records_out_total": int64(5),
  257. "op_5_project_0_exceptions_total": int64(0),
  258. "op_5_project_0_process_latency_us": int64(0),
  259. "op_5_project_0_records_in_total": int64(5),
  260. "op_5_project_0_records_out_total": int64(5),
  261. "sink_mockSink_0_exceptions_total": int64(0),
  262. "sink_mockSink_0_records_in_total": int64(5),
  263. "sink_mockSink_0_records_out_total": int64(5),
  264. "source_demo_0_exceptions_total": int64(0),
  265. "source_demo_0_records_in_total": int64(5),
  266. "source_demo_0_records_out_total": int64(5),
  267. "op_2_window_0_exceptions_total": int64(0),
  268. "op_2_window_0_process_latency_us": int64(0),
  269. "op_2_window_0_records_in_total": int64(5),
  270. "op_2_window_0_records_out_total": int64(5),
  271. "op_3_aggregate_0_exceptions_total": int64(0),
  272. "op_3_aggregate_0_process_latency_us": int64(0),
  273. "op_3_aggregate_0_records_in_total": int64(5),
  274. "op_3_aggregate_0_records_out_total": int64(5),
  275. "op_4_order_0_exceptions_total": int64(0),
  276. "op_4_order_0_process_latency_us": int64(0),
  277. "op_4_order_0_records_in_total": int64(5),
  278. "op_4_order_0_records_out_total": int64(5),
  279. },
  280. }, {
  281. Name: `TestWindowRule5`,
  282. Sql: `SELECT count(temp), window_start() as ws, window_end() FROM sessionDemo GROUP BY SessionWindow(ss, 2, 1) `,
  283. R: [][]map[string]interface{}{
  284. {{
  285. "count": float64(2),
  286. "ws": float64(1541152486013),
  287. "window_end": float64(1541152487823), // timeout
  288. }}, {{
  289. "count": float64(3),
  290. "ws": float64(1541152487932),
  291. "window_end": float64(1541152490000), // tick
  292. }}, {{
  293. "count": float64(5),
  294. "ws": float64(1541152490000),
  295. "window_end": float64(1541152494000), // tick
  296. }}, {{
  297. "count": float64(1),
  298. "ws": float64(1541152494000),
  299. "window_end": float64(1541152495112), // timeout
  300. }},
  301. },
  302. M: map[string]interface{}{
  303. "op_1_preprocessor_sessionDemo_0_exceptions_total": int64(0),
  304. "op_1_preprocessor_sessionDemo_0_process_latency_us": int64(0),
  305. "op_1_preprocessor_sessionDemo_0_records_in_total": int64(11),
  306. "op_1_preprocessor_sessionDemo_0_records_out_total": int64(11),
  307. "op_3_project_0_exceptions_total": int64(0),
  308. "op_3_project_0_process_latency_us": int64(0),
  309. "op_3_project_0_records_in_total": int64(4),
  310. "op_3_project_0_records_out_total": int64(4),
  311. "sink_mockSink_0_exceptions_total": int64(0),
  312. "sink_mockSink_0_records_in_total": int64(4),
  313. "sink_mockSink_0_records_out_total": int64(4),
  314. "source_sessionDemo_0_exceptions_total": int64(0),
  315. "source_sessionDemo_0_records_in_total": int64(11),
  316. "source_sessionDemo_0_records_out_total": int64(11),
  317. "op_2_window_0_exceptions_total": int64(0),
  318. "op_2_window_0_process_latency_us": int64(0),
  319. "op_2_window_0_records_in_total": int64(11),
  320. "op_2_window_0_records_out_total": int64(4),
  321. },
  322. }, {
  323. Name: `TestWindowRule6`,
  324. Sql: `SELECT window_end(), max(temp) as m, count(color) as c, window_start() FROM demo INNER JOIN demo1 ON demo.ts = demo1.ts GROUP BY SlidingWindow(ss, 1)`,
  325. R: [][]map[string]interface{}{
  326. {{
  327. "m": 25.5,
  328. "c": float64(1),
  329. "window_start": float64(1541152485115),
  330. "window_end": float64(1541152486115),
  331. }}, {{
  332. "m": 25.5,
  333. "c": float64(1),
  334. "window_start": float64(1541152485822),
  335. "window_end": float64(1541152486822),
  336. }}, {{
  337. "m": 25.5,
  338. "c": float64(1),
  339. "window_start": float64(1541152485903),
  340. "window_end": float64(1541152486903),
  341. }}, {{
  342. "m": 28.1,
  343. "c": float64(1),
  344. "window_start": float64(1541152486702),
  345. "window_end": float64(1541152487702),
  346. }}, {{
  347. "m": 28.1,
  348. "c": float64(1),
  349. "window_start": float64(1541152487442),
  350. "window_end": float64(1541152488442),
  351. }}, {{
  352. "m": 28.1,
  353. "c": float64(2),
  354. "window_start": float64(1541152487605),
  355. "window_end": float64(1541152488605),
  356. }}, {{
  357. "m": 27.4,
  358. "c": float64(1),
  359. "window_start": float64(1541152488252),
  360. "window_end": float64(1541152489252),
  361. }}, {{
  362. "m": 27.4,
  363. "c": float64(2),
  364. "window_start": float64(1541152488305),
  365. "window_end": float64(1541152489305),
  366. }},
  367. },
  368. M: map[string]interface{}{
  369. "op_1_preprocessor_demo_0_exceptions_total": int64(0),
  370. "op_1_preprocessor_demo_0_process_latency_us": int64(0),
  371. "op_1_preprocessor_demo_0_records_in_total": int64(5),
  372. "op_1_preprocessor_demo_0_records_out_total": int64(5),
  373. "op_2_preprocessor_demo1_0_exceptions_total": int64(0),
  374. "op_2_preprocessor_demo1_0_process_latency_us": int64(0),
  375. "op_2_preprocessor_demo1_0_records_in_total": int64(5),
  376. "op_2_preprocessor_demo1_0_records_out_total": int64(5),
  377. "op_5_project_0_exceptions_total": int64(0),
  378. "op_5_project_0_process_latency_us": int64(0),
  379. "op_5_project_0_records_in_total": int64(8),
  380. "op_5_project_0_records_out_total": int64(8),
  381. "sink_mockSink_0_exceptions_total": int64(0),
  382. "sink_mockSink_0_records_in_total": int64(8),
  383. "sink_mockSink_0_records_out_total": int64(8),
  384. "source_demo_0_exceptions_total": int64(0),
  385. "source_demo_0_records_in_total": int64(5),
  386. "source_demo_0_records_out_total": int64(5),
  387. "source_demo1_0_exceptions_total": int64(0),
  388. "source_demo1_0_records_in_total": int64(5),
  389. "source_demo1_0_records_out_total": int64(5),
  390. "op_3_window_0_exceptions_total": int64(0),
  391. "op_3_window_0_process_latency_us": int64(0),
  392. "op_3_window_0_records_in_total": int64(10),
  393. "op_3_window_0_records_out_total": int64(10),
  394. "op_4_join_0_exceptions_total": int64(0),
  395. "op_4_join_0_process_latency_us": int64(0),
  396. "op_4_join_0_records_in_total": int64(10),
  397. "op_4_join_0_records_out_total": int64(8),
  398. },
  399. }, {
  400. Name: `TestWindowRule7`,
  401. Sql: `SELECT * FROM demoError GROUP BY HOPPINGWINDOW(ss, 2, 1)`,
  402. R: [][]map[string]interface{}{
  403. {{
  404. "error": "error in preprocessor: invalid data type for color, expect string but found int(3)",
  405. }},
  406. {{
  407. "color": "blue",
  408. "size": float64(6),
  409. "ts": float64(1541152486822),
  410. }},
  411. {{
  412. "color": "blue",
  413. "size": float64(6),
  414. "ts": float64(1541152486822),
  415. }, {
  416. "color": "blue",
  417. "size": float64(2),
  418. "ts": float64(1541152487632),
  419. }},
  420. {{
  421. "error": "error in preprocessor: invalid data type for color, expect string but found int(7)",
  422. }},
  423. {{
  424. "color": "blue",
  425. "size": float64(2),
  426. "ts": float64(1541152487632),
  427. }},
  428. {{
  429. "error": "error in preprocessor: invalid data type for size, expect bigint but found string(blue)",
  430. }},
  431. },
  432. M: map[string]interface{}{
  433. "op_1_preprocessor_demoError_0_exceptions_total": int64(3),
  434. "op_1_preprocessor_demoError_0_process_latency_us": int64(0),
  435. "op_1_preprocessor_demoError_0_records_in_total": int64(5),
  436. "op_1_preprocessor_demoError_0_records_out_total": int64(2),
  437. "op_3_project_0_exceptions_total": int64(3),
  438. "op_3_project_0_process_latency_us": int64(0),
  439. "op_3_project_0_records_in_total": int64(6),
  440. "op_3_project_0_records_out_total": int64(3),
  441. "sink_mockSink_0_exceptions_total": int64(0),
  442. "sink_mockSink_0_records_in_total": int64(6),
  443. "sink_mockSink_0_records_out_total": int64(6),
  444. "source_demoError_0_exceptions_total": int64(0),
  445. "source_demoError_0_records_in_total": int64(5),
  446. "source_demoError_0_records_out_total": int64(5),
  447. "op_2_window_0_exceptions_total": int64(3),
  448. "op_2_window_0_process_latency_us": int64(0),
  449. "op_2_window_0_records_in_total": int64(5),
  450. "op_2_window_0_records_out_total": int64(3),
  451. },
  452. }, {
  453. Name: `TestWindowRule8`,
  454. Sql: `SELECT color, window_end(), ts, count(*) as c, window_start() FROM demo where size > 2 GROUP BY tumblingwindow(ss, 1) having c > 1`,
  455. R: [][]map[string]interface{}{
  456. {{
  457. "color": "red",
  458. "ts": float64(1541152486013),
  459. "c": float64(2),
  460. "window_start": float64(1541152486000),
  461. "window_end": float64(1541152487000),
  462. }},
  463. },
  464. M: map[string]interface{}{
  465. "op_1_preprocessor_demo_0_exceptions_total": int64(0),
  466. "op_1_preprocessor_demo_0_process_latency_us": int64(0),
  467. "op_1_preprocessor_demo_0_records_in_total": int64(5),
  468. "op_1_preprocessor_demo_0_records_out_total": int64(5),
  469. "op_5_project_0_exceptions_total": int64(0),
  470. "op_5_project_0_process_latency_us": int64(0),
  471. "op_5_project_0_records_in_total": int64(1),
  472. "op_5_project_0_records_out_total": int64(1),
  473. "sink_mockSink_0_exceptions_total": int64(0),
  474. "sink_mockSink_0_records_in_total": int64(1),
  475. "sink_mockSink_0_records_out_total": int64(1),
  476. "source_demo_0_exceptions_total": int64(0),
  477. "source_demo_0_records_in_total": int64(5),
  478. "source_demo_0_records_out_total": int64(5),
  479. "op_3_window_0_exceptions_total": int64(0),
  480. "op_3_window_0_process_latency_us": int64(0),
  481. "op_3_window_0_records_in_total": int64(3),
  482. "op_3_window_0_records_out_total": int64(2),
  483. "op_2_filter_0_exceptions_total": int64(0),
  484. "op_2_filter_0_process_latency_us": int64(0),
  485. "op_2_filter_0_records_in_total": int64(5),
  486. "op_2_filter_0_records_out_total": int64(3),
  487. "op_4_having_0_exceptions_total": int64(0),
  488. "op_4_having_0_process_latency_us": int64(0),
  489. "op_4_having_0_records_in_total": int64(2),
  490. "op_4_having_0_records_out_total": int64(1),
  491. },
  492. }, {
  493. Name: `TestWindowRule9`,
  494. Sql: `SELECT color, window_start(), window_end() FROM demo GROUP BY HOPPINGWINDOW(ss, 2, 1) FILTER( WHERE size > 2)`,
  495. R: [][]map[string]interface{}{
  496. {{
  497. "color": "red",
  498. "window_start": float64(1541152485000),
  499. "window_end": float64(1541152487000),
  500. }},
  501. {{
  502. "color": "red",
  503. "window_start": float64(1541152486000),
  504. "window_end": float64(1541152488000),
  505. }},
  506. {{
  507. "color": "yellow",
  508. "window_start": float64(1541152487000),
  509. "window_end": float64(1541152489000),
  510. }},
  511. {{
  512. "color": "yellow",
  513. "window_start": float64(1541152488000),
  514. "window_end": float64(1541152490000),
  515. }},
  516. },
  517. M: map[string]interface{}{
  518. "op_1_preprocessor_demo_0_exceptions_total": int64(0),
  519. "op_1_preprocessor_demo_0_process_latency_us": int64(0),
  520. "op_1_preprocessor_demo_0_records_in_total": int64(5),
  521. "op_1_preprocessor_demo_0_records_out_total": int64(5),
  522. "op_4_project_0_exceptions_total": int64(0),
  523. "op_4_project_0_process_latency_us": int64(0),
  524. "op_4_project_0_records_in_total": int64(4),
  525. "op_4_project_0_records_out_total": int64(4),
  526. "sink_mockSink_0_exceptions_total": int64(0),
  527. "sink_mockSink_0_records_in_total": int64(4),
  528. "sink_mockSink_0_records_out_total": int64(4),
  529. "source_demo_0_exceptions_total": int64(0),
  530. "source_demo_0_records_in_total": int64(5),
  531. "source_demo_0_records_out_total": int64(5),
  532. "op_3_window_0_exceptions_total": int64(0),
  533. "op_3_window_0_process_latency_us": int64(0),
  534. "op_3_window_0_records_in_total": int64(3),
  535. "op_3_window_0_records_out_total": int64(4),
  536. },
  537. }, {
  538. Name: `TestCountWindowRule1`,
  539. Sql: `SELECT collect(*)[0]->color as c FROM demo GROUP BY COUNTWINDOW(3)`,
  540. R: [][]map[string]interface{}{
  541. {{
  542. "c": "red",
  543. }},
  544. },
  545. M: map[string]interface{}{
  546. "op_1_preprocessor_demo_0_exceptions_total": int64(0),
  547. "op_1_preprocessor_demo_0_process_latency_us": int64(0),
  548. "op_1_preprocessor_demo_0_records_in_total": int64(5),
  549. "op_1_preprocessor_demo_0_records_out_total": int64(5),
  550. "op_3_project_0_exceptions_total": int64(0),
  551. "op_3_project_0_process_latency_us": int64(0),
  552. "op_3_project_0_records_in_total": int64(1),
  553. "op_3_project_0_records_out_total": int64(1),
  554. "sink_mockSink_0_exceptions_total": int64(0),
  555. "sink_mockSink_0_records_in_total": int64(1),
  556. "sink_mockSink_0_records_out_total": int64(1),
  557. "source_demo_0_exceptions_total": int64(0),
  558. "source_demo_0_records_in_total": int64(5),
  559. "source_demo_0_records_out_total": int64(5),
  560. "op_2_window_0_exceptions_total": int64(0),
  561. "op_2_window_0_process_latency_us": int64(0),
  562. "op_2_window_0_records_in_total": int64(5),
  563. "op_2_window_0_records_out_total": int64(1),
  564. },
  565. }, {
  566. Name: `TestWindowRule10`,
  567. Sql: `SELECT deduplicate(color, false)->color as c FROM demo GROUP BY SlidingWindow(hh, 1)`,
  568. R: [][]map[string]interface{}{
  569. {{
  570. "c": "red",
  571. }}, {{
  572. "c": "blue",
  573. }}, {{}}, {{
  574. "c": "yellow",
  575. }}, {{}},
  576. },
  577. M: map[string]interface{}{
  578. "op_1_preprocessor_demo_0_exceptions_total": int64(0),
  579. "op_1_preprocessor_demo_0_process_latency_us": int64(0),
  580. "op_1_preprocessor_demo_0_records_in_total": int64(5),
  581. "op_1_preprocessor_demo_0_records_out_total": int64(5),
  582. "op_3_project_0_exceptions_total": int64(0),
  583. "op_3_project_0_process_latency_us": int64(0),
  584. "op_3_project_0_records_in_total": int64(5),
  585. "op_3_project_0_records_out_total": int64(5),
  586. "sink_mockSink_0_exceptions_total": int64(0),
  587. "sink_mockSink_0_records_in_total": int64(5),
  588. "sink_mockSink_0_records_out_total": int64(5),
  589. "source_demo_0_exceptions_total": int64(0),
  590. "source_demo_0_records_in_total": int64(5),
  591. "source_demo_0_records_out_total": int64(5),
  592. "op_2_window_0_exceptions_total": int64(0),
  593. "op_2_window_0_process_latency_us": int64(0),
  594. "op_2_window_0_records_in_total": int64(5),
  595. "op_2_window_0_records_out_total": int64(5),
  596. },
  597. }, {
  598. Name: `TestWindowRule11`,
  599. Sql: `SELECT color, name, window_start(), window_end() FROM demo INNER JOIN table1 on demo.ts = table1.id where demo.size > 2 and table1.size > 1 GROUP BY tumblingwindow(ss, 1)`,
  600. R: [][]map[string]interface{}{
  601. {{
  602. "color": "red",
  603. "name": "name1",
  604. "window_start": float64(1541152486000),
  605. "window_end": float64(1541152487000),
  606. }},
  607. },
  608. M: map[string]interface{}{
  609. //"op_4_project_0_exceptions_total": int64(0),
  610. //"op_4_project_0_process_latency_us": int64(0),
  611. //"op_4_project_0_records_in_total": int64(2),
  612. //"op_4_project_0_records_out_total": int64(2),
  613. "op_3_window_0_exceptions_total": int64(0),
  614. "op_3_window_0_process_latency_us": int64(0),
  615. "op_3_window_0_records_in_total": int64(3),
  616. "op_3_window_0_records_out_total": int64(2),
  617. "op_2_filter_0_exceptions_total": int64(0),
  618. "op_2_filter_0_process_latency_us": int64(0),
  619. "op_2_filter_0_records_in_total": int64(5),
  620. "op_2_filter_0_records_out_total": int64(3),
  621. "op_1_preprocessor_demo_0_exceptions_total": int64(0),
  622. "op_1_preprocessor_demo_0_records_in_total": int64(5),
  623. "op_1_preprocessor_demo_0_records_out_total": int64(5),
  624. "op_4_tableprocessor_table1_0_exceptions_total": int64(0),
  625. "op_4_tableprocessor_table1_0_records_in_total": int64(4),
  626. "op_4_tableprocessor_table1_0_records_out_total": int64(1),
  627. "op_5_filter_0_exceptions_total": int64(0),
  628. "op_5_filter_0_records_in_total": int64(1),
  629. "op_5_filter_0_records_out_total": int64(1),
  630. "op_6_join_aligner_0_records_in_total": int64(3),
  631. "op_6_join_aligner_0_records_out_total": int64(2),
  632. "op_7_join_0_exceptions_total": int64(0),
  633. "op_7_join_0_records_in_total": int64(2),
  634. "op_7_join_0_records_out_total": int64(1),
  635. "op_8_project_0_exceptions_total": int64(0),
  636. "op_8_project_0_records_in_total": int64(1),
  637. "op_8_project_0_records_out_total": int64(1),
  638. "sink_mockSink_0_exceptions_total": int64(0),
  639. "sink_mockSink_0_records_in_total": int64(1),
  640. "sink_mockSink_0_records_out_total": int64(1),
  641. "source_demo_0_exceptions_total": int64(0),
  642. "source_demo_0_records_in_total": int64(5),
  643. "source_demo_0_records_out_total": int64(5),
  644. "source_table1_0_exceptions_total": int64(0),
  645. "source_table1_0_records_in_total": int64(4),
  646. "source_table1_0_records_out_total": int64(4),
  647. },
  648. },
  649. }
  650. HandleStream(true, streamList, t)
  651. options := []*api.RuleOption{
  652. {
  653. BufferLength: 100,
  654. SendError: true,
  655. }, {
  656. BufferLength: 100,
  657. SendError: true,
  658. Qos: api.AtLeastOnce,
  659. CheckpointInterval: 5000,
  660. }, {
  661. BufferLength: 100,
  662. SendError: true,
  663. Qos: api.ExactlyOnce,
  664. CheckpointInterval: 5000,
  665. },
  666. }
  667. for j, opt := range options {
  668. DoRuleTest(t, tests, j, opt, 15)
  669. }
  670. }
  671. func TestEventWindow(t *testing.T) {
  672. //Reset
  673. streamList := []string{"demoE", "demoErr", "demo1E", "sessionDemoE"}
  674. HandleStream(false, streamList, t)
  675. var tests = []RuleTest{
  676. {
  677. Name: `TestEventWindowRule1`,
  678. Sql: `SELECT * FROM demoE GROUP BY HOPPINGWINDOW(ss, 2, 1)`,
  679. R: [][]map[string]interface{}{
  680. {{
  681. "color": "red",
  682. "size": float64(3),
  683. "ts": float64(1541152486013),
  684. }},
  685. {{
  686. "color": "red",
  687. "size": float64(3),
  688. "ts": float64(1541152486013),
  689. }, {
  690. "color": "blue",
  691. "size": float64(2),
  692. "ts": float64(1541152487632),
  693. }},
  694. {{
  695. "color": "blue",
  696. "size": float64(2),
  697. "ts": float64(1541152487632),
  698. }, {
  699. "color": "yellow",
  700. "size": float64(4),
  701. "ts": float64(1541152488442),
  702. }}, {{
  703. "color": "yellow",
  704. "size": float64(4),
  705. "ts": float64(1541152488442),
  706. }, {
  707. "color": "red",
  708. "size": float64(1),
  709. "ts": float64(1541152489252),
  710. }}, {{
  711. "color": "red",
  712. "size": float64(1),
  713. "ts": float64(1541152489252),
  714. }},
  715. },
  716. M: map[string]interface{}{
  717. "op_1_preprocessor_demoE_0_exceptions_total": int64(0),
  718. "op_1_preprocessor_demoE_0_process_latency_us": int64(0),
  719. "op_1_preprocessor_demoE_0_records_in_total": int64(6),
  720. "op_1_preprocessor_demoE_0_records_out_total": int64(6),
  721. "op_3_project_0_exceptions_total": int64(0),
  722. "op_3_project_0_process_latency_us": int64(0),
  723. "op_3_project_0_records_in_total": int64(5),
  724. "op_3_project_0_records_out_total": int64(5),
  725. "sink_mockSink_0_exceptions_total": int64(0),
  726. "sink_mockSink_0_records_in_total": int64(5),
  727. "sink_mockSink_0_records_out_total": int64(5),
  728. "source_demoE_0_exceptions_total": int64(0),
  729. "source_demoE_0_records_in_total": int64(6),
  730. "source_demoE_0_records_out_total": int64(6),
  731. "op_2_window_0_exceptions_total": int64(0),
  732. "op_2_window_0_process_latency_us": int64(0),
  733. "op_2_window_0_records_in_total": int64(6),
  734. "op_2_window_0_records_out_total": int64(5),
  735. },
  736. }, {
  737. Name: `TestEventWindowRule2`,
  738. Sql: `SELECT window_start(), window_end(), color, ts FROM demoE where size > 2 GROUP BY tumblingwindow(ss, 1)`,
  739. R: [][]map[string]interface{}{
  740. {{
  741. "window_start": float64(1541152486000),
  742. "window_end": float64(1541152487000),
  743. "color": "red",
  744. "ts": float64(1541152486013),
  745. }},
  746. {{
  747. "window_start": float64(1541152488000),
  748. "window_end": float64(1541152489000),
  749. "color": "yellow",
  750. "ts": float64(1541152488442),
  751. }},
  752. },
  753. M: map[string]interface{}{
  754. "op_1_preprocessor_demoE_0_exceptions_total": int64(0),
  755. "op_1_preprocessor_demoE_0_process_latency_us": int64(0),
  756. "op_1_preprocessor_demoE_0_records_in_total": int64(6),
  757. "op_1_preprocessor_demoE_0_records_out_total": int64(6),
  758. "op_4_project_0_exceptions_total": int64(0),
  759. "op_4_project_0_process_latency_us": int64(0),
  760. "op_4_project_0_records_in_total": int64(2),
  761. "op_4_project_0_records_out_total": int64(2),
  762. "sink_mockSink_0_exceptions_total": int64(0),
  763. "sink_mockSink_0_records_in_total": int64(2),
  764. "sink_mockSink_0_records_out_total": int64(2),
  765. "source_demoE_0_exceptions_total": int64(0),
  766. "source_demoE_0_records_in_total": int64(6),
  767. "source_demoE_0_records_out_total": int64(6),
  768. "op_2_window_0_exceptions_total": int64(0),
  769. "op_2_window_0_process_latency_us": int64(0),
  770. "op_2_window_0_records_in_total": int64(6),
  771. "op_2_window_0_records_out_total": int64(4),
  772. "op_3_filter_0_exceptions_total": int64(0),
  773. "op_3_filter_0_process_latency_us": int64(0),
  774. "op_3_filter_0_records_in_total": int64(4),
  775. "op_3_filter_0_records_out_total": int64(2),
  776. },
  777. }, {
  778. Name: `TestEventWindowRule3`,
  779. Sql: `SELECT color, temp, demoE.ts FROM demoE INNER JOIN demo1E ON demoE.ts = demo1E.ts GROUP BY SlidingWindow(ss, 1)`,
  780. R: [][]map[string]interface{}{
  781. {{
  782. "color": "red",
  783. "temp": 25.5,
  784. "ts": float64(1541152486013),
  785. }}, {{
  786. "color": "red",
  787. "temp": 25.5,
  788. "ts": float64(1541152486013),
  789. }}, {{
  790. "color": "blue",
  791. "temp": 28.1,
  792. "ts": float64(1541152487632),
  793. }}, {{
  794. "color": "blue",
  795. "temp": 28.1,
  796. "ts": float64(1541152487632),
  797. }, {
  798. "color": "yellow",
  799. "temp": 27.4,
  800. "ts": float64(1541152488442),
  801. }}, {{
  802. "color": "yellow",
  803. "temp": 27.4,
  804. "ts": float64(1541152488442),
  805. }, {
  806. "color": "red",
  807. "temp": 25.5,
  808. "ts": float64(1541152489252),
  809. }},
  810. },
  811. M: map[string]interface{}{
  812. "op_1_preprocessor_demoE_0_exceptions_total": int64(0),
  813. "op_1_preprocessor_demoE_0_process_latency_us": int64(0),
  814. "op_1_preprocessor_demoE_0_records_in_total": int64(6),
  815. "op_1_preprocessor_demoE_0_records_out_total": int64(6),
  816. "op_2_preprocessor_demo1E_0_exceptions_total": int64(0),
  817. "op_2_preprocessor_demo1E_0_process_latency_us": int64(0),
  818. "op_2_preprocessor_demo1E_0_records_in_total": int64(6),
  819. "op_2_preprocessor_demo1E_0_records_out_total": int64(6),
  820. "op_5_project_0_exceptions_total": int64(0),
  821. "op_5_project_0_process_latency_us": int64(0),
  822. "op_5_project_0_records_in_total": int64(5),
  823. "op_5_project_0_records_out_total": int64(5),
  824. "sink_mockSink_0_exceptions_total": int64(0),
  825. "sink_mockSink_0_records_in_total": int64(5),
  826. "sink_mockSink_0_records_out_total": int64(5),
  827. "source_demoE_0_exceptions_total": int64(0),
  828. "source_demoE_0_records_in_total": int64(6),
  829. "source_demoE_0_records_out_total": int64(6),
  830. "source_demo1E_0_exceptions_total": int64(0),
  831. "source_demo1E_0_records_in_total": int64(6),
  832. "source_demo1E_0_records_out_total": int64(6),
  833. "op_3_window_0_exceptions_total": int64(0),
  834. "op_3_window_0_process_latency_us": int64(0),
  835. "op_3_window_0_records_in_total": int64(12),
  836. "op_3_window_0_records_out_total": int64(5),
  837. "op_4_join_0_exceptions_total": int64(0),
  838. "op_4_join_0_process_latency_us": int64(0),
  839. "op_4_join_0_records_in_total": int64(5),
  840. "op_4_join_0_records_out_total": int64(5),
  841. },
  842. }, {
  843. Name: `TestEventWindowRule4`,
  844. Sql: `SELECT window_start() as ws, color, window_end() as we FROM demoE GROUP BY SlidingWindow(ss, 2), color ORDER BY color`,
  845. R: [][]map[string]interface{}{
  846. {{
  847. "color": "red",
  848. "ws": float64(1541152484013),
  849. "we": float64(1541152486013),
  850. }}, {{
  851. "color": "blue",
  852. "ws": float64(1541152485632),
  853. "we": float64(1541152487632),
  854. }, {
  855. "color": "red",
  856. "ws": float64(1541152485632),
  857. "we": float64(1541152487632),
  858. }}, {{
  859. "color": "blue",
  860. "ws": float64(1541152486442),
  861. "we": float64(1541152488442),
  862. }, {
  863. "color": "yellow",
  864. "ws": float64(1541152486442),
  865. "we": float64(1541152488442),
  866. }}, {{
  867. "color": "blue",
  868. "ws": float64(1541152487252),
  869. "we": float64(1541152489252),
  870. }, {
  871. "color": "red",
  872. "ws": float64(1541152487252),
  873. "we": float64(1541152489252),
  874. }, {
  875. "color": "yellow",
  876. "ws": float64(1541152487252),
  877. "we": float64(1541152489252),
  878. }},
  879. },
  880. M: map[string]interface{}{
  881. "op_1_preprocessor_demoE_0_exceptions_total": int64(0),
  882. "op_1_preprocessor_demoE_0_process_latency_us": int64(0),
  883. "op_1_preprocessor_demoE_0_records_in_total": int64(6),
  884. "op_1_preprocessor_demoE_0_records_out_total": int64(6),
  885. "op_5_project_0_exceptions_total": int64(0),
  886. "op_5_project_0_process_latency_us": int64(0),
  887. "op_5_project_0_records_in_total": int64(4),
  888. "op_5_project_0_records_out_total": int64(4),
  889. "sink_mockSink_0_exceptions_total": int64(0),
  890. "sink_mockSink_0_records_in_total": int64(4),
  891. "sink_mockSink_0_records_out_total": int64(4),
  892. "source_demoE_0_exceptions_total": int64(0),
  893. "source_demoE_0_records_in_total": int64(6),
  894. "source_demoE_0_records_out_total": int64(6),
  895. "op_2_window_0_exceptions_total": int64(0),
  896. "op_2_window_0_process_latency_us": int64(0),
  897. "op_2_window_0_records_in_total": int64(6),
  898. "op_2_window_0_records_out_total": int64(4),
  899. "op_3_aggregate_0_exceptions_total": int64(0),
  900. "op_3_aggregate_0_process_latency_us": int64(0),
  901. "op_3_aggregate_0_records_in_total": int64(4),
  902. "op_3_aggregate_0_records_out_total": int64(4),
  903. "op_4_order_0_exceptions_total": int64(0),
  904. "op_4_order_0_process_latency_us": int64(0),
  905. "op_4_order_0_records_in_total": int64(4),
  906. "op_4_order_0_records_out_total": int64(4),
  907. },
  908. }, {
  909. Name: `TestEventWindowRule5`,
  910. Sql: `SELECT temp FROM sessionDemoE GROUP BY SessionWindow(ss, 2, 1) `,
  911. R: [][]map[string]interface{}{
  912. {{
  913. "temp": 25.5,
  914. }}, {{
  915. "temp": 28.1,
  916. }, {
  917. "temp": 27.4,
  918. }, {
  919. "temp": 25.5,
  920. }}, {{
  921. "temp": 26.2,
  922. }, {
  923. "temp": 26.8,
  924. }, {
  925. "temp": 28.9,
  926. }, {
  927. "temp": 29.1,
  928. }, {
  929. "temp": 32.2,
  930. }}, {{
  931. "temp": 30.9,
  932. }},
  933. },
  934. M: map[string]interface{}{
  935. "op_1_preprocessor_sessionDemoE_0_exceptions_total": int64(0),
  936. "op_1_preprocessor_sessionDemoE_0_process_latency_us": int64(0),
  937. "op_1_preprocessor_sessionDemoE_0_records_in_total": int64(12),
  938. "op_1_preprocessor_sessionDemoE_0_records_out_total": int64(12),
  939. "op_3_project_0_exceptions_total": int64(0),
  940. "op_3_project_0_process_latency_us": int64(0),
  941. "op_3_project_0_records_in_total": int64(4),
  942. "op_3_project_0_records_out_total": int64(4),
  943. "sink_mockSink_0_exceptions_total": int64(0),
  944. "sink_mockSink_0_records_in_total": int64(4),
  945. "sink_mockSink_0_records_out_total": int64(4),
  946. "source_sessionDemoE_0_exceptions_total": int64(0),
  947. "source_sessionDemoE_0_records_in_total": int64(12),
  948. "source_sessionDemoE_0_records_out_total": int64(12),
  949. "op_2_window_0_exceptions_total": int64(0),
  950. "op_2_window_0_process_latency_us": int64(0),
  951. "op_2_window_0_records_in_total": int64(12),
  952. "op_2_window_0_records_out_total": int64(4),
  953. },
  954. }, {
  955. Name: `TestEventWindowRule6`,
  956. Sql: `SELECT max(temp) as m, count(color) as c FROM demoE INNER JOIN demo1E ON demoE.ts = demo1E.ts GROUP BY SlidingWindow(ss, 1)`,
  957. R: [][]map[string]interface{}{
  958. {{
  959. "m": 25.5,
  960. "c": float64(1),
  961. }}, {{
  962. "m": 25.5,
  963. "c": float64(1),
  964. }}, {{
  965. "m": 28.1,
  966. "c": float64(1),
  967. }}, {{
  968. "m": 28.1,
  969. "c": float64(2),
  970. }}, {{
  971. "m": 27.4,
  972. "c": float64(2),
  973. }},
  974. },
  975. M: map[string]interface{}{
  976. "op_1_preprocessor_demoE_0_exceptions_total": int64(0),
  977. "op_1_preprocessor_demoE_0_process_latency_us": int64(0),
  978. "op_1_preprocessor_demoE_0_records_in_total": int64(6),
  979. "op_1_preprocessor_demoE_0_records_out_total": int64(6),
  980. "op_2_preprocessor_demo1E_0_exceptions_total": int64(0),
  981. "op_2_preprocessor_demo1E_0_process_latency_us": int64(0),
  982. "op_2_preprocessor_demo1E_0_records_in_total": int64(6),
  983. "op_2_preprocessor_demo1E_0_records_out_total": int64(6),
  984. "op_5_project_0_exceptions_total": int64(0),
  985. "op_5_project_0_process_latency_us": int64(0),
  986. "op_5_project_0_records_in_total": int64(5),
  987. "op_5_project_0_records_out_total": int64(5),
  988. "sink_mockSink_0_exceptions_total": int64(0),
  989. "sink_mockSink_0_records_in_total": int64(5),
  990. "sink_mockSink_0_records_out_total": int64(5),
  991. "source_demoE_0_exceptions_total": int64(0),
  992. "source_demoE_0_records_in_total": int64(6),
  993. "source_demoE_0_records_out_total": int64(6),
  994. "source_demo1E_0_exceptions_total": int64(0),
  995. "source_demo1E_0_records_in_total": int64(6),
  996. "source_demo1E_0_records_out_total": int64(6),
  997. "op_3_window_0_exceptions_total": int64(0),
  998. "op_3_window_0_records_in_total": int64(12),
  999. "op_3_window_0_records_out_total": int64(5),
  1000. "op_4_join_0_exceptions_total": int64(0),
  1001. "op_4_join_0_process_latency_us": int64(0),
  1002. "op_4_join_0_records_in_total": int64(5),
  1003. "op_4_join_0_records_out_total": int64(5),
  1004. },
  1005. }, {
  1006. Name: `TestEventWindowRule7`,
  1007. Sql: `SELECT * FROM demoErr GROUP BY HOPPINGWINDOW(ss, 2, 1)`,
  1008. R: [][]map[string]interface{}{
  1009. {{
  1010. "error": "error in preprocessor: invalid data type for color, expect string but found int(2)",
  1011. }},
  1012. {{
  1013. "color": "red",
  1014. "size": float64(3),
  1015. "ts": float64(1541152486013),
  1016. }},
  1017. {{
  1018. "color": "red",
  1019. "size": float64(3),
  1020. "ts": float64(1541152486013),
  1021. }},
  1022. {{
  1023. "color": "yellow",
  1024. "size": float64(4),
  1025. "ts": float64(1541152488442),
  1026. }}, {{
  1027. "color": "yellow",
  1028. "size": float64(4),
  1029. "ts": float64(1541152488442),
  1030. }, {
  1031. "color": "red",
  1032. "size": float64(1),
  1033. "ts": float64(1541152489252),
  1034. }}, {{
  1035. "color": "red",
  1036. "size": float64(1),
  1037. "ts": float64(1541152489252),
  1038. }},
  1039. },
  1040. M: map[string]interface{}{
  1041. "op_1_preprocessor_demoErr_0_exceptions_total": int64(1),
  1042. "op_1_preprocessor_demoErr_0_process_latency_us": int64(0),
  1043. "op_1_preprocessor_demoErr_0_records_in_total": int64(6),
  1044. "op_1_preprocessor_demoErr_0_records_out_total": int64(5),
  1045. "op_3_project_0_exceptions_total": int64(1),
  1046. "op_3_project_0_process_latency_us": int64(0),
  1047. "op_3_project_0_records_in_total": int64(6),
  1048. "op_3_project_0_records_out_total": int64(5),
  1049. "sink_mockSink_0_exceptions_total": int64(0),
  1050. "sink_mockSink_0_records_in_total": int64(6),
  1051. "sink_mockSink_0_records_out_total": int64(6),
  1052. "source_demoErr_0_exceptions_total": int64(0),
  1053. "source_demoErr_0_records_in_total": int64(6),
  1054. "source_demoErr_0_records_out_total": int64(6),
  1055. "op_2_window_0_exceptions_total": int64(1),
  1056. "op_2_window_0_process_latency_us": int64(0),
  1057. "op_2_window_0_records_in_total": int64(6),
  1058. "op_2_window_0_records_out_total": int64(5),
  1059. },
  1060. }, {
  1061. Name: `TestEventWindowRule8`,
  1062. Sql: `SELECT temp, window_start(), window_end() FROM sessionDemoE GROUP BY SessionWindow(ss, 2, 1) `,
  1063. R: [][]map[string]interface{}{
  1064. {{
  1065. "temp": 25.5,
  1066. "window_start": float64(1541152486013),
  1067. "window_end": float64(1541152487013),
  1068. }}, {{
  1069. "temp": 28.1,
  1070. "window_start": float64(1541152487932),
  1071. "window_end": float64(1541152490000),
  1072. }}, {{
  1073. "temp": 26.2,
  1074. "window_start": float64(1541152490000),
  1075. "window_end": float64(1541152494000),
  1076. }}, {{
  1077. "temp": 30.9,
  1078. "window_start": float64(1541152494000),
  1079. "window_end": float64(1541152495112),
  1080. }},
  1081. },
  1082. M: map[string]interface{}{
  1083. "op_1_preprocessor_sessionDemoE_0_exceptions_total": int64(0),
  1084. "op_1_preprocessor_sessionDemoE_0_process_latency_us": int64(0),
  1085. "op_1_preprocessor_sessionDemoE_0_records_in_total": int64(12),
  1086. "op_1_preprocessor_sessionDemoE_0_records_out_total": int64(12),
  1087. "op_3_project_0_exceptions_total": int64(0),
  1088. "op_3_project_0_process_latency_us": int64(0),
  1089. "op_3_project_0_records_in_total": int64(4),
  1090. "op_3_project_0_records_out_total": int64(4),
  1091. "sink_mockSink_0_exceptions_total": int64(0),
  1092. "sink_mockSink_0_records_in_total": int64(4),
  1093. "sink_mockSink_0_records_out_total": int64(4),
  1094. "source_sessionDemoE_0_exceptions_total": int64(0),
  1095. "source_sessionDemoE_0_records_in_total": int64(12),
  1096. "source_sessionDemoE_0_records_out_total": int64(12),
  1097. "op_2_window_0_exceptions_total": int64(0),
  1098. "op_2_window_0_process_latency_us": int64(0),
  1099. "op_2_window_0_records_in_total": int64(12),
  1100. "op_2_window_0_records_out_total": int64(4),
  1101. },
  1102. }, {
  1103. Name: `TestEventWindowRule9`,
  1104. Sql: `SELECT window_end(), color, window_start() FROM demoE GROUP BY HOPPINGWINDOW(ss, 2, 1)`,
  1105. R: [][]map[string]interface{}{
  1106. {{
  1107. "color": "red",
  1108. "window_start": float64(1541152485000),
  1109. "window_end": float64(1541152487000),
  1110. }},
  1111. {{
  1112. "color": "red",
  1113. "window_start": float64(1541152486000),
  1114. "window_end": float64(1541152488000),
  1115. }},
  1116. {{
  1117. "color": "blue",
  1118. "window_start": float64(1541152487000),
  1119. "window_end": float64(1541152489000),
  1120. }}, {{
  1121. "color": "yellow",
  1122. "window_start": float64(1541152488000),
  1123. "window_end": float64(1541152490000),
  1124. }}, {{
  1125. "color": "red",
  1126. "window_start": float64(1541152489000),
  1127. "window_end": float64(1541152491000),
  1128. }},
  1129. },
  1130. M: map[string]interface{}{
  1131. "op_1_preprocessor_demoE_0_exceptions_total": int64(0),
  1132. "op_1_preprocessor_demoE_0_process_latency_us": int64(0),
  1133. "op_1_preprocessor_demoE_0_records_in_total": int64(6),
  1134. "op_1_preprocessor_demoE_0_records_out_total": int64(6),
  1135. "op_3_project_0_exceptions_total": int64(0),
  1136. "op_3_project_0_process_latency_us": int64(0),
  1137. "op_3_project_0_records_in_total": int64(5),
  1138. "op_3_project_0_records_out_total": int64(5),
  1139. "sink_mockSink_0_exceptions_total": int64(0),
  1140. "sink_mockSink_0_records_in_total": int64(5),
  1141. "sink_mockSink_0_records_out_total": int64(5),
  1142. "source_demoE_0_exceptions_total": int64(0),
  1143. "source_demoE_0_records_in_total": int64(6),
  1144. "source_demoE_0_records_out_total": int64(6),
  1145. "op_2_window_0_exceptions_total": int64(0),
  1146. "op_2_window_0_process_latency_us": int64(0),
  1147. "op_2_window_0_records_in_total": int64(6),
  1148. "op_2_window_0_records_out_total": int64(5),
  1149. },
  1150. },
  1151. }
  1152. HandleStream(true, streamList, t)
  1153. options := []*api.RuleOption{
  1154. {
  1155. BufferLength: 100,
  1156. SendError: true,
  1157. IsEventTime: true,
  1158. LateTol: 1000,
  1159. }, {
  1160. BufferLength: 100,
  1161. SendError: true,
  1162. Qos: api.AtLeastOnce,
  1163. CheckpointInterval: 5000,
  1164. IsEventTime: true,
  1165. LateTol: 1000,
  1166. }, {
  1167. BufferLength: 100,
  1168. SendError: true,
  1169. Qos: api.ExactlyOnce,
  1170. CheckpointInterval: 5000,
  1171. IsEventTime: true,
  1172. LateTol: 1000,
  1173. },
  1174. }
  1175. for j, opt := range options {
  1176. DoRuleTest(t, tests, j, opt, 10)
  1177. }
  1178. }
  1179. func TestWindowError(t *testing.T) {
  1180. //Reset
  1181. streamList := []string{"ldemo", "ldemo1"}
  1182. HandleStream(false, streamList, t)
  1183. var tests = []RuleTest{
  1184. {
  1185. Name: `TestWindowErrorRule1`,
  1186. Sql: `SELECT size * 3 FROM ldemo GROUP BY TUMBLINGWINDOW(ss, 2)`,
  1187. R: [][]map[string]interface{}{
  1188. {{
  1189. "error": "run Select error: invalid operation string(string) * int64(3)",
  1190. }}, {{
  1191. "kuiper_field_0": float64(6),
  1192. }, {}},
  1193. },
  1194. M: map[string]interface{}{
  1195. "op_1_preprocessor_ldemo_0_exceptions_total": int64(0),
  1196. "op_1_preprocessor_ldemo_0_process_latency_us": int64(0),
  1197. "op_1_preprocessor_ldemo_0_records_in_total": int64(5),
  1198. "op_1_preprocessor_ldemo_0_records_out_total": int64(5),
  1199. "op_3_project_0_exceptions_total": int64(1),
  1200. "op_3_project_0_process_latency_us": int64(0),
  1201. "op_3_project_0_records_in_total": int64(2),
  1202. "op_3_project_0_records_out_total": int64(1),
  1203. "sink_mockSink_0_exceptions_total": int64(0),
  1204. "sink_mockSink_0_records_in_total": int64(2),
  1205. "sink_mockSink_0_records_out_total": int64(2),
  1206. "source_ldemo_0_exceptions_total": int64(0),
  1207. "source_ldemo_0_records_in_total": int64(5),
  1208. "source_ldemo_0_records_out_total": int64(5),
  1209. "op_2_window_0_exceptions_total": int64(0),
  1210. "op_2_window_0_process_latency_us": int64(0),
  1211. "op_2_window_0_records_in_total": int64(5),
  1212. "op_2_window_0_records_out_total": int64(2),
  1213. },
  1214. }, {
  1215. Name: `TestWindowErrorRule2`,
  1216. Sql: `SELECT color, ts FROM ldemo where size > 2 GROUP BY tumblingwindow(ss, 1)`,
  1217. R: [][]map[string]interface{}{
  1218. {{
  1219. "error": "run Where error: invalid operation string(string) > int64(2)",
  1220. }}, {{
  1221. "color": "red",
  1222. "ts": float64(1541152486013),
  1223. }}, {{
  1224. "ts": float64(1541152487632),
  1225. }},
  1226. },
  1227. M: map[string]interface{}{
  1228. "op_1_preprocessor_ldemo_0_exceptions_total": int64(0),
  1229. "op_1_preprocessor_ldemo_0_process_latency_us": int64(0),
  1230. "op_1_preprocessor_ldemo_0_records_in_total": int64(5),
  1231. "op_1_preprocessor_ldemo_0_records_out_total": int64(5),
  1232. "op_4_project_0_exceptions_total": int64(1),
  1233. "op_4_project_0_process_latency_us": int64(0),
  1234. "op_4_project_0_records_in_total": int64(3),
  1235. "op_4_project_0_records_out_total": int64(2),
  1236. "sink_mockSink_0_exceptions_total": int64(0),
  1237. "sink_mockSink_0_records_in_total": int64(3),
  1238. "sink_mockSink_0_records_out_total": int64(3),
  1239. "source_ldemo_0_exceptions_total": int64(0),
  1240. "source_ldemo_0_records_in_total": int64(5),
  1241. "source_ldemo_0_records_out_total": int64(5),
  1242. "op_3_window_0_exceptions_total": int64(1),
  1243. "op_3_window_0_process_latency_us": int64(0),
  1244. "op_3_window_0_records_in_total": int64(3),
  1245. "op_3_window_0_records_out_total": int64(2),
  1246. "op_2_filter_0_exceptions_total": int64(1),
  1247. "op_2_filter_0_process_latency_us": int64(0),
  1248. "op_2_filter_0_records_in_total": int64(5),
  1249. "op_2_filter_0_records_out_total": int64(2),
  1250. },
  1251. }, {
  1252. Name: `TestWindowErrorRule3`,
  1253. Sql: `SELECT color, temp, ts FROM ldemo INNER JOIN ldemo1 ON ldemo.ts = ldemo1.ts GROUP BY SlidingWindow(ss, 1)`,
  1254. R: [][]map[string]interface{}{
  1255. {{
  1256. "color": "red",
  1257. "temp": 25.5,
  1258. "ts": float64(1541152486013),
  1259. }}, {{
  1260. "color": "red",
  1261. "temp": 25.5,
  1262. "ts": float64(1541152486013),
  1263. }}, {{
  1264. "color": "red",
  1265. "temp": 25.5,
  1266. "ts": float64(1541152486013),
  1267. }}, {{
  1268. "temp": 28.1,
  1269. "ts": float64(1541152487632),
  1270. }}, {{
  1271. "temp": 28.1,
  1272. "ts": float64(1541152487632),
  1273. }}, {{
  1274. "error": "run Join error: invalid operation int64(1541152487632) = string(1541152488442)",
  1275. }}, {{
  1276. "error": "run Join error: invalid operation int64(1541152488442) = string(1541152488442)",
  1277. }}, {{
  1278. "error": "run Join error: invalid operation int64(1541152488442) = string(1541152488442)",
  1279. }},
  1280. },
  1281. M: map[string]interface{}{
  1282. "op_1_preprocessor_ldemo_0_exceptions_total": int64(0),
  1283. "op_1_preprocessor_ldemo_0_process_latency_us": int64(0),
  1284. "op_1_preprocessor_ldemo_0_records_in_total": int64(5),
  1285. "op_1_preprocessor_ldemo_0_records_out_total": int64(5),
  1286. "op_2_preprocessor_ldemo1_0_exceptions_total": int64(0),
  1287. "op_2_preprocessor_ldemo1_0_process_latency_us": int64(0),
  1288. "op_2_preprocessor_ldemo1_0_records_in_total": int64(5),
  1289. "op_2_preprocessor_ldemo1_0_records_out_total": int64(5),
  1290. "op_5_project_0_exceptions_total": int64(3),
  1291. "op_5_project_0_process_latency_us": int64(0),
  1292. "op_5_project_0_records_in_total": int64(8),
  1293. "op_5_project_0_records_out_total": int64(5),
  1294. "sink_mockSink_0_exceptions_total": int64(0),
  1295. "sink_mockSink_0_records_in_total": int64(8),
  1296. "sink_mockSink_0_records_out_total": int64(8),
  1297. "source_ldemo_0_exceptions_total": int64(0),
  1298. "source_ldemo_0_records_in_total": int64(5),
  1299. "source_ldemo_0_records_out_total": int64(5),
  1300. "source_ldemo1_0_exceptions_total": int64(0),
  1301. "source_ldemo1_0_records_in_total": int64(5),
  1302. "source_ldemo1_0_records_out_total": int64(5),
  1303. "op_3_window_0_exceptions_total": int64(0),
  1304. "op_3_window_0_process_latency_us": int64(0),
  1305. "op_3_window_0_records_in_total": int64(10),
  1306. "op_3_window_0_records_out_total": int64(10),
  1307. "op_4_join_0_exceptions_total": int64(3),
  1308. "op_4_join_0_process_latency_us": int64(0),
  1309. "op_4_join_0_records_in_total": int64(10),
  1310. "op_4_join_0_records_out_total": int64(5),
  1311. },
  1312. }, {
  1313. Name: `TestWindowErrorRule4`,
  1314. Sql: `SELECT color FROM ldemo GROUP BY SlidingWindow(ss, 2), color having collect(size)[0] >= 2 order by color`,
  1315. R: [][]map[string]interface{}{
  1316. {{
  1317. "color": "red",
  1318. }}, {{
  1319. "error": "run Having error: invalid operation string(string) >= int64(2)",
  1320. }}, {{
  1321. "error": "run Having error: invalid operation string(string) >= int64(2)",
  1322. }}, {{
  1323. "error": "run Having error: invalid operation string(string) >= int64(2)",
  1324. }}, {{
  1325. "color": float64(49),
  1326. }, {}},
  1327. },
  1328. M: map[string]interface{}{
  1329. "op_1_preprocessor_ldemo_0_exceptions_total": int64(0),
  1330. "op_1_preprocessor_ldemo_0_process_latency_us": int64(0),
  1331. "op_1_preprocessor_ldemo_0_records_in_total": int64(5),
  1332. "op_1_preprocessor_ldemo_0_records_out_total": int64(5),
  1333. "op_6_project_0_exceptions_total": int64(3),
  1334. "op_6_project_0_process_latency_us": int64(0),
  1335. "op_6_project_0_records_in_total": int64(5),
  1336. "op_6_project_0_records_out_total": int64(2),
  1337. "sink_mockSink_0_exceptions_total": int64(0),
  1338. "sink_mockSink_0_records_in_total": int64(5),
  1339. "sink_mockSink_0_records_out_total": int64(5),
  1340. "source_ldemo_0_exceptions_total": int64(0),
  1341. "source_ldemo_0_records_in_total": int64(5),
  1342. "source_ldemo_0_records_out_total": int64(5),
  1343. "op_2_window_0_exceptions_total": int64(0),
  1344. "op_2_window_0_process_latency_us": int64(0),
  1345. "op_2_window_0_records_in_total": int64(5),
  1346. "op_2_window_0_records_out_total": int64(5),
  1347. "op_3_aggregate_0_exceptions_total": int64(0),
  1348. "op_3_aggregate_0_process_latency_us": int64(0),
  1349. "op_3_aggregate_0_records_in_total": int64(5),
  1350. "op_3_aggregate_0_records_out_total": int64(5),
  1351. "op_4_having_0_exceptions_total": int64(3),
  1352. "op_4_having_0_process_latency_us": int64(0),
  1353. "op_4_having_0_records_in_total": int64(5),
  1354. "op_4_having_0_records_out_total": int64(2),
  1355. },
  1356. }, {
  1357. Name: `TestWindowErrorRule5`,
  1358. Sql: `SELECT color, size FROM ldemo GROUP BY tumblingwindow(ss, 1) ORDER BY size`,
  1359. R: [][]map[string]interface{}{
  1360. {{
  1361. "error": "run Order By error: incompatible types for comparison: int and string",
  1362. }}, {{
  1363. "size": float64(3),
  1364. }}, {{
  1365. "color": float64(49),
  1366. "size": float64(2),
  1367. }}, {{
  1368. "color": "red",
  1369. }},
  1370. },
  1371. M: map[string]interface{}{
  1372. "op_1_preprocessor_ldemo_0_exceptions_total": int64(0),
  1373. "op_1_preprocessor_ldemo_0_process_latency_us": int64(0),
  1374. "op_1_preprocessor_ldemo_0_records_in_total": int64(5),
  1375. "op_1_preprocessor_ldemo_0_records_out_total": int64(5),
  1376. "op_4_project_0_exceptions_total": int64(1),
  1377. "op_4_project_0_process_latency_us": int64(0),
  1378. "op_4_project_0_records_in_total": int64(4),
  1379. "op_4_project_0_records_out_total": int64(3),
  1380. "sink_mockSink_0_exceptions_total": int64(0),
  1381. "sink_mockSink_0_records_in_total": int64(4),
  1382. "sink_mockSink_0_records_out_total": int64(4),
  1383. "source_ldemo_0_exceptions_total": int64(0),
  1384. "source_ldemo_0_records_in_total": int64(5),
  1385. "source_ldemo_0_records_out_total": int64(5),
  1386. "op_2_window_0_exceptions_total": int64(0),
  1387. "op_2_window_0_process_latency_us": int64(0),
  1388. "op_2_window_0_records_in_total": int64(5),
  1389. "op_2_window_0_records_out_total": int64(4),
  1390. "op_3_order_0_exceptions_total": int64(1),
  1391. "op_3_order_0_process_latency_us": int64(0),
  1392. "op_3_order_0_records_in_total": int64(4),
  1393. "op_3_order_0_records_out_total": int64(3),
  1394. },
  1395. },
  1396. }
  1397. HandleStream(true, streamList, t)
  1398. DoRuleTest(t, tests, 0, &api.RuleOption{
  1399. BufferLength: 100,
  1400. SendError: true,
  1401. }, 0)
  1402. }