tyarn.lua.in (21264B)
1 #!/usr/bin/env ##LUA_INTERP## 2 -- 3 -- Copyright © 2019 - 2021 Richard Ipsum 4 -- 5 -- This program is free software: you can redistribute it and/or modify 6 -- it under the terms of the GNU General Public License as published by 7 -- the Free Software Foundation, version 3 of the License. 8 -- 9 -- This program is distributed in the hope that it will be useful, 10 -- but WITHOUT ANY WARRANTY; without even the implied warranty of 11 -- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12 -- GNU General Public License for more details. 13 -- 14 -- You should have received a copy of the GNU General Public License 15 -- along with this program. If not, see <http://www.gnu.org/licenses/>. 16 -- 17 18 local tyarn = require 'tyarn' 19 20 local scenario_env = {} 21 local parsed_args = {} 22 local parsed_env = {} 23 24 local DEBUG = nil 25 local DEFAULT_SHELL = "sh" 26 27 function debug(...) 28 if parsed_args["debug"] or DEBUG then 29 print(...) 30 end 31 end 32 33 function lstrip(s) 34 return string.gsub(s, '%s+', '', 1) 35 end 36 37 function normalise_scenario_line(str, step_type) 38 normalised, _ = string.gsub(str, '^' .. step_type .. '%s+', step_type .. ' ', 1) 39 return normalised 40 end 41 42 function parse_implementations(filepath, implementations) 43 local line_no = 0 44 local file, err = io.open(filepath) 45 if file == nil then 46 io.stderr:write(string.format("Couldn't open implementation file: %s\n", filepath, err)) 47 os.exit(1) 48 end 49 50 for line in file:lines() do 51 line_no = line_no + 1 52 53 if tyarn.re_match(line, "^\t+") then 54 io.stderr:write(string.format( 55 "Warning `%s' in implementation file %s (line %s) " .. 56 "starts with tabs instead of spaces, ignoring\n", 57 line, filepath, line_no)) 58 end 59 60 -- ignore lines that are not indented or blank 61 matched, matches = tyarn.re_match(line, "^( )+[^ \t\n\r\f\v]+") 62 line = string.gsub(line, ' ', '', 1) -- strip first 4 spaces 63 64 if matched then 65 if string.find(line, "IMPLEMENTS GIVEN") then 66 key = string.gsub(line, "IMPLEMENTS%s+", "") 67 key = normalise_scenario_line(key, "GIVEN") 68 implementations[key] = {} 69 elseif string.find(line, "IMPLEMENTS WHEN") then 70 key = string.gsub(line, "IMPLEMENTS%s+", "") 71 key = normalise_scenario_line(key, "WHEN") 72 implementations[key] = {} 73 elseif string.find(line, "IMPLEMENTS THEN") then 74 key = string.gsub(line, "IMPLEMENTS%s+", "") 75 key = normalise_scenario_line(key, "THEN") 76 implementations[key] = {} 77 elseif string.find(line, "IMPLEMENTS ASSUMING") then 78 key = string.gsub(line, "IMPLEMENTS%s+", "") 79 key = normalise_scenario_line(key, "ASSUMING") 80 implementations[key] = {} 81 elseif string.find(line, "IMPLEMENTS FINALLY") then 82 key = string.gsub(line, "IMPLEMENTS%s+", "") 83 key = normalise_scenario_line(key, "FINALLY") 84 implementations[key] = {} 85 elseif key ~= nil then 86 -- in the middle of a step's implementation 87 --print(string.format('debug: %s', string.gsub(key, "IMPLEMENTS%s+", ""))) 88 table.insert(implementations[key], line) 89 end 90 end 91 end 92 93 return implementations 94 end 95 96 function validate_scenario(steps_seen, scenario_name, scenario_line_no) 97 local seen_when = false 98 local seen_then = false 99 local i 100 101 for n, step_type in ipairs(steps_seen) do 102 if step_type == "WHEN" then 103 seen_when = true 104 elseif step_type == "THEN" then 105 seen_then = true 106 end 107 end 108 109 if not seen_when then 110 io.stderr:write(string.format('Scenario "%s" (line %d) has no WHEN step\n', 111 scenario_name, scenario_line_no)) 112 os.exit(1) 113 end 114 115 if not seen_then then 116 io.stderr:write(string.format('Scenario "%s" (line %d) has no THEN step\n', 117 scenario_name, scenario_line_no)) 118 os.exit(1) 119 end 120 121 if steps_seen[#steps_seen] == "FINALLY" then 122 i = #steps_seen - 1 123 while steps_seen[i] == "FINALLY" do 124 i = i - 1 125 end 126 127 if steps_seen[i] ~= "THEN" then 128 errmsg = 'Scenario "%s" (line %d) does not have a THEN step before FINALLY step\n' 129 io.stderr:write(string.format(errmsg, scenario_name, scenario_line_no)) 130 os.exit(1) 131 end 132 elseif steps_seen[#steps_seen] ~= "THEN" then 133 io.stderr:write(string.format('Scenario "%s" (line %d) does not end with THEN or FINALLY step\n', 134 scenario_name, scenario_line_no)) 135 os.exit(1) 136 else 137 i = #steps_seen 138 end 139 140 while steps_seen[i] == "THEN" do 141 i = i - 1 142 end 143 144 if steps_seen[i] ~= "WHEN" then 145 errmsg = 'Scenario "%s" (line %d) does not have a WHEN step before THEN step\n' 146 io.stderr:write(string.format(errmsg, scenario_name, scenario_line_no)) 147 os.exit(1) 148 end 149 end 150 151 function parse_scenario_line(scenario, scenario_name, steps_seen, line, line_no) 152 stripped_line = lstrip(line) 153 154 if tyarn.re_match(line, "^( )[ \t]*GIVEN") then 155 table.insert(scenario, normalise_scenario_line(stripped_line, "GIVEN")) 156 table.insert(steps_seen, "GIVEN") 157 elseif tyarn.re_match(line, "^( )[ \t]*WHEN") then 158 table.insert(scenario, normalise_scenario_line(stripped_line, "WHEN")) 159 table.insert(steps_seen, "WHEN") 160 elseif tyarn.re_match(line, "^( )[ \t]*THEN") then 161 table.insert(scenario, normalise_scenario_line(stripped_line, "THEN")) 162 table.insert(steps_seen, "THEN") 163 elseif tyarn.re_match(line, "^( )[ \t]*ASSUMING") then 164 table.insert(scenario, normalise_scenario_line(stripped_line, "ASSUMING")) 165 table.insert(steps_seen, "ASSUMING") 166 elseif tyarn.re_match(line, "^( )[ \t]*AND") then 167 if #steps_seen == 0 then 168 io.stderr:write(string.format( 169 'Scenario "%s" (line %d) has AND as first step: %s\n', 170 scenario_name, line_no, stripped_line)) 171 os.exit(1) 172 end 173 174 processed_line = string.gsub(stripped_line, "AND", steps_seen[#steps_seen]) 175 table.insert(steps_seen, steps_seen[#steps_seen]) 176 177 if steps_seen[#steps_seen - 1] == "FINALLY" then 178 table.insert(scenario["FINALLY"], processed_line) 179 else 180 table.insert(scenario, processed_line) 181 end 182 elseif tyarn.re_match(line, "^( )[ \t]*FINALLY") then 183 table.insert(steps_seen, "FINALLY") 184 table.insert(scenario["FINALLY"], normalise_scenario_line(stripped_line, "FINALLY")) 185 elseif tyarn.re_match(line, "^( )[ \t]*\\.\\.\\.") then 186 if #steps_seen == 0 then 187 io.stderr:write(string.format( 188 'Scenario "%s" (line %d) has ... as first step: %s\n', 189 scenario_name, line_no, stripped_line)) 190 os.exit(1) 191 end 192 193 -- continuation of previous scenario line 194 scenario[#scenario] = scenario[#scenario] .. ' ' .. lstrip(string.gsub(stripped_line, '^...', '', 1)) 195 elseif tyarn.re_match(line, "^( )[ \t]*.+") then 196 io.stderr:write(string.format( 197 'Scenario "%s" (line %d) invalid scenario line: %s\n', 198 scenario_name, line_no, stripped_line)) 199 os.exit(1) 200 end 201 end 202 203 function _parse_scenarios(scenario_list, scenarios, filepath, 204 file, scenario_name, scenario_line_no) 205 debug('Parsing scenario', scenario_name) 206 scenario = {} 207 scenario["FINALLY"] = {} 208 line_no = scenario_line_no 209 steps_seen = {} 210 in_scenario = true 211 212 for line in file:lines() do 213 line_no = line_no + 1 214 215 if tyarn.re_match(line, "^\t+") then 216 io.stderr:write(string.format( 217 "Warning `%s' in scenario file %s (line %s) " .. 218 "starts with tabs instead of spaces, ignoring\n", 219 line, filepath, line_no)) 220 end 221 222 if string.len(line) == 0 then 223 -- blank line 224 in_scenario = false 225 end 226 227 if in_scenario then 228 parse_scenario_line(scenario, scenario_name, steps_seen, line, line_no) 229 end 230 231 -- ignore lines that aren't indented by one level 232 matched, matches = tyarn.re_match(line, "^( )SCENARIO ([^ \t\n\r\f\v]+.*)") 233 234 if matched then 235 -- we've hit the end of the current scenario 236 -- validate, and store current scenario 237 validate_scenario(steps_seen, scenario_name, scenario_line_no) 238 scenarios[scenario_name] = scenario 239 table.insert(scenario_list, scenario_name) 240 241 -- now onto the next scenario 242 scenario_name = matches[2] 243 _parse_scenarios(scenario_list, scenarios, filepath, 244 file, scenario_name, line_no) 245 return 246 end 247 end 248 249 -- hit the end of the file, validate the current scenario 250 -- assuming there is a current scenario 251 if scenario_name then 252 validate_scenario(steps_seen, scenario_name, scenario_line_no) 253 scenarios[scenario_name] = scenario 254 table.insert(scenario_list, scenario_name) 255 end 256 end 257 258 function parse_scenarios(filepath) 259 local scenario_list = {} 260 local scenarios = {} 261 local scenario_name = nil 262 local line_no = 0 263 264 file, err = io.open(filepath) 265 if file == nil then 266 io.stderr:write(string.format("Couldn't open scenario file: %s\n", err)) 267 os.exit(1) 268 end 269 270 -- parse the first scenario line, as the base case 271 for line in file:lines() do 272 line_no = line_no + 1 273 274 matched, matches = tyarn.re_match(line, "^( )SCENARIO ([^ \t\n\r\f\v]+.*)") 275 276 if matched then 277 scenario_name = matches[2] 278 break 279 end 280 end 281 282 if scenario_name == nil then 283 return nil, nil -- no scenario 284 end 285 286 -- now we have the base case, begin the recursion 287 _parse_scenarios(scenario_list, scenarios, filepath, file, scenario_name, line_no) 288 289 return scenario_list, scenarios 290 end 291 292 function find_matching_implementation(implementations, step) 293 -- In the event we have more than one matching implementation 294 -- pick the implementation that has the longest pattern. 295 296 local ret_impl = nil 297 local ret_matches = nil 298 local ret_patt = "" 299 300 for patt, impl in pairs(implementations) do 301 matched, matches = tyarn.re_match(step, string.format("^%s$", patt)) 302 if matched and string.len(patt) > string.len(ret_patt) then 303 ret_impl = impl 304 ret_matches = matches 305 ret_patt = patt 306 end 307 end 308 309 return ret_impl, ret_matches 310 end 311 312 function load_shell_library(path) 313 fh, err = io.open(path) 314 315 if fh == nil then 316 io.stderr:write(string.format("tyarn: Couldn't open shell library: %s\n", err)) 317 os.exit(1) 318 end 319 320 return fh:read('*all') 321 end 322 323 function cleanenv() 324 return { 325 ['TERM'] = 'dumb', 326 ['SHELL'] = '/bin/sh', 327 ['LC_ALL'] = 'C', 328 ['USER'] = 'tomjon', 329 ['USERNAME'] = 'tomjon', 330 ['LOGNAME'] = 'tomjon', 331 ['TZ'] = '' 332 } 333 end 334 335 function env_from_table(env) 336 local e = {} 337 for k, v in pairs(env) do 338 table.insert(e, string.format('%s=%s', k, v)) 339 end 340 return e 341 end 342 343 function write_env_to_file(datadir, envvars) 344 local path = datadir .. "/ENV" 345 local file, err = io.open(path, "w") 346 if file == nil then 347 io.stderr:write(string.format("Couldn't open `%s': %s\n", path, err)) 348 os.exit(1) 349 end 350 for _, var in pairs(envvars) do 351 file:write(var .. '\n') 352 end 353 file:close() 354 end 355 356 function run_step(scenario_dir, datadir, implementations, scenario_key, step, shell_prelude) 357 success = true 358 skip_scenario = false 359 env = cleanenv() 360 shell_script_lines = {} 361 env['DATADIR'] = datadir 362 env['SRCDIR'] = tyarn.getcwd() 363 debug(string.format("Run step %s", step)) 364 365 step_impl, step_captures = find_matching_implementation(implementations, step) 366 if step_impl == nil then 367 error(string.format("No matching implementations for step '%s'", step)) 368 end 369 370 for _, capture in pairs(step_captures) do 371 debug(string.format("capture: %s", capture)) 372 end 373 374 for n, capture in ipairs(step_captures) do 375 env[string.format('MATCH_%d', n)] = capture 376 end 377 378 -- Add any environment variables passed via command line 379 for k, v in pairs(parsed_env) do 380 env[k] = v 381 end 382 383 for _, impl_line in ipairs(step_impl) do 384 debug('impl_line', impl_line) 385 table.insert(shell_script_lines, impl_line) 386 end 387 388 shell_script_str = table.concat(shell_script_lines, '\n') 389 path, err = tyarn.mkstemp(string.format("%s/%s", scenario_dir, "tyarn_shell_script.XXXXXXX")) 390 if path == nil then 391 io.stderr:write(string.format("tyarn: Error creating shell script: %s\n", err)) 392 os.exit(1) 393 end 394 395 debug("datadir:", datadir) 396 debug("path:", path) 397 398 shell_script, err = io.open(path, "w") 399 if shell_script == nil then 400 io.stderr:write(string.format("tyarn: Couldn't open implementation shell script: %s\n", err)) 401 os.exit(1) 402 end 403 404 shell_script:write(shell_prelude) 405 shell_script:write(shell_script_str) 406 shell_script:close() 407 debug('shell_script_str:', shell_script_str) 408 409 cmd = {"/usr/bin/env", parsed_args["shell"] or DEFAULT_SHELL, path} 410 envvars = env_from_table(env) 411 write_env_to_file(datadir, envvars) 412 ret, exit_code, stdout, stderr = tyarn.exec(cmd, envvars, datadir) 413 if parsed_args["show_stdout"] and string.len(stdout) > 0 then 414 io.stderr:write("STDOUT:\n" .. stdout) 415 end 416 if parsed_args["show_stderr"] and string.len(stderr) > 0 then 417 io.stderr:write("STDERR:\n" .. stderr) 418 end 419 debug('ret', ret, 'exit_code', exit_code) 420 if ret == -1 then 421 error(string.format("Failed to exec step: %s", impl_step)) 422 end 423 424 if parsed_args["debug"] then 425 print(string.format("Logging stdout from shell command:\n%s\n", stdout)) 426 print(string.format("Logging stderr from shell command:\n%s\n", stderr)) 427 end 428 429 if exit_code ~= 0 then 430 if string.sub(step, 1, string.len("ASSUMING")) == "ASSUMING" then 431 io.stderr:write(string.format('Skipping "%s" because "%s" failed\n', scenario_key, step)) 432 skip_scenario = true 433 else 434 io.stderr:write(string.format(' ERROR: In scenario: "%s" ', scenario_key)) 435 io.stderr:write(string.format('step "%s" failed with exit code %d\n', step, exit_code)) 436 if parsed_args["no_cleanup"] then 437 io.stderr:write(string.format('DATADIR: "%s"\n', datadir)) 438 end 439 io.stderr:write(string.format("Standard output from shell command:\n%s\n", stdout)) 440 io.stderr:write(string.format("Standard error from shell command:\n%s\n", stderr)) 441 442 success = false 443 end 444 end 445 446 if not parsed_args["no_cleanup"] then 447 tyarn.unlink(path) 448 end 449 450 return success, skip_scenario 451 end 452 453 function fs_quote(s) 454 -- Make string more nicely represented as a path 455 -- Replace ' ' with '_', and lower case the entire string. 456 457 -- FIXME: potential bug here, multiple scenario keys can map to the same path! 458 -- e.g. scenario_key /okay/ and (okay) both map to _okay_ 459 return s:gsub("[ /'\"()]", "_"):lower() 460 end 461 462 local maxline = 0 463 464 function write_progress(s) 465 local ws = '' 466 467 if parsed_args["verbose"] or parsed_args["debug"] or DEBUG then 468 print(s) 469 return 470 end 471 472 if string.len(s) > maxline then 473 maxline = string.len(s) 474 end 475 476 io.stderr:write(s .. string.rep(' ', maxline - string.len(s)) .. '\r') 477 end 478 479 function write_progress_final(s) 480 write_progress(s) 481 if parsed_args["verbose"] or parsed_args["debug"] or DEBUG then 482 return 483 end 484 print('') 485 end 486 487 function run_scenario(scenarios, implementations, scenario_key, shell_lib_path) 488 local scenario = scenarios[scenario_key] 489 local tmpdir = nil 490 local failed_step = nil 491 492 debug('parsed_args["tmpdir"]', parsed_args["tmpdir"]) 493 494 if parsed_args["tmpdir"] ~= nil then 495 debug('path exists?', tyarn.path_exists(parsed_args["tmpdir"])) 496 if tyarn.path_exists(parsed_args["tmpdir"]) then 497 tmpdir = parsed_args["tmpdir"] 498 else 499 tmpdir, errno = tyarn.mkdtemp(parsed_args["tmpdir"]) 500 end 501 else 502 tmpdir, errno = tyarn.mkdtemp("/tmp/temp.XXXXXX") 503 end 504 505 debug('tmpdir', tmpdir) 506 507 scenario_fs_key = fs_quote(scenario_key) 508 509 if shell_lib_path then 510 shell_prelude = load_shell_library(shell_lib_path) 511 debug('shell_prelude', shell_prelude) 512 else 513 shell_prelude = '' 514 end 515 516 if tmpdir == nil then 517 if errno == nil then 518 errmsg = "???" 519 else 520 errmsg = tyarn.strerror(errno) 521 end 522 io.stderr:write(string.format("Failed to make tmpdir: %s\n", errmsg)) 523 os.exit(1) 524 end 525 526 scenario_dir = string.format("%s/%s", tmpdir, scenario_fs_key) 527 ret, err = tyarn.mkdir(scenario_dir) 528 if ret == -1 then 529 io.stderr:write(string.format('tyarn: mkdir "%s" failed: %s\n', scenario_dir, err)) 530 os.exit(1) 531 end 532 533 datadir = string.format("%s/DATADIR", scenario_dir) 534 ret, err = tyarn.mkdir(datadir) 535 if ret == -1 then 536 io.stderr:write(string.format('tyarn: mkdir "%s" failed: %s\n', datadir, err)) 537 os.exit(1) 538 end 539 540 scenario_passed = true 541 for n, step in ipairs(scenario) do 542 if parsed_args['verbose'] then 543 print('Running step', step) 544 end 545 success, skip_scenario = run_step(scenario_dir, datadir, implementations, 546 scenario_key, step, shell_prelude) 547 548 if not success then 549 scenario_passed = false 550 failed_step = step 551 break 552 end 553 554 if skip_scenario then 555 -- ASSUME step failed, skip the rest of the steps in the scenario 556 break 557 end 558 end 559 560 for _, step in pairs(scenario["FINALLY"]) do 561 success, _ = run_step(scenario_dir, datadir, implementations, 562 scenario_key, step, shell_prelude) 563 564 if not success then 565 scenario_passed = false 566 break 567 end 568 end 569 570 if not parsed_args["no_cleanup"] then 571 tyarn.rmutil(datadir) 572 end 573 574 575 return scenario_passed, failed_step 576 end 577 578 function main() 579 parsed_args, parsed_env = tyarn.parse_args(arg) 580 581 if parsed_args["help"] then 582 tyarn.help() 583 return 1 584 end 585 586 if #parsed_args < 2 then 587 tyarn.usage() 588 return 1 589 end 590 591 if parsed_args["debug"] or DEBUG then 592 print("Args:") 593 for k, v in ipairs(parsed_args) do 594 print(k, v) 595 end 596 597 print("Parsed environment options:") 598 for k, v in pairs(parsed_env) do 599 print(k, v) 600 end 601 end 602 603 scenario_filepath = parsed_args[1] 604 scenario_list, scenarios = parse_scenarios(scenario_filepath) 605 606 implementations = {} 607 seen = {} 608 failed = {} 609 nfailed = 0 610 611 if scenario_list == nil then 612 io.stderr:write(string.format("No scenarios found in '%s'\n", scenario_filepath)) 613 return 1 614 end 615 616 for i = 2, #parsed_args do 617 parse_implementations(parsed_args[i], implementations) 618 end 619 620 for _, scenario_name in ipairs(scenario_list) do 621 if seen[scenario_name] then 622 io.stderr:write(string.format("Duplicate scenario: '%s'\n", scenario_name)) 623 return 1 624 end 625 626 seen[scenario_name] = true 627 end 628 629 for n, scenario_name in ipairs(scenario_list) do 630 write_progress(string.format("%d/%d: %s", n, #scenario_list, scenario_name)) 631 passed, failed_step = run_scenario(scenarios, implementations, 632 scenario_name, parsed_args['shell_lib']) 633 634 if not passed then 635 if parsed_args["exit_early"] then 636 print(string.format("%d/%d: %s: FAILED", 637 n, #scenario_list, scenario_name)) 638 return 1 639 end 640 641 failed[scenario_name] = failed_step 642 nfailed = nfailed + 1 643 end 644 645 seen_scenario = true 646 end 647 648 if not seen_scenario then 649 io.stderr:write("No scenarios\n") 650 return 1 651 end 652 653 if nfailed > 0 then 654 write_progress_final("Failed scenarios:") 655 for scenario, step in pairs(failed) do 656 print(string.format(" - %s", scenario)) 657 end 658 if nfailed > 1 then 659 print(string.format("ERROR: Test suite FAILED in %d scenarios", nfailed)) 660 else 661 print(string.format("ERROR: Test suite FAILED in %d scenario", nfailed)) 662 end 663 return 1 664 end 665 666 write_progress_final('Scenario test suite PASS') 667 return 0 668 end 669 670 os.exit(main())