irace/0000755000176200001440000000000015060507630011337 5ustar liggesusersirace/tests/0000755000176200001440000000000014736526233012512 5ustar liggesusersirace/tests/testthat/0000755000176200001440000000000015060507630014341 5ustar liggesusersirace/tests/testthat/good_scenario.txt0000644000176200001440000000004114736526233017721 0ustar liggesusersparameterFile = "parameters.txt" irace/tests/testthat/test-path.R0000644000176200001440000001657014736526233016417 0ustar liggesuserstest_path_rel2abs <- function(testcases) { for (i in 1:nrow(testcases)) { orig <- testcases[i,1L] cwd <- testcases[i,2L] res <- path_rel2abs(testcases[i,1L], cwd) if (testcases[i,3L] == "Sys.which") { exp <- fs::path_abs(Sys.which(testcases[i,1L])) } else { exp <- gsub("\\", "/", fs::path_expand(testcases[i,3L]), fixed = TRUE) } if (res == exp) { #cat("[OK] ", i, ": path_rel2abs(\"", orig, "\", \"", cwd, "\") -> ", res, "\n", sep="") } else { cat(sep="", "[FAILED] ", i, ": path_rel2abs(\"", orig, "\", \"", cwd, "\") -> ", res, " but expected: ", exp, "\n") } expect_match(res, exp, fixed = TRUE) } } test_that("test path_rel2abs", { # Try to set wd; otherwise fail silently. old.cwd <- getwd() skip_if(is.null(old.cwd)) withr::defer(setwd(old.cwd)) tryCatch(setwd("/tmp"), error = function(e) { skip(e) }) testcases <- read.table(text=' "." "/tmp" "/tmp" ".." "/tmp" "/" "../" "/tmp" "/" "../." "/tmp" "/" "../.." "/tmp" "/" "../../" "/tmp" "/" "../../x.r" "/tmp" "/x.r" "../leslie/" "/tmp" "/leslie" "../leslie/x.r" "/tmp" "/leslie/x.r" "../x.r" "/tmp" "/x.r" "..irace" "/tmp" "/tmp/..irace" "./" "/tmp" "/tmp" "./." "/tmp" "/tmp" "./" "/tmp/" "/tmp" "./." "/tmp/" "/tmp" "././x.r" "/tmp" "/tmp/x.r" "./irace/../x.r" "/tmp" "/tmp/x.r" "./x.r" "/tmp" "/tmp/x.r" ".x.R" "/tmp" "/tmp/.x.R" "/./x.r" "/tmp" "/x.r" "/home" "/tmp" "/home" "/home/leslie/././x.r" "/tmp" "/home/leslie/x.r" "/home/leslie/~/x.r" "/tmp" "/home/leslie/~/x.r" "/~/x.r" "/tmp" "/~/x.r" "leslie/leslie/../../irace" "/tmp" "/tmp/irace" "x.r" "/tmp" "/tmp/x.r" "~/irace/../x.r" "/tmp" "~/x.r" "~/x.r" "/tmp" "~/x.r" "../../../data" "./" "/data" "../../../data" "/tmp/a/b/c/" "/tmp/data" "..//a" ".//" "/a" ', stringsAsFactors=FALSE) test_path_rel2abs(testcases) }) test_that("test path_rel2abs without /tmp", { testcases <- read.table(text=' . N:\\\\tmp N:/tmp .. N:\\\\tmp N:/ ..\\\\ N:\\\\tmp N:/ ..\\\\. N:\\\\tmp N:/ ..\\\\.. N:\\\\tmp N:/ ..\\\\..\\\\ N:\\\\tmp N:/ ..\\\\..\\\\x.r N:\\\\tmp N:/x.r ..\\\\leslie\\\\ N:\\\\tmp N:/leslie ..\\\\leslie\\\\x.r N:\\\\tmp N:/leslie/x.r ..\\\\x.r N:\\\\tmp N:/x.r ..irace N:\\\\tmp N:/tmp/..irace .\\\\ N:\\\\tmp N:/tmp .\\\\. N:\\\\tmp N:/tmp .\\\\ N:\\\\tmp\\\\ N:/tmp .\\\\. N:\\\\tmp\\\\ N:/tmp .\\\\.\\\\x.r N:\\\\tmp N:/tmp/x.r .\\\\x.r N:\\\\tmp N:/tmp/x.r .\\\\irace\\\\..\\\\x.r N:\\\\tmp N:/tmp/x.r .x.R N:\\\\tmp N:/tmp/.x.R . N:\\tmp N:/tmp .. N:\\tmp N:/ ..\\ N:\\tmp N:/ ..\\. N:\\tmp N:/ ..\\.. N:\\tmp N:/ ..\\..\\ N:\\tmp N:/ ..\\..\\x.r N:\\tmp N:/x.r ..\\leslie\\ N:\\tmp N:/leslie ..\\leslie\\x.r N:\\tmp N:/leslie/x.r ..\\x.r N:\\tmp N:/x.r ..irace N:\\tmp N:/tmp/..irace .\\ N:\\tmp N:/tmp .\\. N:\\tmp N:/tmp .\\ N:\\tmp\\ N:/tmp .\\. N:\\tmp\\ N:/tmp .\\.\\x.r N:\\tmp N:/tmp/x.r .\\irace\\..\\x.r N:\\tmp N:/tmp/x.r .\\x.r N:\\tmp N:/tmp/x.r .x.R N:\\tmp N:/tmp/.x.R . N: N:/ .. N: N:/ ..\\\\ N: N:/ ..\\\\. N: N:/ ..\\\\.. N: N:/ ..\\\\..\\\\ N: N:/ ..\\\\..\\\\x.r N: N:/x.r ..\\\\leslie\\\\ N: N:/leslie ..\\\\leslie\\\\x.r N: N:/leslie/x.r ..\\\\x.r N: N:/x.r ..\\ N: N:/ ..\\. N: N:/ ..\\.. N: N:/ ..\\..\\ N: N:/ ..\\..\\x.r N: N:/x.r ..\\leslie\\ N: N:/leslie ..\\leslie\\x.r N: N:/leslie/x.r ..\\x.r N: N:/x.r ..irace N: N:/..irace .\\\\ N: N:/ .\\\\. N: N:/ .\\\\ N:\\\\ N:/ .\\\\. N:\\\\ N:/ .\\\\.\\\\x.r N: N:/x.r .\\\\irace\\\\..\\\\x.r N: N:/x.r .\\\\x.r N: N:/x.r .\\ N: N:/ .\\. N: N:/ .\\ N:\\ N:/ .\\. N:\\ N:/ .\\.\\x.r N: N:/x.r .\\irace\\..\\x.r N: N:/x.r .\\x.r N: N:/x.r .x.R N: N:/.x.R . N:/tmp N:/tmp .. N:/tmp N:/ ../ N:/tmp N:/ ../. N:/tmp N:/ ../.. N:/tmp N:/ ../../ N:/tmp N:/ ../../x.r N:/tmp N:/x.r ../leslie/ N:/tmp N:/leslie ../leslie/x.r N:/tmp N:/leslie/x.r ../x.r N:/tmp N:/x.r ..irace N:/tmp N:/tmp/..irace ./ N:/tmp N:/tmp ./. N:/tmp N:/tmp ./ N:/tmp/ N:/tmp ./. N:/tmp/ N:/tmp ././x.r N:/tmp N:/tmp/x.r ./irace/../x.r N:/tmp N:/tmp/x.r ./x.r N:/tmp N:/tmp/x.r .x.R N:/tmp N:/tmp/.x.R D:/./x.r N:/tmp D:/x.r D:\\\\.\\\\x.r N:/tmp D:/x.r D:\\.\\x.r N:/tmp D:/x.r D: N:/tmp D:/ D:\\\\ N:/tmp D:/ D:/ N:/tmp D:/ D:/leslie/././x.r N:/tmp D:/leslie/x.r D:/leslie/~/x.r N:/tmp D:/leslie/~/x.r e:/home/leslie/x.r /tmp E:/home/leslie/x.r leslie/leslie/../../irace N:/tmp N:/tmp/irace x.r N:/tmp N:/tmp/x.r ~/irace/../x.r N:/tmp ~/x.r ~/x.r N:/tmp ~/x.r "R" "/tmp/" "Sys.which" ', stringsAsFactors=FALSE) test_path_rel2abs(testcases) }) test_that("test path_rel2abs with symlink", { # Try to set wd; otherwise fail silently. old.cwd <- getwd() skip_if(is.null(old.cwd)) withr::defer(setwd(old.cwd)) tryCatch({ tmp <- withr::local_tempdir() setwd(tmp) fs::dir_create("a") fs::file_create("a/b") fs::link_create(fs::path_abs("a"), "c") }, error = function(e) { skip(e) }) testcases <- data.frame(p = "c/b", wd = ".", res = file.path(tmp, "c/b"), stringsAsFactors=FALSE) test_path_rel2abs(testcases) }) irace/tests/testthat/test-similar.R0000644000176200001440000000237114736526233017115 0ustar liggesuserstest_that("similarConfigurations", { parameters <- readParameters(text = ' n1 "" r (0,1) n2 "" r (0,1) n3 "" r (0,1) c1 "" c ("a","b") c2 "" c ("a","b") c3 "" c ("a","b") ') confs <- read.table(header = TRUE, stringsAsFactors = FALSE, text = ' .ID. n1 n2 n3 c1 c2 c3 1 0.5 0.5 0.5 "a" "a" "a" 2 NA 0.5 0.5 "a" "a" "a" 3 0.5 0.5 0.5 "a" "a" "b" 4 0.5 0.501 0.5 "a" "a" "a" 5 0.5 0.5 0.499 "a" "a" "a" 6 0.5 0.5 0.5 "a" "a" "a" 7 0.5 0.5 0.5 "a" NA "a" 8 0.5 0.1 0.5 "a" "a" "a" 9 0.5 0.49 0.5 "a" "a" "a" 10 0.5 0.5 0.5 "a" "a" NA 11 0.5 0.5 0.5 "a" "a" NA 12 NA 0.5 0.5 "a" "a" "a" ') expect_identical( irace:::similarConfigurations(confs[1:10,], parameters, threshold = 0.001), as.integer(c(1,6))) expect_identical( irace:::similarConfigurations(confs[1:10,], parameters, threshold = 0.01), as.integer(c(1,4,5,6))) expect_identical( irace:::similarConfigurations(confs[1:10,], parameters, threshold = 0.1), as.integer(c(1,4,5,6,9))) expect_identical( # FIXME: The output should be already sorted sort(irace:::similarConfigurations(confs, parameters, threshold = 0.001)), as.integer(c(1,2,6,10,11,12))) }) irace/tests/testthat/test-bug-10.R0000644000176200001440000000415014736526233016445 0ustar liggesuserswithr::with_output_sink("test-bug-10.Rout", { parameters_txt <- ' algorithm "--" c (as,mmas,eas,ras,acs) localsearch "--localsearch " c (0, 1, 2, 3) alpha "--alpha " r (0.00, 5.00) beta "--beta " r (0.00, 10.00) rho "--rho " r (0.01, 1.00) ants "--ants " i (5, 100) q0 "--q0 " r (0.0, 1.0) | algorithm == "acs" rasrank "--rasranks " i (1, 100) | algorithm == "ras" elitistants "--elitistants " i (1, 750) | algorithm == "eas" nnls "--nnls " i (5, 50) | localsearch %in% c(1,2,3) dlb "--dlb " c (0, 1) | localsearch %in% c(1,2,3) ' parameters <- readParameters(text=parameters_txt) test_that("bug blocksize", { skip_on_cran() target_runner <- function(experiment, scenario) list(cost = 100 + rnorm(1, 0, 0.1)) withr::with_options(list(warning=2), { scenario <- list(targetRunner = target_runner, instances=1:5, firstTest=5*5, eachTest=5, sampleInstances=FALSE, maxExperiments = 5000, logFile = "", elitistNewInstances = 1, elitist = TRUE, parameters = parameters) scenario <- checkScenario (scenario) confs <- irace(scenario = scenario) expect_false(is.null(confs)) }) }) # https://github.com/MLopez-Ibanez/irace/issues/10 test_that("bug 10", { skip_on_cran() target_runner <- function(experiment, scenario) list(cost = 100, call = toString(experiment)) withr::with_options(list(warning=2), { scenario <- list(targetRunner = target_runner, instances=1:10, maxExperiments = 5000, logFile = "", deterministic = TRUE, elitistNewInstances = 0, elitistLimit = 0, elitist = 0, parameters = parameters) scenario <- checkScenario (scenario) confs <- irace(scenario = scenario) expect_false(is.null(confs)) }) }) }) # withr::with_output_sink irace/tests/testthat/test-forbidden.R0000644000176200001440000000340414736526233017407 0ustar liggesuserswithr::with_output_sink("test-forbidden.Rout", { test_that("checkForbidden", { test.checkForbidden <- function(param.file) { params <- readParameters(param.file) confs <- readConfigurationsFile("configurations.txt", params) exp.confs <- readConfigurationsFile(text=' param1 param2 mode real mutation 5 NA "x2" 4.0 "low" 1 NA "x2" 4.0 "low" 5 6 "x1" 3.5 "low" NA NA "x3" 4.5 "low" ', parameters = params) confs <- irace:::checkForbidden(confs, params$forbidden) rownames(confs) <- rownames(exp.confs) <- NULL expect_equal(confs, exp.confs) } test.checkForbidden(test_path("parameters.txt")) test.checkForbidden(test_path("logparameters.txt")) }) test_that("filter_forbidden", { test_filter_forbidden <- function(param_file) { params <- readParameters(param_file) confs <- readConfigurationsFile("configurations.txt", params) exp_confs <- data.table::as.data.table(readConfigurationsFile(text=' param1 param2 mode real mutation 5 NA "x2" 4.0 "low" 1 NA "x2" 4.0 "low" 5 6 "x1" 3.5 "low" NA NA "x3" 4.5 "low" ', parameters = params)) confs <- irace:::filter_forbidden(data.table::as.data.table(confs), params$forbidden) expect_equal(confs, exp_confs) } test_filter_forbidden(test_path("parameters.txt")) test_filter_forbidden(test_path("logparameters.txt")) }) test_that("retries", { params <- readParameters(text=' p0 "" r (0,1) [forbidden] p0 > 0.9 ') confs <- irace:::sampleUniform(params, 50) expect_equal(nrow(confs), 50L) }) test_that("max retries", { params <- readParameters(text=' p0 "" r (0,1) [forbidden] p0 <= 0.5 p0 > 0.5 ') expect_error(irace:::sampleUniform(params, 1), "perhaps your constraints are too strict") }) }) irace/tests/testthat/test-raceconfs.R0000644000176200001440000000241414736526233017416 0ustar liggesuserswithr::with_output_sink("test-raceconfs.Rout", { parameters_txt <- ' launch_method "--launch_method=" c (L-BFGS-B,SLSQP) visit_method "--visit_method=" c (DA, DE) global_method "--global_method=" c (CMAES,DE) ' confs_txt <- ' launch_method visit_method global_method L-BFGS-B DA CMAES SLSQP DA CMAES L-BFGS-B DE CMAES SLSQP DE CMAES L-BFGS-B DA DE SLSQP DA DE L-BFGS-B DE DE SLSQP DE DE ' target_runner <- function(experiment, scenario) list(cost = 100, call = toString(experiment)) withr::with_options(list(warning=2), { parameters <- readParameters(text=parameters_txt) initconfs <- readConfigurationsFile(text=confs_txt, parameters=parameters) scenario <- list(targetRunner = target_runner, instances = 1:10, nbConfigurations = 8, maxExperiments = 96, logFile = "", initConfigurations = initconfs, parameters = parameters) scenario <- checkScenario (scenario) confs <- irace(scenario = scenario) confs <- data.table::as.data.table(removeConfigurationsMetaData(confs)) initconfs <- data.table::as.data.table(initconfs) expect_equal(nrow(data.table::fsetdiff(confs, initconfs)), 0L) }) }) # withr::with_output_sink() irace/tests/testthat/test-get-functions.R0000644000176200001440000000060214737241452020233 0ustar liggesuserswithr::with_output_sink("test-get-functions.Rout", { test_that("getConfigurationById()", { log <- read_logfile(system.file(package="irace", "exdata", "irace-acotsp.Rdata", mustWork=TRUE)) ids <- sample(log$allConfigurations[[".ID."]], size=3) ids <- c(ids, rev(ids)) sel_ids <- getConfigurationById(log, ids = ids)[[".ID."]] expect_identical(sel_ids, ids) }) }) irace/tests/testthat/teardown.R0000644000176200001440000000002214736526233016312 0ustar liggesusersoptions(old_opts) irace/tests/testthat/test-psrace.R0000644000176200001440000000307714736526233016736 0ustar liggesuserswithr::with_output_sink("test-psrace.Rout", { test_that("max_experiments=0.1", { skip_on_cran() generate_set_seed() irace_log <- read_logfile(system.file(package="irace", "exdata", "sann.rda")) # Use a temporary file to not change the original "sann.rda". psrace_logFile <- withr::local_tempfile(fileext = ".Rdata") # Execute the post-selection after the execution of irace. Use 10% of the total budget. psRace(irace_log, max_experiments=0.1, psrace_logFile = psrace_logFile) irace_log <- read_logfile(psrace_logFile) budget <- nrow(irace_log$state$experiment_log[iteration == max(iteration)]) expect_gt(budget, 50L) # It should be equal but elitist_race sometimes fails to consume all the budget. expect_lte(budget, irace_log$psrace_log$max_experiments) }) test_that("max_experiments=101", { skip_on_cran() generate_set_seed() irace_log <- read_logfile(system.file(package="irace", "exdata", "sann.rda")) conf_ids <- unlist(tail(irace_log$allElites, n=1L)) # Use a temporary file to not change the original "sann.rda". psrace_logFile <- withr::local_tempfile(fileext = ".Rdata") # Execute the post-selection after the execution of irace. psRace(irace_log, max_experiments=101, conf_ids = conf_ids, psrace_logFile = psrace_logFile) irace_log <- read_logfile(psrace_logFile) budget <- nrow(irace_log$state$experiment_log[iteration == max(iteration)]) expect_gt(budget, 10L) # It should be equal but elitist_race sometimes fails to consume all the budget. expect_lte(budget, irace_log$psrace_log$max_experiments) }) }) # withr::with_output_sink() irace/tests/testthat/dependencies2.txt0000644000176200001440000000070714736526233017627 0ustar liggesusersparam1 "--param1 " i (1, "real") | mode %in% c("x1", "x2") param2 "--param2 " i ("real", 10) | mode %in% c("x1", "x3") & real > 2.5 & real <= 3.5 mode "--" c ("x1" ,"x2", "x3") real "--paramreal=" r (1.5, 7.5) mutation "--mutation=" o ("none", "very low", "low", "medium", "high", "very high", "all") #unused "-u " c (1, 2, 10, 20) irace/tests/testthat/configurations.txt0000644000176200001440000000056714556723674020165 0ustar liggesusersparam1 param2 mode real mutation 5 NA "x2" 4.0 "low" 1 NA "x1" 4.0 "low" 1 NA "x2" 4.0 "low" 5 7 "x1" 3.5 "low" 5 6 "x1" 3.5 "low" NA 7 "x3" 3.5 "low" NA 6 "x3" 3.5 "low" 5 NA "x2" 2.0 "low" NA NA "x3" 1.5 "low" NA NA "x3" 4.5 "low" irace/tests/testthat/test-readParameters.R0000644000176200001440000001536314736526233020421 0ustar liggesuserswithr::with_output_sink("test-readParameters.Rout", { test_that("optimize conditions", { p <- readParameters(text=' ROOT "" c ("S") ROOT_T "" c ("FIC") | ROOT %in% c("S") ROOT_T_FIC.sum "" r (-50.0, 50.0) | ROOT_T %in% c("FIC") ROOT_T.FFI "" c ("true") | ROOT_T == "FIC" ROOT_T.im "" c ("FFI", "FIC") | ROOT == "S" & ROOT_T.FFI == "true" ROOT_E.FFI "" c ("false") | ROOT_T_FIC.sum < 0 ROOT_E.FIC "" i (0,100) | ROOT_E.FFI == "false" ') expect_identical(p$conditions, list(ROOT=TRUE, ROOT_T=TRUE, ROOT_T_FIC.sum=TRUE, ROOT_T.FFI=TRUE, ROOT_T.im=TRUE, ROOT_E.FFI=expression(ROOT_T_FIC.sum < 0), ROOT_E.FIC=expression(ROOT_E.FFI == "false"))) }) test_that("error checking", { ref <- parametersNew(param_real(name = "tmp", lower = 0, upper=1, label = "", digits = 5), forbidden = "tmp == 0") expect_identical(ref, readParameters(text=' tmp "" r (0, 1) [global] digits = 5 [forbidden] tmp == 0 ')) expect_identical(ref, readParameters(text=' tmp "" r (0, 1) [forbidden] tmp == 0 [global] digits = 5 ')) expect_identical(ref, readParameters(text=' tmp "" r (0, 1) [forbidden] tmp == 0 [global] digits = 5 ')) expect_error(readParameters(text = ' rest_t "--restart_temp_ratio=" r,log (0.00001, 0.9) [global] digits = 4 '), "Domain bounds") expect_no_error(readParameters(text = ' rest_t "--restart_temp_ratio=" r,log (0.00001, 0.9) [global] digits = 5 ')) expect_error(readParameters(text = ' temp "" r (0, 10) t-max "" i (1, 50) '), "name must be alphanumeric") expect_error(readParameters(text = ' temp "" r (0, 10) [forbidden] tmp == 0 '), "contains unknown parameter(s): tmp", fixed = TRUE) expect_error(readParameters(text = ' temp "" r (0, 10) temp "" i (1, 50) '), "Duplicated parameter name") expect_error(readParameters(text = ' temp "" c ("a", "b", "a") '), "duplicated values") expect_error(readParameters(text = ' temp --temp i (0, 10) '), "Parameter label (switch) must be a double-quoted string", fixed = TRUE) expect_error(readParameters(text = ' temp "--temp" i,lag (0, 10) '), "Parameter type must be a single character in .'c','i','r','o'.") expect_error(readParameters(text = ' temp "--temp" i c(0, 10) '), "Allowed values must be a list within parenthesis at line 2") expect_error(readParameters(text = 'param1 "--param1 " c "a,b,c"'), "Allowed values must be a list within parenthesis at line 1") expect_error(readParameters(text = 'param1 "--param1 " i (1,2,3 )'), "Incorrect numeric range") expect_error(readParameters(text = 'param1 "--param1 " i ( 1,10) |'), "Expected condition before '|'") expect_error(readParameters(text = 'param1 "--param1 " i ( 1, 10) param1 < param2'), "Expected condition after '|'") expect_error(readParameters(text = ' param1 "--param1 " i (0,2) param2 "--param2 " r (0,1) | param1 <> 1'), "Invalid condition 'param1 <> 1': ") expect_error(readParameters(text = ' param1 "--param1 " i (0,2) param2 "--param2 " r (0,2) | param1 < param2'), "Cycle detected.+param2") expect_error(readParameters(text = ' param1 "--param1 " i (0,2) param2 "--param2 " r (0,param2)'), "Cycle detected.+param2") expect_error(readParameters(text = ' param1 "--param1 " i (0,2) param2 "--param2 " r (0,2) | param1 < param3'), "Parameter 'param2' depends on 'param3' which is not defined") expect_error(readParameters(text = ' param1 "--param1 " i (0,2) param2 "--param2 " r (0,param3)'), "'param3' cannot be found in the scenario parameters") expect_error(readParameters(text = ' param1 "--param1 " c (a,b) param2 "--param2 " r (0,param1)'), "Domain of parameter 'param2' depends on non-numerical parameters: param1") expect_error(readParameters(text = 'param1 "--param1 " i (1.1,2.0)'), "For parameter 'param1' of type 'i' values must be integers") expect_error(readParameters(text = 'param1 "--param1 " r (0.01, 0.001)'), "Lower bound must be smaller than upper bound in numeric range") expect_error(readParameters(text = 'param1 "--param1 " i (1, 1)'), "Lower bound must be smaller than upper bound in numeric range") expect_error(readParameters(text = ' # # '), "No parameter definition found") test_that("ordinal out of order", { expect_error(readParameters(text = ' param "" o (0, 2, 1) '), "the values are not in increasing order") }) expect_error(readParameters(text = 'param1 "--param1 " r,log (0, 100)'), "of parameter of type 'log' contains non-positive values") expect_error(readParameters(text = 'param1 "--param1 " r,log (0.0001, 0.99999)', digits=3), "must be representable within the given 'digits=3'; you would need at least 'digits=5'") expect_error(param_cat(name = "algorithm", values = c("as", "mmas", "eas", "ras", "acs"), label = "--", condition = "a == 0 && b == 2"), "Please use '&' and '|' instead of '&&' and '|' in: a == 0 && b == 2", fixed = TRUE) expect_error(param_cat(name = "algorithm", values = c("as", "mmas", "eas", "ras", "acs"), label = "--", condition = quote(a == 0 || b == 2)), "Please use '&' and '|' instead of '&&' and '|' in: a == 0 || b == 2", fixed = TRUE) expect_error(param_cat(name = "algorithm", values = c("as", "mmas", "eas", "ras", "acs"), label = "--", condition = 0), "Invalid condition", fixed = TRUE) expect_error(parametersNew(param_real(name = "alpha", lower = 0.0, upper=5.0), param_real(name = "beta", lower = 0.0, upper = 10.0), forbidden = expression((alpha == 0) || (beta == 0))), "Please use '&' and '|' instead of '&&' and '|' in: (alpha == 0) || (beta == 0)", fixed = TRUE) expect_error(parametersNew(param_real(name = "alpha", lower = 0.0, upper=5.0), param_real(name = "beta", lower = 0.0, upper = 10.0), forbidden = quote((alpha == 0) && (beta == 0))), "Please use '&' and '|' instead of '&&' and '|' in: (alpha == 0) && (beta == 0)", fixed = TRUE) expect_error(parametersNew(param_real(name = "alpha", lower = 0.0, upper=5.0), param_real(name = "beta", lower = 0.0, upper = 10.0), forbidden = TRUE), "Invalid forbidden expression", fixed = TRUE) expect_identical(param_real(name = "p", lower = "-1", upper = "10")$domain, c(-1,10)) expect_identical(param_int(name = "p", lower = "-1", upper = "10")$domain, c(-1L,10L)) expect_identical(param_int(name = "p", lower = -1, upper = 10)$domain, c(-1L,10L)) expect_identical(param_int(name = "p", lower = "-1", upper = expression(a))$domain[[1L]], -1L) expect_identical( param_int(name = "p", lower = "-1", upper= "a"), param_int(name = "p", lower = "-1", upper= expression(a))) expect_identical( param_int(name = "p", lower = -1, upper= "a"), param_int(name = "p", lower = "-1", upper= "a")) }) }) irace/tests/testthat/test-repair.R0000644000176200001440000000440514736526233016737 0ustar liggesuserswithr::with_output_sink("test-repair.Rout", { repair_irace <- function(targetRunner, repair) { weights <- rnorm(200, mean = 0.9, sd = 0.02) parameters <- readParameters(text = ' p1 "" r (0,1) p2 "" r (0,1) p3 "" r (0,1) dummy "" c ("d1", "d2") ') scenario <- list(targetRunner = targetRunner, repairConfiguration = repair, instances = weights, maxExperiments=180, seed = 1234567, parameters = parameters) scenario <- checkScenario (scenario) expect_true(irace:::checkTargetFiles(scenario = scenario)) confs <- irace(scenario = scenario) expect_gt(nrow(confs), 0L) confs } target_sum2one <- function(experiment, scenario) { configuration <- experiment$configuration p1 <- configuration[["p1"]] p2 <- configuration[["p2"]] p3 <- configuration[["p3"]] stopifnot(isTRUE(all.equal(p1+p2+p3, 1.0))) list(cost = -p1, call = toString(experiment)) } repair_sum2one <- function(configuration, parameters) { isreal <- parameters$names[parameters$types == "r"] digits <- sapply(isreal, function(x) parameters$get(x)[["digits"]]) c_real <- unlist(configuration[isreal]) c_real <- c_real / sum(c_real) c_real[-1] <- round(c_real[-1], digits[-1]) c_real[1L] <- 1 - sum(c_real[-1]) configuration[isreal] <- c_real configuration } target_order <- function(experiment, scenario) { configuration <- experiment$configuration p1 <- configuration[["p1"]] p2 <- configuration[["p2"]] p3 <- configuration[["p3"]] stopifnot(p1 <= p2 && p2 <= p3) list(cost = -p1, call = toString(experiment)) } repair_order <- function(configuration, parameters) { columns <- c("p1","p2","p3") #cat("Before"); print(configuration) configuration[columns] <- sort(unlist(configuration[columns], use.names=FALSE)) #cat("After"); print(configuration) configuration } test_that("repair: sum to one", { generate_set_seed() confs <- repair_irace(target_sum2one, repair_sum2one) expect_equal(unique(apply(confs[, c("p1", "p2", "p3")], 1L, sum)), 1) }) test_that("repair: increasing order", { generate_set_seed() confs <- repair_irace(target_order, repair_order) expect_true(all(apply(confs[, c("p1", "p2", "p3")], 1L, diff) >= 0)) }) }) # withr::with_output_sink() irace/tests/testthat/test-bug-55.R0000644000176200001440000000101614736526233016454 0ustar liggesusers# https://github.com/MLopez-Ibanez/irace/issues/55 withr::with_output_sink("test-bug-55.Rout", { test_that("bug-55", { parameters_txt <- ' foo "--foo " i (0, 1) foo2 "--foo2 " c (true, false) | foo == 0 ' params <- readParameters(text=parameters_txt) configurations_txt <- ' foo foo2 1 0 false 2 1 ' confs <- readConfigurationsFile(text=configurations_txt, parameters = params) expect_equal(confs, data.frame(foo=c(0L,1L), foo2=c("false",NA), row.names=c("1", "2"), stringsAsFactors=FALSE)) }) }) irace/tests/testthat/test-sann-irace.R0000644000176200001440000000653114736526233017477 0ustar liggesuserswithr::with_output_sink("test-sann-irace.Rout", { ## Functions ########################################################## f_rosenbrock <- function (x) { d <- length(x) z <- x + 1 hz <- z[1:(d - 1)] tz <- z[2:d] sum(100 * (hz^2 - tz)^2 + (hz - 1)^2) } f_rastrigin <- function (x) sum(x * x - 10 * cos(2 * pi * x) + 10) ## target runner ########################################################### target_runner <- function(experiment, scenario) { debugLevel <- scenario$debugLevel configuration_id <- experiment$id_configuration instance_id <- experiment$id_instance seed <- experiment$seed configuration <- experiment$configuration instance <- experiment$instance D <- 3 par <- runif(D, min = -1, max = 1) fn <- function(x) (instance * f_rastrigin(x) + (1 - instance) * f_rosenbrock(x)) tmax = 1 + configuration[["tmax"]] temp = 11.0 + configuration[["temp"]] stopifnot(tmax > 0) stopifnot(temp > 0) res <- withr::with_seed(seed, optim(par, fn, method = "SANN", control = list(maxit = 10, tmax = tmax, temp = temp)) ) list(cost = res$value, call = toString(experiment)) } ## target runner ########################################################### target_runner_reject <- function(experiment, scenario) { if (runif(1) <= 0.05) return (list(cost = -Inf, call = toString(experiment))) target_runner(experiment, scenario) } ## Run function ######################################################## sann_irace <- function(log.param=FALSE, ...) { args <- list(...) # tmax and temp must be > 0 if (log.param) parameters_table <- ' tmax "" i,log (1, 5000) temp "" r,log (1, 100) ' else parameters_table <- ' tmax "" i (1, 5000) temp "" r (1, 100) ' parameters <- readParameters(text = parameters_table) scenario <- list(targetRunner = target_runner, maxExperiments = 1000, seed = 1234567, parameters = parameters) scenario <- modifyList(scenario, args) scenario <- checkScenario (scenario) confs <- irace(scenario = scenario) best.conf <- getFinalElites(scenario$logFile, n = 1, drop.metadata = TRUE) expect_identical(removeConfigurationsMetaData(confs[1, , drop = FALSE]), best.conf) } test_that("parallel", { skip_on_cran() # Reproducible results generate_set_seed() weights <- rnorm(200, mean = 0.9, sd = 0.02) sann_irace(instances = weights, parallel = test_irace_detectCores()) }) test_that("parallel reject", { # Reproducible results generate_set_seed() weights <- rnorm(200, mean = 0.9, sd = 0.02) sann_irace(instances = weights, parallel = test_irace_detectCores(), targetRunner = target_runner_reject) }) test_that("deterministic", { skip_on_cran() # Reproducible results generate_set_seed() weights <- rnorm(200, mean = 0.9, sd = 0.02) sann_irace(deterministic = TRUE, instances = weights[1:7]) }) test_that("log", { skip_on_cran() # Reproducible results generate_set_seed() weights <- rnorm(200, mean = 0.9, sd = 0.02) sann_irace(log.param=TRUE, instances = weights) }) test_that("large newInstances", { skip_on_cran() # Reproducible results generate_set_seed() weights <- rnorm(200, mean = 0.9, sd = 0.02) sann_irace(instances = weights, elitistNewInstances = 6, elitistLimit = 2) }) }) # withr::with_output_sink() irace/tests/testthat/logparameters.txt0000644000176200001440000000161314736526233017761 0ustar liggesusersparam1 "--param1 " i,log (1, 10) | mode %in% c("x1", "x2") param2 "--param2 " i (1, 10) | mode %in% c("x1", "x3") & real > 2.5 & real <= 3.5 mode "--" c ("x1" ,"x2", "x3") real "--paramreal=" r,log (1.5, 4.5) mutation "--mutation=" o ("none", "very low", "low", "medium", "high", "very high", "all") #unused "-u " c (1, 2, 10, 20) [forbidden] ## The format is one constraint per line. Each constraint is a logical ## expression (in R syntax). If a parameter configuration ## is generated that makes the logical expression evaluate to TRUE, ## then the configuration is discarded. ## ## Examples of valid logical operators are: == != >= <= > < & | ! %in% param1 < 5 & mode == "x1" (param2 > 6 & mode == "x1") | (param2 <= 6 & mode == "x3") real < 4 & mode %in% c("x2", "x3") irace/tests/testthat/test-argparser.R0000644000176200001440000000046614736526233017446 0ustar liggesuserswithr::with_output_sink("test-argparser.Rout", { test_that("argparse", { params_def <- data.frame(name =".param", type ="s", short = "-p", long=NA, default=NA, domain=NA, description="") parser <- irace:::CommandArgsParser$new("-p 'something something'", params_def) expect_length(parser$argv, 2L) }) }) irace/tests/testthat/test-bug-87.R0000644000176200001440000000350415060050551016447 0ustar liggesuserswithr::with_output_sink("test-bug-87.Rout", { test_that("bug-87", { parameters <- parametersNew( param_cat(name = "algorithm", values = c("as", "mmas", "eas", "ras", "acs"), label = "--"), param_ord(name = "localsearch", values = c("0", "1", "2", "3"), label = "--localsearch "), param_real(name = "alpha", lower = 0.0, upper = 5.0, label = "--alpha "), param_real(name = "beta", lower = 0.0, upper = 10.0, label = "--beta "), param_real(name = "rho", lower = 0.01, upper = 1.00, label = "--rho "), param_int(name = "ants", lower = 5, upper = 100, transf = "log", label = "--ants "), param_real(name = "q0", label = "--q0 ", lower = 0.0, upper = 1.0, condition = expression(algorithm == "acs")), param_int(name = "rasrank", label = "--rasranks ", lower = 1, upper = quote(min(ants, 10)), condition = 'algorithm == "ras"'), param_int(name = "elitistants", label = "--elitistants ", lower = 1, upper = expression(ants), condition = 'algorithm == "eas"'), param_int(name = "nnls", label = "--nnls ", lower = 5, upper = 50, condition = expression(localsearch %in% c("1", "2", "3"))), param_cat(name = "dlb", label = "--dlb ", values = c("0", "1"), condition = "localsearch %in% c('1','2','3')"), forbidden = "(alpha == 0) & (beta == 0)" ) logFile <- withr::local_tempfile(fileext=".Rdata") scenario <- defaultScenario(list(parameters = parameters, instances = 1, maxExperiments = 300, logFile = logFile, targetRunner = function(experiment, scenario) { list(cost = experiment$configuration$alpha * experiment$configuration$beta) })) confs <- irace(scenario) best_conf <- getFinalElites(scenario$logFile, n = 1L, drop.metadata = TRUE) expect_identical(removeConfigurationsMetaData(confs[1L, , drop = FALSE]), best_conf) }) }) # withr::with_output_sink() irace/tests/testthat/test-sobol.R0000644000176200001440000000755414745735066016611 0ustar liggesuserswithr::with_output_sink("test-sobol.Rout", { test_that("bug with conditional dependent", { parameters <- parametersNew(param_cat(name = "algorithm", values = c("as", "mmas", "ras", "acs")), param_real(name = "alpha", lower = 0.0, upper=5.0), param_real(name = "beta", lower = 0.0, upper = 10.0), param_int(name = "ants", lower = 2, upper = 100), param_real(name = "q0", lower=0.0, upper=1.0, condition = expression(algorithm == "acs")), param_int(name = "rasrank", lower=1, upper=quote(min(ants, 10)), condition = 'algorithm == "ras"'), param_int(name = "eants", lower=0, upper=expression(rasrank)), param_cat(name = "dlb", values = c(0,1), condition = "localsearch == 1"), param_int(name = "nnls", lower = 5, upper = 50, condition = expression(dlb == 1)), param_ord(name = "localsearch", values = c("0", "1")), param_cat(name = "fixed", values = "0")) confs <- irace:::sampleSobol(parameters, 1000L) expect_equal(nrow(confs), 1000L) expect_valid_configurations(confs, parameters) }) test_that("bug with dependent fixed", { params <- readParameters(text=' ROOT "ROOT=" c ("SimpleAlgorithm") ROOT_SimpleAlgorithm.constructive "ROOT_SimpleAlgorithm.constructive=" c ("FasterInvertedConstructive", "SlowConstructive") | ROOT %in% c("SimpleAlgorithm") ROOT_SimpleAlgorithm.constructive_FasterInvertedConstructive.sumThis "ROOT_SimpleAlgorithm.constructive_FasterInvertedConstructive.sumThis=" r (-50.0, 50.0) | ROOT_SimpleAlgorithm.constructive %in% c("FasterInvertedConstructive") ROOT_SimpleAlgorithm.constructive_SlowConstructive.sumThis "ROOT_SimpleAlgorithm.constructive_SlowConstructive.sumThis=" i (-10, 10) | ROOT_SimpleAlgorithm.constructive %in% c("SlowConstructive") ROOT_SimpleAlgorithm.improver "ROOT_SimpleAlgorithm.improver=" c ("FlippyFlopImprover") | ROOT %in% c("SimpleAlgorithm") ROOT_SimpleAlgorithm.improver_FlippyFlopImprover.enabled "ROOT_SimpleAlgorithm.improver_FlippyFlopImprover.enabled=" c ("true", "false") | ROOT_SimpleAlgorithm.improver %in% c("FlippyFlopImprover") ROOT_SimpleAlgorithm.improver_FlippyFlopImprover.sleepy "ROOT_SimpleAlgorithm.improver_FlippyFlopImprover.sleepy=" c ("8", "6", "12", "11", "7", "5", "4", "10", "1", "9", "2", "3", "13") | ROOT_SimpleAlgorithm.improver %in% c("FlippyFlopImprover") ') confs <- irace:::sampleSobol(params, 10L) expect_equal(nrow(confs), 10L) expect_valid_configurations(confs, params) }) test_that("bug with dependent fixed #2", { params <- readParameters(text=' ROOT "ROOT=" c ("SimpleAlgorithm", "ComplexAlgorithms") ROOT_SimpleAlgorithm.constructive "ROOT_SimpleAlgorithm.constructive=" c ("FasterInvertedConstructive", "SlowConstructive") | ROOT %in% c("SimpleAlgorithm") ROOT_SimpleAlgorithm.constructive_FasterInvertedConstructive.sumThis "ROOT_SimpleAlgorithm.constructive_FasterInvertedConstructive.sumThis=" r (-50.0, 50.0) | ROOT_SimpleAlgorithm.constructive %in% c("FasterInvertedConstructive") ROOT_SimpleAlgorithm.constructive_SlowConstructive.sumThis "ROOT_SimpleAlgorithm.constructive_SlowConstructive.sumThis=" i (-10, 10) | ROOT_SimpleAlgorithm.constructive %in% c("SlowConstructive") ROOT_SimpleAlgorithm.improver "ROOT_SimpleAlgorithm.improver=" c ("FlippyFlopImprover") | ROOT %in% c("SimpleAlgorithm") ROOT_SimpleAlgorithm.improver_FlippyFlopImprover.enabled "ROOT_SimpleAlgorithm.improver_FlippyFlopImprover.enabled=" c ("true", "false") | ROOT_SimpleAlgorithm.improver %in% c("FlippyFlopImprover") ROOT_SimpleAlgorithm.improver_FlippyFlopImprover.sleepy "ROOT_SimpleAlgorithm.improver_FlippyFlopImprover.sleepy=" c ("8", "6", "12", "11", "7", "5", "4", "10", "1", "9", "2", "3", "13") | ROOT_SimpleAlgorithm.improver %in% c("FlippyFlopImprover") ') confs <- irace:::sampleSobol(params, 10) expect_equal(nrow(confs), 10) expect_valid_configurations(confs, params) }) }) irace/tests/testthat/test-target-runner-dummy.R0000644000176200001440000001476115060056174021402 0ustar liggesuserswithr::with_output_sink("test-target-runner-dummy.Rout", { skip_on_cran() get_executable <- function(filename) { filename <- paste0(filename, if (system_os_is_windows()) ".exe" else "") p <- if (.Platform$r_arch == "") file.path( "bin", filename) else file.path("bin", .Platform$r_arch, filename) system.file(p, package="irace", mustWork = TRUE) } runexe <- function(exe, args) { err <- NULL output <- withCallingHandlers( tryCatch(system2(exe, args, stdout = TRUE, stderr = TRUE), error = function(e) { err <<- c(err, paste(conditionMessage(e), collapse="\n")) NULL }), warning = function(w) { err <<- c(err, paste(conditionMessage(w), collapse="\n")) invokeRestart("muffleWarning") }) if (!is.null(err)) { err <- paste(err, collapse = "\n") if (!is.null(attr(output, "errmsg"))) err <- paste(sep = "\n", err, attr(output, "errmsg")) stop(err) } if (is.null(output)) output <- "" return(output) } test_that("irace exe works", { skip_if_local_test() iraceexe <- get_executable("irace") expect_true(file.exists(iraceexe)) # FIXME: For some reason, this does not generate any output on Windows output <- expect_silent(runexe(iraceexe, "--help")) ## cat("irace --help\n") ## print(output) expected_output <- if (system_os_is_windows()) "^$" else "irace: An implementation in R.*called with: --help" expect_match(paste(collapse="", output), expected_output) }) test_that("ablation exe works", { skip_if_local_test() ablationexe <- get_executable("ablation") expect_true(file.exists(ablationexe)) # FIXME: For some reason, this does not generate any output on Windows output <- expect_silent(runexe(ablationexe, "--help")) ## cat("ablation --help\n") ## print(output) expected_output <- if (system_os_is_windows()) "^$" else "ablation: An implementation in R of Ablation Analysis.*called with: --help" expect_match(paste(collapse="", output), expected_output) }) run_cmdline <- function(parameters, args) { parameters_file <- tempfile("dummy-parameters", fileext = ".txt") withr::local_file(parameters_file) cat(parameters, file=parameters_file) train_instances_file <- tempfile("dummy-train-instances", fileext = ".txt") withr::local_file(train_instances_file) cat("1\n", file=train_instances_file) irace_cmdline(paste0(args, ' --debug-level 3 --parameter-file=', parameters_file, ' --train-instances-dir= --train-instances-file=', train_instances_file, ' --target-runner=', target_runner_dummy)) } skip_if_local_test() target_runner_dummy <- get_executable("target-runner-dummy") expect_true(file.exists(target_runner_dummy)) test_that("--check", { expect_warning( run_cmdline(paste0('p_int "--p_int " i (1, 10)\n', 'p_real "--p_real " r (1, 10)\n'), '--check --max-experiments 500'), "No scenario file given") }) test_that("--max-time", { expect_warning( run_cmdline(paste0('p_int "--p_int " c (1)\n', 'p_real "--p_real " r (0, 1)\n', 'dummy1 "--dummy1 " r (0, 1)\n', 'time "--time " c (1)\n'), '--max-time 2500'), "No scenario file given") }) test_that("boundMax is too large", { expect_warning( expect_warning( run_cmdline(paste0('p_int "--p_int " i (1, 10)\n', 'p_real "--p_real " r (0, 1)\n', 'time "--time " c (1)\n', 'capping "--opt-time " c (1)\n'), '--max-time 1000 --bound-max 100 --capping 1'), "is too large"), "No scenario file given") }) test_that("--capping", { expect_warning( expect_warning( run_cmdline(paste0('p_int "--p_int " i (1, 10)\n', 'p_real "--p_real " r (1, 10)\n', 'dummy1 "--dummy1 " r (1, 10)\n', 'dummy2 "--dummy2 " r (1, 10)\n', 'dummy3 "--dummy3 " r (1, 10)\n', 'dummy4 "--dummy4 " r (1, 10)\n', 'dummy5 "--dummy5 " r (1, 10)\n', 'dummy6 "--dummy6 " r (1, 10)\n', 'dummy7 "--dummy7 " r (1, 10)\n', 'dummy8 "--dummy8 " r (1, 10)\n', 'dummy9 "--dummy9 " r (1, 10)\n', 'dummy11 "--dummy11 " r (1, 10)\n', 'dummy12 "--dummy12 " r (1, 10)\n', 'dummy13 "--dummy13 " r (1, 10)\n', 'dummy14 "--dummy14 " r (1, 10)\n', 'dummy15 "--dummy15 " r (1, 10)\n', 'dummy16 "--dummy16 " r (1, 10)\n', 'dummy17 "--dummy17 " r (1, 10)\n', 'dummy18 "--dummy18 " r (1, 10)\n', 'dummy19 "--dummy19 " r (1, 10)\n', 'time "--time " c (1)\n', 'capping "--opt-time " c (1)\n'), '--max-time 500 --bound-max 1 --capping 1 '), "With the current settings and estimated time per run"), "No scenario file given") }) test_that("Error cost time", { expect_error( expect_warning( run_cmdline(paste0('p_int "--p_int " i (1, 10)\n', 'p_real "--p_real " r (1, 10)\n'), '--max-time 100 '), "No scenario file given"), "The output of targetRunner must be two numbers 'cost time'", fixed = TRUE) }) test_that("low --max-experiments", { expect_warning( expect_error( run_cmdline(paste0('p_int "--p_int " i (1, 10)\n', 'p_real "--p_real " r (1, 10)\n', 'time "--time " c (1)\n'), '--max-experiments 50'), "With the current settings"), "No scenario file given") }) test_that("--min-experiments", { expect_no_warning( expect_warning( run_cmdline(paste0('p_int "--p_int " i (1, 10)\n', 'p_real "--p_real " r (1, 10)\n', 'time "--time " c (1)\n'), '--min-experiments 50'), "No scenario file given")) }) }) # withr::with_output_sink() irace/tests/testthat/test-ablation.R0000644000176200001440000002415514745735066017260 0ustar liggesuserswithr::with_output_sink("test-ablation.Rout", { skip_on_cran() withr::local_options(warn=2) test_that("generateAblation", { parameters <- parametersNew(param_cat(name = "algorithm", values = c("as", "mmas", "ras", "acs")), param_real(name = "alpha", lower = 0.0, upper=5.0), param_real(name = "beta", lower = 0.0, upper = 10.0), param_int(name = "ants", lower = 2, upper = 100), param_real(name = "q0", lower=0.0, upper=1.0, condition = expression(algorithm == "acs")), param_int(name = "rasrank", lower=1, upper=quote(min(ants, 10)), condition = 'algorithm == "ras"'), param_int(name = "eants", lower=0, upper=expression(rasrank)), param_cat(name = "dlb", values = c(0,1), condition = "localsearch == 1"), param_int(name = "nnls", lower = 5, upper = 50, condition = expression(dlb == 1)), param_ord(name = "localsearch", values = c("0", "1")), param_cat(name = "fixed", values = "0"), forbidden = "(alpha == 0) & (beta == 0)") confs <- readConfigurationsFile(parameters = parameters, text =' algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch ras 1 0 5 NA 5 5 1 5 1 ras 0.5 0 5 NA 2 2 1 5 1 ras 0.5 0 4 NA 2 2 1 5 1 acs 0 1 2 0.5 NA NA NA NA 0 mmas 1 1 6 NA NA NA 0 NA 1 ') confs[[".ID."]] <- seq_len(nrow(confs)) colClasses <- c(localsearch="character", q0="numeric", rasrank="integer", eants="integer", dlb="character", nnls="integer") check_generate_ablation <- function(src, target, configurations_table, changed) { aux <- irace:::generate_ablation(confs[src, , drop=FALSE], confs[target, , drop=FALSE], parameters, param_names = parameters$names_variable) expect_valid_configurations(aux$configurations, parameters) configurations <- read.table(header=TRUE, colClasses=colClasses, text=configurations_table) configurations[["fixed"]] <- "0" configurations[[".ID."]] <- src configurations[[".PARENT."]] <- src expect_equal(aux, list(configurations=configurations, changed_params = changed)) } check_generate_ablation(1L, 2L, ' algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch ras 0.5 0 5 NA 5 5 1 5 1 ras 1.0 0 5 NA 2 2 1 5 1 ras 1.0 0 5 NA 5 2 1 5 1 ', changed = list("alpha", c("rasrank", "eants"), "eants")) check_generate_ablation(1L, 3L, ' algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch ras 0.5 0 5 NA 5 5 1 5 1 ras 1.0 0 4 NA 2 2 1 5 1 ras 1.0 0 5 NA 2 2 1 5 1 ras 1.0 0 5 NA 5 2 1 5 1 ', changed = list("alpha", c("ants", "rasrank", "eants"), c("rasrank", "eants"),"eants")) check_generate_ablation(1L, 4L, ' algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch acs 1.0 0 5 0.5 NA NA 1 5 1 ras 1.0 1.0 5 NA 5 5 1 5 1 ras 1.0 0 5 NA 5 5 NA NA 0 ', changed = list(c("algorithm", "q0", "rasrank", "eants"), "beta", c("localsearch", "dlb", "nnls"))) check_generate_ablation(1L, 5L, ' algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch mmas 1.0 0 5 NA NA NA 1 5 1 ras 1.0 1.0 5 NA 5 5 1 5 1 ras 1 0 6 NA 5 5 1 5 1 ras 1.0 0 5 NA 5 5 0 NA 1 ', changed = list(c("algorithm", "rasrank", "eants"), "beta", "ants", c("dlb", "nnls"))) check_generate_ablation(2L, 1L, ' algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch ras 1.0 0 5 NA 2 2 1 5 1 ras 0.5 0 5 NA 5 2 1 5 1 ', changed = list("alpha", c("rasrank"))) check_generate_ablation(3L, 1L, ' algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch ras 1.0 0 4 NA 2 2 1 5 1 ras 0.5 0 5 NA 2 2 1 5 1 ', changed = list("alpha", c("ants"))) check_generate_ablation(4L, 1L, ' algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch acs 1 1 2 0.5 NA NA NA NA 0 acs 0 1 5 0.5 NA NA NA NA 0 acs 0 1 2 0.5 NA NA 1 5 1 ', changed = list("alpha", "ants", c("localsearch", "dlb", "nnls"))) check_generate_ablation(5L, 1L, ' algorithm alpha beta ants q0 rasrank eants dlb nnls localsearch ras 1 1 6 NA 5 5 0 NA 1 mmas 1 0 6 NA NA NA 0 NA 1 mmas 1 1 5 NA NA NA 0 NA 1 mmas 1 1 6 NA NA NA 1 5 1 ', changed = list(c("algorithm", "rasrank", "eants"), "beta", "ants", c("dlb", "nnls"))) }) test_that("--help", { expect_output(ablation_cmdline("--help")) }) outfile <- withr::local_tempfile(pattern = "log-ablation", fileext = ".Rdata") logfile <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") parameters <- parametersNew( param_cat("cat", values = c("0", "1", "2", "3", "4")), param_real("real", lower = 0.0, upper=1.0), param_int("int", lower = 100, upper = 500), param_cat("bool", values = c("0", "1"))) default <- data.frame(cat="4", real=1.0, int=500L, bool = "1") target_runner <- function(experiment, scenario) { conf <- experiment$configuration instance <- experiment$instance seed <- experiment$seed k <- if (as.logical(as.integer(conf[["bool"]]))) 1000 else 100 list(cost = instance + 1/seed + k * (conf[["int"]] + as.integer(conf[["cat"]]) + (conf[["real"]]-0.5)^2)) } check_log <- function(log) { instances_log <- log$state$instances_log instances_log[, instance:=.I] experiment_log <- log$state$experiment_log[instances_log, on="instance"] experiment_log[["instance_value"]] <- log$scenario$instances[experiment_log[["instanceID"]]] experiment_log <- experiment_log[log$allConfigurations, on = c(configuration=".ID.")] experiments <- log$experiments experiments = data.table( instance = rep(seq_len(nrow(experiments)), ncol(experiments)), configuration = rep(seq_len(ncol(experiments)), each = nrow(experiments)), cost3 = c(experiments) ) experiments <- experiments[!is.na(experiments$cost3),] experiment_log <- experiment_log[experiments, on=.NATURAL] experiment_log[, cost2:=instance_value + 1/seed + fifelse(as.logical(as.integer(bool)), 1000, 100) * (int + as.integer(cat) + (real - 0.5)^2)] if ("bound" %in% colnames(experiment_log)) { experiment_log[, cost2 := pmin.int(cost2, bound)] experiment_log[, cost3 := pmin.int(cost3, bound)] } expect_equal(experiment_log[["cost"]], experiment_log[["cost2"]]) expect_equal(experiment_log[["cost"]], experiment_log[["cost3"]]) } src_file <- withr::local_tempfile(pattern="src", fileext=".txt", lines=c("cat real int bool", "4 1.0 500 1")) target_file <- withr::local_tempfile(pattern="target", fileext=".txt", lines=c("cat real int bool", "0 0.0 100 0")) test_that("ablation maxTime", { target_runner_time <- function(experiment, scenario) list(cost = target_runner(experiment, scenario)$cost, time = runif(1, min=0.1, max=1)) scenario <- list(targetRunner = target_runner_time, instances = seq(1000, 10000, 1000), seed = 42, maxTime = 1000, initConfigurations = default, logFile = logfile, parameters = parameters) scenario <- checkScenario (scenario) irace(scenario = scenario) check_log(read_logfile(logfile)) res <- ablation(logfile, ablationLogFile = outfile) check_log(res) expect_true(res$complete) res <- ablation(logfile, ablationLogFile = outfile, type = "racing") check_log(res) expect_true(res$complete) res <- ablation(logfile, ablationLogFile = outfile, src = src_file, target = target_file) check_log(res) expect_true(res$complete) }) test_that("ablation capping", { target_runner_capping <- function(experiment, scenario) { cost <- min(experiment$bound, target_runner(experiment, scenario)$cost) list(cost = cost, time = cost) } boundMax <- 1000 + (1000 * 505) scenario <- list(targetRunner = target_runner_capping, instances = seq(1000, 10000, 1000), seed = 42, maxTime = 100 * boundMax, boundMax = boundMax, initConfigurations = default, logFile = logfile, parameters = parameters) scenario <- checkScenario(scenario) expect_warning(irace(scenario = scenario), "is too large") check_log(read_logfile(logfile)) res <- ablation(logfile, ablationLogFile = outfile) check_log(res) expect_true(res$complete) res <- ablation(logfile, ablationLogFile = outfile, type = "racing") check_log(res) expect_true(res$complete) }) test_that("ablation maxExperiments", { scenario <- list(targetRunner = target_runner, instances = seq(1000, 10000, 1000), maxExperiments = 1000, seed = 42, initConfigurations = default, logFile = logfile, parameters = parameters) scenario <- checkScenario (scenario) irace(scenario = scenario) check_log(read_logfile(logfile)) res <- ablation(logfile, ablationLogFile = outfile) check_log(res) expect_true(res$complete) res <- ablation(logfile, ablationLogFile = outfile, type = "racing") check_log(res) expect_true(res$complete) res <- ablation(logfile, ablationLogFile = outfile, src = src_file, target = target_file) check_log(res) expect_true(res$complete) plotfile <- withr::local_tempfile(pattern = "ablation", fileext = ".pdf") res <- ablation_cmdline(paste0("--log-file=", logfile, " -o ", outfile, " -p ", plotfile)) check_log(res) expect_true(res$complete) }) }) # withr::with_output_sink() irace/tests/testthat/dependencies.txt0000644000176200001440000000057114735305474017545 0ustar liggesusersparam1 "--param1 " i (1, "real") param2 "--param2 " i ("real", 10) mode "--" c ("x1" ,"x2", "x3") real "--paramreal=" r (1.5, 7.5) mutation "--mutation=" o ("none", "very low", "low", "medium", "high", "very high", "all") #unused "-u " c (1, 2, 10, 20) irace/tests/testthat/test-targeteval.R0000644000176200001440000000722215060050754017602 0ustar liggesuserswithr::with_output_sink("test-targeteval.Rout", { target_evaluator <- function(experiment, num_configurations, all_conf_id, scenario, target_runner_call) { withr::local_seed(experiment$seed) list(cost = runif(1), call = deparse1(experiment)) } parameters <- readParameters(text = ' algorithm "--" c (as,mmas,eas,ras,acs) ') test_that("target_evaluator", { target_runner <- function(experiment, scenario) { list(call = deparse1(experiment)) } seed <- sample.int(min(2147483647L, .Machine$integer.max), size = 1L, replace = TRUE) instances <- 1:10 limit <- 58L maxExperiments <- 200 logFile <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") scenario <- checkScenario(list( targetRunner = target_runner, targetEvaluator = target_evaluator, maxExperiments = maxExperiments, instances = instances, logFile = logFile, seed = seed, quiet = TRUE, parameters = parameters)) expect_true(irace:::checkTargetFiles(scenario = scenario)) scenario <- checkScenario(list( targetRunner = target_runner, targetEvaluator = target_evaluator, maxExperiments = maxExperiments, instances = instances, logFile = logFile, seed = seed, quiet = TRUE, parameters = parameters)) expect_silent(confs <- irace(scenario = scenario)) expect_gt(nrow(confs), 0L) scenario$targetRunner <- wrap_target_runner_error(target_runner, limit) # Otherwise, the tests are too fast. with_mocked_bindings({ expect_error(irace(scenario = scenario), "== irace == The cost returned by targetRunner is not numeric") }, .get_time_next_save = function(now) now ) logFile_new <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") scenario <- modifyList(scenario, list( targetRunner = wrap_target_runner_counter(target_runner), recoveryFile = logFile, seed = NA, quiet = FALSE, logFile = logFile_new)) recover_confs <- irace(scenario = scenario) expect_identical(confs, recover_confs) expect_equal(environment(scenario$targetRunner)$counter, maxExperiments - limit + 1L) }) test_that("target_evaluator maxTime", { target_runner <- function(experiment, scenario) { withr::local_seed(experiment$seed) list(time = min(experiment$bound, as.integer(1 + 10*runif(1)))) } seed <- sample.int(min(2147483647L, .Machine$integer.max), size = 1L, replace = TRUE) instances <- 1:10 limit <- 100L logFile <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") scenario <- checkScenario(list( targetRunner = target_runner, targetEvaluator = target_evaluator, maxTime = 2000, boundMax = 10, instances = instances, logFile = logFile, seed = seed, quiet = TRUE, parameters = parameters)) expect_true(scenario$capping) expect_silent(confs <- irace(scenario = scenario)) expect_gt(nrow(confs), 0L) scenario$targetRunner <- wrap_target_runner_error(target_runner, limit) # Otherwise, the tests are too fast. with_mocked_bindings({ expect_error(irace(scenario = scenario), "== irace == The cost returned by targetRunner is not numeric") }, .get_time_next_save = function(now) now ) logFile_new <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") scenario <- modifyList(scenario, list( targetRunner = wrap_target_runner_counter(target_runner), recoveryFile = logFile, seed = NA, quiet = FALSE, logFile = logFile_new)) recover_confs <- irace(scenario = scenario) expect_identical(confs, recover_confs) expect_lt(environment(scenario$targetRunner)$counter, sum(!is.na(read_logfile(logFile_new)$experiments)) - 10L) }) }) # withr::with_output_sink() irace/tests/testthat/dummy_wrapper.py0000755000176200001440000000435314735305474017630 0ustar liggesusers#!/usr/bin/env python3 # encoding: utf-8 ''' DummyWrapper -- Dummy wrapper for unit testing This requires the installation of https://github.com/automl/GenericWrapper4AC @author: Manuel López-Ibáñez @copyright: 2018 @license: BSD @contact: manuel.lopez-ibanez@manchester.ac.uk ''' import json import traceback from genericWrapper4AC.generic_wrapper import AbstractWrapper class DummyWrapper(AbstractWrapper): ''' Dummy wrapper for unit testing ''' def __init__(self): AbstractWrapper.__init__(self) self._return_value = None def get_command_line_args(self, runargs, config): ''' ''' runargs.update(config) return "echo '" + json.dumps(runargs) + "'" def process_results(self, filepointer, exit_code): ''' Parse a results file to extract the run's status (SUCCESS/CRASHED/etc) and other optional results. Args: filepointer: a pointer to the file containing the solver execution standard out. exit_code : exit code of target algorithm ''' statuses = ['SUCCESS', 'TIMEOUT', 'CRASHED', 'ABORT'] # If something fails, we a serious problem output = dict(status='ABORT') for line in filepointer: try: argmap = json.loads(str(line.decode('UTF-8')).replace("\n","")) ins = argmap["instance"] seed = argmap["seed"] cost = float(argmap["-cost"]) runtime = float(argmap["-runtime"]) status = statuses[seed % len(statuses)] if ins == "cost": output = dict(status = status, cost = cost) elif ins == "time": cutoff = float(argmap["cutoff"]) output = dict(status=status, runtime=min(cutoff,runtime)) elif ins == "cost+time": cutoff = float(argmap["cutoff"]) output = dict(status=status, cost=cost, runtime=min(cutoff,runtime)) except ValueError: traceback.print_exc() pass return output if __name__ == "__main__": wrapper = DummyWrapper() wrapper.main() irace/tests/testthat/bad_scenario.txt0000644000176200001440000000004214736526233017520 0ustar liggesusersparametersFile = "parameters.txt" irace/tests/testthat/test-multi_irace.R0000644000176200001440000001167514736526233017761 0ustar liggesuserswithr::with_output_sink("test-multi_irace.Rout", { # FIXME: Use temporary files and directories. make.target.runner <- function(parameters) { function(experiment, scenario) { cost <- max(1, abs(rnorm(1, mean=sum(unlist(experiment$configuration[parameters]))))) list(cost = cost, call = toString(experiment)) } } make.parameters.table <- function(parameters) readParameters(text = sprintf('%s "" r (0, 10)', parameters)) make.scenario <- function(targetRunner, maxExperiments = 1000) { list(targetRunner = targetRunner, maxExperiments = maxExperiments, instances = rnorm(200, mean = 0.9, sd = 0.02)) } target.runner.1 <- make.target.runner(c("x1", "x2")) target.runner.2 <- make.target.runner(c("x2", "x3")) target.runner.3 <- make.target.runner(c("x3", "x1")) parameters.table.1 <- make.parameters.table(c("x1", "x2")) parameters.table.2 <- make.parameters.table(c("x2", "x3")) parameters.table.3 <- make.parameters.table(c("x3", "x1")) check.default.logFiles <- function(n) { dir <- sprintf("run_%02d", 1:n) expect_equal(dir.exists(dir), rep_len(TRUE, n)) expect_equal(file.exists(file.path(dir, "irace.Rdata")), rep_len(TRUE, n)) } test_that("multiple scenarios, multiple parameters", { skip_on_cran() # Reproducible results generate_set_seed() scenarios <- lapply(list(target.runner.1, target.runner.2, target.runner.3), make.scenario) parameters <- list(parameters.table.1, parameters.table.2, parameters.table.3) runs <- multi_irace(scenarios, parameters) expect_length(runs, 3) check.default.logFiles(3) }) test_that("one scenario, multiple parameters", { skip_on_cran() # Reproducible results generate_set_seed() scenarios <- list(make.scenario(target.runner.1)) dummy_parameters_names <- c("dummy1", "dummy2", "dummy3") parameter_names <- lapply(dummy_parameters_names, function(dummy) c("x1", "x2", dummy)) parameters <- lapply(parameter_names, make.parameters.table) runs <- multi_irace(scenarios, parameters) expect_length(runs, 3) check.default.logFiles(3) }) test_that("multiple scenarios, one parameters", { skip_on_cran() # Reproducible results generate_set_seed() scenarios <- lapply(list(100, 500, 1000), function(maxExperiments) make.scenario(target.runner.1, maxExperiments) ) parameters <- list(parameters.table.1) runs <- multi_irace(scenarios, parameters) expect_length(runs, 3) check.default.logFiles(3) }) test_that("one scenario, one parameters, multiple n", { skip_on_cran() # Reproducible results generate_set_seed() scenarios <- list(make.scenario(target.runner.1)) parameters <- list(parameters.table.1) runs <- multi_irace(scenarios, parameters, n = 3) expect_length(runs, 3) check.default.logFiles(3) }) test_that("multiple scenarios, multiple parameters, multiple n", { skip_on_cran() # Reproducible results generate_set_seed() scenarios <- lapply(list(target.runner.1, target.runner.2, target.runner.3), make.scenario) parameters <- list(parameters.table.1, parameters.table.2, parameters.table.3) runs <- multi_irace(scenarios, parameters, n = 3) expect_length(runs, 9) check.default.logFiles(9) }) test_that("logFile not in execDir", { skip_on_cran() # Reproducible results generate_set_seed() execDir <- path_rel2abs("./multi_irace_execDir") logFileDir <- path_rel2abs("./multi_irace_logFileDir") dir.create(execDir, showWarnings = FALSE) dir.create(logFileDir, showWarnings = FALSE) scenarios <- lapply(list(target.runner.1, target.runner.2, target.runner.3), make.scenario) for (i in seq_along(scenarios)) { scenarios[[i]]$logFile <- file.path(logFileDir, "irace.Rdata") scenarios[[i]]$execDir <- execDir } parameters <- list(parameters.table.1, parameters.table.2, parameters.table.3) expect_error(multi_irace(scenarios, parameters)) # expect_length(runs, 3) # for (i in 1:3) { # logFile <- file.path(logFileDir, sprintf("irace_%02d.Rdata", i)) # expect_true(file.exists(logFile)) # } }) test_that("global seed", { skip_on_cran() scenarios <- lapply(list(target.runner.1, target.runner.2, target.runner.3), make.scenario) parameters <- list(parameters.table.1, parameters.table.2, parameters.table.3) runs.1 <- multi_irace(scenarios, parameters, global_seed = 42) runs.2 <- multi_irace(scenarios, parameters, global_seed = 42) expect_equal(runs.1, runs.2) }) test_that("sequential and parallel identical", { skip_on_cran() skip_on_os("windows") ncores <- test_irace_detectCores() skip_if(ncores <= 1L, message = "This test only makes sense if multiple cores are available") scenarios <- lapply(list(target.runner.1, target.runner.2, target.runner.3), make.scenario) parameters <- list(parameters.table.1, parameters.table.2, parameters.table.3) runs.sequential <- multi_irace(scenarios, parameters, global_seed = 42) runs.parallel <- multi_irace(scenarios, parameters, global_seed = 42, parallel = ncores) expect_equal(runs.sequential, runs.parallel) }) }) # withr::with_output_sink() irace/tests/testthat/helper-common.R0000644000176200001440000001430515060056174017236 0ustar liggesusers# This file is loaded automatically by testthat. generate_set_seed <- function() { seed <- sample.int(min(2147483647L, .Machine$integer.max), size = 1L, replace = TRUE) cat("Seed: ", seed, "\n") set.seed(seed) } test_irace_detectCores <- function() { if (!identical(Sys.getenv("NOT_CRAN"), "true")) return(1L) # FIXME: covr takes a very long time on github actions otherwise. if (identical(Sys.getenv("COVR_COVERAGE"), "true")) return(1L) x <- Sys.getenv("_R_CHECK_LIMIT_CORES_", "") if (nzchar(x) && x == "TRUE") return(2L) parallel::detectCores() } skip_if_local_test <- function() { # testthat puts us in tests/testthat test_dir <- testthat::test_path() pkg_root <- normalizePath(file.path(test_dir, "..", "..")) # dev mode: DESCRIPTION file lives here if (file.exists(file.path(pkg_root, "DESCRIPTION"))) skip("Not run when testing in local mode.") else invisible() } skip_on_coverage <- function() { if (identical(Sys.getenv("COVR_COVERAGE"), "true")) skip("Not run during coverage.") else invisible() } system_os_is_windows <- function() .Platform$OS.type == "windows" expect_valid_configurations <- function(configurations, parameters) { configurations <- irace:::removeConfigurationsMetaData(as.data.frame(configurations)) output <- capture.output(print(configurations, width=5000L, row.names = FALSE)) output <- paste0(output, "\n", collapse="") confs2 <- readConfigurationsFile(text=output, parameters=parameters) expect_equal(configurations, confs2) } ## Functions ########################################################## f_ackley <- function (x,y, nsize = 0.01) { # Transformation of parameter values # from [0,1] to [vmin,vmax] vmin <- -5 vmax <- 5 x <- (x*(vmax-vmin)) + vmin y <- (y*(vmax-vmin)) + vmin a <- -20 * exp (-0.2 * sqrt(0.5 * (x^2 + y^2))) b <- exp(0.5 * (cos(2*pi*x) + cos(2*pi*y))) f <- a - b + exp(1) + 20 # Simulating stochasticity noise <- runif(1, min = (1-nsize), max = (1+nsize)) f <- f*noise # Transform result from [fmin,fmax] # to [0,100] fmin <- 0 fmax <- 15*(1+nsize) ((f - fmin) / (fmax-fmin)) * (100-0) + 0 } f_goldestein_price <- function (x,y, nsize = 0.01) { # Transformation of parameter values # from [0,1] to [vmin,vmax] vmin <- -2 vmax <- 2 x <- (x*(vmax-vmin)) + vmin y <- (y*(vmax-vmin)) + vmin a <- 1 + ((x + y + 1)^2) * (19 - 14*x + 3*x^2 - 14*y + 6*x*y + 3*y^2) b <- 30 + ((2*x - 3*y)^2) * (18 - 32*x + 12*x^2 + 48*y - 36*x*y + 27*y^2) f <- a*b # Simulating stochasticity noise <- runif(1, min = (1-nsize), max = (1+nsize)) f <- f*noise # Transform result from [fmin,fmax] # to [0,100] fmin <- 0 fmax <- 1000000*(1+nsize) ((f - fmin) / (fmax-fmin)) * (100-0) + 0 } f_matyas <- function (x,y, nsize = 0.01) { # Trasfomation of parameter values # from [0,1] to [vmin,vmax] vmin <- -10 vmax <- 10 x <- (x*(vmax-vmin)) + vmin y <- (y*(vmax-vmin)) + vmin f <- 0.26 * (x^2 + y^2) - (0.48*x*y) # Simulating stochasticity noise <- runif(1, min = (1-nsize), max = (1+nsize)) f <- f*noise # Transform result from [fmin,fmax] # to [0,100] fmin <- 0 fmax <- 100*(1+nsize) ((f - fmin) / (fmax-fmin)) * (100-0) + 0 } f_himmelblau <- function (x,y, nsize = 0.01) { # Trasfomation of parameter values # from [0,1] to [vmin,vmax] vmin <- -5 vmax <- 5 x <- (x*(vmax-vmin)) + vmin y <- (y*(vmax-vmin)) + vmin f <- (x^2 + y - 11)^2 + (x + y^2 - 7)^2 # Simulating stochasticity noise <- runif(1, min = (1-nsize), max = (1+nsize)) f <- f*noise # Transform result from [fmin,fmax] # to [0,100] fmin <- 0 fmax <- 2000*(1+nsize) ((f - fmin) / (fmax-fmin)) * (100-0) + 0 } target_runner_capping_xy <- function(experiment, scenario) { configuration <- experiment$configuration instance <- experiment$instance bound <- experiment$bound x <- configuration[["x"]] y <- configuration[["y"]] value <- switch(instance, ackley = f_ackley(x, y), goldestein = f_goldestein_price(x, y), matyas = f_matyas(x, y), himmelblau = f_himmelblau(x, y)) # Simulate execution bound list(cost = value, time=min(value + 0.1, bound)) } irace_capping_xy <- function(..., targetRunner = force(target_runner_capping_xy), parallel = test_irace_detectCores()) { # Silence Error in `save(iraceResults, file = logfile, version = 3L)`: (converted from warning) 'package:irace' may not be available when loading # See https://github.com/r-lib/testthat/issues/2044 if (!is.null(attr(environment(targetRunner), "name", exact=TRUE))) { environment(targetRunner) <- globalenv() } args <- list(...) parameters_table <- ' x "" r (0, 1.00) y "" r (0, 1.00) reject "" c (0,1)' parameters <- readParameters(text = parameters_table) logFile <- withr::local_tempfile(fileext=".Rdata") scenario <- list(instances = c("ackley", "goldestein", "matyas", "himmelblau"), targetRunner = targetRunner, capping = TRUE, boundMax = 80, testType = "t-test", logFile = logFile, parallel = parallel, parameters = parameters) scenario <- modifyList(scenario, args) scenario <- checkScenario (scenario) expect_true(irace:::checkTargetFiles(scenario = scenario)) confs <- irace(scenario = scenario) best_conf <- getFinalElites(scenario$logFile, n = 1L, drop.metadata = TRUE) expect_identical(removeConfigurationsMetaData(confs[1L, , drop = FALSE]), best_conf) } # Useful for testing recovery. wrap_target_runner_error <- function(target_runner, limit) { counter <- force(limit) target_runner <- force(target_runner) fun <- function(experiment, scenario) { counter <<- counter - 1L if (counter <= 0L) return(list(cost=NA)) target_runner(experiment, scenario) } parent.env(environment(fun)) <- globalenv() fun } wrap_target_runner_counter <- function(target_runner) { counter <- 0L target_runner <- force(target_runner) fun <- function(experiment, scenario) { counter <<- counter + 1L target_runner(experiment, scenario) } parent.env(environment(fun)) <- globalenv() fun } irace/tests/testthat/test-dependencies.R0000644000176200001440000001071714736526233020106 0ustar liggesuserswithr::with_output_sink("test-dependencies.Rout", { test_that("param depend error checking", { expect_error_readParameters <- function(text, error) expect_error(readParameters(text=text), error) expect_error_readParameters(' p1 "" r (0, 1) p2 "" r (p3, 1) ', "parameter 'p2' is not valid: 'p3' cannot be found") expect_error_readParameters(' p1 "" c (0, 1) p2 "" r (1, p1) ', "parameter 'p2' depends on non-numerical parameters") expect_error_readParameters(' p1 "" r (0, 1) p2 "" r (1, foobar(p1)) ', "parameter 'p2' uses function") expect_error_readParameters(' p1 "" r (0, p3) p2 "" r (1, p1) p3 "" r (p2, 1) ', "Cycle detected") expect_error_readParameters(' p1 "" r (0, 1) p2 "" i (0.1, p1) ', "values must be integers") }) checkConditionalAndDependency <- function(configuration, parameters) { for (param in parameters$get_ordered()) { p <- param[["name"]] if (!irace:::conditionsSatisfied(param[["condition"]], configuration)) { expect(is.na(configuration[[p]]), paste0("Conditional parameter '", p, "' is not active but it has a value '", configuration[[p]], "' assigned.")) } else if (param[["is_dependent"]]) { bounds <- irace:::getDependentBound(param, configuration) if (anyNA(bounds)) { expect(is.na(configuration[[p]]), paste0("Dependent parameter '", p, "' has a value '", configuration[[p]], "' but it should be inactive.")) expect_true(anyNA(configuration[parameters$depends[[p]]])) } else { expect (configuration[[p]] >= bounds[1], paste0("Parameter '", p, "=", configuration[[p]], "' does not comply with dependency: ", parameters$depends[[p]], " and lower bound: ", bounds[1])) expect (configuration[[p]] <= bounds[2], paste0("Parameter '", p, " = ", configuration[[p]], "' does not comply with dependency: ", parameters$depends[[p]], " and upper bound: ", bounds[2])) } } } } test_that("test inactive dependent", { parameters <- readParameters(text=' p1 "" r (0,1) p2 "" r (0, p1) | p1 < 0.5 p3 "" r (0, p2) ', digits = 2) confs <- irace:::sampleUniform(parameters, 50) confs <- as.data.frame(confs) for (i in seq_len(nrow(confs))) { checkConditionalAndDependency(confs[i,], parameters) } expect_error(readConfigurationsFile(parameters = parameters, text = ' p1 p2 p3 0.4 0.4 0.4 0.4 0.5 0.4 '), "Configuration number 2 is invalid because the value") expect_error(readConfigurationsFile(parameters = parameters, text = ' p1 p2 p3 0.4 0.3 0.2 0.4 0.3 0.4 '), "Configuration number 2 is invalid because the value") readConfigurationsFile(parameters = parameters, text = ' p1 p2 p3 0.4 0.3 0.2 0.5 NA NA 0.4 0.4 0.4 ') }) test_checkDependencies <- function(parameterFile, ...) { args <- list(...) target_runner <- function(experiment, scenario) { configuration <- experiment$configuration tmax <- configuration[["real"]] stopifnot(is.numeric(tmax)) if (configuration[["mode"]] %in% c("x1", "x2")) temp <- configuration[["param1"]] else temp <- 1 stopifnot(is.numeric(temp)) time <- max(1, abs(rnorm(1, mean=(tmax+temp)/10))) list(cost = time, time = time, call = toString(experiment)) } weights <- rnorm(200, mean = 0.9, sd = 0.02) parameters <- readParameters(parameterFile) scenario <- list(targetRunner = target_runner, instances = weights, seed = 1234567, maxExperiments = 200, parameters = parameters) scenario <- modifyList(scenario, args) scenario <- checkScenario (scenario) nconf <- 100 conf <- irace:::sampleUniform(parameters, nconf) expect_equal(nconf, nrow(conf)) conf$.ID. <- seq_len(nconf) conf <- as.data.frame(conf) for (i in seq_len(nconf)) checkConditionalAndDependency(conf[i,], parameters) model <- irace:::initialiseModel(parameters, conf) conf2 <- irace:::sampleModel(parameters, conf, model, nconf) conf2 <- as.data.frame(conf2) for (i in seq_len(nconf)) checkConditionalAndDependency(conf2[i,], parameters) confs <- irace(scenario = scenario) for (i in seq_len(nrow(confs))) { checkConditionalAndDependency(confs[i,], parameters) } } test_that("checkDependencies", { test_checkDependencies(parameterFile="dependencies.txt") }) test_that("checkDependencies2", { test_checkDependencies(parameterFile="dependencies2.txt") }) }) # withr::with_output_sink() irace/tests/testthat/test-bad_scenario.R0000644000176200001440000000067714736526233020075 0ustar liggesuserswithr::with_output_sink("test-bad_scenario.Rout", { test_that("bad scenario: unknown variables", { skip_on_cran() bad_scenario <- test_path("bad_scenario.txt") expect_error(irace_cmdline(paste0("--scenario ", bad_scenario)), "unknown variables") good_scenario <- test_path("good_scenario.txt") expect_error(irace_cmdline(paste0("--scenario ", good_scenario, " --unknown")), "Unknown command-line options: --unknown") }) }) irace/tests/testthat/test-maxTime.R0000644000176200001440000000326314736526233017062 0ustar liggesuserswithr::with_output_sink("test-maxTime.Rout", { target_runner <- function(experiment, scenario) { configuration <- experiment$configuration tmax <- configuration[["tmax"]] temp <- configuration[["temp"]] time <- max(1, abs(rnorm(1, mean=(tmax+temp)/10))) list(cost = time, time = time) } time_irace <- function(...) { args <- list(...) parameters <- readParameters(text = ' tmax "" i (1, 50) temp "" r (0, 10) dummy "" c ("dummy") ') scenario <- list(targetRunner = target_runner, instances = 1:10, testInstances = 11:20, seed = 1234567, parameters = parameters) scenario <- modifyList(scenario, args) scenario <- checkScenario (scenario) expect_true(irace:::checkTargetFiles(scenario = scenario)) confs <- irace(scenario = scenario) final_ids <- sort(as.character(confs$.ID.[1:scenario$testNbElites])) expect_gt(nrow(confs), 0L) testing_fromlog(scenario$logFile) iraceResults <- read_logfile(scenario$logFile) if (scenario$testIterationElites) { # FIXME: We could test here that the correct configurations are tested. expect_gte(ncol(iraceResults$testing$experiments), scenario$testNbElites) } else { test_ids <- sort(colnames(iraceResults$testing$experiments)) expect_equal(final_ids, test_ids) } confs } test_that("maxTime=500 testNbElites=2 testIterationElites=FALSE", { generate_set_seed() time_irace(maxTime = 500, testNbElites=2) }) test_that("maxTime=1111 testNbElites=3 testIterationElites=TRUE", { skip_on_cran() generate_set_seed() time_irace(maxTime = 1111, testNbElites=3, testIterationElites=TRUE) }) }) # withr::with_output_sink() irace/tests/testthat/test-bug-44.R0000644000176200001440000000064614736526233016462 0ustar liggesusers# https://github.com/MLopez-Ibanez/irace/issues/44 withr::with_output_sink("test-bug-44.Rout", { test_that("bug 44", { readParameters(text='a "a" r (0.01, 0.99)', digits=2) readParameters(text='a "a" r (0, 1)', digits=1) expect_error(readParameters(text='a "a" r (0.01, 0.99)', digits=1), "must be representable") expect_error(readParameters(text='a "a" r,log (1e-8, 1)', digits=4), "must be representable") }) }) irace/tests/testthat/test-read_pcs_file.R0000644000176200001440000000251514736526233020234 0ustar liggesuserswithr::with_output_sink("test-read_pcs_file.Rout", { test_that("read_pcs_file", { pcs_table <- ' # name domain algorithm {as,mmas,eas,ras,acs}[as] localsearch {0, 1, 2, 3}[0] alpha [0.00, 5.00][1] beta [0.00, 10.00][1] rho [0.01, 1.00][0.95]l ants [1, 100][10]il q0 [0.0, 1.0][0] rasrank [1, 100][1]i elitistants [1, 750][1]i nnls [5, 50][5]i dlb {0, 1}[1] Conditionals: q0 | algorithm in {acs} rasrank | algorithm in {ras} elitistants | algorithm in {eas} nnls | localsearch in {1,2,3} dlb | localsearch in {1,2,3} {alpha=0, beta= 0.0} {ants=1,algorithm=eas} {ants=1,algorithm="ras"}' parameters_table <- ' # name domain algorithm "algorithm" c (as,mmas,eas,ras,acs) localsearch "localsearch" c (0, 1, 2, 3) alpha "alpha" r (0.00, 5.00) beta "beta" r (0.00, 10.00) rho "rho" r,log (0.01, 1.00) ants "ants" i,log (1, 100) q0 "q0" r (0.0, 1.0) | algorithm == "acs" rasrank "rasrank" i (1, 100) | algorithm == "ras" elitistants "elitistants" i (1, 750) | algorithm == "eas" nnls "nnls" i (5, 50) | localsearch %in% c("1","2","3") dlb "dlb" c (0, 1) | localsearch %in% c("1","2","3") [forbidden] (alpha == 0) & (beta == 0.0) (ants == 1) & (algorithm == "eas") (ants == 1) & (algorithm == "ras") ' expect_equal(parameters_table, read_pcs_file(text=pcs_table)) }) }) irace/tests/testthat/test-GenericWrapper4AC.R0000644000176200001440000001467214736526233020671 0ustar liggesusers# This requires the installation of https://github.com/automl/GenericWrapper4AC test_that("GenericWrapper4AC", { skip_on_cran() skip_on_coverage() skip_on_ci() skip_on_travis() skip_on_os("mac") # FIXME: how to check that a python package is installed? parameters <- readParameters(text = ' cost "cost" r (0.1, 1.00) runtime "runtime" r (0.1, 1.00)', digits = 15L) configurations <- data.frame(.ID. = 1L, cost = 0.5, runtime = 0.8) instances = c("time", "cost", "cost+time") names(instances) = instances dummy_wrapper <- test_path("dummy_wrapper.py") scenario <- list(targetRunner = dummy_wrapper, instances = instances, maxExperiments = 1000, aclib = TRUE, parameters = parameters) scenario <- checkScenario (scenario) experiments <- irace:::createExperimentList(configurations, parameters = parameters, instances = scenario$instances, instances_ID = rep("cost", 2), seeds = 0:1, bounds = NULL) race_state <- irace:::RaceState$new(scenario) output <- irace:::execute_experiments(race_state, experiments, scenario) expect_equal(output[[1]]$status, "SUCCESS") expect_equal(output[[1]]$cost, 0.5) expect_true(is.null(output[[1]]$error)) expect_equal(output[[2]]$status, "TIMEOUT") expect_equal(output[[2]]$cost, 0.5) expect_true(is.null(output[[2]]$error)) experiments <- irace:::createExperimentList(configurations, parameters = parameters, instances = scenario$instances, instances_ID = rep("cost", 2), seeds = 2, bounds = NULL) expect_error(irace:::execute_experiments(race_state, experiments, scenario), "CRASHED") experiments <- irace:::createExperimentList(configurations, parameters = parameters, instances = scenario$instances, instances_ID = rep("cost", 2), seeds = 3, bounds = NULL) expect_error(irace:::execute_experiments(race_state, experiments, scenario), "ABORT") scenario <- modifyList(scenario, list(capping = TRUE, boundMax = 1, maxTime = 10, maxExperiments = NULL)) scenario <- checkScenario (scenario) experiments <- irace:::createExperimentList(configurations, parameters = parameters, instances = scenario$instances, instances_ID = rep("time", 2), seeds = 0:1, bounds = scenario$boundMax) output <- irace:::execute_experiments(race_state, experiments, scenario) expect_equal(output[[1]]$status, "SUCCESS") expect_equal(output[[1]]$cost, 0.8) expect_equal(output[[1]]$time, 0.8) expect_match(output[[1]]$call, "--cutoff") expect_true(is.null(output[[1]]$error)) expect_equal(output[[2]]$status, "TIMEOUT") expect_equal(output[[2]]$cost, 0.8) expect_equal(output[[2]]$time, 0.8) expect_match(output[[2]]$call, "--cutoff") expect_true(is.null(output[[2]]$error)) experiments <- irace:::createExperimentList(configurations, parameters = parameters, instances = scenario$instances, instances_ID = rep("time", 2), seeds = 2, bounds = scenario$boundMax) expect_error(irace:::execute_experiments(race_state, experiments, scenario), "CRASHED") experiments <- irace:::createExperimentList(configurations, parameters = parameters, instances = scenario$instances, instances_ID = rep("time", 2), seeds = 3, bounds = scenario$boundMax) expect_error(irace:::execute_experiments(race_state, experiments, scenario), "ABORT") experiments <- irace:::createExperimentList(configurations, parameters = parameters, instances = scenario$instances, instances_ID = rep("cost+time", 2), seeds = 0:1, bounds = scenario$boundMax) output <- irace:::execute_experiments(race_state, experiments, scenario) expect_equal(output[[1]]$status, "SUCCESS") expect_equal(output[[1]]$cost, 0.5) expect_equal(output[[1]]$time, 0.8) expect_match(output[[1]]$call, "--cutoff") expect_true(is.null(output[[1]]$error)) expect_equal(output[[2]]$status, "TIMEOUT") expect_equal(output[[2]]$cost, 0.5) expect_equal(output[[2]]$time, 0.8) expect_match(output[[2]]$call, "--cutoff") expect_true(is.null(output[[2]]$error)) experiments <- irace:::createExperimentList(configurations, parameters = parameters, instances = scenario$instances, instances_ID = rep("cost+time", 2), seeds = 2, bounds = scenario$boundMax) expect_error(irace:::execute_experiments(race_state, experiments, scenario), "CRASHED") experiments <- irace:::createExperimentList(configurations, parameters = parameters, instances = scenario$instances, instances_ID = rep("cost+time", 2), seeds = 3, bounds = scenario$boundMax) expect_error(irace:::execute_experiments(race_state, experiments, scenario), "ABORT") }) irace/tests/testthat/test-bug71.R0000644000176200001440000000113614736526233016400 0ustar liggesuserswithr::with_output_sink("test-bug71.Rout", { test_that("bug71.evaluator", { target.runner <- function(experiment, scenario) list(cost = runif(1), call = toString(experiment)) parameters <- readParameters(text = ' algorithm "--" c (as,mmas,eas,ras,acs) ') scenario <- checkScenario(list( targetRunner = target.runner, maxExperiments = 200, instances = runif(10), initConfigurations = data.frame(algorithm="as"), parameters = parameters)) expect_true(irace:::checkTargetFiles(scenario = scenario)) }) }) # withr::with_output_sink() irace/tests/testthat/parameters.txt0000644000176200001440000000166514736526233017266 0ustar liggesusersparam1 "--param1 " i (1, 10) | mode %in% c("x1", "x2") param2 "--param2 " i (1, 10) | mode %in% c("x1", "x3") & real > 2.5 & real <= 3.5 mode "--" c ("x1" ,"x2", "x3") real "--paramreal=" r (1.5, 4.5) mutation "--mutation=" o ("none", "very low", "low", "medium", "high", "very high", "all") fixed "--fixed " c ("fixed") #unused "-u " c (1, 2, 10, 20) [forbidden] ## The format is one constraint per line. Each constraint is a logical ## expression (in R syntax). If a parameter configuration ## is generated that makes the logical expression evaluate to TRUE, ## then the configuration is discarded. ## ## Examples of valid logical operators are: == != >= <= > < & | ! %in% param1 < 5 & mode == "x1" (param2 > 6 & mode == "x1") | (param2 <= 6 & mode == "x3") real < 4 & mode %in% c("x2", "x3") irace/tests/testthat/test-recovery.R0000644000176200001440000000654515060050217017303 0ustar liggesuserswithr::with_output_sink("test-recovery.Rout", { test_that("recovery works", { target_runner_xy <- function(experiment, scenario) { configuration <- experiment$configuration instance <- experiment$instance x <- configuration[["x"]] y <- configuration[["y"]] value <- switch(instance, ackley = f_ackley(x, y), goldestein = f_goldestein_price(x, y), matyas = f_matyas(x, y), himmelblau = f_himmelblau(x, y)) list(cost = value) } parameters_table <- ' x "" r (0, 1.00) y "" r (0, 1.00) ' parameters <- readParameters(text = parameters_table) logFile <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") seed <- sample.int(min(2147483647L, .Machine$integer.max), size = 1, replace = TRUE) maxExperiments <- 500L limit <- 100L scenario <- list( instances = c("ackley", "goldestein", "matyas", "himmelblau"), parameters = parameters, targetRunner = target_runner_xy, logFile = logFile, seed = seed, quiet = TRUE, maxExperiments = maxExperiments) expect_silent(confs <- irace(scenario = scenario)) scenario$targetRunner <- wrap_target_runner_error(target_runner_xy, limit) # Otherwise, the tests are too fast. with_mocked_bindings({ expect_error(irace(scenario = scenario), "== irace == The cost returned by targetRunner is not numeric") }, .get_time_next_save = function(now) now ) logFile_new <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") scenario <- modifyList(scenario, list( targetRunner = wrap_target_runner_counter(target_runner_xy), recoveryFile = logFile, seed = NA, quiet = FALSE, logFile = logFile_new)) recover_confs <- irace(scenario = scenario) expect_identical(confs, recover_confs) expect_lt(environment(scenario$targetRunner)$counter, maxExperiments - 50) }) test_that("recovery maxTime", { target_runner <- function(experiment, scenario) { configuration <- experiment$configuration tmax <- configuration[["tmax"]] temp <- configuration[["temp"]] time <- max(1, abs(rnorm(1, mean=(tmax+temp)/10))) list(cost = time, time = time) } parameters <- readParameters(text = ' tmax "" i (1, 50) temp "" r (0, 10) ') logFile <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") seed <- 1234567 scenario <- list(targetRunner = target_runner, instances = 1:10, seed = seed, maxTime = 500, logFile = logFile, quiet = TRUE, parameters = parameters) expect_silent(confs <- irace(scenario = scenario)) scenario$targetRunner <- wrap_target_runner_error(target_runner, 200L) # Otherwise, the tests are too fast. with_mocked_bindings({ expect_error(irace(scenario = scenario), "== irace == The cost returned by targetRunner is not numeric") }, .get_time_next_save = function(now) now ) logFile_new <- withr::local_tempfile(pattern = "irace", fileext = ".Rdata") scenario <- modifyList(scenario, list( targetRunner = wrap_target_runner_counter(target_runner), recoveryFile = logFile, seed = NA, quiet = FALSE, logFile = logFile_new)) recover_confs <- irace(scenario = scenario) expect_identical(confs, recover_confs) expect_lt(environment(scenario$targetRunner)$counter, nrow(read_logfile(logFile_new)$state$experiment_log) - 150L) }) }) # withr::with_output_sink() irace/tests/testthat/test-blocksize.R0000644000176200001440000000660414736526233017445 0ustar liggesuserswithr::with_output_sink("test-blocksize.Rout", { cap_irace <- function(..., targetRunner = force(target_runner_capping_xy)) { # Silence Error in `save(iraceResults, file = logfile, version = 3L)`: (converted from warning) 'package:irace' may not be available when loading # See https://github.com/r-lib/testthat/issues/2044 if (!is.null(attr(environment(targetRunner), "name", exact=TRUE))) { environment(targetRunner) <- globalenv() } args <- list(...) parameters_table <- ' x "" r (0, 1.00) y "" r (0, 1.00)' parameters <- readParameters(text = parameters_table) logFile <- withr::local_tempfile(fileext=".Rdata") scenario <- list(instances = c("ackley", "goldestein", "matyas", "himmelblau"), targetRunner = targetRunner, capping = TRUE, blockSize = 4, boundMax = 80, logFile = logFile, testType = "t-test", parameters = parameters) scenario <- modifyList(scenario, args) scenario <- checkScenario (scenario) confs <- irace(scenario = scenario) best_conf <- getFinalElites(scenario$logFile, n = 1L, drop.metadata = TRUE) expect_identical(removeConfigurationsMetaData(confs[1L, , drop = FALSE]), best_conf) invisible(read_logfile(scenario$logFile)) } target_runner_time <- function(experiment, scenario) { configuration <- experiment$configuration tmax <- configuration[["tmax"]] temp <- configuration[["temp"]] time <- max(1, abs(rnorm(1, mean=(tmax+temp)/10))) list(cost = time, time = time) } time_irace <- function(...) { args <- list(...) parameters <- readParameters(text = ' tmax "" i (-10, 10) temp "" r (0, 10) ') scenario <- list(targetRunner = target_runner_time, instances = c("ackley", "goldestein", "matyas"), blockSize=3, parameters = parameters) scenario <- modifyList(scenario, args) scenario <- checkScenario (scenario) confs <- irace(scenario = scenario) best_conf <- getFinalElites(scenario$logFile, n = 1L, drop.metadata = TRUE) expect_identical(removeConfigurationsMetaData(confs[1L, , drop = FALSE]), best_conf) invisible(read_logfile(scenario$logFile)) } check_blocksize <- function(results) { expect_equal(rowMeans(matrix(get_instanceID_seed_pairs(results)[["instanceID"]],nrow=results$scenario$blockSize)), rowMeans(matrix(seq_along(results$scenario$instances), nrow=results$scenario$blockSize))) expect_equal(sum(colSums(!is.na(results$experiments)) %% results$scenario$blockSize), 0) } test_that("blockSize error", { expect_error(cap_irace(maxExperiments = 1000, blockSize=3), "must be a multiple of 'blockSize") }) test_that("blockSize cap_irace maxExperiments = 1000", { generate_set_seed() expect_warning(check_blocksize(cap_irace(maxExperiments = 1000, debugLevel = 3)), "Assuming 'mu = firstTest * blockSize' because 'mu' cannot be smaller", fixed = TRUE) }) test_that("blockSize maxTime=1000", { generate_set_seed() check_blocksize(time_irace(maxTime = 1000)) }) test_that("blockSize maxTime=1000 elitistNewInstances", { skip_on_cran() generate_set_seed() check_blocksize(time_irace(maxTime = 1000, instances = letters[1:9], elitistNewInstances = 2, elitistLimit = 2)) }) }) # withr::with_output_sink() irace/tests/testthat/test-readconfs.R0000644000176200001440000000651515024013102017376 0ustar liggesuserswithr::with_output_sink("test-readconfs.Rout", { params <- readParameters("parameters.txt") expect_error_readconfs <- function(params, table, exp) expect_error(irace::readConfigurationsFile(text=table, parameters = params), exp) test_that("no error", { x <- irace::readConfigurationsFile(text=' param1 param2 mode real mutation 5 NA "x2" 4.0 "low" 1 NA "x2" 4.0 "low" NA NA "x3" 4.5 "low"', parameters=params) expect_equal(nrow(x), 3L) expect_equal(ncol(x), 6L) # It adds the fixed column }) test_that("wrong fixed", { expect_error_readconfs(params, ' param1 param2 mode real mutation fixed 5 NA "x2" 4.0 "low" fixed 1 NA "x2" 4.0 "low" fixed NA NA "x3" 4.5 "low" wrong', "is not among the valid values") }) test_that("checkDuplicates", { expect_error_readconfs(params, ' param1 param2 mode real mutation 5 NA "x2" 4.0 "low" 1 NA "x2" 4.0 "low" 5 NA "x2" 4.0 "low" NA NA "x3" 4.5 "low" ', "Duplicated") expect_error_readconfs(params, ' param1 param2 mode real mutation 5 NA "x2" 4.0 "low" 1 NA "x2" 4.0 "low" 5 NA "x2" 4.0 "low" 1 NA "x2" 4.0 "low" ', "Duplicated") }) test_that("parameter values", { expect_error_readconfs(params, ' param1 param2 mode real mutation 5 NA "x2" 4.0 "low" 1 NA "x2" 4.0 "low" 11 NA "x2" 4.0 "low" ', "is not within the valid range") expect_error_readconfs(params, ' param1 param2 mode real mutation 5 NA "x2" 4.0 "low" 1 NA "x2" 4.5001 "low" ', "is not within the valid range") expect_error_readconfs(params, ' param1 param2 mode real mutation 5 NA "x2" 4.0 "low" 1.1 NA "x2" 4.5 "low" ', "is not an integer") expect_error_readconfs(params, ' param1 param2 mode real mutation 5 NA "x2" 4.0 "low" 1 NA "x2" 4.5 "lower" ', "is not among the valid values") expect_error_readconfs(params, ' param1 param2 mode real mutation 5 NA "x3" 4.0 "low" 1 NA "x2" 4.5 "low" ', "is not enabled") }) test_that("parameter names", { expect_error_readconfs(params, ' param1 param0 mode real mutation 5 NA "x2" 4.0 "low" 1 NA "x2" 4.0 "low" ', "do not match") expect_error_readconfs(params, ' param1 mode real mutation 5 "x2" 4.0 "low" 1 "x2" 4.0 "low" ', "are missing") expect_error_readconfs(params, ' param1 param2 mode real mutation param3 5 NA "x2" 4.0 "low" NA 1 NA "x2" 4.0 "low" NA ', "do not match") }) test_that("conditional fixed", { params <- readParameters(text=' param1 "" c ("fixed") param2 "" c ("cond_fixed") | param3 == "on" param3 "" c ("on", "off") ') x <- irace::readConfigurationsFile(text=' param1 param3 fixed off ', parameters=params) expect_equal(x[["param2"]], NA_character_) x <- irace::readConfigurationsFile(text=' param1 param2 param3 fixed NA off ', parameters=params) x <- irace::readConfigurationsFile(text=' param1 param3 fixed on ', parameters=params) expect_equal(x[["param2"]], "cond_fixed") expect_error_readconfs(params, ' param1 param2 param3 fixed cond_fixed off ', "is not enabled because of condition") }) }) # withr::with_output_sink irace/tests/testthat/test-targetRunnerParallel.R0000644000176200001440000000412014736526233021604 0ustar liggesuserswithr::with_output_sink("test-targetRunnerParallel.Rout", { test_that("targetRunnerParallel", { setClasses <- function(x, classes) { class(x) = classes x } # test that targetRunnerParallel works with arbitrary instance objects. targetRunnerParallel <- function(experiment, exec_target_runner, scenario, target_runner) { # get our param settings that irace should try cands = lapply(experiment, "[[", "configuration") # the instance is always the same for all different param setting theinst = experiment[[1L]]$instance # we check that we have instances of correct class expect_s3_class(theinst, "foo", exact = TRUE) # fabricate some random fitness vals ys = rnorm(length(cands)) lapply(ys, function(y) list(cost = y, time = NA_real_)) } n.inst = 7L instances = replicate(n.inst, setClasses(list(x=123), classes = "foo"), simplify = FALSE) parameters = readParameters(text='x1 "" r (0,1)') log.file = tempfile() tuner_config = list(maxExperiments = 40L, nbIterations = 1L, minNbSurvival = 1L, targetRunnerParallel = targetRunnerParallel, instances = instances, logFile = log.file) confs <- irace(scenario = tuner_config) expect_gt(nrow(confs), 0L) }) test_that("targetRunnerData", { targetRunnerParallel <- function(experiments, exec_target_runner, scenario, target_runner) { cat("a = ", scenario$targetRunnerData$a, ", b = ", scenario$targetRunnerData$b, "\n", sep = "") # get our param settings that irace should try cands = lapply(experiments, "[[", "configuration") # fabricate some random fitness vals ys = rnorm(length(cands)) ys = lapply(ys, function(y) list(cost = y, time = NA_real_)) return(ys) } parameters = readParameters(text=' x "x" r (1,2) ') expect_output( irace(scenario = list(targetRunnerParallel = targetRunnerParallel, instances = replicate(5, list(10)), targetRunnerData = list(a=1, b=2), maxExperiments = 42L, parameters = parameters)), "a = 1, b = 2") }) }) # with_output_sink() irace/tests/testthat/test-capping.R0000644000176200001440000000354314745735066017106 0ustar liggesuserswithr::with_output_sink("test-capping.Rout", { ## target runner ########################################################### target_runner_reject <- function(experiment, scenario) { if (experiment$configuration[["reject"]] == "1" && runif(1) <= 0.01) return(list(cost = -Inf, time = experiment$bound)) target_runner_capping_xy(experiment, scenario) } test_that("irace_capping_xy maxExperiments = 1000", { generate_set_seed() irace_capping_xy(maxExperiments = 1000) }) test_that("irace_capping_xy maxExperiments = 1000 cappingAfterFirstTest", { generate_set_seed() irace_capping_xy(maxExperiments = 1000, cappingAfterFirstTest=1) }) test_that("irace_capping_xy maxTime = 1000", { generate_set_seed() expect_warning(irace_capping_xy(maxTime = 1000), "boundMax=80 is too large, using 5 instead") }) test_that("irace_capping_xy targetRunner = target_runner_reject, maxTime = 1000", { skip_on_cran() skip_on_coverage() # This test sometimes fails randomly generate_set_seed() irace_capping_xy(targetRunner = target_runner_reject, maxTime = 1000, boundMax = 5, debugLevel = 3, # FIXME: target_runner_reject does not work in parallel on Windows. parallel = if (system_os_is_windows()) 1L else test_irace_detectCores()) }) test_that("capping default value", { parameters_table <- ' x "" r (0, 1.00) y "" r (0, 1.00) reject "" c (0,1)' parameters <- readParameters(text = parameters_table) def_scenario <- list(instances = c("ackley", "goldestein", "matyas", "himmelblau"), targetRunner = target_runner_reject, maxTime = 1200, parameters = parameters) scenario <- checkScenario(def_scenario) expect_false(scenario$capping) def_scenario$boundMax <- 80 scenario <- checkScenario(def_scenario) expect_true(scenario$capping) }) }) # withr::with_output_sink() irace/tests/testthat/setup.R0000644000176200001440000000027414736526233015640 0ustar liggesusersold_opts = options( warnPartialMatchArgs = TRUE, warnPartialMatchAttr = TRUE, warnPartialMatchDollar = TRUE ) old_opts = lapply(old_opts, function(x) if (is.null(x)) FALSE else x) irace/tests/testthat.R0000644000176200001440000000056214736526233014500 0ustar liggesusers# This file is part of the standard setup for testthat. # It is recommended that you do not modify it. # # Where should you do additional test configuration? # Learn more about the roles of various files in: # * https://r-pkgs.org/tests.html # * https://testthat.r-lib.org/reference/test_package.html#special-files library(testthat) library(irace) test_check("irace") irace/.Rinstignore0000644000176200001440000000010714736526233013652 0ustar liggesusers\.github ^GenericWrapper4AC _snaps vignettes/section/irace-options.tex irace/MD50000644000176200001440000003130315060507630011647 0ustar liggesusers0ef82bb86664db41147caed9eda18f88 *DESCRIPTION b7738f5a508f22779b74733abc034786 *NAMESPACE 0d40861d52258afc46c65cbe078753e1 *NEWS.md 53e3ad1374412f5e5b836795bf9a193a *R/aaa.R 006a116a9218bd0228401734ee4e12c3 *R/ablation.R 5e7738e78b65ea41cfcb43fbf5a08d34 *R/argparser.R 34fbb61abcb2fc74ec11918f14653035 *R/cluster.R 70d588290f1097835f9ee893c1a4247e *R/configurations.R 898d058ec054b2b26bec3f0f9e56de73 *R/generation.R 313c8948f5533525372d0dc0af5058aa *R/irace-options.R 1f98f6158b4d713cd233c8bf0752cfb8 *R/irace-package.R 8f3e4255357b4ffdd68e022a2ab45af3 *R/irace.R 1cf273fe37c5662ad9c6ded6c1690a2b *R/irace_summarise.R 720de1df587d31c2f26f0dccb3644b85 *R/main.R d0e49cf3a5f60810630ddb55ee0f135b *R/model.R 0e5b0a80321f5d9c849378bb5f9a332a *R/multi_irace.R 0d9c21eb7bd2f6ec40df8fa05636619e *R/parameterAnalysis.R 5971b567bb7bc5c0ec07fbc3ce42c0f2 *R/parameters.R fc9f3c7970c55df0c8b521fba54ae104 *R/path_rel2abs.R 77729498466b297a7f5ca73550ecba84 *R/psRace.R a1c67d3e5baaca6da775489851a4a328 *R/race-wrapper.R e06155a1b4a6bfac5f115943976c2eb7 *R/race.R 0fa377394e236b8160c3ff6d6189fbbf *R/race_state.R 1d4a29f5811864c1ee0956f4619d5062 *R/random_seed.R 84c5ae42f120ff3b0b638c711cc10a69 *R/readConfiguration.R df00857874f80f81a879a7f16cafb881 *R/readParameters.R ea7357b20c352b9569c8d4af8fef8787 *R/scenario.R 1d227a1312b5c1e2e74ac2cd5c456305 *R/testing.R e7db059f3f4a16184a579ee1aedb5733 *R/timer.R 3d9278311be3b74d2efae99f9663e250 *R/tnorm.R 611025ddf1c4accd57794e1be06bfd42 *R/utils.R a96f953ab214bb90f1470b4383d94999 *R/version.R 2a118b7354fdedff5c74246d8423e5ad *R/zzz.R 6f3b067388fc95ea11303f7dc9726638 *README.md f7bf3d04e51c232b5d4afc76a41d7a6a *build/partial.rdb 66bd0cdd5e2192f491c905e29f040774 *build/vignette.rds 90f388627b81fe6ba8219fb8d4b06a3c *cleanup c62ed27a7d824faa54b9bade601f0086 *inst/CITATION 4fafc43ccb5f8f46ea49dba2cb5d93e7 *inst/bin/parallel-irace 3e8f7689974b059d5a3fa356c7d8a231 *inst/bin/parallel-irace-mpi bd8592671bd7acc42ea91a49a5fc7304 *inst/bin/parallel-irace-qsub 3484f8d53fb8b7eb78ba2ad56171a525 *inst/bin/parallel-irace-slurm 25bd69b39616e59e18676afdaf8c32b9 *inst/doc/irace-package.R 9d17c11573b3dfa93504c08071aa9efd *inst/doc/irace-package.Rnw 352b99d7906b7e0df53a0196fca3a529 *inst/doc/irace-package.pdf 6f5adad387e00e68a7802e8d543b18ce *inst/examples/README.md 067fe2821c16cf8262dece7a4c36144e *inst/examples/Spear/README 4a9044d0341a321c352ec901cb97a8f0 *inst/examples/Spear/parameters-cat.txt cb2da2a31c96ffc3a4274e9a8ef3832a *inst/examples/Spear/parameters-mixed.txt deda433ece24d324cbc1eef067cb43d7 *inst/examples/Spear/scenario.txt 039051117ef71079e1d2ff106d6c4e7a *inst/examples/Spear/target-runner 7d36a29ebc23a8f813ec3a0c46eb1e84 *inst/examples/acotsp/README d14e5bb16ddb374e204cabfc0be4545d *inst/examples/acotsp/default.txt ec439a0bc50219bd37b71c7fb6e67dd0 *inst/examples/acotsp/parameters-acotsp.txt 296613c78543f277ad03f13dd45632b6 *inst/examples/acotsp/scenario.txt 3b2e0404fedaf8c5771da68553626557 *inst/examples/acotsp/target-runner b7d067aba94e5b7fc1e92237a6f0a1b4 *inst/examples/batchmode-cluster/README d960f69eeadff7405e4452f8f50c710a *inst/examples/batchmode-cluster/irace-sge-cluster 0b3da55eb761846dcf247a23060ffb0a *inst/examples/batchmode-cluster/target-evaluator cb7e8b282835fd66d0745aa16dfdecbf *inst/examples/batchmode-cluster/target-runner-htcondor 682e5b6a084e3e94e0ae045b60de8f9f *inst/examples/batchmode-cluster/target-runner-pbs 3cd7828292c3927406d34868ad9266b0 *inst/examples/batchmode-cluster/target-runner-sge 40fbfe617cda99828251f4b2128a4076 *inst/examples/batchmode-cluster/target-runner-slurm 3523d53e5d616ae1343cbb787b703b5c *inst/examples/hypervolume/README 5d16974368bcb97b8dcf9594b19152f2 *inst/examples/hypervolume/target-evaluator 74406c35bc1a89a6e202f9aa27f48817 *inst/examples/hypervolume/target-runner b99b382d0af20c11d595c4f7a2135379 *inst/examples/matlab/RUN.m ef092e94145f01eea192cef2f1cdf3d8 *inst/examples/matlab/parameters.txt bb65d3d452fd6b608be1aa46fe262705 *inst/examples/matlab/target-runner c3a9d6c129e565e00d6ef25ad8be0f90 *inst/examples/matlabr/Main.m 3b0332e02daabf31651a5a0d81ba830a *inst/examples/matlabr/instances.txt 570fbc605de77856756be963e4fb8c91 *inst/examples/matlabr/parameters.txt a924aed58b5df474a286eb17e8dfb7d6 *inst/examples/matlabr/scenario.txt 117ed65e989d5b1d901431e35c392b00 *inst/examples/moaco/README c1f850010a57a1ef001062ccb67a5195 *inst/examples/moaco/parameters.txt f852be06460223000aa582c4e749bb1f *inst/examples/moaco/scenario.txt 5157ef3fc2efb9230f76aef1cfe73291 *inst/examples/moaco/target-evaluator e250b38c64ed8030bc71ee66ec4f8941 *inst/examples/moaco/target-runner 8652813c19e9266c3e1849a06fd04907 *inst/examples/slurm/job.sh 388f9371ca0d455e51e200879b1772f0 *inst/examples/slurm/run_irace.R 03229b37f1f15e8e94a0c7444436f438 *inst/examples/target-runner-c/instances.txt 50d6243d8eb29e9a104461aa9657e2e3 *inst/examples/target-runner-c/parameters.txt 795af3a128d0d1266b90dde627d43b9c *inst/examples/target-runner-c/scenario.txt bb9dabb68e2467077c87d796db8d33a1 *inst/examples/target-runner-c/target-runner.c 3a66bdf4ccc99e46369ac61adcd68a71 *inst/examples/target-runner-python/target-runner-acotsp.py 3181689c55fc627ccc96e5f9980484f0 *inst/examples/target-runner-python/target-runner-advanced.py fc5b988b87d11f4e54a72ced66b72fb2 *inst/examples/target-runner-python/target-runner-python-win.bat 2286cd7e0e526fa1526ea38a612d476b *inst/examples/target-runner-python/trivial/instances.txt e3f5b6aee118bd70096440b0d648e812 *inst/examples/target-runner-python/trivial/parameters.txt e35dda63b5e6a98c1c15b392693ac633 *inst/examples/target-runner-python/trivial/scenario.txt b23ae5fbdad504a0373d8a4147b379e9 *inst/examples/target-runner-python/trivial/target-runner.py c868589f82ab78c538bb73b63ba5ed59 *inst/exdata/irace-acotsp.Rdata 86bac7b38049082b3a0f73634144fd1f *inst/exdata/log-ablation.Rdata 52feab4bf6777dc9bdd387df9703a628 *inst/exdata/sann.rda a523dd89b6ec7ba2adc26e2ed3a3a179 *inst/irace.sindef 5718f32d8359ec715f2d8bc1a0ec02b3 *inst/templates/configurations.txt.tmpl af9376428b2c2d911710fd6e51ecf784 *inst/templates/instances-list.txt.tmpl 5223b5c5b935d24826f19a63540b59c7 *inst/templates/parameters.txt.tmpl 1029819534f830899eaeed554f80f930 *inst/templates/scenario.txt.tmpl 0185c79ae7b6f5994f89da433acd7b94 *inst/templates/target-evaluator.tmpl 0c9ea3e193e137a921c5e1567433a112 *inst/templates/target-runner.tmpl 4850d7fbd68fbc7c06af53d3c510734f *inst/templates/windows/target-runner.bat b8c1358b4dcc1157dcded4358327a365 *man/ablation.Rd ad88cc23f0dd138ede5ecbbd4348ef39 *man/ablation_cmdline.Rd 97df995452432aedab6501c447222bee *man/buildCommandLine.Rd 22c55e3778d336a7c65dcc0fad4b8be0 *man/checkIraceScenario.Rd ca4dea9aebb234d9db7afd9742bf2b39 *man/checkParameters.Rd 78d5c354d12f0b0d166cbbc69f6590f0 *man/checkScenario.Rd 08eaf87dd5077a7a2e51debba1327160 *man/check_output_target_runner.Rd 84bffa7e4008c648aa6093ac6b521a0e *man/configurations_print.Rd 20974d74befdc45e32057cc236b250af *man/configurations_print_command.Rd aa9f362031321f1959b94d430b53d07d *man/defaultScenario.Rd b872bbcb3cab0c61fea1051a491dcafd *man/getConfigurationById.Rd e0516ae2bbf6a95946468342474980ca *man/getConfigurationByIteration.Rd 4379d0202d10764f6582d4c77816e516 *man/getFinalElites.Rd 8962b34f3d99b7400b943a72483b7584 *man/get_instanceID_seed_pairs.Rd 14a6c3069e36e86aab218d5f3881620f *man/has_testing_data.Rd 1c89a853ee2bfa26f82518f61e642a12 *man/irace-package.Rd 8a01d51299214920726f2cca2d8dde83 *man/irace.Rd 99ab3e426427a328185d62e9a9cc7c72 *man/irace_cmdline.Rd c5028f112fcba7a0357272df300bfeef *man/irace_license.Rd 798efa16aa50ce4056ba58c2338c93b3 *man/irace_main.Rd c53417b5fbb84a928125a2a63217e2a7 *man/irace_summarise.Rd 1cb0277b653693a2d927d4799e6c82f3 *man/irace_version.Rd b19078bd26ec7ffd9821e44d369afe18 *man/multi_irace.Rd 28b6c3cc7b4bfbad36a9cdeaeda2f148 *man/parameters.Rd 98f45d9d1092dc87591febe99be3b2a7 *man/path_rel2abs.Rd 34ca000f9e09e1e33e569af65355be75 *man/plotAblation.Rd b94901cc7ab0506875410d2684b7bd50 *man/printParameters.Rd 6e4645d0710c8013dd8d1970bf629fb9 *man/printScenario.Rd 6ed47d873d3e4a37f389477163ccf417 *man/psRace.Rd 83683f19856a4b7bdbce1345dab2c539 *man/random_seed.Rd 44ec2759fb9575dd6cd000fe8a94b107 *man/readConfigurationsFile.Rd e0989af0e7d48c44d0574ac8d51bce58 *man/readParameters.Rd 9806771a8e8e3bfcf551cd52902c3ea5 *man/readScenario.Rd 3b1091c12c43f96f4d7622ee269b739a *man/read_ablogfile.Rd 06e4d5d2b1eeddcc2400726e6aa3d62f *man/read_logfile.Rd f700572809f0a7f8ee398e7f18358e13 *man/read_pcs_file.Rd 6ca7619bd07a92f1cca2e228ab184e18 *man/removeConfigurationsMetaData.Rd 7f7e1e4184d1996f3ca4dc3ea153e002 *man/save_irace_logfile.Rd 874142b614625461f91ab9bb1c89c543 *man/scenario_update_paths.Rd 71e04de7d540160a730e85ceb0d5130f *man/target_evaluator_default.Rd 43929367039d9a6095051329fd262831 *man/target_runner_default.Rd 3815d6b4c28da8acb256ee0ff00a251a *man/testConfigurations.Rd fa1eb0e5d9f8e82e066aefab0a50ef0e *man/testing_fromfile.Rd 05fdbf65834ad9d1f1e82cf6cc64b1e7 *man/testing_fromlog.Rd a886f8cb75354603c3204d2cebc07888 *src/Makevars a226bdfce980e2c61c21c03a6442f322 *src/Makevars-common ca5fd252fe6221d98e26e1eab7b8d9c6 *src/Makevars.win 1398b2fd4ff03df145e7be99716fb5c3 *src/dummy/target-runner-dummy.c 70c3abdb60157b72947499b5110cb964 *src/install.libs.R aa3c94c075de8e74d82fa9dbac430ae0 *src/iracebin/ablation.c 7d0d72752d80088203df170808753c12 *src/iracebin/irace.c 21a6f2c3b1b9d8f6f577ed0350f3be5d *src/iracebin/irace.h dd5bdc751bb1cbdd2dd55e82e89a7ee8 *src/iracebin/whereami.c 6c3b99e57a7cd7611fb4c079fafb2cd6 *src/iracebin/whereami.h 5a48d8cf11a02c33827a139b0a225fff *tests/testthat.R b735d8cdd247647949863dc0db5242c5 *tests/testthat/bad_scenario.txt 0c70d9f2df1bdaae7a5427383deae61f *tests/testthat/configurations.txt 5392765ea169c97660c1a8f5e38962d5 *tests/testthat/dependencies.txt cb3f44bc6c27b7761045e0c433c2797d *tests/testthat/dependencies2.txt a96f6d23df9bca94fd33c47b8fe54835 *tests/testthat/dummy_wrapper.py 802b1377a53b451f13bc190dd692eb9e *tests/testthat/good_scenario.txt ef4e5ad623fdbf2ef505edf92db600f5 *tests/testthat/helper-common.R 6fda56b55b2ddcd5255cc1f0c38d6081 *tests/testthat/logparameters.txt b5281655464b518a9905e360f72172d0 *tests/testthat/parameters.txt fa5811cf38687ea0ba22857d07e43708 *tests/testthat/setup.R 390e7a8b3437a82b11f614fd1c1976f1 *tests/testthat/teardown.R cc1537221892ea4f0c8169bb85a3b5a3 *tests/testthat/test-GenericWrapper4AC.R 9d28323afacc3d6342da7e21d5f9e2eb *tests/testthat/test-ablation.R dffb23cb04b2410a861012138443c85d *tests/testthat/test-argparser.R 130d5905ba8b3e11af3759d64b826dbb *tests/testthat/test-bad_scenario.R 1d63bd03609be7dfffc15bea8e1900b1 *tests/testthat/test-blocksize.R a17f8f4a5337eac7a4792ca4c657cee6 *tests/testthat/test-bug-10.R c23c303b507d81f3446f93f786838217 *tests/testthat/test-bug-44.R 700aa77339ccdda457068bef5c348ff2 *tests/testthat/test-bug-55.R cfe3de6faeec01edc29f688baa2e4894 *tests/testthat/test-bug-87.R ffdf4325c439bc2cfc01dc7dfd9902d4 *tests/testthat/test-bug71.R de0cd2d1ade2d1cfe4fc38e4ecfe3f35 *tests/testthat/test-capping.R 882d8851c39a2da5e124c8d4b21999dc *tests/testthat/test-dependencies.R efd1eef47896bc45b8bd9c5f0a98439b *tests/testthat/test-forbidden.R a1da29cd4b5fdad9f96e202a8479e278 *tests/testthat/test-get-functions.R f1e8529b31b7da07bee6238d6b830598 *tests/testthat/test-maxTime.R c56de868281cbf7d393875cade6271d0 *tests/testthat/test-multi_irace.R a8a04422d027bbfb1834ba727612e56d *tests/testthat/test-path.R 103ee22c9b050575083a8320ee3c552d *tests/testthat/test-psrace.R 1102ec28db5067837c544b3f3ab527fd *tests/testthat/test-raceconfs.R d976745248ffdf56fc99c36853735a22 *tests/testthat/test-readParameters.R 105d18d04d2c1b8f7781f4b6991c291c *tests/testthat/test-read_pcs_file.R e938ca5e1e917aa10ce7d10c948b3316 *tests/testthat/test-readconfs.R 0b28c0899b59ba9aabfa8f9cf10c481a *tests/testthat/test-recovery.R 26a45177ed6f9aaff18f2e33a65b740f *tests/testthat/test-repair.R dd476a54bd1c9dadec0789c184ac36fd *tests/testthat/test-sann-irace.R 6b5366dcf14f8a98e8c30f923fb06a95 *tests/testthat/test-similar.R 8782bffc2d9c5ab48419fed065d1bf25 *tests/testthat/test-sobol.R 2ec9431415e52e9b29754508e56783ba *tests/testthat/test-target-runner-dummy.R 4439f315ad740b55aa598777f5753ad0 *tests/testthat/test-targetRunnerParallel.R 8870110605b5fdd100ce72953cc7d721 *tests/testthat/test-targeteval.R 289ab2a15fa9dca53bb4a94dff1817e3 *vignettes/Warning-icon.png 92cb3ab29da2cefc7dfc3a5cc5f1ec46 *vignettes/examples.Rdata 4784f9bce400ae4624ca874739cd7c4b *vignettes/fig1u-acotsp-instances.pdf 14eff35115c1805af0e8a9bb74298b19 *vignettes/irace-acotsp-stdout.txt 8130717f1dab1c71475aed5f4dca45cb *vignettes/irace-acotsp-testing.txt 9d17c11573b3dfa93504c08071aa9efd *vignettes/irace-package.Rnw 94237e6e090105a0aab22b0559917ff2 *vignettes/irace-package.bib f44025a8b672c55f7c68873d309336ad *vignettes/irace-scheme.pdf 0b5a399b2478c44749ab63304194f6a7 *vignettes/light-bulb-icon.png 53ec1112017113469f7427837fd3cf72 *vignettes/section/irace-options.Rnw d41d8cd98f00b204e9800998ecf8427e *vignettes/section/irace-options.tex irace/R/0000755000176200001440000000000015060500572011536 5ustar liggesusersirace/R/psRace.R0000644000176200001440000003627114736526233013122 0ustar liggesusers#' Post-selection race #' #' \code{psRace} performs a post-selection race of a set of configurations. #' #' @inheritParams has_testing_data #' #' @param max_experiments `numeric(1)`\cr Number of experiments for the #' post-selection race. If it is equal to or smaller than 1, then it is a #' fraction of the total budget given by #' `iraceResults$scenario$maxExperiments` or `iraceResults$scenario$maxTime / #' iraceResults$state$boundEstimate`. #' @param conf_ids IDs of the configurations in `iraceResults$allConfigurations` to be used for the post-selection. #' If `NULL`, then the configurations are automatically selected. #' @param iteration_elites If `FALSE`, give priority to selecting the configurations that were elite in the last iteration. #' If `TRUE`, select from all elite configurations of all iterations. This parameter only has an effect when `conf_ids` is not `NULL`. #' @param psrace_logFile `character(1)`\cr Log file to save the post-selection race log. If `NULL`, the log is saved in `iraceResults$scenario$logFile`. #' #' @return The elite configurations after the post-selection. In addition, if `iraceResults$scenario$logFile` is defined, #' it saves an updated copy of `iraceResults` in that file, where `iraceResults$psrace_log` is a list with the following elements: #' \describe{ #' \item{configurations}{Configurations used in the post-selection race.} #' \item{instances}{Data frame with the instances used in the post-selection race. First column has the #' instances IDs from `iraceResults$scenario$instances`, second column the seed assigned to the instance.} #' \item{max_experiments}{Configuration budget assigned to the post-selection race.} #' \item{experiments}{Matrix of results generated by the post-selection race, in the same format as the matrix \code{iraceResults$experiments}. Column names are the configuration IDs and row names are the instance IDs.} #' \item{elites}{Best configurations found in the experiments.} #' } #' #' @examples #' \donttest{ #' irace_log <- read_logfile(system.file(package="irace", "exdata", "sann.rda")) #' # Use a temporary file to not change the original "sann.rda". #' psrace_logFile <- withr::local_tempfile(fileext = ".Rdata") #' # Execute the post-selection after the execution of irace. Use 10% of the total budget. #' psRace(irace_log, max_experiments=0.1, psrace_logFile = psrace_logFile) #' # Print psrace_log #' irace_log <- read_logfile(psrace_logFile) #' str(irace_log$psrace_log) #' } #' @author Leslie Pérez Cáceres and Manuel López-Ibáñez #' @export psRace <- function(iraceResults, max_experiments, conf_ids = NULL, iteration_elites = FALSE, psrace_logFile = NULL) { irace_note("Starting post-selection:\n") if (missing(iraceResults)) stop("argument 'iraceResults' is missing.") iraceResults <- read_logfile(iraceResults) scenario <- iraceResults$scenario if (!is.null(psrace_logFile)) scenario$logFile <- psrace_logFile race_state <- iraceResults$state race_state$initialize(scenario, new = FALSE) # Restores the random seed if (max_experiments <= 0) stop("'max_experiments' must be positive.") if (max_experiments <= 1) { budget <- if (scenario$maxTime == 0L) scenario$maxExperiments else scenario$maxTime / race_state$boundEstimate max_experiments <- as.integer(ceiling(max_experiments * budget)) if (scenario$maxTime == 0L) cat(sep="", "# scenario maxExperiments:", scenario$maxExperiments, "\n") else cat(sep="", "# scenario maxTime:", scenario$maxTime, "\n") } # Get selected configurations. if (is.null(conf_ids)) { # FIXME: Handle scenario$maxTime > 0 irace_assert(scenario$maxTime == 0) which_max_last <- function(x) 1L + length(x) - which.max(rev(x)) # We want to race at least two configurations, so we generate all # integers between 3 and 2^max_confs - 1L that are not powers of 2, then convert # it to a bit-string then to a logical vector. generate_combs_2 <- function(max_confs) { max_pow <- as.integer(2^max_confs) pow_2 <- as.integer(2L^(0L:(max_confs-1L))) s <- seq.int(3L, max_pow - 1L, 2L) if (max_pow >= 8L) s <- c(s, setdiff(seq.int(6L, max_pow - 2L, 2L), pow_2[-1L:-3L])) lapply(s, function(z) as.logical((z %/% pow_2) %% 2L)) } # A version of the above that is not limited to two configurations. generate_combs_1 <- function(max_confs) { max_pow <- as.integer(2^max_confs) pow_2 <- as.integer(2L^(0L:(max_confs-1L))) lapply(seq.int(1L, max_pow - 1L, 1L), function(z) as.logical((z %/% pow_2) %% 2L)) } # FIXME: Is this really faster than head(x, n=max_len) ? truncate_conf_needs <- function(x, max_len) { if (length(x) <= max_len) return(x) x[seq_len(max_len)] } get_confs_for_psrace <- function(iraceResults, iteration_elites, max_experiments, conf_ids, rejected_ids) { scenario <- iraceResults$scenario report_selected <- if (scenario$quiet) do_nothing else function(conf_ids, conf_needs) cat(sep="", "# Configurations selected: ", paste0(collapse=", ", conf_ids), ".\n# Pending instances: ", paste0(collapse=", ", conf_needs), ".\n") allElites <- iraceResults$allElites experiments <- iraceResults$experiments conf_ids <- if (iteration_elites) unlist(rev(allElites)) else allElites[[length(allElites)]] conf_ids <- unique(c(conf_ids, iraceResults$allConfigurations[[".ID."]])) # NA may be generated if we skipped iterations. if (anyNA(conf_ids)) conf_ids <- conf_ids[!is.na(conf_ids)] # Remove rejected configurations. if (length(rejected_ids)) conf_ids <- setdiff(conf_ids, rejected_ids) experiments <- experiments[, conf_ids, drop = FALSE] conf_needs <- matrixStats::colCounts(experiments, value = NA, useNames = TRUE) n_done <- nrow(experiments) - min(conf_needs) # Remove any configuration that needs more than max_experiments. conf_needs <- conf_needs[conf_needs <= max_experiments] if (length(conf_needs) == 1L) { if (!scenario$quiet) cat("# Insufficient budget to race more than one configuration!\n") return(names(conf_needs)) } if (!scenario$deterministic || n_done < length(scenario$instances)) { # We want to evaluate in at least n_new instances more. n_new <- scenario$eachTest * scenario$blockSize n_confs <- floor(max_experiments / n_new) # If we have n_confs that have been evaluated in all instances, select those. if (n_confs > 1L && sum(conf_needs == 0L) >= n_confs) { conf_needs <- conf_needs[conf_needs == 0L][1:n_confs] conf_ids <- names(conf_needs) report_selected(conf_ids, conf_needs) irace_assert(length(conf_ids) > 1L, eval_after = { cat("n_confs: ", n_confs, "\nn_new:", n_new, "\n") print(conf_needs) save(iraceResults, file="bug-conf_ids.Rdata") }) return(conf_ids) } # Let's try first to evaluate on new instances. conf_needs_new <- truncate_conf_needs(conf_needs + n_new, max_len = 16L) irace_assert(length(conf_needs_new) > 1L, eval_after={ cat("max_experiments: ", max_experiments, "\nn_new: ", n_new, "\nconf_needs:") print(conf_needs) save(iraceResults, file="bug-conf_ids.Rdata") }) combs <- generate_combs_2(length(conf_needs_new)) left <- sapply(combs, function(x) max_experiments - sum(conf_needs_new[x]), USE.NAMES=FALSE) irace_assert(!is.null(left) && length(left) > 0L && !anyNA(left), eval_after= { cat("max_experiments: ", max_experiments, "\n") cat("length(left): ", length(left), "\n") cat("sum(is.na(left)): ", sum(is.na(left)), "\n") save(iraceResults, file="bug-conf_ids.Rdata") }) if (any(left >= 0L)) { # We have enough budget to evaluate on a new instance. # Select the combination that will allow to evaluate the most configurations. combs <- combs[left >= 0] n <- sapply(combs, sum, USE.NAMES=FALSE) winner <- which.max(n) conf_needs_new <- conf_needs_new[combs[[winner]]] conf_ids <- names(conf_needs_new) report_selected(conf_ids, conf_needs_new) irace_assert(length(conf_ids) > 1L, eval_after = { cat("max_experiments: ", max_experiments, "\n") cat("length(left): ", length(left), "\n") cat("sum(is.na(left)): ", sum(is.na(left)), "\n") cat("winner: ", winner, "\n") cat("combs[[winner]]: ", paste0(collapse=",", combs[[winner]]), "\n") save(iraceResults, file="bug-conf_ids.Rdata") }) return(conf_ids) } } # We do not have enough budget to see new instances, so we need consider # at least 1 configuration that has NA values, but we still include the # configurations that have been evaluated on all instances. conf_needs_zero <- conf_needs[conf_needs == 0L] conf_needs <- truncate_conf_needs(conf_needs[conf_needs > 0L], 16L) combs <- generate_combs_1(length(conf_needs)) left <- sapply(combs, function(x) max_experiments - sum(conf_needs[x]), USE.NAMES=FALSE) irace_assert(any(left >= 0), eval_after=save(iraceResults, file="bug-conf_ids.Rdata")) # Select the combination that will allow us to evaluate the most configurations. combs <- combs[left >= 0] n <- sapply(combs, sum, USE.NAMES=FALSE) winner <- which.max(n) conf_needs <- c(conf_needs_zero, conf_needs[combs[[winner]]]) conf_ids <- names(conf_needs) report_selected(conf_ids, conf_needs) irace_assert(length(conf_ids) > 1L, eval_after = { print(conf_needs) cat("winner: ", winner, "\n") cat("combs[[winner]]: ", paste0(collapse=",", combs[[winner]]), "\n") save(iraceResults, file="bug-conf_ids.Rdata") }) return(conf_ids) } conf_ids <- get_confs_for_psrace(iraceResults, iteration_elites, max_experiments, conf_ids = conf_ids, rejected_ids = race_state$rejected_ids) irace_assert(length(conf_ids) >= 1L, eval_after = { # Debug what happened if the assert failed. rejected_ids <- race_state$rejected_ids cat("blockSize:", scenario$blockSize, "\niteration_elites: ", as.character(iteration_elites), "\nrejected: ", paste0(collapse=", ", rejected_ids), "\n") allElites <- iraceResults$allElites experiments <- iraceResults$experiments conf_ids <- if (iteration_elites) unlist(rev(allElites)) else allElites[[length(allElites)]] cat("conf_ids1:", paste0(collapse=", ", conf_ids), "\n") conf_ids <- unique(c(conf_ids, iraceResults$allConfigurations[[".ID."]])) cat("conf_ids2:", paste0(collapse=", ", conf_ids), "\n") # NA may be generated if we skipped iterations. if (anyNA(conf_ids)) conf_ids <- conf_ids[!is.na(conf_ids)] cat("conf_ids3:", paste0(collapse=", ", conf_ids), "\n") # Remove rejected configurations. if (length(rejected_ids)) conf_ids <- setdiff(conf_ids, rejected_ids) cat("conf_ids4:", paste0(collapse=", ", conf_ids), "\n") conf_needs <- matrixStats::colCounts(experiments[, conf_ids, drop = FALSE], value = NA) cat("conf_needs:", paste0(collapse=", ", conf_needs), "\n") }) if (length(conf_ids) == 1L) { # If we cannot run post-selection, we just return the elite configurations. allElites <- iraceResults$allElites return(allElites[[length(allElites)]]) } } else if (length(conf_ids) <= 1L) { irace_error ("The number configurations provided should be larger than 1.") } else if (length(race_state$rejected_ids) && any(conf_ids %in% race_state$rejected_ids)) { irace_error ("Some configuration IDs provided were rejected in the previous run: ", paste0(collapse=", ", intersect(conf_ids, race_state$rejected_ids)), ".") } if (!all(conf_ids %in% iraceResults$allConfigurations[[".ID."]])) { irace_error("Some configuration IDs provided cannot be found in the configurations: ", paste0(collapse=", ", setdiff(conf_ids, iraceResults$allConfigurations[[".ID."]])), ".") } elite_configurations <- iraceResults$allConfigurations[iraceResults$allConfigurations[[".ID."]] %in% conf_ids, , drop=FALSE] # Generate new instances. generateInstances(race_state, scenario, max_experiments / nrow(elite_configurations), update = TRUE) elite_data <- iraceResults$experiments[, as.character(elite_configurations[[".ID."]]), drop=FALSE] race_state$next_instance <- nrow(elite_data) + 1L irace_note("seed: ", race_state$seed, "\n# Configurations: ", nrow(elite_configurations), "\n# Available experiments: ", max_experiments, "\n# minSurvival: 1\n") # We do not want to stop because of this limit. scenario$elitistLimit <- 0L # FIXME: elitist_race should not require setting this, but it currently does. scenario$elitist <- TRUE raceResults <- elitist_race(race_state, maxExp = max_experiments, minSurvival = 1L, configurations = elite_configurations, scenario = scenario, elite_data = elite_data, elitist_new_instances = 0L, # FIXME: This should be nrow(elite_data) + 1L if we have budget to evaluate on new instances. firstTest = nrow(elite_data) / scenario$blockSize) elite_configurations <- extractElites(raceResults$configurations, nbElites = race_state$minSurvival, debugLevel = scenario$debugLevel) irace_note("Elite configurations (first number is the configuration ID;", " listed from best to worst according to the ", test.type.order.str(scenario$testType), "):\n") if (!scenario$quiet) configurations_print(elite_configurations, metadata = scenario$debugLevel >= 1L) if (!is.null(scenario$logFile)) { elapsed <- race_state$time_elapsed() if (!scenario$quiet) cat("# Total CPU user time: ", elapsed["user"], ", CPU sys time: ", elapsed["system"], ", Wall-clock time: ", elapsed["wallclock"], "\n", sep="") indexIteration <- 1L + max(race_state$experiment_log[["iteration"]]) # We add indexIteration as an additional column. set(raceResults$experiment_log, j = "iteration", value = indexIteration) race_state$experiment_log <- rbindlist(list(race_state$experiment_log, raceResults$experiment_log), fill=TRUE, use.names=TRUE) # Merge new results. iraceResults$experiments <- merge_matrix(iraceResults$experiments, raceResults$experiments) iraceResults$iterationElites[indexIteration] <- elite_configurations[[".ID."]][1L] iraceResults$allElites[[indexIteration]] <- elite_configurations[[".ID."]] iraceResults$scenario <- scenario iraceResults$state <- race_state # FIXME: This log should contain only information of what was done in the # psRace and avoid duplicating info from iraceResults. iraceResults$psrace_log <- list(configurations = elite_configurations, instances = race_state$instances_log[seq_len(nrow(raceResults$experiments)), , drop = FALSE], max_experiments = max_experiments, experiments = raceResults$experiments, elites = elite_configurations[[".ID."]]) save_irace_logfile(iraceResults, logfile = scenario$logFile) } elite_configurations } irace/R/random_seed.R0000644000176200001440000000404614736526233014160 0ustar liggesusers#' Get, set and restore the state of the random number generator state. #' #' @details These functions originate from the [`withr`][withr::with_seed()] package. #' #' @param seed (`list()`|`integer(1)`)\cr Either an integer or the list returned gy [get_random_seed()]. #' @return [get_random_seed()] returns a list with two components `random_seed` and `rng_kind` or NULL if no seed was set; [set_random_seed()] and [restore_random_seed()] do not return anything. #' @name random_seed #' @examples #' old_seed <- get_random_seed() #' on.exit(restore_random_seed(old_seed)) #' set_random_seed(42) #' value1 <- runif(1) #' set_random_seed(42) #' value2 <- runif(1) #' stopifnot(all.equal(value1,value2)) #' #' @export get_random_seed <- function() { if (has_random_seed()) { return(list(random_seed = get(".Random.seed", globalenv(), mode = "integer", inherits = FALSE), rng_kind = RNGkind())) } NULL } #' @rdname random_seed #' @export set_random_seed <- function(seed) { if (is.list(seed)) { RNGkind(seed$rng_kind[[1L]], normal.kind = seed$rng_kind[[2L]], sample.kind = seed$rng_kind[[3L]]) assign(".Random.seed", seed$random_seed, globalenv()) } else { set.seed(seed) } } #' @rdname random_seed #' @export restore_random_seed <- function(seed) { if (is.null(seed)) { rm_random_seed() } else { set_random_seed(seed) } } rm_random_seed <- function () { if (has_random_seed()) { set.seed(seed = NULL) rm(".Random.seed", envir = globalenv()) } } has_random_seed <- function() { exists(".Random.seed", globalenv(), mode = "integer", inherits = FALSE) } # @return [gen_random_seeds()] returns a list of `n` seeds that were generated from the `global_seed`. # The generated seeds can then e.g. be used to seed thread-local RNGs. gen_random_seeds <- function(n, global_seed = NULL) { # Use a random global seed if not set. if (!is.null(global_seed)) set_random_seed(global_seed) # Generate 'n' seeds using the 'global_seed'. trunc(runif(n, 1, .Machine$integer.max)) } irace/R/generation.R0000644000176200001440000002755015060056077014033 0ustar liggesusers####################################### ## GENERATE CONFIGURATIONS ####################################### ## When called with an unconditional parameter, it must return TRUE. conditionsSatisfied <- function(condition, partialConfiguration) { # If there is no condition, do not waste time evaluating it. if (isTRUE(condition)) return(TRUE) v <- eval(condition, as.list(partialConfiguration)) # Return TRUE if TRUE, FALSE if FALSE or NA ## FIXME: If we byte-compile the condition, then we should incorporate the ## following into the condition directly. !is.na(v) && v } repairConfigurations <- function(x, parameters, repair) { if (!is.null(repair)) { # FIXME: Pass the whole newConfigurations to repair and let it handle each row. j <- parameters$names df <- as.data.frame(x) for (i in seq_nrow(x)) set(x, i = i, j = j, value = repair(df[i, parameters$names], parameters)) } x } is_within_dependent_bound <- function(param, configuration, value) { domain <- unlist(lapply(param[["domain"]], eval, configuration)) # Value gets truncated (defined from robotics initial requirements) if (param[["type"]] == "i") domain <- as.integer(domain) domain[[1L]] <= value && value <= domain[[2L]] } ## Calculates the parameter bounds when parameters domain is dependent getDependentBound <- function(param, configuration) { domain <- param[["domain"]] if (is.expression(domain)) { deps <- all.vars(domain) # If it depends on a parameter that is disabled, then this is disabled. if (anyNA(configuration[deps])) return(NA) domain <- sapply(domain, eval, configuration) irace_assert(all(is.finite(domain))) # Value gets truncated (defined from robotics initial requirements) if (param[["type"]] == "i") domain <- as.integer(domain) if (domain[[1L]] > domain[[2L]]) { irace_error ("Invalid domain (", paste0(domain, collapse=", "), ") generated for parameter '", param[["name"]], "' that depends on parameters (", paste0(deps, collapse=", "), "). This is NOT a bug in irace. Check the definition of these parameters.") } } domain } ## Calculates the parameter bounds when the parameter is dependent. get_dependent_domain <- function(param, configuration) { # FIXME: Make this function handle a data.table and return a list of domains. configuration <- as.list(configuration) domain <- param[["domain"]] deps <- all.vars(domain) # FIXME: This function should not be called if the parent is disabled. # If it depends on a parameter that is disabled, then this is disabled. if (anyNA(configuration[deps])) return(NA) irace_assert(is.expression(domain)) domain <- sapply(domain, eval, configuration, USE.NAMES=FALSE) irace_assert(all(is.finite(domain))) # Value gets truncated (defined from robotics initial requirements) if (param[["type"]] == "i") domain <- as.integer(domain) if (domain[[1L]] > domain[[2L]]) { # FIXME: Add test for this error. irace_error ("Invalid domain (", paste0(domain, collapse=", "), ") generated for parameter '", param[["name"]], "' that depends on parameters (", paste0(deps, collapse=", "), "). This is NOT a bug in irace. Check the definition of these parameters.") } domain } generate_sobol <- function(parameters, n, repair = NULL) { confs <- spacefillr::generate_sobol_set(n, dim = parameters$nbVariable, seed = runif_integer(size = 1L)) confs <- data.table(confs) setnames(confs, parameters$names_variable) hierarchy <- parameters$hierarchy nodep_names <- names(hierarchy)[(hierarchy == 1L) & !parameters$isFixed] # FIXME: How to do this faster using data.table? for (x in nodep_names) set(confs, j = x, value = param_quantile(parameters$get(x), confs[[x]])) for (x in parameters$names_fixed) set(confs, j = x, value = parameters$domains[[x]]) setcolorder(confs, parameters$names) max_level <- max(hierarchy) if (max_level > 1L) { .NEWVALUE <- .DOMAIN <- NULL # To silence CRAN warnings. for (k in seq_len(max_level - 1L)) { prev_names <- names(hierarchy)[hierarchy <= k] dep_names <- names(hierarchy)[hierarchy == k+1L] for (p in dep_names) { param <- parameters$get(p) if (param$isFixed) { if (!isTRUE(param[["condition"]])) # If somehow this fixed parameter was not satisfied sometimes, just set its value to NA. set(confs, which(!eval(param[["condition"]], confs)), j = p, value = NA_character_) next } idx_satisfied <- which_satisfied(confs, param[["condition"]]) if (length(idx_satisfied)) { if (param[["is_dependent"]]) { confs[idx_satisfied, let(.DOMAIN = list(get_dependent_domain(param, .SD))), by=.I, .SDcols=prev_names] confs[idx_satisfied, .NEWVALUE := param_quantile(param, .SD, domain = unlist(.DOMAIN)), by=.I, .SDcols=p] set(confs, j = ".DOMAIN", value = NULL) } else { confs[idx_satisfied, .NEWVALUE := param_quantile(param, unlist(.SD)), .SDcols=p] } set(confs, j = c(p, ".NEWVALUE"), value = list(confs[[".NEWVALUE"]], NULL)) } else { set(confs, j = p, value = .param_na_value_type[[ param[["type"]] ]]) } } } } repairConfigurations(confs, parameters, repair) confs } sampleSobol <- function(parameters, n, repair = NULL) { newConfigurations <- generate_sobol(parameters, n, repair) newConfigurations <- unique(newConfigurations) forbidden <- parameters$forbidden newConfigurations <- filter_forbidden(newConfigurations, forbidden) have <- nrow(newConfigurations) if (have < n) { needed <- max(ceiling(n + (n - have) * (2 - have / n)), min(parameters$nbVariable * 5L, 100L)) newConfigurations <- generate_sobol(parameters, needed, repair) newConfigurations <- unique(newConfigurations) newConfigurations <- filter_forbidden(newConfigurations, forbidden) if (nrow(newConfigurations) == 0L) { irace_error("irace tried to sample a configuration not forbidden without success, perhaps your constraints are too strict?") } newConfigurations <- truncate_rows(newConfigurations, n) } set(newConfigurations, j = ".PARENT.", value = NA_integer_) newConfigurations } generate_uniform <- function(parameters, nbConfigurations, repair = NULL) { newConfigurations <- configurations_alloc(parameters[["names"]], nrow = nbConfigurations, parameters = parameters) # We sample in the order of the conditions. for (param in parameters$get_ordered()) { pname <- param[["name"]] idx <- which_satisfied(newConfigurations, param[["condition"]]) if (length(idx) == 0L) next if (param[["isFixed"]]) { # We don't need to sample, there is only one value. set(newConfigurations, i = idx, j = pname, value = param[["domain"]]) next } if (param[["is_dependent"]]) { sample_dep_unif <- function(x) sample_unif(param, n = 1L, domain = get_dependent_domain(param, x)) newConfigurations[idx, c(pname) := sample_dep_unif(.SD), by=.I, .SDcols=parameters$depends[[pname]]] ## newVals <- sapply(idx, function(i) { ## domain <- get_dependent_domain(param, newConfigurations[i,]) ## runif(param, n = 1L, domain = domain) ## }) } else { set(newConfigurations, i = idx, j = pname, value = sample_unif(param, n = length(idx))) } } repairConfigurations(newConfigurations, parameters, repair) newConfigurations } ### Uniform sampling for the initial generation sampleUniform <- function(parameters, nbConfigurations, repair = NULL) { newConfigurations <- generate_uniform(parameters, nbConfigurations, repair) forbidden <- parameters$forbidden if (!is.null(forbidden)) { retries <- 100L repeat { newConfigurations <- filter_forbidden(newConfigurations, forbidden) needed <- nbConfigurations - nrow(newConfigurations) if (needed == 0L) break newConfigurations <- rbindlist(list(newConfigurations, generate_uniform(parameters, needed, repair = repair))) retries <- retries - 1L if (retries == 0L) { irace_error("irace tried 100 times to sample uniformly a configuration not forbidden without success, perhaps your constraints are too strict?") } } } set(newConfigurations, j = ".PARENT.", value = NA_integer_) newConfigurations } sample_from_model <- function(parameters, eliteConfigurations, model, nbNewConfigurations, repair = NULL) { # FIXME: We only need .WEIGHT. from eliteConfigurations. ids_elites <- names(model[[1L]]) irace_assert(identical(as.integer(ids_elites), as.integer(eliteConfigurations[[".ID."]])), { print(utils::str(ids_elites)) print(utils::str(eliteConfigurations[[".ID."]])) }) newConfigurations <- configurations_alloc(parameters$names, nrow = nbNewConfigurations, parameters) idx_elites <- sample.int(n = length(ids_elites), size = nbNewConfigurations, prob = eliteConfigurations[[".WEIGHT."]], replace = TRUE) .PARENT. <- NULL # Silence CRAN warning. set(newConfigurations, j = ".PARENT.", value = ids_elites[idx_elites]) # Sample a value for every parameter of the new configuration. for (param in parameters$get_ordered()) { idx_satisfied <- which_satisfied(newConfigurations, param[["condition"]]) if (length(idx_satisfied) == 0L) next if (param[["isFixed"]]) { # We don't need to sample, there is only one value. set(newConfigurations, i = idx_satisfied, j = param[["name"]], value = param[["domain"]]) next } pname <- param[["name"]] this_model <- model[[pname]] # Avoid the cost of method lookup at every call of dep_rmodel(). this_sample_model <- utils::getS3method("sample_model", class(param)[1L]) if (param[["is_dependent"]]) { dep_rmodel <- function(x, sd_mean) { domain <- get_dependent_domain(param, x) if (is.na(domain[[1L]])) return(NA) # If parameters are dependent standard deviation must be computed # based on the current domain sd <- (domain[[2L]] - domain[[1L]]) * sd_mean[[1L]] if (sd < .Machine$double.eps) return(domain[[1L]]) this_sample_model(param, n = 1L, model = c(sd, sd_mean[[2L]]), domain = domain) } newConfigurations[idx_satisfied, c(pname) := dep_rmodel(.SD, this_model[[.PARENT.]]), by=.I, .SDcols=parameters$depends[[pname]]] next # We are done with this parameter. } # .BY is a list, so take the first argument. newConfigurations[idx_satisfied, c(pname) := list(this_sample_model(param, .N, this_model[[ .BY[[1L]] ]])), by = .PARENT.] } set(newConfigurations, j = ".PARENT.", value = as.integer(newConfigurations[[".PARENT."]])) repairConfigurations(newConfigurations, parameters, repair) newConfigurations } sampleModel <- function(parameters, eliteConfigurations, model, nbNewConfigurations, repair = NULL) { if (nbNewConfigurations <= 0) irace_error ("The number of configurations to generate appears to be negative or zero.") newConfigurations <- sample_from_model(parameters, eliteConfigurations, model, nbNewConfigurations, repair) forbidden <- parameters$forbidden if (!is.null(forbidden)) { retries <- 100L repeat { newConfigurations <- filter_forbidden(newConfigurations, forbidden) needed <- nbNewConfigurations - nrow(newConfigurations) if (needed == 0L) break tmp <- sample_from_model(parameters, eliteConfigurations, model, needed, repair = repair) newConfigurations <- rbindlist(list(newConfigurations, tmp)) retries <- retries - 1L if (retries == 0L) { irace_error("irace tried 100 times to sample from the model a configuration not forbidden without success, perhaps your constraints are too strict?") } } } newConfigurations } irace/R/irace.R0000644000176200001440000015412714745735066012777 0ustar liggesusers# FIXME: Restoring occurs after reading the command-line/scenario file. At # least for the irace command-line parameters (scenario), it should occur # before. We would need to: # # 1) Read recovery file settings from command-line/scenario file # # 2) if set, then recover irace scenario # 3) then read other settings from command-line/scenario file being # careful to not override with defaults whatever the recovery has set. # # 4) checkScenario() # # A work-around is to modify the recovery file (you can load it in R, # modify scenario then save it again). recoverFromFile <- function(filename, scenario = list()) { iraceResults <- read_logfile(filename) if (iraceResults$irace_version != irace::irace_version) irace_error("Recovery file '", filename, "' was generated by a version of irace (", iraceResults$irace_version, ") different from this version of irace (", irace::irace_version, ").") # Restore part of scenario but not all. for (name in .irace.params.recover) scenario[[name]] <- iraceResults$scenario[[name]] # We call checkScenario() again to fix any inconsistencies in the recovered data. # FIXME: Do not call checkScenario earlier and instead do the minimum to check recoveryFile. scenario <- checkScenario(scenario) race_state <- iraceResults$state$clone() race_state$initialize(scenario, recover = TRUE) race_state } ## ## Numerical configurations similarity function ## # FIXME: This function is too slow and it shows up in profiles. numeric.configurations.equal <- function(x, configurations, parameters, threshold, param.names) { d <- numeric(nrow(configurations)) isSimilar <- matrix(TRUE, nrow = nrow(configurations), ncol = length(param.names)) selected <- seq_nrow(configurations) for (i in seq_along(param.names)) { pname <- param.names[i] param <- parameters$get(pname) is_dep <- param[["is_dependent"]] x_domain <- if (is_dep) getDependentBound(param, x) else param[["domain"]] x_range <- x_domain[[2L]] - x_domain[[1L]] X <- x[[pname]] # FIXME: Since at the end we select a subset of configurations, we could use selected here. y <- configurations[[pname]] ## FIXME: This can probably done much faster by doing a matrix operation that updates ## isSimilar[, i] in one step instead of the for-loop. ## We would need to handle the NAs first. for (j in seq_len(nrow(isSimilar))) { # Configurations loop Y <- y[selected[j]] if (is.na (X) && is.na(Y)) { # Both NA, just ignore this pname next } else if (xor(is.na (X), is.na(Y))) { # Distance is 1.0, so not equal isSimilar[j,i] <- FALSE } else { # FIXME: Why is this updating d[j]? It seems that if the difference is # large for one configuration, then it will be assumed to be large for # the rest. if (is_dep) { # Compare dependent domains by normalising their values to their own ranges first # and calculating the difference. (When possible) y_domain <- getDependentBound(param, configurations[selected[j],]) y_range <- y_domain[[2L]] - y_domain[[1L]] dx <- if (x_range == 0) 0 else (as.numeric(X) - x_domain[[1L]]) / x_range dy <- if (y_range == 0) 0 else (as.numeric(Y) - y_domain[[1L]]) / y_range d[j] <- max(d[j], abs(dx - dy)) } else { # FIXME: We should calculate (X - x.domain[1]) / x.range once for all configurations # and all parameters, then calculate the differences using vectorization. d[j] <- max(d[j], abs((as.numeric(X) - as.numeric(Y)) / x_range)) } if (d[j] > threshold) isSimilar[j,i] <- FALSE } } index <- which(rowAlls(isSimilar)) isSimilar <- isSimilar[index, , drop=FALSE] d <- d[index] selected <- selected[index] if (nrow(isSimilar) == 0L) break } if (length(selected) == 0L) return(NULL) c(x[[".ID."]], configurations[selected,".ID."]) } ## ## Identify which configurations are similar. ## # FIXME: It would be nice to print the minimum similarity found to the user. similarConfigurations <- function(configurations, parameters, threshold) { # FIXME: Use data.table debug.level <- getOption(".irace.debug.level", 0) if (debug.level >= 1) irace_note ("Computing similarity of configurations .") # Create vectors of categorical and numerical p <- parameters$types %in% c("c","o") vecCat <- parameters$names[p & !parameters$isFixed] vecNum <- parameters$names[!p & !parameters$isFixed] irace_assert(all(parameters$types[vecCat] %in% c("c","o"))) irace_assert(all(parameters$types[vecNum] %not_in% c("c","o"))) irace_assert(length(intersect(vecCat, vecNum)) == 0) nbCater <- length(vecCat) nbNumer <- length(vecNum) ### Categorical/Ordinal filtering #### if (nbCater > 0) { ## Build a vector with the categorical appended together in a string strings <- do.call(paste, c(configurations[, vecCat, drop=FALSE], sep = " ; ")) if (nbNumer != 0) configurations <- configurations[, c(".ID.", vecNum)] ord.strings <- order(strings) configurations <- configurations[ord.strings, ] strings <- strings[ord.strings] ## keep similar (index i == true means is the same as i + 1) similarIdx <- strings[-length(strings)] == strings[-1] ## Now let's get just a FALSE if we remove it, TRUE otherwise: keepIdx <- c(similarIdx[1], (similarIdx[-1] | similarIdx[-length(similarIdx)]), similarIdx[length(similarIdx)]) ## filtering them out: configurations <- configurations [keepIdx, , drop=FALSE] ## filtering their strings out (to use them to define blocks): strings <- strings[keepIdx] ## if everything is already filtered out, return if (nrow(configurations) == 0) { if (debug.level >= 1) cat(" DONE\n") return(NULL) } } ### Numerical parameters within blocks of the same string ### if (nbNumer > 0) { similar <- c() if (nbCater > 0) { ## In this case the object "string" is available to define blocks ## Loop over blocks: beginBlock <- 1L while (beginBlock < nrow(configurations)) { ## The current block is made of all configurations that have the same ## categorical string as the one of configuration[beginBlock, ] blockIds <- which(strings == strings[beginBlock]) endBlock <- blockIds[length(blockIds)] irace_assert (endBlock > beginBlock) ## Loop inside blocks: for (i in seq(beginBlock, endBlock - 1L)) { ## Compare configuration i with all the ones that are after in the block similar <- c(similar, numeric.configurations.equal(configurations[i, ], configurations[(i+1L):endBlock,], parameters, threshold = threshold, param.names = vecNum)) if (debug.level >= 1) cat(".") } beginBlock <- endBlock + 1L # Next block starts after the end of the current one } } else { ## No categorical, so no blocks, just do the basic check without blocks for (i in seq_len(nrow(configurations) - 1L)) { similar <- c(similar, numeric.configurations.equal(configurations[i, ], configurations[(i+1L):nrow(configurations),], parameters, threshold = threshold, param.names = vecNum)) if (debug.level >= 1) cat(".") } } # FIXME: We have to use unique because we return the same configuration # more than once in different calls to numeric.configurations.equal. # Currently, we compare each configuration k=1...n with every configuration # k+1...n. Instead, we should compare k=1...n with ((k+1...n) notin # similar). It may happen that A ~ B and A ~ C and B /= C, but this is OK # because we still return A, B and C. It may also happen that A ~ B, B ~ C # and A /= C, but this is also OK because we will compare A with B,C then B # with C. similar <- unique(similar) configurations <- configurations[configurations[[".ID."]] %in% similar, ] } if (debug.level >= 1) cat(" DONE\n") if (nrow(configurations) == 0L) return(NULL) configurations[[".ID."]] } ## Number of iterations. computeNbIterations <- function(nbParameters) (2 + log2(nbParameters)) ## Computational budget at each iteration. computeComputationalBudget <- function(remainingBudget, indexIteration, nbIterations) floor (remainingBudget / (nbIterations - indexIteration + 1L)) ## The number of configurations computeNbConfigurations <- function(currentBudget, indexIteration, mu, eachTest, blockSize, nElites = 0L, nOldInstances = 0L, newInstances = 0L, maxConfigurations = 1024L) { # FIXME: This is slightly incorrect, because we may have elites that have not # been executed on all nOldInstances. Thus, we need to pass explicitly the # budget that we save (that is, number of entries that are not NA). savedBudget <- nElites * nOldInstances eachTest <- eachTest * blockSize n <- max (mu + eachTest * min(5L, indexIteration), round_to_next_multiple(nOldInstances + newInstances, eachTest)) min (floor ((currentBudget + savedBudget) / n), maxConfigurations) } ## Termination of a race at each iteration. The race will stop if the ## number of surviving configurations is equal or less than this number. computeTerminationOfRace <- function(nbParameters) (2 + log2(nbParameters)) ## Compute the minimum budget required, and exit early in case the ## budget given by the user is insufficient. computeMinimumBudget <- function(scenario, minSurvival, nbIterations, elitist_new_instances) { blockSize <- scenario$blockSize eachTest <- blockSize * scenario$eachTest Tnew <- elitist_new_instances # This is already multiplied by blockSize. mu <- scenario$mu # This is computed from the default formulas as follows: # B_1 = B / I # B_2 = B - (B/I) / (I - 1) = B / I # B_3 = B - 2(B/I) / (I - 2) = B / I # thus # B_i = B / I # and # C_i = B_i / T_i = B / (I * T_i). # # We want to enforce that C_i >= min_surv + 1, thus # B / (I * T_i) >= min_surv + 1 (1) # becomes # B >= (min_surv + 1) * I * T_i # # This is an over-estimation, since actually B_1 = floor(B/I) and if # floor(B/I) < B/I, then B_i < B/I, and we could still satisfy Eq. (1) # with a smaller budget. However, the exact formula requires computing B_i # taking into account the floor() function, which is not obvious. minimumBudget <- (minSurvival + 1L) * nbIterations # We need to compute T_i: if (scenario$elitist) { # T_i = max(mu + Teach * min (5, i), # ceiling((T_{i-1} + Tnew) / Teach) * Teach) # T_1 = mu + Teach # T_2 ~ mu + Teach + max (Teach, Tnew) # T_3 ~ max(mu + 3 * Teach, # mu + Teach + max(Teach, Tnew) + T_new) # = mu + Teach + max(Teach + max(Teach, Tnew), 2 * Tnew) # if Teach > Tnew then 2*Teach > 2*Tnew then max = 2*Teach # if Teach < Tnew then Teach + Tnew < 2*Tnew then max = 2*Tnew # hence: T_3 = mu + Teach + 2 * max(Teach, Tnew) # T_4 = max(mu + 4 * Teach, # ceiling((mu + Teach + 2 * max(Teach, Tnew)) + Tnew) / Teach) * Teach) # ~ mu + Teach + max(2 * Teach + max(Teach, Tnew), 3 * Tnew) # = mu + Teach + 3 * max(Teach, Tnew) # T_i = mu + Teach + (i - 1) * max(Teach, Tnew) # T_6 = max (mu + 5*Teach, # mu + Teach + 5 * max(Teach, Tnew) + Tnew) # = mu + Teach + Tnew + 5 * max (Teach, Tnew) # T_i = mu + Teach + max(I-5, 0) * Tnew + 5 * max (Teach, Tnew) if (nbIterations > 5L) { minimumBudget <- minimumBudget * (mu + eachTest + (nbIterations - 5L) * Tnew + 5L * max(eachTest, Tnew)) } else { minimumBudget <- minimumBudget * (mu + eachTest + (nbIterations - 1L) * max(eachTest, Tnew)) } } else { # T_i = mu + T_each * min (5, i) # and the most strict value is for i >= 5, thus # B >= (min_surv + 1) * I * (mu + 5 * T_each) minimumBudget <- minimumBudget * (mu + 5L * eachTest) } minimumBudget } checkMinimumBudget <- function(scenario, remainingBudget, minSurvival, nbIterations, boundEstimate, timeUsed, elitist_new_instances) { minimumBudget <- computeMinimumBudget(scenario, minSurvival, nbIterations, elitist_new_instances) if (remainingBudget >= minimumBudget) return(TRUE) if (scenario$maxTime == 0) { irace_error("Insufficient budget: ", "With the current settings, irace will require a value of ", "'maxExperiments' of at least '", minimumBudget, "'.") } else if (nbIterations == 1L) { irace_error("Insufficient budget: ", "With the current settings and estimated time per run (", boundEstimate, ") irace will require a value of 'maxTime' of at least '", ceiling((minimumBudget * boundEstimate) + timeUsed), "'.") } FALSE } ## Generate instances + seed. generateInstances <- function(race_state, scenario, n, update = FALSE) { if (!update) race_state$instances_log <- NULL # If we are adding and the scenario is deterministic, we have already added all instances. else if (scenario$deterministic) return(race_state$instances_log) instances <- scenario$instances # Number of times that we need to repeat the set of instances given by the user. ntimes <- if (scenario$deterministic) 1L else # "Upper bound"" of instances needed # FIXME: We could bound it even further if maxExperiments >> nInstances ceiling(n / length(instances)) # Get instances order if (scenario$sampleInstances) { blockSize <- scenario$blockSize n_blocks <- length(instances) / blockSize # Sample instances index in groups (ntimes) selected_blocks <- unlist(lapply(rep.int(n_blocks, ntimes), sample.int, replace = FALSE)) sindex <- c(outer(seq_len(blockSize), (selected_blocks - 1L) * blockSize, "+")) } else { sindex <- rep.int(seq_along(instances), ntimes) } # Sample seeds. race_state$instances_log <- rbindlist(use.names = TRUE, list( race_state$instances_log, data.table(instanceID = sindex, seed = runif_integer(length(sindex))))) race_state$instances_log } do_experiments <- function(race_state, configurations, ninstances, scenario, iteration) { instances <- seq_len(ninstances) output <- race_wrapper_helper(race_state = race_state, configurations = configurations, instance_idx = instances, bounds = rep(scenario$boundMax, nrow(configurations)), is_exe = rep_len(TRUE, nrow(configurations) * ninstances), scenario = scenario) set(output, j = "iteration", value = iteration) Results <- race_state$update_experiment_log(output, instances = instances, scenario = scenario) rejected_ids <- configurations[[".ID."]][colAnys(is.infinite(Results))] scenario$parameters$forbid_configurations( race_state$update_rejected(rejected_ids, configurations) ) Results } ## Initialize allConfigurations with any initial configurations provided. allConfigurationsInit <- function(scenario) { initConfigurations <- scenario$initConfigurations confs_from_file <- NULL if (!is.null.or.empty(scenario$configurationsFile)) { confs_from_file <- readConfigurationsFile(scenario$configurationsFile, scenario$parameters, scenario$debugLevel) } if (is.null.or.empty(initConfigurations)) { initConfigurations <- confs_from_file } else { if (!is.null.or.empty(scenario$configurationsFile) && !identical(initConfigurations, confs_from_file)) irace_warning("'initConfigurations' provided in 'scenario',", " thus ignoring configurations from file '", scenario$configurationsFile, "'.") cat("# Adding", nrow(initConfigurations), "initial configuration(s)\n") initConfigurations <- fix_configurations(initConfigurations, scenario$parameters, debugLevel = scenario$debugLevel) } if (is.null.or.empty(initConfigurations)) { allConfigurations <- configurations_alloc(c(".ID.", scenario$parameters$names, ".PARENT."), nrow = 0L, parameters = scenario$parameters) setDF(allConfigurations) } else { allConfigurations <- cbind(.ID. = seq_nrow(initConfigurations), initConfigurations, .PARENT. = NA_integer_) rownames(allConfigurations) <- allConfigurations[[".ID."]] num <- nrow(allConfigurations) allConfigurations <- checkForbidden(allConfigurations, scenario$parameters$forbidden) if (nrow(allConfigurations) < num) { irace_warning(num - nrow(allConfigurations), " of the ", num, " initial configurations were forbidden", " and, thus, discarded.") } } allConfigurations } ## extractElites # Input: the configurations with the .RANK. field filled. # the number of elites wished # Output: nbElites elites, sorted by ranks, with the weights assigned. extractElites <- function(configurations, nbElites, debugLevel) { irace_assert(nbElites > 0L) # Keep only alive configurations. elites <- as.data.table(configurations) before <- nrow(elites) # Remove duplicated. Duplicated configurations may be generated, however, it # is too slow to check at generation time. Nevertheless, we can check now # since we typically have very few elites. elites <- unique(elites, by=which(!startsWith(colnames(elites), "."))) after <- nrow(elites) if (debugLevel >= 2L && after < before) irace_note("Dropped ", before - after, " duplicated elites.\n") after <- min(after, nbElites) setorderv(elites, cols=".RANK.") selected <- seq_len(after) elites <- elites[selected, ] set(elites, j = ".WEIGHT.", value = ((after + 1L) - selected) / (after * (after + 1L) / 2)) setDF(elites) rownames(elites) <- elites[[".ID."]] elites } #' Execute one run of the Iterated Racing algorithm. #' #' The function `irace` implements the Iterated Racing procedure for parameter #' tuning. It receives a configuration scenario and a parameter space to be #' tuned, and returns the best configurations found, namely, the elite #' configurations obtained from the last iterations. As a first step, it checks #' the correctness of `scenario` using [checkScenario()] and recovers a #' previous execution if `scenario$recoveryFile` is set. A R data file log of #' the execution is created in `scenario$logFile`. #' #' The execution of this function is reproducible under some conditions. See #' the FAQ section in the [User #' Guide](https://cran.r-project.org/package=irace/vignettes/irace-package.pdf). #' #' @inheritParams defaultScenario #' #' @template return_irace #' #' @examples #' \dontrun{ #' # In general, there are three steps: #' scenario <- readScenario(filename = "scenario.txt") #' irace(scenario = scenario) #' } #' ####################################################################### #' # This example illustrates how to tune the parameters of the simulated #' # annealing algorithm (SANN) provided by the optim() function in the #' # R base package. The goal in this example is to optimize instances of #' # the following family: #' # f(x) = lambda * f_rastrigin(x) + (1 - lambda) * f_rosenbrock(x) #' # where lambda follows a normal distribution whose mean is 0.9 and #' # standard deviation is 0.02. f_rastrigin and f_rosenbrock are the #' # well-known Rastrigin and Rosenbrock benchmark functions (taken from #' # the cmaes package). In this scenario, different instances are given #' # by different values of lambda. #' ####################################################################### #' ## First we provide an implementation of the functions to be optimized: #' f_rosenbrock <- function (x) { #' d <- length(x) #' z <- x + 1 #' hz <- z[1L:(d - 1L)] #' tz <- z[2L:d] #' sum(100 * (hz^2 - tz)^2 + (hz - 1)^2) #' } #' f_rastrigin <- function (x) { #' sum(x * x - 10 * cos(2 * pi * x) + 10) #' } #' #' ## We generate 20 instances (in this case, weights): #' weights <- rnorm(20, mean = 0.9, sd = 0.02) #' #' ## On this set of instances, we are interested in optimizing two #' ## parameters of the SANN algorithm: tmax and temp. We setup the #' ## parameter space as follows: #' parameters_table <- ' #' tmax "" i,log (1, 5000) #' temp "" r (0, 100) #' ' #' ## We use the irace function readParameters to read this table: #' parameters <- readParameters(text = parameters_table) #' #' ## Next, we define the function that will evaluate each candidate #' ## configuration on a single instance. For simplicity, we restrict to #' ## three-dimensional functions and we set the maximum number of #' ## iterations of SANN to 1000. #' target_runner <- function(experiment, scenario) #' { #' instance <- experiment$instance #' configuration <- experiment$configuration #' #' D <- 3 #' par <- runif(D, min=-1, max=1) #' fn <- function(x) { #' weight <- instance #' return(weight * f_rastrigin(x) + (1 - weight) * f_rosenbrock(x)) #' } #' # For reproducible results, we should use the random seed given by #' # experiment$seed to set the random seed of the target algorithm. #' res <- withr::with_seed(experiment$seed, #' stats::optim(par,fn, method="SANN", #' control=list(maxit=1000 #' , tmax = as.numeric(configuration[["tmax"]]) #' , temp = as.numeric(configuration[["temp"]]) #' ))) #' ## This list may also contain: #' ## - 'time' if irace is called with 'maxTime' #' ## - 'error' is a string used to report an error #' ## - 'outputRaw' is a string used to report the raw output of calls to #' ## an external program or function. #' ## - 'call' is a string used to report how target_runner called the #' ## external program or function. #' return(list(cost = res$value)) #' } #' #' ## We define a configuration scenario by setting targetRunner to the #' ## function define above, instances to the first 10 random weights, and #' ## a maximum budget of 'maxExperiments' calls to targetRunner. #' scenario <- list(targetRunner = target_runner, #' instances = weights[1:10], #' maxExperiments = 500, #' # Do not create a logFile #' logFile = "", #' parameters = parameters) #' #' ## We check that the scenario is valid. This will also try to execute #' ## target_runner. #' checkIraceScenario(scenario) #' #' \donttest{ #' ## We are now ready to launch irace. We do it by means of the irace #' ## function. The function will print information about its #' ## progress. This may require a few minutes, so it is not run by default. #' tuned_confs <- irace(scenario = scenario) #' #' ## We can print the best configurations found by irace as follows: #' configurations_print(tuned_confs) #' #' ## We can evaluate the quality of the best configuration found by #' ## irace versus the default configuration of the SANN algorithm on #' ## the other 10 instances previously generated. #' test_index <- 11:20 #' test_seeds <- sample.int(2147483647L, size = length(test_index), replace = TRUE) #' test <- function(configuration) #' { #' res <- lapply(seq_along(test_index), #' function(x) target_runner( #' experiment = list(instance = weights[test_index[x]], #' seed = test_seeds[x], #' configuration = configuration), #' scenario = scenario)) #' return (sapply(res, getElement, name = "cost")) #' } #' ## To do so, first we apply the default configuration of the SANN #' ## algorithm to these instances: #' default <- test(data.frame(tmax=10, temp=10)) #' #' ## We extract and apply the winning configuration found by irace #' ## to these instances: #' tuned <- test(removeConfigurationsMetaData(tuned_confs[1,])) #' #' ## Finally, we can compare using a boxplot the quality obtained with the #' ## default parametrization of SANN and the quality obtained with the #' ## best configuration found by irace. #' boxplot(list(default = default, tuned = tuned)) #' } #' @seealso #' \describe{ #' \item{[irace_main()]}{a higher-level interface to [irace()].} #' \item{[irace_cmdline()]}{a command-line interface to [irace()].} #' \item{[readScenario()]}{for reading a configuration scenario from a file.} #' \item{[readParameters()]}{read the target algorithm parameters from a file.} #' \item{[defaultScenario()]}{returns the default scenario settings of \pkg{irace}.} #' \item{[checkScenario()]}{to check that the scenario is valid.} #' } #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @concept running #' @export irace <- function(scenario) irace_common(scenario, simple = TRUE) irace_common <- function(scenario, simple, output.width = 9999L) { if (!simple) { withr::local_options(width = output.width) # Do not wrap the output. } scenario <- checkScenario(scenario) debugLevel <- scenario$debugLevel if (debugLevel >= 1L) { op <- list(warning.length = 8170L) if (!base::interactive()) op <- c(op, list(error = irace_dump_frames)) withr::local_options(op) printScenario (scenario) } elite_configurations <- irace_run(scenario = scenario) if (simple) return(elite_configurations) if (!scenario$quiet) { order_str <- test.type.order.str(scenario$testType) cat("# Best configurations (first number is the configuration ID;", " listed from best to worst according to the ", order_str, "):\n", sep = "") configurations_print(elite_configurations) cat("# Best configurations as commandlines (first number is the configuration ID;", " listed from best to worst according to the ", order_str, "):\n", sep = "") configurations_print_command (elite_configurations, scenario$parameters) } testing_fromlog(logFile = scenario$logFile) invisible(elite_configurations) } irace_run <- function(scenario) { # Recover state from file? if (is.null.or.empty(scenario$recoveryFile)) { race_state <- RaceState$new(scenario) } else { irace_note ("Recovering from file: '", scenario$recoveryFile,"'\n") race_state <- recoverFromFile(scenario$recoveryFile, scenario = scenario) } quiet <- scenario$quiet catInfo <- if (quiet) do_nothing else function(..., verbose = TRUE) { irace_note (..., "\n") if (verbose) { cat ("# Iteration: ", indexIteration, "\n", "# nbIterations: ", nbIterations, "\n", "# experimentsUsed: ", experimentsUsed, "\n", "# timeUsed: ", timeUsed, "\n", "# remainingBudget: ", remainingBudget, "\n", "# currentBudget: ", currentBudget, "\n", "# number of elites: ", nrow(elite_configurations), "\n", "# nbConfigurations: ", nbConfigurations, "\n", sep = "") } } # FIXME: use this from psrace? irace_finish <- function(iraceResults, scenario, reason) { elapsed <- race_state$time_elapsed() if (!scenario$quiet) cat("# Total CPU user time: ", elapsed["user"], ", CPU sys time: ", elapsed["system"], ", Wall-clock time: ", elapsed["wallclock"], "\n", sep="") # FIXME: Do we need to clone? race_state$completed <- reason iraceResults$state <- race_state save_irace_logfile(iraceResults, logfile = scenario$logFile) # FIXME: Handle scenario$maxTime > 0 if (scenario$postselection && scenario$maxTime == 0 && floor(remainingBudget / max(scenario$blockSize, scenario$eachTest)) > 1L) psRace(iraceResults, max_experiments = remainingBudget, iteration_elites = TRUE) else elite_configurations } debugLevel <- scenario$debugLevel # Set options controlling debug level. # FIXME: This should be the other way around, the options set the debugLevel. options(.race.debug.level = debugLevel) options(.irace.debug.level = debugLevel) firstRace <- TRUE # Create a data frame of all configurations ever generated. allConfigurations <- allConfigurationsInit(scenario) irace_assert(is.integer(allConfigurations[[".ID."]])) nbUserConfigurations <- nrow(allConfigurations) # To save the logs iraceResults <- list( scenario = scenario, irace_version = irace_version, iterationElites = c(), allElites = list(), experiments = matrix(nrow = 0L, ncol = 0L)) model <- NULL nbConfigurations <- 0L elite_configurations <- data.frame(stringsAsFactors=FALSE) nbIterations <- if (scenario$nbIterations == 0) computeNbIterations(scenario$parameters$nbVariable) else scenario$nbIterations nbIterations <- floor(nbIterations) minSurvival <- if (scenario$minNbSurvival == 0) computeTerminationOfRace(scenario$parameters$nbVariable) else scenario$minNbSurvival minSurvival <- floor(minSurvival) # FIXME: Do this initialization within race_state. race_state$minSurvival <- minSurvival # Generate initial instance + seed list generateInstances(race_state, scenario, n = if (scenario$maxExperiments != 0) ceiling(scenario$maxExperiments / minSurvival) else max(scenario$firstTest, length(scenario$instances))) indexIteration <- 1L experimentsUsed <- 0L timeUsed <- 0 boundEstimate <- NA race_state$start_parallel(scenario) on.exit(race_state$stop_parallel(), add = TRUE) if (scenario$maxTime == 0) { if (is.na(scenario$minExperiments)) { remainingBudget <- scenario$maxExperiments } else { remainingBudget <- max(scenario$minExperiments, computeMinimumBudget(scenario, minSurvival, nbIterations, race_state$elitist_new_instances)) } } else { ## Estimate time when maxTime is defined. ## IMPORTANT: This is firstTest because these configurations will be ## considered elite later, thus preserved up to firstTest, which is ## fine. If a larger number of instances is used, it would prevent ## discarding these configurations. # Get the number of instances to be used. ninstances <- scenario$firstTest * scenario$blockSize estimationTime <- ceiling(scenario$maxTime * scenario$budgetEstimation) irace_note("Estimating execution time using ", 100 * scenario$budgetEstimation, "% of ", scenario$maxTime, " = ", estimationTime, "\n") # Estimate the number of configurations to be used nconfigurations <- max(2L, floor(scenario$parallel / ninstances)) next_configuration <- 1L nruns <- nconfigurations * ninstances boundEstimate <- if (is.null(scenario$boundMax)) 1.0 else scenario$boundMax if (estimationTime < boundEstimate * nruns) { boundEstimate <- max(ceiling_digits(estimationTime / nruns, scenario$boundDigits), scenario$minMeasurableTime) if (!is.null(scenario$boundMax)) { irace_warning("boundMax=", scenario$boundMax, " is too large, using ", boundEstimate, " instead.\n") # FIXME: We should not modify the scenario scenario$boundMax <- boundEstimate } } repeat { # Sample new configurations if needed if (nrow(allConfigurations) < nconfigurations) { newConfigurations <- sampleSobol(scenario$parameters, nconfigurations - nrow(allConfigurations), repair = scenario$repairConfiguration) set(newConfigurations, j = ".ID.", value = max(0L, vlast(allConfigurations[[".ID."]])) + seq_nrow(newConfigurations)) setcolorder(newConfigurations, ".ID.", before=1L) setDF(newConfigurations) # FIXME: use rbindlist(use.names=TRUE) allConfigurations <- rbind(allConfigurations, newConfigurations) rownames(allConfigurations) <- allConfigurations[[".ID."]] # We may have generated less than the number requested if there were duplicates. nconfigurations <- nrow(allConfigurations) } # Estimate the mean execution time. # FIXME: Shouldn't we pass the bounds? experiments <- do_experiments(race_state, configurations = allConfigurations[next_configuration:nconfigurations, ], ninstances = ninstances, scenario = scenario, # These experiments are assigned iteration 0. iteration = 0L) # FIXME: Here we should check if everything timed out and increase the bound dynamically. iraceResults$experiments <- merge_matrix(iraceResults$experiments, experiments) rownames(iraceResults$experiments) <- seq_nrow(iraceResults$experiments) # For the used time, we count the time reported in all configurations # including rejected ones. timeUsed <- sum(race_state$experiment_log[["time"]], na.rm = TRUE) experimentsUsed <- nrow(race_state$experiment_log) # User should return time zero for rejected_ids. boundEstimate <- timeUsed / experimentsUsed boundEstimate <- max(ceiling_digits(boundEstimate, scenario$boundDigits), scenario$minMeasurableTime) next_configuration <- nconfigurations + 1L # Calculate how many new configurations: # 1. We do not want to overrun estimationTime new_conf <- floor(((estimationTime - timeUsed) / boundEstimate) / ninstances) # 2. But there is no point in executing more configurations than those # that we can execute in parallel. new_conf <- min(new_conf, max(1L, floor(scenario$parallel / ninstances))) if (timeUsed >= estimationTime || new_conf == 0L || nconfigurations == 1024L) break else nconfigurations <- min(1024L, nconfigurations + new_conf) } # end of repeat if (length(race_state$rejected_ids)) irace_note ("Immediately rejected configurations: ", paste0(race_state$rejected_ids, collapse = ", ") , "\n") # Update budget remainingBudget <- round((scenario$maxTime - timeUsed) / boundEstimate) elite_configurations <- allConfigurations[allConfigurations[[".ID."]] %not_in% race_state$rejected_ids, , drop = FALSE] irace_assert(is.integer(elite_configurations[[".ID."]])) # Without elitist, the racing does not re-use the results computed during # the estimation. This means that the time used during estimation needs # to be spent again during racing, thus leaving less time for racing. We # want to avoid having less time for racing, and this is an # implementation detail, thus we assume that the time was not actually # wasted. if (!scenario$elitist) timeUsed <- 0 irace_note("Estimated execution time is ", boundEstimate, " based on ", next_configuration - 1L, " configurations and ", ninstances," instances. Used time: ", timeUsed, ", remaining time: ", (scenario$maxTime - timeUsed), ", remaining budget (experiments): ", remainingBudget, "\n") if (!is.null(scenario$boundMax) && 2 * boundEstimate < scenario$boundMax) { irace_warning("boundMax=", scenario$boundMax, " is much larger than estimated execution time, using ", 2 * boundEstimate, " instead.\n") scenario$boundMax <- 2 * boundEstimate } } # end of time estimation # Compute the total initial budget, that is, the maximum number of # experiments that we can perform. currentBudget <- if (scenario$nbExperimentsPerIteration == 0L) computeComputationalBudget(remainingBudget, indexIteration, nbIterations) else scenario$nbExperimentsPerIteration # Check that the budget is enough. For the time estimation case we reduce # the number of iterations. warn_msg <- NULL while (!checkMinimumBudget(scenario, remainingBudget, minSurvival, nbIterations, boundEstimate, timeUsed, race_state$elitist_new_instances)) { if (is.null(warn_msg)) warn_msg <- paste0("With the current settings and estimated time per run (", boundEstimate, ") irace will not have enough budget to execute the minimum", " number of iterations (", nbIterations, "). ", "Execution will continue by assuming that the estimated time", " is too high and reducing the minimum number of iterations,", " however, if the estimation was correct or too low,", " results might not be better than random sampling.\n") nbIterations <- nbIterations - 1L scenario$nbConfigurations <- if (scenario$nbConfigurations > 0L) min(minSurvival * 2L, scenario$nbConfigurations) else minSurvival * 2L } if (!is.null(warn_msg)) irace_warning(warn_msg) catInfo("Initialization\n", if (scenario$elitist) paste0("# Elitist race\n", "# Elitist new instances: ", race_state$elitist_new_instances, "\n", "# Elitist limit: ", scenario$elitistLimit, "\n") else paste0("# Non-elitist race\n"), "# nbIterations: ", nbIterations, "\n", "# minNbSurvival: ", minSurvival, "\n", "# nbParameters: ", scenario$parameters$nbVariable, "\n", "# seed: ", race_state$seed, "\n", "# confidence level: ", scenario$confidence, "\n", "# budget: ", remainingBudget, "\n", if (scenario$maxTime == 0) "" else paste0("# time budget: ", scenario$maxTime - timeUsed, "\n"), "# mu: ", scenario$mu, "\n", "# deterministic: ", scenario$deterministic, "\n", if (scenario$capping) paste0("# capping: ", scenario$cappingType, "\n", "# type bound: ", scenario$boundType, "\n", "# boundMax: ", scenario$boundMax, "\n", "# par bound: ", scenario$boundPar, "\n", "# bound digits: ", scenario$boundDigits, "\n") else if (!is.null(scenario$boundMax)) paste0("# boundMax: ", scenario$boundMax, "\n"), verbose = FALSE) blockSize <- scenario$blockSize repeat { # FIXME: We could directly use race_state$timeUsed everywhere. race_state$timeUsed <- timeUsed ## Save to the log file. iraceResults$allConfigurations <- allConfigurations race_state$save_recovery(iraceResults, logfile = scenario$logFile) # With elitist=TRUE and without targetEvaluator we should never re-run the # same configuration on the same (instance,seed) pair. if (scenario$elitist) { irace_assert(sum(!is.na(iraceResults$experiments)) == experimentsUsed) if (is.null(scenario$targetEvaluator)) irace_assert(experimentsUsed == nrow(race_state$experiment_log)) } if (remainingBudget <= 0) { catInfo("Stopped because budget is exhausted") return(irace_finish(iraceResults, scenario, reason = "Budget exhausted")) } if (scenario$maxTime > 0 && timeUsed >= scenario$maxTime) { catInfo("Stopped because time budget is exhausted") return(irace_finish(iraceResults, scenario, reason = "Time budget exhausted")) } if (indexIteration > nbIterations) { if (scenario$nbIterations == 0L) { nbIterations <- indexIteration } else { if (debugLevel >= 1L) catInfo("Limit of iterations reached", verbose = FALSE) return(irace_finish(iraceResults, scenario, reason = "Limit of iterations reached")) } } # Compute the current budget (nb of experiments for this iteration), # or take the value given as parameter. currentBudget <- if (scenario$nbExperimentsPerIteration == 0L) computeComputationalBudget(remainingBudget, indexIteration, nbIterations) else scenario$nbExperimentsPerIteration # Compute the number of configurations for this race. if (scenario$elitist && !firstRace) { nbConfigurations <- computeNbConfigurations(currentBudget, indexIteration, mu = scenario$mu, eachTest = scenario$eachTest, blockSize = blockSize, nElites = nrow(elite_configurations), nOldInstances = nrow(iraceResults$experiments), newInstances = race_state$elitist_new_instances) # If we don't have enough budget, do not evaluate new instances. if (nbConfigurations <= minSurvival) { race_state$elitist_new_instances <- 0L nbConfigurations <- computeNbConfigurations(currentBudget, indexIteration, mu = scenario$mu, eachTest = scenario$eachTest, blockSize = blockSize, nElites = nrow(elite_configurations), nOldInstances = nrow(iraceResults$experiments), newInstances = race_state$elitist_new_instances) } # If still not enough budget, then try to do at least one test. if (nbConfigurations <= minSurvival) { nbConfigurations <- computeNbConfigurations(currentBudget, indexIteration = 1L, mu = 1L, eachTest = scenario$eachTest, blockSize = blockSize, nElites = nrow(elite_configurations), nOldInstances = nrow(iraceResults$experiments), newInstances = 0L) } } else { nbConfigurations <- computeNbConfigurations(currentBudget, indexIteration, mu = scenario$mu, eachTest = scenario$eachTest, blockSize = blockSize, nElites = 0L, nOldInstances = 0L, newInstances = 0L) } # If a value was given as a parameter, then this value limits the maximum, # but if we have budget only for less than this, then we have run out of # budget. if (scenario$nbConfigurations > 0L) { if (scenario$nbConfigurations <= nbConfigurations) { nbConfigurations <- scenario$nbConfigurations } else if (currentBudget < remainingBudget) { # We skip one iteration catInfo("Not enough budget for this iteration, skipping to the next one.") indexIteration <- indexIteration + 1L next } else { catInfo("Stopped because ", "there is not enough budget to enforce the value of nbConfigurations.") return(irace_finish(iraceResults, scenario, reason = "Not enough budget to enforce the value of nbConfigurations")) } } # Stop if the number of configurations to test is NOT larger than the minimum. if (nbConfigurations <= minSurvival) { catInfo("Stopped because there is not enough budget left to race more than ", "the minimum (", minSurvival,").\n", "# You may either increase the budget or set 'minNbSurvival' to a lower value.") return(irace_finish(iraceResults, scenario, reason = "Not enough budget to race more than the minimum configurations")) } # If we have too many elite_configurations, reduce their number. This can # happen before the first race due to the initial budget estimation. if (firstRace) { if (nbConfigurations < nrow(elite_configurations)) { eliteRanks <- overall_ranks(iraceResults$experiments, test = scenario$testType) elite_configurations <- elite_configurations[order(eliteRanks), ] elite_configurations <- elite_configurations[seq_len(nbConfigurations), ] } } else if (nbConfigurations <= nrow(elite_configurations)) { # Stop if the number of configurations to produce is not greater than # the number of elites. catInfo("Stopped because ", "there is not enough budget left to race newly sampled configurations.") #(number of elites + 1) * (mu + min(5, indexIteration)) > remainingBudget" return(irace_finish(iraceResults, scenario, reason = "Not enough budget left to race newly sampled configurations")) } if (scenario$elitist) { # The non-elite have to run up to the first test. The elites consume # budget at most up to the new instances. if ((nbConfigurations - nrow(elite_configurations)) * scenario$mu + nrow(elite_configurations) * min(race_state$elitist_new_instances, scenario$mu) > currentBudget) { catInfo("Stopped because there is not enough budget left to race all configurations up to the first test (or mu).") return(irace_finish(iraceResults, scenario, reason = "Not enough budget to race all configurations up to the first test (or mu)")) } } else if (nbConfigurations * scenario$mu > currentBudget) { catInfo("Stopped because there is not enough budget left to race all configurations up to the first test (or mu).") return(irace_finish(iraceResults, scenario, reason = "Not enough budget to race all configurations up to the first test (or mu)")) } catInfo("Iteration ", indexIteration, " of ", nbIterations, "\n", "# experimentsUsed: ", experimentsUsed, "\n", if (scenario$maxTime == 0) "" else paste0("# timeUsed: ", timeUsed, "\n", "# boundEstimate: ", boundEstimate, "\n"), "# remainingBudget: ", remainingBudget, "\n", "# currentBudget: ", currentBudget, "\n", "# nbConfigurations: ", nbConfigurations, verbose = FALSE) iraceResults$softRestart[indexIteration] <- FALSE # Sample for the first time. if (firstRace) { # If we need more configurations, sample uniformly. nbNewConfigurations <- nbConfigurations - sum(allConfigurations[[".ID."]] %not_in% race_state$rejected_ids) if (nbNewConfigurations > 0L) { # Sample new configurations. if (debugLevel >= 1L) { catInfo("Sample ", nbNewConfigurations, " configurations from Sobol distribution", verbose = FALSE) } newConfigurations <- sampleSobol(scenario$parameters, nbNewConfigurations, repair = scenario$repairConfiguration) # We could get fewer than we asked for due to removing duplicates and forbidden. nbNewConfigurations <- nrow(newConfigurations) set(newConfigurations, j= ".ID.", value = max(0L, vlast(allConfigurations[[".ID."]])) + seq_nrow(newConfigurations)) setcolorder(newConfigurations, ".ID.", before=1L) setDF(newConfigurations) allConfigurations <- rbind(allConfigurations, newConfigurations) rownames(allConfigurations) <- allConfigurations[[".ID."]] raceConfigurations <- allConfigurations[allConfigurations[[".ID."]] %not_in% race_state$rejected_ids, , drop = FALSE] } else if (nbNewConfigurations <= 0L) { # We let the user know that not all configurations will be used. if (nbUserConfigurations > nbConfigurations) { catInfo("Only ", nbConfigurations, " from the initial configurations will be used", verbose = FALSE) } # This is made only in case that the number of configurations used in # the time estimation is more than needed. if (nrow(elite_configurations) == nbConfigurations) { raceConfigurations <- elite_configurations } else { raceConfigurations <- allConfigurations[allConfigurations[[".ID."]] %not_in% race_state$rejected_ids, , drop = FALSE] raceConfigurations <- raceConfigurations[seq_len(nbConfigurations), , drop = FALSE] } } # end of indexIteration == 1 } else { # How many new configurations should be sampled? nbNewConfigurations <- nbConfigurations - nrow(elite_configurations) # Update the model based on elites configurations if (debugLevel >= 1L) irace_note("Update model\n") model <- updateModel(scenario$parameters, elite_configurations, model, indexIteration, nbIterations, nbNewConfigurations, elitist = scenario$elitist) if (debugLevel >= 2L) printModel (model) if (debugLevel >= 1L) irace_note("Sample ", nbNewConfigurations, " configurations from model\n") irace_assert(is.integer(elite_configurations[[".ID."]])) newConfigurations <- sampleModel(scenario$parameters, elite_configurations, model, nbNewConfigurations, repair = scenario$repairConfiguration) # Set ID of the new configurations. set(newConfigurations, j = ".ID.", value = vlast(allConfigurations[[".ID."]]) + seq_nrow(newConfigurations)) setcolorder(newConfigurations, ".ID.", before=1L) setDF(newConfigurations) raceConfigurations <- rbind(elite_configurations[, colnames(newConfigurations)], newConfigurations) rownames(raceConfigurations) <- raceConfigurations[[".ID."]] if (scenario[["softRestart"]]) { # Rprof("profile.out") restart_ids <- similarConfigurations (raceConfigurations, scenario$parameters, threshold = scenario$softRestartThreshold) # Rprof(NULL) if (!is.null(restart_ids)) { if (debugLevel >= 1L) irace_note("Soft restart: ", paste(collapse = " ", restart_ids), " !\n") model <- restartModel(model, raceConfigurations, restart_ids, scenario$parameters, nbNewConfigurations) iraceResults$softRestart[indexIteration] <- TRUE if (debugLevel >= 2L) { printModel (model) } # Re-sample after restart like above #cat("# ", format(Sys.time(), usetz=TRUE), " sampleModel()\n") newConfigurations <- sampleModel(scenario$parameters, elite_configurations, model, nbNewConfigurations, repair = scenario$repairConfiguration) #cat("# ", format(Sys.time(), usetz=TRUE), " sampleModel() DONE\n") # Set ID of the new configurations. # Set ID of the new configurations. set(newConfigurations, j = ".ID.", value = vlast(allConfigurations[[".ID."]]) + seq_nrow(newConfigurations)) setcolorder(newConfigurations, ".ID.", before=1L) setDF(newConfigurations) raceConfigurations <- rbind(elite_configurations[, colnames(newConfigurations)], newConfigurations) rownames(raceConfigurations) <- raceConfigurations[[".ID."]] } } # Append these configurations to the global table. allConfigurations <- rbind(allConfigurations, newConfigurations) rownames(allConfigurations) <- allConfigurations[[".ID."]] } if (debugLevel >= 2L) { irace_note("Configurations for the race n ", indexIteration, " (elite configurations listed first, then new configurations):\n") configurations_print(raceConfigurations, metadata = TRUE) } # Get data from previous races. elite_data <- if (scenario$elitist && nrow(elite_configurations)) iraceResults$experiments[, as.character(elite_configurations[[".ID."]]), drop=FALSE] else NULL race_state$next_instance <- nrow(iraceResults$experiments) + 1L # Add instances if needed. # Calculate budget needed for old instances assuming non elitist irace. if ((nrow(race_state$instances_log) - (race_state$next_instance - 1L)) < ceiling(remainingBudget / minSurvival)) { generateInstances(race_state, scenario, n = ceiling(remainingBudget / minSurvival), update = TRUE) } if (debugLevel >= 1L) irace_note("Launch race\n") raceResults <- elitist_race (race_state, scenario = scenario, configurations = raceConfigurations, maxExp = currentBudget, minSurvival = minSurvival, elite_data = elite_data, elitist_new_instances = if (firstRace) 0L else race_state$elitist_new_instances) # We add indexIteration as an additional column. set(raceResults$experiment_log, j = "iteration", value = indexIteration) # FIXME: There is a chance that the process stops after we remove # race_experiment_log in elitist_race(), but before we update # race_state$experiment_log here. Doing the two steps in a different order # would be more robust but would need a smarter recovery routine that # checks for duplicates. race_state$experiment_log <- rbindlist(list(race_state$experiment_log, raceResults$experiment_log), use.names=TRUE) # Merge new results. iraceResults$experiments <- merge_matrix(iraceResults$experiments, raceResults$experiments) # Update remaining budget. experimentsUsed <- experimentsUsed + raceResults$experimentsUsed if (scenario$maxTime > 0L) { timeUsed <- timeUsed + sum(raceResults$experiment_log[["time"]], na.rm = TRUE) boundEstimate <- timeUsed / experimentsUsed remainingBudget <- round((scenario$maxTime - timeUsed) / boundEstimate) } else { remainingBudget <- remainingBudget - raceResults$experimentsUsed } if (debugLevel >= 3L) { irace_note("Results for the race of iteration ", indexIteration, " (from best to worst, according to the ", test.type.order.str(scenario$testType), "):\n") configurations_print(raceResults$configurations, metadata = TRUE) } if (debugLevel >= 1L) irace_note("Extracting elites\n") elite_configurations <- extractElites(raceResults$configurations, nbElites = minSurvival, debugLevel = scenario$debugLevel) irace_note("Elite configurations (first number is the configuration ID;", " listed from best to worst according to the ", test.type.order.str(scenario$testType), "):\n") if (!quiet) configurations_print(elite_configurations, metadata = debugLevel >= 1L) iraceResults$iterationElites[indexIteration] <- elite_configurations[[".ID."]][1L] iraceResults$allElites[[indexIteration]] <- elite_configurations[[".ID."]] if (firstRace) { if (debugLevel >= 1L) irace_note("Initialise model\n") model <- initialiseModel(scenario$parameters, elite_configurations) if (debugLevel >= 2L) printModel (model) firstRace <- FALSE } if (debugLevel >= 1L) { irace_note("End of iteration ", indexIteration, "\n") if (debugLevel >= 3L) { irace_note("All configurations (sampling order):\n") configurations_print(allConfigurations, metadata = TRUE) irace_note("Memory used in irace():\n") race_state$print_mem_used() } } indexIteration <- indexIteration + 1L } # end of repeat } irace/R/version.R0000644000176200001440000000016515060500541013344 0ustar liggesusers#' A character string containing the version of `irace` including git SHA. #' @export irace_version <- '4.3.c5b213a' irace/R/timer.R0000644000176200001440000000076714736526233013026 0ustar liggesusersTimer <- R6::R6Class("Timer", cloneable = TRUE, lock_class = TRUE, lock_objects = TRUE, public = list( start = NULL, initialize = function() { self$start <- proc.time() }, elapsed = function() { x <- proc.time() - self$start if (!is.na(x[[4L]])) x[[1L]] <- x[[1L]] + x[[4L]] if (!is.na(x[[5L]])) x[[2L]] <- x[[2L]] + x[[5L]] c(user=x[[1L]], system=x[[2L]], wallclock=x[[3L]]) }, wallclock = function() { proc.time()[[3L]] - self$start[[3L]] }) ) irace/R/tnorm.R0000644000176200001440000001040014736526233013026 0ustar liggesusers## The naive method would be: ## ## rtnorm.naive <- function(n, mean = 0, sd = 1, lower, upper) ## { ## lower <- (lower - mean) / sd ## Algorithm works on mean 0, sd 1 scale ## upper <- (upper - mean) / sd ## x <- rnorm(n) ## repeat { ## w <- which( x > upper | x < lower) ## if (length(w) == 0) break ## x[w] <- rnorm(length(w)) ## } ## return(x * sd + mean) ## } ## ## There is also https://cran.r-project.org/web/packages/truncnorm/ ## Function to sample according to a truncated normal distribution ## This function comes from the R package 'msm' maintained by ## Christopher Jackson # # Package: msm # Version: 1.7.1 # Date: 2023-03-23 # Title: Multi-State Markov and Hidden Markov Models in Continuous Time # Author: Christopher Jackson # Maintainer: Christopher Jackson # Description: Functions for fitting continuous-time Markov and hidden # Markov multi-state models to longitudinal data. Designed for # processes observed at arbitrary times in continuous time (panel data) # but some other observation schemes are supported. Both Markov # transition rates and the hidden Markov output process can be modelled # in terms of covariates, which may be constant or piecewise-constant # in time. # License: GPL (>= 2) # Imports: # survival,mvtnorm,expm # Suggests: # mstate,minqa,doParallel,foreach,numDeriv,testthat,flexsurv # URL: https://github.com/chjackson/msm # # Repository: CRAN ## Rejection sampling algorithm by Robert (Stat. Comp (1995), 5, 121-5) ## for simulating from the truncated normal distribution. rtnorm <- function (n, mean = 0, sd = 1, lower, upper) { stopifnot(length(n) == 1L && length(mean) == 1L && length(sd) == 1L && length(lower) == 1L && length(upper) == 1L) stopifnot(is.finite(lower) && is.finite(upper)) lower <- (lower - mean) / sd # Algorithm works on mean 0, sd 1 scale upper <- (upper - mean) / sd sdzero <- (sd < .Machine$double.eps) nas <- is.na(mean) || is.na(sd) || (sdzero && (lower > mean || mean > upper)) ret <- rep_len(NaN, n) if (nas) { warning("NAs produced") return(ret) # return NaN } else if (lower > upper) { return(ret) # return NaN } else if (sdzero) { return(rep_len(mean, n)) # SD zero, so set the sampled value to the mean. } else if ((lower < 0) && (upper > 0) && (upper - lower > sqrt(2*pi))) { # standard "simulate from normal and reject if outside limits" method. Use if bounds are wide. ind.no <- seq_len(n) while (length(ind.no) > 0) { y <- rnorm(length(ind.no)) done <- (y >= lower & y <= upper) ret[ind.no[done]] <- y[done] ind.no <- ind.no[!done] } } else if (lower >= 0 && (upper > lower + 2*sqrt(exp(1)) / (lower + sqrt(lower^2 + 4)) * exp((lower*2 - lower*sqrt(lower^2 + 4)) / 4))) { # rejection sampling with exponential proposal. Use if lower >> mean ind.expl <- seq_len(n) a <- (sqrt(lower^2 + 4) + lower) / 2 while (length(ind.expl) > 0) { z <- rexp(length(ind.expl), a) + lower u <- runif(length(ind.expl)) done <- (u <= exp(-(z - a)^2 / 2)) & (z <= upper) ret[ind.expl[done]] <- z[done] ind.expl <- ind.expl[!done] } } else if (upper <= 0 && (-lower > -upper + 2*sqrt(exp(1)) / (-upper + sqrt(upper^2 + 4)) * exp((upper*2 - -upper*sqrt(upper^2 + 4)) / 4))) { # rejection sampling with exponential proposal. Use if upper << mean. ind.expu <- seq_len(n) a <- (sqrt(upper^2 +4) - upper) / 2 while (length(ind.expu) > 0) { z <- rexp(length(ind.expu), a) - upper u <- runif(length(ind.expu)) done <- (u <= exp(-(z - a)^2 / 2)) & (z <= -lower) ret[ind.expu[done]] <- -z[done] ind.expu <- ind.expu[!done] } } else { # rejection sampling with uniform proposal. Use if bounds are narrow and central. ind.u <- seq_len(n) K <- if (lower > 0) lower^2 else if (upper < 0) upper^2 else 0 while (length(ind.u) > 0) { z <- runif(length(ind.u), lower, upper) rho <- exp((K - z^2) / 2) u <- runif(length(ind.u)) done <- (u <= rho) ret[ind.u[done]] <- z[done] ind.u <- ind.u[!done] } } ret*sd + mean } irace/R/path_rel2abs.R0000644000176200001440000000274214736603236014246 0ustar liggesusers#' Converts a relative path to an absolute path. #' #' If the path passed corresponds to an executable, it tries to find its path #' using [Sys.which()]. Expansion of `'~'` in Windows follows the definition #' of [fs::path_expand()] rather than [base::path.expand()]. This function #' tries really hard to create canonical paths. #' #' @param path (`character(1)`) Character string representing a relative path. #' @param cwd (`character(1)`) Current working directory. #' #' @return (`character(1)`) Character string representing the absolute path #' #' @examples #' path_rel2abs("..") #' @export path_rel2abs <- function (path, cwd = getwd()) { if (path == "") return("") # FIXME: split this function into two, one for executable files than handles # finding executables in the PATH with Sys.which() and another for everything # else that doesn't try to be that smart. # We need to do this even with the path returned by `getwd()`. cwd <- fs::path_norm(fs::path_expand(cwd)) if (fs::is_absolute_path(path)) { # Possibly expand ~/path to /home/user/path. path <- fs::path_expand(path) } else if (!startsWith(path, ".")) { # it may be a command in the path. sys_path <- suppressWarnings(Sys.which(path)) if (sys_path != "" && fs::file_access(sys_path, "execute") # fs::is_dir() is broken: https://github.com/r-lib/fs/issues/440 && !file.info(sys_path)$isdir) { path <- as.vector(sys_path) } } as.character(fs::path_abs(path, start = cwd)) } irace/R/aaa.R0000644000176200001440000000213715060047632012411 0ustar liggesusersupdate_package_version <- function() { dcf <- read.dcf(file = "DESCRIPTION", fields=c("Version", "RemoteSha")) if (NROW(dcf) < 1L) return(invisible()) dcf <- as.list(dcf[1L, ]) git_rev <- dcf$RemoteSha if (is.na(git_rev)) git_rev <- "unknown" git <- Sys.which("git") if (git != "" && fs::file_exists(".git") && grepl("[0-9a-z]+$", system2(git, "describe --first-parent --always", stdout = TRUE), perl=TRUE)) { git_rev <- system2(git, "describe --dirty --first-parent --always --exclude '*'", stdout = TRUE) } if (git_rev != "unknown") { realversion <- paste0(dcf$Version, ".", git_rev) cat(file='./R/version.R', sep='', "#' A character string containing the version of `irace` including git SHA.\n#' @export\nirace_version <- '", realversion, "'\n") } invisible() } update_package_version() # We define this tentatively to avoid: undefined exports: irace_version irace_version <- "unknown" .irace_tolerance <- sqrt(.Machine$double.eps) .get_time_next_save <- function(now) now + 60 # seconds # Prefix for printing messages to the user. .irace_msg_prefix <- "== irace == " irace/R/ablation.R0000644000176200001440000007702215055371305013466 0ustar liggesusers.ablation.params.def <- utils::read.table(header=TRUE, stringsAsFactors = FALSE, text=" name ab type short long default description iraceResults 0 p -l --log-file NA 'Path to the (.Rdata) file created by irace from which the \"iraceResults\" object will be loaded.' src 1 i -S --src 1 'Source configuration ID or the path to a file containing the configuration.' target 1 i -T --target NA 'Target configuration ID (by default the best configuration found by irace) or the path to a file containing the configuration.' ab_params 1 s -P --params '' 'Specific parameter names to be used for the ablation (separated with commas). By default use all' type 1 s -t --type 'full' 'Type of ablation to perform: \"full\" will execute each configuration on all \"--n-instances\" to determine the best-performing one; \"racing\" will apply racing to find the best configurations.' nrep 1 i -n --nrep 1 'Number of replications per instance used in \"full\" ablation.' seed 1 i '' --seed 1234567 'Integer value to use as seed for the random number generation.' ablationLogFile 1 p -o --output-file 'log-ablation.Rdata' 'Log file to save the ablation log. If \"\", the results are not saved to a file.' instancesFile 1 s '' --instances-file 'train' 'Instances file used for ablation: \"train\", \"test\" or a filename containing the list of instances.' plot 0 s -p --plot '' 'Output filename (.pdf) for the plot. If not given, no plot is created.' plot_type 0 s -O --plot-type 'mean' 'Type of plot. Supported values are \"mean\", \"boxplot\", \"rank\" or \"rank,boxplot\".' old_path 0 p '' --old-path NA 'Old path found in the log-file (.Rdata) given as input to be replaced by --new-path.' new_path 0 p '' --new-path NA 'New path to replace the path found in the log-file (.Rdata) given as input.' execDir 0 p -e --exec-dir NA 'Directory where the target runner will be run.' scenarioFile 0 p -s --scenario NA 'Scenario file to override the scenario given in the log-file (.Rdata)' parallel 0 i '' --parallel NA 'Number of calls to targetRunner to execute in parallel. Values 0 or 1 mean no parallelization.' ") ## __VERSION__ below will be replaced by the version defined in R/version.R ## This avoids constant conflicts within this file. cat_ablation_license <- function() { ablation_license <- '#------------------------------------------------------------------------------ # ablation: An implementation in R of Ablation Analysis # Version: __VERSION__ # Copyright (C) 2020--2025 # Manuel Lopez-Ibanez # Leslie Perez Caceres # # This is free software, and you are welcome to redistribute it under certain # conditions. See the GNU General Public License for details. There is NO # WARRANTY; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. #------------------------------------------------------------------------------ ' cat(sub("__VERSION__", irace_version, ablation_license, fixed=TRUE)) } #' Launch ablation with command-line options. #' #' Launch [ablation()] with the same command-line options as the command-line #' executable (`ablation.exe` in Windows). #' #' @param argv `character()`\cr The arguments #' provided on the R command line as a character vector, e.g., #' `c("-i", "irace.Rdata", "--src", 1)`. #' #' @details The function reads the parameters given on the command line #' used to invoke R, launches [ablation()] and possibly [plotAblation()]. #' #' List of command-line options: #' ```{r echo=FALSE,comment=NA} #' cmdline_usage(.ablation.params.def) #' ``` #' @return A list containing the following elements: #' \describe{ #' \item{allConfigurations}{Configurations tested in the ablation.} #' \item{state}{State of the ablation process.} #' \item{experiments}{A matrix with the results of the experiments (columns are configurations, rows are instances).} #' \item{scenario}{Scenario object with the settings used for the experiments.} #' \item{trajectory}{IDs of the best configurations at each step of the ablation.} #' \item{best}{Best configuration found in the experiments.} #' \item{complete}{`TRUE` if the ablation process was completed.} #' } #' #' @seealso [plotAblation()] [ablation()] #' @examples #' ablation_cmdline("--help") #' # Find the ablation command-line executable: #' Sys.glob(file.path(system.file(package="irace", "bin"), "ablation*")) #' @author Manuel López-Ibáñez #' @concept ablation #' @export ablation_cmdline <- function(argv = commandArgs(trailingOnly = TRUE)) { withr::local_options(list(width = 9999L)) # Do not wrap the output. cat_ablation_license() cat ("# installed at: ", system.file(package="irace"), "\n", "# called with: ", paste(argv, collapse = " "), "\n", sep = "") parser <- CommandArgsParser$new(argv = argv, argsdef = .ablation.params.def) if (!is.null(parser$readArg (short = "-h", long = "--help"))) { parser$cmdline_usage() return(invisible(NULL)) } if (!is.null(parser$readArg (short = "-v", long = "--version"))) { print(utils::citation(package="irace")) return(invisible(NULL)) } params <- parser$readAll() # TODO: Send the other options to the irace command-line parser so the user # can override options in scenario via the command-line. if (length(parser$argv) > 0L) stop("Unknown command-line options: ", paste(parser$argv, collapse = " ")) if (is.null(params$iraceResults)) { irace_error("You must provide the path to the '.Rdata' file that contains the 'iraceResults' object generated by irace.") return(invisible(NULL)) } iraceResults <- read_logfile(params$iraceResults) if (is.null(params$old_path) != is.null(params$new_path)) { irace_error("To update paths you must provide both --old-path and --new-path.") return(invisible(NULL)) } else if (!is.null(params$old_path)) { iraceResults$scenario <- scenario_update_paths(iraceResults$scenario, params$old_path, params$new_path) } scenario <- list() if (!is.null(params$scenarioFile)) { scenario <- readScenario(params$scenarioFile) # We do not want this seed value to override the the command-line. scenario$seed <- NULL } for (p in c("execDir", "parallel")) { if (!is.null(params[[p]])) scenario[[p]] <- params[[p]] } if (is_null_or_empty_or_na(trim(params$ablationLogFile))) { params$ablationLogFile <- NULL } else { params$ablationLogFile <- path_rel2abs(params$ablationLogFile) } if (!is.null(params$ab_params)) params$ab_params <- trimws(strsplit(params$ab_params, ",", fixed=TRUE)[[1L]]) # The shell may introduce extra quotes, remove them. params$plot_type <- trimws(gsub("[\"']", "", params$plot_type)) # We want to select elements that actually appear in params, otherwise we get NA names. ablation_params <- intersect(.ablation.params.def[.ablation.params.def$ab == 1, "name", drop=TRUE], names(params)) ablog <- do.call(ablation, args = c(list(iraceResults = iraceResults), params[ablation_params], scenario)) if (!is.null(params[["plot"]]) || base::interactive()) { plotAblation(ablog, pdf_file = params[["plot"]], type = params$plot_type) } invisible(ablog) } fix_dependents <- function(parameters, dep_names, new_conf, final_conf, changed) { # ... check every dependent parameter... for (dep in dep_names) { # If the conditions are satisfied and it doesn't have a value but it # should, then change it. If the conditions are not satisfied and it # has a value but it shouldn't, then change it. ok <- conditionsSatisfied(parameters$conditions[[dep]], new_conf) change <- ok == is.na(new_conf[[dep]]) dep_param <- parameters$get(dep) if (change) { # If it doesn't have a value but it should, take it from final. if (ok) { if (dep_param[["is_dependent"]]) { # If the parameter is domain-dependent, then: if (anyNA(new_conf[, .SD, .SDcols=all.vars(dep_param$domain)])) { # If it depends on a parameter that is disabled, then this is disabled, so no need to change. next } else if (is.na(final_conf[[dep]]) # If the final value is NA, we cannot fix it, so skip it. # If parameter X enables dep but dep has domain that depends on # parameter Y, then Y in new_conf may have a value that makes # final_conf[[dep]] invalid. We cannot fix this, so skip. || !is_within_dependent_bound(dep_param, new_conf, value = final_conf[[dep]])) return(NULL) } set(new_conf, j = dep, value = final_conf[[dep]]) } else { set(new_conf, j = dep, value = NA) } changed <- c(changed, dep) } else if (dep_param[["is_dependent"]] && !is.na(new_conf[[dep]])) { # The condition may not have changed but the domain may depend on a parameter that has been changed. if (anyNA(new_conf[, .SD, .SDcols=all.vars(dep_param$domain)])) { set(new_conf, j = dep, value = NA) changed <- c(changed, dep) } else if (!is_within_dependent_bound(dep_param, new_conf, value = new_conf[[dep]])) { # If the current value is not within bound and the final value is NA, we cannot fix it, so skip it. if (is.na(final_conf[[dep]]) # If parameter X enables dep but dep has domain that depends on # parameter Y, then Y in new_conf may have a value that makes # final_conf[[dep]] invalid. We cannot fix this, so skip. || !is_within_dependent_bound(dep_param, new_conf, value = final_conf[[dep]])) return(NULL) set(new_conf, j = dep, value = final_conf[[dep]]) changed <- c(changed, dep) } } } changed } ## Function that generates the configurations of the ablation path ## between initial_configuration and final_configuration. ## parameters can be selected by specifying them in para.names. generate_ablation <- function(initial_configuration, final_configuration, parameters, param_names) { # Only change variable parameters param_names <- setdiff(param_names, parameters$names_fixed) # Follow the hierarchy order. hierarchy <- sort(parameters$hierarchy) param_names <- intersect(names(parameters$hierarchy), param_names) configurations <- list() changed_params <- list() # skip if the value is the same or NA (not active) skip <- (initial_configuration[param_names] == final_configuration[param_names]) skip <- skip[1L, ] # Drop to a vector with names skip <- skip | is.na(skip) for (pname in param_names) { # If parameter is not active, it will only change if its condition becomes true. if (skip[[pname]]) next param <- parameters$get(pname) # If the parameter is domain-dependent, then we can only change its # value if the new value is within the domain. if (param[["is_dependent"]] && !is_within_dependent_bound(param, initial_configuration, value = final_configuration[[pname]])) next new_configuration <- as.data.table(initial_configuration) set(new_configuration, j = pname, value = final_configuration[[pname]]) changed <- pname # If other parameters depend on this one then... if (any(sapply(parameters$depends, function(x) any(pname %in% x)))) { changed <- fix_dependents(parameters, dep_names = names(hierarchy)[hierarchy > hierarchy[[pname]]], new_conf = new_configuration, final_conf = final_configuration, changed = changed) if (is.null(changed)) next } new_configuration <- filter_forbidden(new_configuration, parameters$forbidden) if (nrow(new_configuration) == 0L) next changed_params <- c(changed_params, list(changed)) configurations <- c(configurations, list(new_configuration)) } configurations <- rbindlist(configurations) set(configurations, j = ".PARENT.", value = configurations[[".ID."]]) setDF(configurations) rownames(configurations) <- NULL list(configurations=configurations, changed_params=changed_params) } report_duplicated_results <- function(experiments, configurations) { x <- t(experiments) x <- x[duplicated(x) | duplicated(x, fromLast = TRUE), , drop=FALSE] if (nrow(x) == 0L) return(NULL) # No duplicates dups <- split(rownames(x), apply(x, 1L, paste0, collapse="")) names(dups) <- NULL for (g in dups) { cat("Warning: The following configurations are different but produced the same results:\n") df <- configurations[configurations[[".ID."]] %in% g, , drop=FALSE] print(df) cat("Parameters with different values from the above configurations:\n") df <- removeConfigurationsMetaData(df) print(df[, vapply(df, function(x) length(unique(x)) > 1L, logical(1L)), drop=FALSE]) } dups } ab_generate_instances <- function(race_state, scenario, nrep, type, instancesFile) { nrep <- suppressWarnings(as.integer(nrep)) if (is.na(nrep) || length(nrep) == 0L || nrep <= 0L) stop("'nrep' must be an integer larger than zero") if (nrep != 1L && type == "racing") stop("'nrep' has no effect when type == 'racing'") if (nrep > 1L && scenario$deterministic) stop("'nrep > 1' does not make sense with a deterministic scenario") if (instancesFile == "test") { scenario$instances <- scenario$testInstances } else if (instancesFile != "train") { scenario$instances <- readInstances(instancesFile = path_rel2abs(instancesFile)) } n_inst <- length(scenario$instances) if (type == "full" && n_inst * nrep == 1L) stop("'nrep' must be larger than 1 when type == 'full' and a single instance") generateInstances(race_state, scenario, n_inst * nrep) msg <- if (instancesFile %in% c("train", "test")) paste0("'", instancesFile, "' instances") else paste0("instances from '", instancesFile, "'") if (n_inst > 100L) { n_inst <- 100L msg <- paste0(msg, " (only showing first 100)") } cat(sep="", "# Using ", msg, ":\n", paste0(collapse="\n", scenario$instances[1L:n_inst]), "\n") scenario$instances } #' Performs ablation between two configurations (from source to target). #' #' @description Ablation is a method for analyzing the differences between two configurations. #' #' @inheritParams has_testing_data #' @param src,target `integer(1)|character(1)`\cr Source and target configuration IDs. By default, the first configuration ever evaluated (ID 1) is used as `src` and the best configuration found by irace is used as target. If the argument is a string, it is interpreted as the path to a file, with the format specified by [readConfigurationsFile()], that contains the configuration. #' @param ab_params `character()`\cr Specific parameter names to be used for the ablation. They must be in `parameters$names`. By default, use all parameters. #' @param type `"full"|"racing"`\cr Type of ablation to perform: `"full"` will execute each configuration on all `n_instances` to determine the best-performing one; `"racing"` will apply racing to find the best configurations. #' @param nrep `integer(1)`\cr Number of replications per instance used in `"full"` ablation. When `nrep > 1`, each configuration will be executed `nrep` times on each instance with different random seeds. #' @param seed `integer(1)`\cr Integer value to use as seed for the random number generation. #' @param ablationLogFile `character(1)`\cr Log file to save the ablation log. If `NULL`, the results are not saved to a file. #' @param instancesFile `character(1)`\cr Instances file used for ablation: `'train'`, `'test'` or a filename containing the list of instances. #' @param ... Further arguments to override scenario settings, e.g., `debugLevel`, `parallel`, etc. #' #' @references #' C. Fawcett and H. H. Hoos. Analysing differences between algorithm #' configurations through ablation. Journal of Heuristics, 22(4):431–458, 2016. #' #' @inherit ablation_cmdline return #' @seealso [plotAblation()] [ablation_cmdline()] #' @examples #' \donttest{ #' logfile <- system.file(package="irace", "exdata", "sann.rda") #' # Execute ablation between the first and the best configuration found by irace. #' ablog <- ablation(logfile, ablationLogFile = NULL) #' plotAblation(ablog) #' # Execute ablation between two selected configurations, and selecting only a #' # subset of parameters, directly reading the setup from the irace log file. #' ablog <- ablation(logfile, src = 1, target = 10, #' ab_params = c("temp"), ablationLogFile = NULL) #' plotAblation(ablog, type = "mean") #' } #' #' @author Leslie Pérez Cáceres and Manuel López-Ibáñez #' @concept ablation #' @export ablation <- function(iraceResults, src = 1L, target = NULL, ab_params = NULL, type = c("full", "racing"), nrep = 1L, seed = 1234567L, ablationLogFile = "log-ablation.Rdata", instancesFile="train", ...) { # Input check if (missing(iraceResults) || is.null(iraceResults)) stop("You must provide an 'iraceResults' object generated by irace or the path to the '.Rdata' file that contains this object.") type <- match.arg(type) if (!is.null(ablationLogFile)) { file.check(ablationLogFile, writeable = TRUE, text = 'ablationLogFile') } save_ablog <- function(complete) { ablog <- list(changes = changes, # This name must match the one used in the irace log. allConfigurations = as.data.frame(all_configurations), experiments = experiments, state = race_state, scenario = scenario, trajectory = trajectory, best = best_configuration, complete = complete) if (!is.null(ablationLogFile)) save(ablog, file = ablationLogFile, version = 3L) invisible(ablog) } # Load the data of the log file. iraceResults <- read_logfile(iraceResults) log_version <- get_log_clean_version(iraceResults) if (log_version < "3.9.0") irace_error("The version of the logfile (", log_version, ") is too old for this version of ablation") if (is.null(iraceResults$state$completed) || length(iraceResults$state$completed) != 1L || iraceResults$state$completed == "Incomplete") stop("The 'iraceResults' logfile seems to belong to an incomplete run of irace.") scenario <- update_scenario(scenario = iraceResults$scenario, ...) scenario$logFile <- "" old_seed <- get_random_seed() on.exit(restore_random_seed(old_seed)) # FIXME: We need to overwrite the seed because RaceState takes it from the scenario and calls set_random_seed(). scenario$seed <- seed scenario <- checkScenario(scenario) race_state <- RaceState$new(scenario) # Generate instances scenario$instances <- ab_generate_instances(race_state, scenario, nrep, type, instancesFile) parameters <- scenario$parameters # Process src and target. if (is.character(src) && is.na(suppressWarnings(as.integer(src)))) { src_configuration <- readConfigurationsFile(src, parameters) if (nrow(src_configuration) != 1L) { stop("Argument of src=\"", src, "\" contains more than one configuration!") } src <- 1L + max(iraceResults$allConfigurations$.ID.) src_configuration$.ID. <- src src_configuration$.PARENT. <- NA_integer_ iraceResults$allConfigurations <- rbind.data.frame(iraceResults$allConfigurations, src_configuration) # FIXME: Check for duplicates } else if (src %not_in% iraceResults$allConfigurations[[".ID."]]) stop("Source configuration ID (", src, ") cannot be found!") if (is.null(target)) target <- iraceResults$iterationElites[length(iraceResults$iterationElites)] else if (is.character(target) && is.na(suppressWarnings(as.integer(target)))) { target_configuration <- readConfigurationsFile(target, parameters) if (nrow(target_configuration) != 1L) { stop("Argument of target=\"", target, "\" contains more than one configuration!") } target <- 1L + max(iraceResults$allConfigurations$.ID.) target_configuration$.ID. <- target target_configuration$.PARENT. <- NA_integer_ iraceResults$allConfigurations <- rbind.data.frame(iraceResults$allConfigurations, target_configuration) # FIXME: Check for duplicates } else if (target %not_in% iraceResults$allConfigurations[[".ID."]]) stop("Target configuration ID (", target, ") cannot be found!") if (src == target) stop("Source and target configuration IDs must be different!") irace_note ("Starting ablation from ", src, " to ", target, "\n# Seed: ", race_state$seed, "\n") cat("# Source configuration (row number is ID):\n") src_configuration <- iraceResults$allConfigurations[src, , drop = FALSE] configurations_print(src_configuration) cat("# Target configuration (row number is ID):\n") target_configuration <- iraceResults$allConfigurations[target, , drop = FALSE] configurations_print(target_configuration) # Select the parameters used for ablation if (is.null(ab_params)) { ab_params <- parameters$names } else if (!all(ab_params %in% parameters$names)) { irace_error("Some of the parameters provided (", paste0(setdiff(ab_params, parameters$names), collapse=", "), ") are not defined in the parameter space.") } # Select parameters that are different in both configurations neq_params <- which(src_configuration[,ab_params] != target_configuration[,ab_params]) if (length(neq_params) == 0L) irace_error("src and target configurations are equal considering the parameters selected.\n") param_names <- colnames(src_configuration[,ab_params])[neq_params] # FIXME: Do we really need to override the ID? src_configuration[[".ID."]] <- 1L target_configuration[[".ID."]] <- 2L all_configurations <- rbindlist(list(src_configuration, target_configuration), use.names=TRUE) race_state$start_parallel(scenario) on.exit(race_state$stop_parallel(), add = TRUE) # Execute source and target configurations. ## FIXME: We may already have these experiments in the logFile! irace_note("Executing source and target configurations on the given instances * nrep (", nrow(race_state$instances_log), ")...\n") experiments <- do_experiments(race_state, configurations = rbind.data.frame(src_configuration, target_configuration), ninstances = nrow(race_state$instances_log), scenario = scenario, iteration = 0L) irace_assert(ncol(experiments) == 2L) step <- 1L # Define variables needed trajectory <- 1L names(trajectory) <- "source" # FIXME: changes should only store the changed parameters. changes <- list() best_configuration <- src_configuration save_ablog(complete = FALSE) while (length(param_names) > 1L) { # Generate ablation configurations cat("# Generating configurations (row number is ID):", param_names, "\n") ab_aux <- generate_ablation(best_configuration, target_configuration, parameters, param_names) aconfigurations <- ab_aux$configurations if (nrow(aconfigurations) == 0L) { cat("# Stopping ablation, no parameter change possible.\n") break } ## FIXME: We may already have these configurations in the logFile! # New configurations IDs aconfigurations[[".ID."]] <- max(all_configurations[[".ID."]]) + seq_nrow(aconfigurations) configurations_print(aconfigurations, metadata = FALSE) all_configurations <- rbindlist(list(all_configurations, aconfigurations), use.names=TRUE) irace_note("Ablation (", type, ") of ", nrow(aconfigurations), " configurations on ", nrow(race_state$instances_log), " instances (this may take a while ...).\n") if (type == "full") { step_experiments <- do_experiments(race_state, configurations = aconfigurations, ninstances = nrow(race_state$instances_log), scenario = scenario, iteration = step) } else { # Set variables for the racing procedure # FIXME: what about blockSize? race_state$next_instance <- 1L race_output <- elitist_race(race_state, maxExp = nrow(aconfigurations) * nrow(race_state$instances_log), minSurvival = 1L, elite_data = NULL, configurations = aconfigurations, scenario = scenario, elitist_new_instances = 0L) set(race_output$experiment_log, j = "iteration", value = step) race_state$experiment_log <- rbindlist(list(race_state$experiment_log, race_output$experiment_log), use.names=TRUE) step_experiments <- race_output$experiments } experiments <- merge_matrix(experiments, step_experiments) save_ablog(complete = FALSE) # Get the best configuration based on the criterion of irace cranks <- overall_ranks(experiments[, aconfigurations[[".ID."]], drop=FALSE], test = scenario$testType) best_id <- which.min(cranks)[1L] changes[[step]] <- ab_aux$changed_params best_change <- changes[[step]][[best_id]] trajectory <- c(trajectory, aconfigurations[[".ID."]][best_id]) # Report best. cat("# Best changed parameters:\n") for (i in seq_along(best_change)) { cat("#", best_change[i], ":", best_configuration[,best_change[i]], "->", aconfigurations[best_id, best_change[i]], "\n") } best_configuration <- aconfigurations[best_id,,drop=FALSE] best_id <- best_configuration[[".ID."]] param_names <- param_names[param_names %not_in% best_change] step <- step + 1L } trajectory <- c(trajectory, target_configuration[[".ID."]]) # Get the overall best cranks <- overall_ranks(experiments[,trajectory, drop=FALSE], test = scenario$testType) best_id <- which.min(cranks)[1L] setDF(all_configurations, rownames = all_configurations[[".ID."]]) best_configuration <- all_configurations[trajectory[best_id], , drop=FALSE] irace_note("Final best configuration:\n") configurations_print(best_configuration) # Check for duplicated results: report_duplicated_results(experiments, all_configurations) save_ablog(complete = TRUE) } ablation_labels <- function(trajectory, configurations) { configurations <- removeConfigurationsMetaData(configurations[trajectory, , drop = FALSE]) labels <- names(trajectory) last <- configurations[1L, , drop = FALSE] param_names <- colnames(last) for (i in 2L:length(trajectory)) { current <- configurations[i, , drop = FALSE] # Select everything that is NOT NA now and was different or NA before. select <- !is.na(current) & (is.na(last) | (current != last)) irace_assert(!anyNA(select)) labels[i] <- paste0(param_names[select], "=", current[, select], collapse = "\n") last <- current } labels } #' Create plot from an ablation log #' #' @param ablog (`list()`|`character(1)`) Ablation log object returned by [ablation()]. Alternatively, the path to an `.Rdata` file, e.g., `"log-ablation.Rdata"`, from which the object will be loaded. #' @param pdf_file Output filename. #' @param width Width provided to create the PDF file. #' @param height Height provided to create the PDF file. #' @param type Type of plot. Supported values are `"mean"` and `"boxplot"`. Adding `"rank"` will plot rank per instance instead of raw cost value. #' @param n `integer(1)`\cr Number of parameters included in the plot. By default all parameters are included. #' @param mar Vector with the margins for the ablation plot. #' @param ylab Label of y-axis. #' @param ylim Numeric vector of length 2 giving the y-axis range. #' @param rename_labels `character()`\cr Renaming table for nicer labels. For example, `c("No value"="NA", "LongParameterName"="LPN")`. #' @param ... Further graphical parameters may also be supplied as #' arguments. See [graphics::plot.default()]. #' #' @author Leslie Pérez Cáceres and Manuel López-Ibáñez #' @seealso [ablation()] [ablation_cmdline()] #' @examples #' logfile <- file.path(system.file(package="irace"), "exdata", "log-ablation.Rdata") #' plotAblation(ablog = logfile) #' plotAblation(ablog = logfile, type = "mean") #' plotAblation(ablog = logfile, type = c("rank","boxplot"), rename_labels = c( #' "localsearch"="ls", algorithm="algo", source="default")) #' @concept ablation #' @export plotAblation <- function (ablog, pdf_file = NULL, width = 20, height = 7, type = c("mean", "boxplot", "rank"), n = 0L, mar = NULL, ylab = "Mean configuration cost", ylim = NULL, rename_labels = NULL, ...) { type <- trimws(unlist(strsplit(type, ",", fixed=TRUE))) type <- match.arg(type, several.ok = TRUE) if (missing(ylab) && ("rank" %in% type)) ylab <- "Rank per instance" if (missing(ablog) || is.null(ablog)) { irace_error("You must provide an 'ablog' object generated by ablation() or the path to the '.Rdata' file that contains this object.") } ablog <- read_ablogfile(ablog) if (!ablog$complete) stop("The ablog shows that the ablation procedure did not complete cleanly and only contains partial information") if (!is.null(pdf_file)) { if (!is.file.extension(pdf_file, ".pdf")) pdf_file <- paste0(pdf_file, ".pdf") cat("Creating PDF file '", pdf_file, "'\n", sep="") local_cairo_pdf(pdf_file, width = width, height = height, onefile= TRUE) } configurations <- ablog$allConfigurations # Support irace < 4.2 if (is.null(configurations)) configurations <- ablog$configurations trajectory <- ablog$trajectory if (n > 0L) trajectory <- trajectory[seq_len(n+1L)] # Generate labels labels <- ablation_labels(trajectory, configurations) if (!is.null(rename_labels)) # stringr::str_replace_all() would be better but it has so many dependencies! for (i in seq_along(rename_labels)) labels <- gsub(names(rename_labels)[i], rename_labels[i], labels) experiments <- ablog$experiments if ("rank" %in% type) { experiments <- rowRanks(experiments, ties.method = "average") if (is.null(ylim)) ylim <- c(1L, ncol(experiments)) } costs_avg <- colMeans2(experiments, cols = trajectory) if (is.null(mar)) mar <- par("mar") inches_to_lines <- (mar / par("mai"))[1L] lab_width <- max(strwidth(labels, units = "inches")) * inches_to_lines with_par(list(mar = c(lab_width + 2.1, 4.1, 0.1, 0.1), cex.axis = 1), { # FIXME: We could also show the other alternatives at each step not just the # one selected. See Leonardo Bezerra's thesis. if ("boxplot" %in% type) { bx <- boxplot(experiments[, trajectory], plot=FALSE) if (is.null(ylim)) { ylim <- range(bx$stats[is.finite(bx$stats)], bx$out[is.finite(bx$out)], bx$conf[is.finite(bx$conf)]) } } # Extra whitespace around boxplots. xlim <- c(0.6, length(trajectory) + 0.4) plot(costs_avg, xaxt = "n", xlab = NA, ylab = ylab, xlim = xlim, ylim = ylim, type = "b", pch = 19, ..., panel.first = { grid(nx = NA, ny = NULL, lwd = 2); abline(h = c(costs_avg[1], utils::tail(costs_avg, n = 1L)), col = "lightgray", lty = "dotted", lwd = 2) }) axis(1, at = seq_along(costs_avg), labels = labels, las = 3) if ("boxplot" %in% type) { bxp(bx, show.names = FALSE, add = TRUE) } }) invisible() } #' Read the log file (`log-ablation.Rdata`) produced by [irace::ablation()]. #' #' @param filename `character(1)`\cr Filename that contains the log file saved by [ablation()]. Example: `log-ablation.Rdata`. #' #' @return `list()` #' @concept ablation #' @export read_ablogfile <- function(filename) read_logfile(filename, name = "ablog") irace/R/readConfiguration.R0000644000176200001440000002424415024011076015326 0ustar liggesusers#' Read parameter configurations from a file #' #' Reads a set of target-algorithm configurations from a file and puts them in #' \pkg{irace} format. The configurations are checked to match the parameters #' description provided. #' #' @param filename `character(1)`\cr Filename from which the configurations should be read. #' The contents should be readable by `read.table( , header=TRUE)`. #' @param text `character(1)`\cr If \code{file} is not supplied and this is, #' then configurations are read from the value of \code{text} via a text connection. #' @inheritParams readParameters #' @inheritParams printParameters #' #' @return A data frame containing the obtained configurations. #' Each row of the data frame is a candidate configuration, #' the columns correspond to the parameter names in `parameters`. #' #' @details #' Example of an input file: #' ``` #' # This is a comment line #' param_1 param_2 #' 0.5 "value_1" #' 1.0 NA #' 1.2 "value_3" #' ``` #' The order of the columns does not necessarily have to be the same #' as in the file containing the definition of the parameters. #' #' @seealso #' [readParameters()] to obtain a valid parameter structure from a parameters file. #' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export readConfigurationsFile <- function(filename, parameters, debugLevel = 0L, text) { if (missing(filename) && !missing(text)) { filename <- NULL configurationTable <- utils::read.table(text = text, header = TRUE, na.strings = c("NA", ""), colClasses = "character", stringsAsFactors = FALSE) } else { # Read the file. configurationTable <- utils::read.table(filename, header = TRUE, na.strings = c("NA", ""), colClasses = "character", stringsAsFactors = FALSE) } irace_assert(is.data.frame(configurationTable)) irace_note("Read ", nrow(configurationTable), " configuration(s)", if (is.null(filename)) "\n" else paste0(" from file '", filename, "'\n")) fix_configurations(configurationTable, parameters, debugLevel = debugLevel, filename = filename) } fix_configurations <- function(configurations, parameters, debugLevel = 0L, filename = NULL) { conf_error <- function(k, ...) irace_error("Configuration number ", k, if (is.null(filename)) "" else paste0(" from file '", filename, "'"), ...) if (debugLevel >= 2L) print(configurations, digits=15L) namesParameters <- parameters[["names"]] # This ignores fixed parameters unless they are given with a different value. if (ncol(configurations) != length(namesParameters) || !setequal(colnames(configurations), namesParameters)) { # Column names must match a parameter, including fixed ones. missing <- setdiff(colnames(configurations), namesParameters) if (length(missing) > 0L) { if (is.null(filename)) { irace_error("The parameter names (", strlimit(paste(missing, collapse=", ")), ") do not match the parameter names: ", paste(namesParameters, collapse=", ")) } else { irace_error("The parameter names (", strlimit(paste(missing, collapse=", ")), ") given in the first row of file ", filename, " do not match the parameter names: ", paste(namesParameters, collapse=", ")) } return(NULL) } # All non-fixed parameters must appear in column names. varParameters <- parameters$names_variable missing <- setdiff(varParameters, colnames(configurations)) if (length(missing) > 0) { if (is.null(filename)) { irace_error("The parameter names (", strlimit(paste(missing, collapse=", ")), ") are missing from the configurations provided.") } else { irace_error("The parameter names (", strlimit(paste(missing, collapse=", ")), ") are missing from the first row of file ", filename) } return(NULL) } # Add any missing fixed parameters. missing <- setdiff(namesParameters, colnames(configurations)) if (length(missing) > 0L) { irace_assert(all(parameters$isFixed[missing])) configurations <- cbind.data.frame(configurations, parameters$domains[missing], stringsAsFactors = FALSE) # We can have fixed parameters that are conditional and disabled, # thus their value should be NA. for (pname in missing) { param <- parameters$get(pname) condition <- param[["condition"]] for (k in seq_nrow(configurations)) { if (!conditionsSatisfied(condition, configurations[k, ])) { configurations[k, pname] <- NA } } } } } # Reorder columns. configurations <- configurations[, namesParameters, drop = FALSE] # Loop over all parameters. for (param in parameters$get()) { pname <- param[["name"]] type <- param[["type"]] domain <- param[["domain"]] is_dep_param <- param[["is_dependent"]] condition <- param[["condition"]] # Fix up numeric columns. if (type == "i") { # For integers, only accept an integer. configurations[[pname]] <- suppressWarnings(as.numeric(configurations[[pname]])) # Remove NAs for this check. values <- configurations[[pname]] values[is.na(values)] <- 0L if (any(as.integer(values) != values)) { k <- which(as.integer(values) != values)[1L] conf_error (k, " is invalid because parameter ", pname, " is of type integer but its value ", values[k], " is not an integer") return(NULL) } } else if (type == "r") { configurations[[pname]] <- round( suppressWarnings(as.numeric(configurations[[pname]])), digits = parameters$get(pname)[["digits"]]) } # Loop over all configurations. # FIXME: Vectorize this loop values <- configurations[[pname]] for (k in seq_nrow(configurations)) { currentValue <- values[k] # Check the status of the conditions for this parameter to know whether # it must be enabled. if (conditionsSatisfied(condition, configurations[k, ])) { # Check that the value is among the valid ones. if (is_dep_param) { dep_domain <- getDependentBound(param, configurations[k, ]) if (is.na(dep_domain[1L])) { # Dependencies are not satisfied, so skip if (is.na(currentValue)) next conf_error (k, " is invalid because parameter \"", pname, "\" is not enabled, because its domain ", sub("expression", "", deparse(domain)), " depends on parameters that are not enabled, but its value is \"", currentValue, "\" instead of NA") return(NULL) } lower <- dep_domain[1L] upper <- dep_domain[2L] if (is.na(currentValue) || currentValue < lower || currentValue > upper) { conf_error (k, " is invalid because the value \"", configurations[k, pname], "\" for the parameter ", pname, " is not within the valid range ", sub("expression", "", deparse(domain)), ", that is, [", lower,", ", upper,"]") return(NULL) } } else if (type == "i" || type == "r") { lower <- domain[[1L]] upper <- domain[[2L]] if (is.na(currentValue) || currentValue < lower || currentValue > upper) { conf_error (k, " is invalid because the value \"", configurations[k, pname], "\" for the parameter ", pname, " is not within the valid range [", lower,", ", upper,"]") return(NULL) } # type == "o" or "c" } else if (currentValue %not_in% domain) { conf_error (k, " is invalid because the value \"", currentValue, "\" for the parameter \"", pname, "\" is not among the valid values: (\"", paste0(domain, collapse="\", \""), "\")") return(NULL) } } else if (!is.na(currentValue)) { conf_error (k, " is invalid because parameter \"", pname, "\" is not enabled because of condition \"", param[["condition"]], "\" but its value is \"", currentValue, "\" instead of NA") return(NULL) } } } if (anyDuplicated(configurations)) { irace_error("Duplicated configurations", if (is.null(filename)) "" else paste0(" in file '", filename, "'"), ":\n", paste0(utils::capture.output( configurations[duplicated(configurations), , drop=FALSE]), "\n")) } configurations } compile_forbidden <- function(x) { if (is.null(x) || is.bytecode(x)) return(x) # If we are given an expression, it must be a single one. irace_assert(is.language(x) && (!is.expression(x) || length(x) == 1L)) if (is.expression(x)) x <- x[[1L]] # When a is NA and we check a == 5, we would get NA, which is # always FALSE, when we actually want to be TRUE, so we test # is.na() first below. # We expect that there will be undefined variables, since the expressions # will be evaluated within a data.frame later. expr <- compiler::compile(substitute(is.na(x) | !(x), list(x = x)), options = list(suppressUndefined=TRUE)) attr(expr, "source") <- as.character(as.expression(x)) expr } buildForbiddenExp <- function(configurations) { if (is.null(configurations) || nrow(configurations) == 0L) return(NULL) pnames <- colnames(configurations) lines <- c() # We cannot use apply() because it converts numeric to character. for (k in seq_nrow(configurations)) { values <- as.list(configurations[k, ]) has.value <- !is.na(values) values <- lapply(values[has.value], function(x) deparse(substitute(x, list(x=x)))) lines <- c(lines, paste0("(", pnames[has.value]," == ", values, ")", collapse = "&")) } exps <- parse(text = lines) lapply(exps, compile_forbidden) } irace/R/cluster.R0000644000176200001440000001157614753644141013365 0ustar liggesusers### Submit/wait for jobs in batch clusters. sge.job.finished <- function(jobid) system (paste0("qstat -j ", jobid), ignore.stdout = TRUE, ignore.stderr = TRUE, intern = FALSE, wait = TRUE) pbs.job.finished <- function(jobid) system (paste0("qstat ", jobid), ignore.stdout = TRUE, ignore.stderr = TRUE, intern = FALSE, wait = TRUE) torque.job.finished <- function(jobid) { output <- suppressWarnings(system2("qstat", jobid, stdout = TRUE, stderr = TRUE)) ## 1. If the return code of qstat in not 0, then no such job is in the queue ## anymore. That means that the job was (assumption) successfully submitted, ## executed and completed. Then, for some time the job id was marked as ## completed in the jobs queue and the job status removed from the queue list ## after some time. No job completed status of successfully submitted and ## completed jobs in the queue list happens if a small task was executed ## first and a task of some hours execution time finishes much later. The job ## status of the small task will not be in the queue anymore after some ## minutes. So: If no job id is in the queue list anymore and torq's qstat ## returns an error (return code > 0), then the job has been successfully ## executed (if it has been started successfully before). if (!is.null(attr(output, "status"))) return(TRUE) # 2. If qstat returns OK (return code ==0), then one has to parse qstat's # output. If the 5th token in the last line is a 'C', then the job has # terminated and its output files can be processed. Otherwise the job is not # completed (queued, running, exiting...) any(grepl(paste0(jobid, ".*\\sC\\s"), output)) } slurm.job.finished <- function(jobid) { output <- suppressWarnings(system2("squeue", c("-j", jobid, "--noheader"), stdout = TRUE, stderr = TRUE)) # If the above returns non-zero, either the job terminated or it never # existed. if (!is.null(attr(output, "status"))) return(TRUE) # If may return zero, but the job is not in the system anymore because it # completed. This is different from the Torque case. !any(grepl(paste0("\\s", jobid, "\\s"), output)) } htcondor.job.finished <- function(jobid) { output <- suppressWarnings(system2("condor_q", jobid, stdout = TRUE, stderr = TRUE)) # Check if job is still in the queue, otherwise it is considered finished !any(grepl(paste0("ID:\\s", jobid), output)) } ## Launch a job with qsub and return its jobID. This function does not ## call qsub directly, but instead targetRunner should be a script that ## invokes qsub and returns a jobID. target_runner_qsub <- function(experiment, scenario) { debugLevel <- scenario$debugLevel res <- run_target_runner(experiment, scenario) cmd <- res$cmd output <- res$output args <- res$args jobID <- NULL outputRaw <- output$output err.msg <- output$error if (is.null(err.msg)) { # We cannot use parse.output because that tries to convert to numeric. if (debugLevel >= 2) { cat (outputRaw, sep = "\n") } # Initialize output as raw. If it is empty stays like this. # strsplit crashes if outputRaw == character(0) if (length(outputRaw) > 0) { jobID <- strsplit(trim(outputRaw), "[[:space:]]+")[[1]] } if (length(jobID) != 1) { err.msg <- paste0("The output of targetRunner should be only the jobID!") jobID <- NULL } } list(jobID = jobID, error = err.msg, outputRaw = outputRaw, call = paste(cmd, args)) } cluster_lapply <- function(X, scenario, poll.time = 2) { debugLevel <- scenario$debugLevel cluster.job.finished <- switch(scenario$batchmode, sge = sge.job.finished, pbs = pbs.job.finished, torque = torque.job.finished, slurm = slurm.job.finished, htcondor = htcondor.job.finished, irace_error ("Invalid value of scenario$batchmode = ", scenario$batchmode)) # Parallel controls how many jobs we send at once. Some clusters have low # limits. ## FIXME: It would be better to submit up to the limit, then one by one as jobs finish. chunksize <- scenario$parallel if (chunksize < 0L) { chunksize <- length(X) } chunks <- split(X, ceiling(seq_along(X) / chunksize)) for (chunk in chunks) { if (debugLevel >= 1) { irace_note ("Sending ", length(chunk), " / ", length(X), " jobs\n") } output <- lapply(chunk, exec_target_runner, scenario = scenario, target_runner = target_runner_qsub) jobIDs <- sapply(output, "[[", "jobID") ## Wait for cluster jobs to finish. if (length(jobIDs) > 0L && debugLevel >= 1L) { irace_note("Waiting for jobs ('.' == ", poll.time, " s): ") } for (jobID in jobIDs) { while (!cluster.job.finished(jobID)) { if (debugLevel >= 1) { cat(".") } Sys.sleep(poll.time) } if (debugLevel >= 1) cat(" DONE(", jobID, ") ") } if (debugLevel >= 1) cat("\n") } output } irace/R/model.R0000644000176200001440000001531014741742510012766 0ustar liggesusers#################################################### ## INITIALISE AND UPDATE THE MODEL #################################################### # Initial standard deviation for numerical sampling model. init_sd_numeric <- function(param) { # Dependent parameters define the standard deviation as # a portion of the size of the domain interval. In this case, # 0.5 indicates half of the interval, equivalent to # (domain[2] - domain[1]) * 0.5 if (param[["is_dependent"]] || param[["transform"]] == "log") return(0.5) domain <- param[["domain"]] (domain[[2L]] - domain[[1L]]) * 0.5 } ## Initialisation of the model after the first iteration ## # IN: configurations matrix, parameters datastructure ## # OUTPUT: A list of list of vectors. The higher-level list contains # one element per categorical parameter. Each categorical parameter # contains a list of vector. This list contains elements which are the # .ID. of the configuration. initialiseModel <- function (parameters, configurations) { nbConfigurations <- nrow(configurations) ids <- as.character(configurations[[".ID."]]) param_names <- parameters$names_variable model <- setNames(vector("list", length(param_names)), param_names) for (currentParameter in param_names) { param <- parameters$get(currentParameter) type <- param[["type"]] if (type == "c") { nbValues <- length(param[["domain"]]) value <- rep_len(1. / nbValues, nbValues) param <- rep_len(list(value), nbConfigurations) } else { if (type == "r" || type == "i") { sd <- init_sd_numeric(param) values <- configurations[[currentParameter]] } else if (type == "o") { domain <- param[["domain"]] sd <- (length(domain) - 1L) * 0.5 values <- match(configurations[[currentParameter]], domain) } else { irace_internal_error("Unknown parameter type '", type, "'") } # Assign current parameter value to model. param <- mapply(c, sd, values, SIMPLIFY=FALSE, USE.NAMES=FALSE) } names(param) <- ids model[[currentParameter]] <- param } model } updateModel <- function(parameters, eliteConfigurations, oldModel, indexIteration, nbIterations, nbNewConfigurations, elitist) { dec_factor <- (1 - ((indexIteration - 1) / nbIterations)) add_factor <- ((indexIteration - 1) / nbIterations) num_factor <- ((1 / nbNewConfigurations)^(1 / parameters$nbVariable)) prob_max <- 0.2^(1. / parameters$nbVariable) update_prob <- if (elitist) function(p, idx) { # Decrease first all values in the vector: p <- p * dec_factor p[idx] <- p[idx] + add_factor # Normalize probabilities. p <- p / sum(p) # Prevent probabilities from growing too much. p <- pmin.int(p, prob_max) p <- p / sum(p) p / sum(p) } else function(p, idx) { # Decrease first all values in the vector: p <- p * dec_factor p[idx] <- p[idx] + add_factor # Normalize probabilities. p <- p / sum(p) } param_names <- parameters$names_variable model_ids <- names(oldModel[[1L]]) # If the elite is older than the current iteration, it has its own model # that has evolved with time. If the elite is new (generated in the current # iteration), it does not have any, and we have to copy the one from its # parent. The condition of the IF statement is for checking whether the # configuration already has its model or not. elite_ids <- as.character(eliteConfigurations[[".ID."]]) not_in <- elite_ids %not_in% model_ids ids_in_model <- elite_ids # If a configuration does not have any entry, copy the parent one. ids_in_model[not_in] <- as.character(eliteConfigurations[[".PARENT."]][not_in]) newModel <- setNames(vector("list", length(param_names)), param_names) for (currentParameter in param_names) { param <- parameters$get(currentParameter) irace_assert(all(ids_in_model %in% names(oldModel[[currentParameter]]))) this_model <- oldModel[[currentParameter]][ids_in_model] values <- eliteConfigurations[[currentParameter]] values_not_na <- !is.na(values) values <- values[values_not_na] type <- param[["type"]] if (type == "c") { # Find the value that has been "chosen" to increase its probability. values <- match(values, param[["domain"]]) this_model[values_not_na] <- mapply(update_prob, this_model[values_not_na], values, SIMPLIFY=FALSE) } else { irace_assert(type %in% c("i", "r", "o")) if (type == "o") values <- match(values, param[["domain"]]) this_model[values_not_na] <- mapply(function(p, value) c(p[[1L]] * num_factor, value), this_model[values_not_na], values, SIMPLIFY=FALSE) } names(this_model) <- elite_ids newModel[[currentParameter]] <- this_model } newModel } printModel <- function (model) { cat("# Model:\n") print(model) } restartModel <- function(model, configurations, restart_ids, parameters, nbConfigurations) { back_factor <- nbConfigurations^(2 / parameters$nbVariable) second_factor <- (1 / nbConfigurations)^(1 / parameters$nbVariable) model_ids <- names(model[[1L]]) restart_ids <- as.character(sort.int(as.integer(restart_ids))) not_in <- restart_ids %not_in% model_ids configurations <- configurations[configurations[[".ID."]] %in% restart_ids, c(".ID.", ".PARENT.")] restart_ids[not_in] <- configurations[[".PARENT."]][order(as.integer(configurations[[".ID."]]))][not_in] restart_ids <- as.character(unique(restart_ids)) restart_ids <- restart_ids[!is.na(restart_ids)] for (pname in parameters$names_variable) { model_param <- model[[pname]] irace_assert (all(restart_ids %in% names(model_param)), { cat("Param:", pname, "\n") print(restart_ids) print(model) print(configurations[, c(".ID.", ".PARENT.")]) }) param <- parameters$get(pname) type <- param[["type"]] if (type == "c") { model[[pname]][restart_ids] <- sapply(model_param[restart_ids], function(p) { p <- 0.9 * p + 0.1 * max(p) p / sum(p) }, simplify=FALSE) } else { if (type == "i" || type == "r") { value <- init_sd_numeric(param) } else { irace_assert(type == "o") value <- (length(param[["domain"]]) - 1L) * 0.5 } # Bring back the value 2 iterations or to the second iteration value. value <- value * second_factor model[[pname]][restart_ids] <- sapply( model_param[restart_ids], function(x) c(min(x[[1L]] * back_factor, value), x[[2L]]), simplify=FALSE) } } model } irace/R/irace_summarise.R0000644000176200001440000000473014736526233015050 0ustar liggesusers#' Summarise the results of a run of irace #' #' @inheritParams has_testing_data #' #' @return `list()` #' #' @examples #' irace_results <- read_logfile(system.file("exdata/irace-acotsp.Rdata", #' package="irace", mustWork=TRUE)) #' irace_summarise(irace_results) #' #' @author Manuel López-Ibáñez #' @concept analysis #' @export irace_summarise <- function(iraceResults) { if (missing(iraceResults)) stop("argument 'iraceResults' is missing") iraceResults <- read_logfile(iraceResults) if (is.null(iraceResults$state$elapsed)) { time_cpu_user <- time_cpu_sys <- time_cpu_total <- time_wallclock <- NA } else { time_cpu_user <- iraceResults$state$elapsed[["user"]] time_cpu_sys <- iraceResults$state$elapsed[["system"]] time_cpu_total <- time_cpu_user + time_cpu_sys time_wallclock <- iraceResults$state$elapsed[["wallclock"]] } n_iterations <- length(iraceResults$allElites) experiment_log <- iraceResults$state$experiment_log if (is.null(experiment_log)) { experiment_log <- iraceResults$experimentLog } else if (is.null(experiment_log)) stop("Experiment log is NULL") version <- iraceResults$irace_version if (is.null(version)) version <- iraceResults$irace.version # Here to support older versions of irace. time_targetrunner <- iraceResults$state$recovery_info$timeUsed if (is.null(time_targetrunner)) time_targetrunner <- iraceResults$state$timeUsed rejected_ids <- iraceResults$state$rejected_ids if (is.null(rejected_ids)) rejected_ids <- iraceResults$state$rejectedIDs list( version = version, n_iterations = n_iterations, n_configurations = nrow(iraceResults$allConfigurations), n_initial_configurations = if (is.null(iraceResults$scenario$initConfigurations)) 0L else nrow(iraceResults$scenario$initConfigurations), n_instances = nrow(iraceResults$experiments), n_experiments = if (is.null(iraceResults$scenario$targetEvaluator)) nrow(experiment_log) else sum(!is.na(iraceResults$experiments)), n_elites = length(iraceResults$allElites[[n_iterations]]), n_soft_restarts = sum(iraceResults$softRestart), n_rejected = length(rejected_ids), time_targetrunner = time_targetrunner, time_cpu_user = time_cpu_user, time_cpu_sys = time_cpu_sys, time_cpu_total = time_cpu_total, time_wallclock = time_wallclock, termination_reason = if (is.null(iraceResults$state$completed)) "Missing" else iraceResults$state$completed ) } irace/R/testing.R0000644000176200001440000000721414736526233013355 0ustar liggesusers#' Execute the given configurations on the testing instances specified in the #' scenario #' #' @inheritParams removeConfigurationsMetaData #' @inheritParams defaultScenario #' #' @return A list with the following elements: #' \describe{ #' \item{\code{experiments}}{Experiments results.} #' \item{\code{seeds}}{Array of the instance seeds used in the experiments.} #' } #' #' @details A test instance set must be provided through `scenario[["testInstances"]]`. #' #' @seealso #' [testing_fromlog()] #' #' @author Manuel López-Ibáñez #' @export testConfigurations <- function(configurations, scenario) { # We need to set up a default scenario (and repeat all checks) in case # we are called directly instead of being called after executing irace. scenario <- checkScenario(scenario) testInstances <- scenario[["testInstances"]] instances_id <- names(testInstances) if (length(testInstances) == 0L) irace_error("No test instances given") if (is.null(instances_id)) irace_error("testInstances must have names") # 2147483647 is the maximum value for a 32-bit signed integer. # We use replace = TRUE, because replace = FALSE allocates memory for each possible number. ## FIXME: scenario[["testInstances"]] and scenario$instances behave differently, ## we should unify them so that the seeds are also saved in scenario. instanceSeed <- runif_integer(length(testInstances)) names(instanceSeed) <- instances_id # If there is no ID (e.g., after using readConfigurations), then add it. if (".ID." %not_in% colnames(configurations)) configurations[[".ID."]] <- seq_nrow(configurations) # Create experiment list experiments <- createExperimentList(configurations, parameters = scenario$parameters, instances = testInstances, instances_ID = instances_id, seeds = instanceSeed, # We cannot use rep.int because scenario$boundMax may be NULL. bounds = rep(scenario$boundMax, nrow(configurations))) race_state <- RaceState$new(scenario) if (scenario$debugLevel >= 3L) { irace_note ("Memory used before execute_experiments() in testConfigurations():\n") race_state$print_mem_used() } race_state$start_parallel(scenario) on.exit(race_state$stop_parallel()) # We cannot let targetRunner or targetEvaluator modify our random seed, so we save it. withr::local_preserve_seed() target_output <- execute_experiments(race_state, experiments, scenario) # targetEvaluator may be NULL. If so, target_output must contain the right # output already. if (!is.null(scenario$targetEvaluator)) target_output <- execute_evaluator(race_state$target_evaluator, experiments, scenario, target_output) # FIXME: It would be much faster to convert target_output to a data.table like we do in race_wrapper(), # then dcast() to a matrix like we do elsewhere. testResults <- matrix(NA, ncol = nrow(configurations), nrow = length(testInstances), # dimnames = list(rownames, colnames) dimnames = list(instances_id, configurations$.ID.)) cost <- unlist_element(target_output, "cost") if (scenario$capping) cost <- applyPAR(cost, boundMax = scenario$boundMax, boundPar = scenario$boundPar) # FIXME: Vectorize this loop for (i in seq_along(experiments)) { testResults[rownames(testResults) == experiments[[i]]$id_instance, colnames(testResults) == experiments[[i]]$id_configuration] <- cost[i] } if (scenario$debugLevel >= 3L) { irace_note ("Memory used at the end of testConfigurations():\n") race_state$print_mem_used() } ## FIXME: Shouldn't we record these experiments in experiment_log ? list(experiments = testResults, seeds = instanceSeed) } irace/R/zzz.R0000644000176200001440000000066114736526233012534 0ustar liggesusers# Uses 10 decimal places at most so that there is space for '-', '.', and # 'e-NN' within the 16 spaces. .irace.format.perf <- "%#16.10g" .onLoad <- function(libname, pkgname) { # FIXME: We would like to use %#16.10g but this causes problems with # https://github.com/oracle/fastr/issues/191 R_engine <- R.version$engine if (!is.null(R_engine) && R_engine == "FastR") .irace.format.perf <<- "%16.10g" invisible() } irace/R/readParameters.R0000644000176200001440000005204015060315667014632 0ustar liggesusers#' Reads the parameters to be tuned by \pkg{irace} from a file or from a #' character string. #' #' @param file `character(1)`\cr Filename containing the definitions of #' the parameters to be tuned. #' @param digits `integer(1)`\cr The number of decimal places to be considered for real-valued parameters. #' @param debugLevel `integer(1)`\cr Larger values produce more verbose output. #' @param text `character(1)`\cr If \code{file} is not supplied and this is, #' then parameters are read from the value of \code{text} via a text connection. #' #' @return A list containing the definitions of the parameters read. The list is #' structured as follows: #' \describe{ #' \item{`names`}{Vector that contains the names of the parameters.} #' \item{`types`}{Vector that contains the type of each parameter 'i', 'c', 'r', 'o'. #' Numerical parameters can be sampled in a log-scale with 'i,log' and 'r,log' #' (no spaces).} #' \item{`switches`}{Vector that contains the switches to be used for the #' parameters on the command line.} #' \item{`domain`}{List of vectors, where each vector may contain two #' values (minimum, maximum) for real and integer parameters, or #' possibly more for categorical parameters.} #' \item{`conditions`}{List of R logical expressions, with variables #' corresponding to parameter names.} #' \item{`isFixed`}{Logical vector that specifies which parameter is fixed #' and, thus, it does not need to be tuned.} #' \item{`nbParameters`}{An integer, the total number of parameters.} #' \item{`nbFixed`}{An integer, the number of parameters with a fixed value.} #' \item{`nbVariable`}{Number of variable (to be tuned) parameters.} #' \item{`depends`}{List of character vectors, each vector specifies #' which parameters depend on this one.} #' \item{`is_dependent`}{Logical vector that specifies which parameter has #' a dependent domain.} #' \item{`digits`}{Integer vector that specifies the number of digits per parameter.} #' \item{`forbidden`}{List of expressions that define which parameter configurations are forbidden.} #' } #' #' @details Either `file` or `text` must be given. If `file` is given, the #' parameters are read from the file `file`. If `text` is given instead, #' the parameters are read directly from the `text` character string. #' In both cases, the parameters must be given (in `text` or in the file #' whose name is `file`) in the expected form. See the documentation #' for details. If none of these parameters is given, \pkg{irace} #' will stop with an error. #' #' A fixed parameter is a parameter that should not be sampled but #' instead should be always set to the only value of its domain. In this #' function we set `isFixed` to TRUE only if the parameter is a categorical #' and has only one possible value. If it is an integer and the minimum #' and maximum are equal, or it is a real and the minimum and maximum #' values satisfy `round(minimum, digits) == round(maximum, digits)`, #' then the parameter description is rejected as invalid to identify #' potential user errors. #' #' The order of the parameters determines the order in which parameters are #' given to `targetRunner`. Changing the order may also change the results #' produced by `irace`, even with the same random seed. #' #' @examples #' ## Read the parameters directly from text #' parameters_table <- ' #' # name switch type values [conditions (using R syntax)] #' algorithm "--" c (as,mmas,eas,ras,acs) #' localsearch "--localsearch " o (0, 1, 2, 3) #' alpha "--alpha " r (0.00, 5.00) #' beta "--beta " r (0.00, 10.00) #' rho "--rho " r (0.01, 1.00) #' ants "--ants " i,log (5, 100) #' q0 "--q0 " r (0.0, 1.0) | algorithm == "acs" #' rasrank "--rasranks " i (1, "min(ants, 10)") | algorithm == "ras" #' elitistants "--elitistants " i (1, ants) | algorithm == "eas" #' nnls "--nnls " i (5, 50) | localsearch %in% c(1,2,3) #' dlb "--dlb " c (0, 1) | localsearch %in% c(1,2,3) #' #' [forbidden] #' (alpha == 0.0) & (beta == 0.0) #' [global] #' digits = 4 #' ' #' parameters <- readParameters(text=parameters_table) #' str(parameters) #' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export readParameters <- function (file, digits = 4L, debugLevel = 0L, text) { if (missing(file) && !missing(text)) { filename <- NA_character_ file <- textConnection(text) on.exit(close(file)) } else if (is.character(file)) { filename <- file file.check (file, readable = TRUE, text = "readParameter: parameter file") } else { irace_error("'file' must be a character string") } digits <- as.integer(digits) field_match <- function (line, pattern, delimited = FALSE, sep = "[[:space:]]") { positions <- lapply(seq_along(pattern), function(x) regexpr (paste0("^", pattern[x], sep), line)) if (all(sapply(positions, "[[", 1L) == -1L)) { return (list(match = NULL, line = line)) } pos_matched <- lapply(seq_along(pattern), function(x) regexpr (paste0("^", pattern[x]), line)) if (all(sapply(pos_matched, "[[", 1L) == -1L)) { return (list(match = NULL, line = line)) } position <- which(sapply(pos_matched, `[[`,1L) != -1L) if (length(position) > 1L) { position <- position[1L] } pos_matched <- pos_matched[[position]] delimited <- as.integer(delimited) match <- substr(line, pos_matched[1L] + delimited, attr(pos_matched, "match.length") - delimited) line <- substr(line, pos_matched[1L] + attr(pos_matched, "match.length"), nchar(line)) line <- trim_leading (line) list(match = match, line = line) } string2vector <- function(str) { v <- c() str <- trim(str) while (str != "") { result <- field_match (str, "\"[^\"]*\"", delimited = TRUE, sep="") if (is.null(result$match)) { result <- field_match (str, "[^,]+", sep="") } v <- c(v, result$match) str <- sub(",[[:space:]]*", "", result$line) } v } errReadParameters <- function(filename, line, context, ...) { context <- if (is.null (context)) "" else paste0(" when reading: \"", context, "\"") fileloc <- if (is.na(filename)) "" else paste0("'", filename, "'," ) irace_error(paste0(..., collapse = ""), " at ", fileloc, "line ", line, context) } warnReadParameters <- function(filename, line, context, ...) { context <- if (is.null (context)) "" else paste0(" when reading: \"", context, "\"") fileloc <- if (is.na(filename)) "" else paste0("'", filename, "'," ) irace_warning(paste0(..., collapse = ""), " at ", fileloc, "line ", line, context) } parse_condition <- function(s, filename, nbLines, line, context) { if (grepl("||", s, fixed=TRUE) || grepl("&&", s, fixed=TRUE)) { warnReadParameters (filename, nbLines, line, paste0("Please use '&' and '|' instead of '&&' and '|'", context)) s <- gsub("||", "|", fixed=TRUE, gsub("&&", "&", fixed=TRUE, s)) } str2expression(s) } params <- list() pnames <- c() lines <- readLines(con = file) # Delete comments lines <- trim(sub("#.*$", "", lines)) within_global <- FALSE nbLines <- 0L # Parse [global] first. for (line in lines) { nbLines <- nbLines + 1L if (line == "") next if (grepl("^[[:space:]]*\\[forbidden\\]", line)) { if (within_global) break next } if (within_global) { if (grepl("^[[:space:]]*digits[[:space:]]*=[[:space:]]*[0-9]+[[:space:]]*$", line)) { eval(parse(text=line)) if (!is.wholenumber(digits) || digits > 15 || digits < 1) errReadParameters(filename, nbLines, line, "'digits' must be an integer within [1, 15]") digits <- as.integer(digits) } else errReadParameters(filename, nbLines, line, "Unknown global option") lines[nbLines] <- "" # Do not parse it again. next } if (grepl("^[[:space:]]*\\[global\\]", line)) { within_global <- TRUE lines[nbLines] <- "" # Do not parse it again. next } } forbidden <- NULL within_forbidden <- FALSE nbLines <- 0L for (line in lines) { nbLines <- nbLines + 1L if (line == "") next if (within_forbidden) { # FIXME: Better error reporting. exp <- parse_condition(line, filename, nbLines, line, " for forbidden expressions") forbidden <- c(forbidden, exp) next } if (grepl("^[[:space:]]*\\[forbidden\\]", line)) { within_forbidden <- TRUE next } ## Match name (unquoted alphanumeric string) result <- field_match (line, "[._[:alnum:]]+") name <- result$match line <- result$line if (is.null(result$match)) { errReadParameters (filename, nbLines, line, "Parameter name must be alphanumeric") } if (name %in% pnames) { errReadParameters (filename, nbLines, NULL, "Duplicated parameter name '", name, "'") } ## Match p_switch (quoted string) result <- field_match (line, "\"[^\"]*\"", delimited = TRUE) p_label <- result$match line <- result$line if (is.null(p_label)) { errReadParameters (filename, nbLines, line, "Parameter label (switch) must be a double-quoted string") } ## Match param.type (longer matches must precede shorter ones) result <- field_match (line, c("i,log", "r,log", "c", "i", "r", "o")) param.type <- result$match line <- result$line if (is.null (param.type)) { errReadParameters(filename, nbLines, line, "Parameter type must be a single character in {'c','i','r','o'}, ", "with 'i', 'r' optionally followed by ',log' (no spaces in between) ", "to sample using a logarithmic scale") } else if (param.type == "i,log") { param.type <- "i" param.transform <- "log" } else if (param.type == "r,log") { param.type <- "r" param.transform <- "log" } else { param.transform <- "" } ## Match domain (delimited by parenthesis) # Regexp to detect dependent domains of the type ("min(p1)", 100) result <- field_match (line, "\\([^|]+\\)", delimited = TRUE, sep = "") domain_str <- result$match line <- result$line if (is.null (domain_str)) { errReadParameters (filename, nbLines, line, "Allowed values must be a list within parenthesis") } # For numerical parameters domains could be dependent # thus, we keep the string values in a variable # for example (10, param1+2) if (param.type %in% c("r","i")) { domain <- eval(parse(text=paste0("expression(", domain_str, ")"), keep.source=FALSE)) # Some expressions like -10 may need to be evaluated again. domain <- sapply(domain, function(x) if (!is.expression(x) || length(all.vars(x, unique=FALSE))) x else eval(x), USE.NAMES=FALSE) if (is.list(domain)) domain <- as.expression(domain) # For dependent domains domain will be NA (we will parse it later) if (length(suppressWarnings(as.numeric(sapply(domain, function(x) if(is.language(x)) NA else x)))) != 2L) { errReadParameters (filename, nbLines, NULL, "Incorrect numeric range (", result$match, ") for parameter '", name, "'") } } else { # type %in% c("c", "o") domain <- string2vector(domain_str) } ## Match start of conditions result <- field_match (line, "\\|", sep="") line <- result$line if (!is.null(result$match) && result$match != "") { result <- field_match (line, ".*$", sep="") condition <- result$match if (is.null(result$match) || result$match == "") errReadParameters (filename, nbLines, line, "Expected condition after '|'") line <- result$line } else if (!is.null(result$line) && result$line != "") { errReadParameters (filename, nbLines, line, "Expected '|' before condition") } else { condition <- TRUE } # ***************************************************************** p <- tryCatch( Parameter(name = name, type = param.type, domain = domain, label = p_label, condition = condition, transf = param.transform, digits = digits), invalid_domain = function(c) structure(paste0("For parameter '", name, "' of type 'i' values must be integers (", domain_str, ")"), class = "try-error"), invalid_range = function(c) structure(paste0("Lower bound must be smaller than upper bound in numeric range (", domain_str, ") for parameter '", name, "'"), class="try-error"), error = function(c) structure(conditionMessage(c), class="try-error") ) if (inherits(p, "try-error")) errReadParameters(filename, nbLines, NULL, p) params <- c(params, list(p)) pnames <- c(pnames, name) } # end loop on lines # Check that we have read at least one parameter if (length(params) == 0) { if (is.na(filename)) irace_error("No parameter definition found in the input text") else irace_error("No parameter definition found, check that the parameter file '", filename, "' is not empty.") } if (length(forbidden)) { irace_note(length(forbidden), " expression(s) specifying forbidden configurations read.\n") check_forbidden_params(forbidden, pnames, filename = filename) } parameters <- do.call(parametersNew, c(params, list(forbidden=forbidden, debugLevel = debugLevel))) if (debugLevel >= 2) { print(parameters, digits = 15L) irace_note("Parameters have been read\n") } parameters } #' Read parameters in PCS (AClib) format and write them in irace format. #' #' @inheritParams readParameters #' #' @return A string representing the parameters in irace format. #' #' @details Either `file` or `text` must be given. If `file` is given, the #' parameters are read from the file `file`. If `text` is given instead, #' the parameters are read directly from the `text` character string. #' In both cases, the parameters must be given (in `text` or in the file #' whose name is `file`) in the expected form. See the documentation #' for details. If none of these parameters is given, \pkg{irace} #' will stop with an error. #' #' **FIXME:** Multiple conditions and default configuration are currently ignored. See #' #' @references #' Frank Hutter, Manuel López-Ibáñez, Chris Fawcett, Marius Thomas Lindauer, Holger H. Hoos, Kevin Leyton-Brown, and Thomas Stützle. **AClib: A Benchmark Library for Algorithm Configuration**. In P. M. Pardalos, M. G. C. Resende, C. Vogiatzis, and J. L. Walteros, editors, _Learning and Intelligent Optimization, 8th International Conference, LION 8_, volume 8426 of Lecture Notes in Computer Science, pages 36–40. Springer, Heidelberg, 2014. #' #' @seealso [readParameters()] #' @examples #' ## Read the parameters directly from text #' pcs_table <- ' #' # name domain #' algorithm {as,mmas,eas,ras,acs}[as] #' localsearch {0, 1, 2, 3}[0] #' alpha [0.00, 5.00][1] #' beta [0.00, 10.00][1] #' rho [0.01, 1.00][0.95] #' ants [1, 100][10]il #' q0 [0.0, 1.0][0] #' rasrank [1, 100][1]i #' elitistants [1, 750][1]i #' nnls [5, 50][5]i #' dlb {0, 1}[1] #' Conditionals: #' q0 | algorithm in {acs} #' rasrank | algorithm in {ras} #' elitistants | algorithm in {eas} #' nnls | localsearch in {1,2,3} #' dlb | localsearch in {1,2,3} #' {alpha=0, beta=0}' #' parameters_table <- read_pcs_file(text=pcs_table) #' cat(parameters_table) #' parameters <- readParameters(text=parameters_table) #' str(parameters) #' #' @author Manuel López-Ibáñez #' @export read_pcs_file <- function(file, digits = 4L, debugLevel = 0L, text) { if (missing(file) && !missing(text)) { filename <- paste0("text=", deparse(substitute(text))) file <- textConnection(text) on.exit(close(file)) } else if (is.character(file)) { filename <- file file.check (file, readable = TRUE, text = "read_pcs_file: parameter file") } else { irace_error("'file' must be a character string") } lines <- readLines(con = file) lines <- trim(lines) # Remove leading and trailing whitespace lines <- lines[!grepl("Conditionals:", lines, fixed=TRUE)] # useless line conditions <- list() forbidden <- NULL regex_cond <- "^([^[:space:]]+)[[:space:]]+\\|[[:space:]]+(.+)$" regex_forbidden <- "^{(.+)}$" for (k in seq_along(lines)) { if (grepl(regex_cond, lines[k], perl=TRUE)) { matches <- regmatches(lines[k], regexec(regex_cond, lines[k], perl=TRUE))[[1L]] stopifnot(length(matches) > 0) lines[k] <- NA_character_ conditions[[matches[[2L]]]] <- matches[[3L]] } else if (grepl(regex_forbidden, lines[k], perl=TRUE)) { forbidden <- c(forbidden, sub(regex_forbidden, "\\1", lines[k], perl=TRUE)) lines[k] <- NA_character_ } } parse_pcs_condition <- function(x, types) { if (is.null(x)) return ("") matches <- regmatches(x, regexec("([^[:space:]]+)[[:space:]]+in[[:space:]]+\\{([^}]+)\\}$", x, perl=TRUE))[[1L]] if (length(matches) == 0L) irace_error("unknown condition ", x) param <- matches[[2L]] type <- types[[param]] if (is.null(type)) irace_error("unknown type for ", param, " in condition: ", x) cond <- matches[[3L]] if (type == "c" || type == "o") { cond <- strsplit(cond, ",[[:space:]]*")[[1L]] equal <- (length(cond) == 1L) cond <- paste0('"', cond, '"', collapse=',') } else { equal <- grepl(",", cond, fixed=TRUE) } if (equal) return(paste0(" | ", param, ' == ', cond)) return(paste0(" | ", param, " %in% c(", cond, ")")) } param_types <- list() param_domains <- list() param_comments <- list() lines <- lines[!is.na(lines)] for (line in lines) { if (startsWith(line, "#") || line == "") next # match a parameter matches <- regmatches(line, regexec("^([^[:space:]]+)[[:space:]]+\\[([^,]+),[[:space:]]*([^]]+)\\][[:space:]]*\\[[^]]+\\](i?l?i?)(.*)$", line, perl=TRUE))[[1]] if (length(matches) > 0L) { param_name <- matches[[2L]] param_type <- paste0(if(grepl("i", matches[5L], fixed=TRUE)) "i" else "r", if(grepl("l", matches[5L], fixed=TRUE)) ",log" else "") param_types[[param_name]] <- param_type param_domains[[param_name]] <- paste0("(", matches[3L], ", ", matches[4L], ")") param_comments[[param_name]] <- matches[6L] next } matches <- regmatches(line, regexec("^([^[:space:]]+)[[:space:]]+\\{([^}]+)\\}[[:space:]]*\\[[^]]+\\](.*)$", line, perl=TRUE))[[1L]] if (length(matches) > 0L) { param_name <- matches[[2L]] param_type <- "c" param_types[[param_name]] <- param_type param_types[[param_name]] <- param_type param_domains[[param_name]] <- paste0("(", matches[3L], ")") param_comments[[param_name]] <- matches[4L] next } } output <- "" for (line in lines) { if (startsWith(line, "#") || line == "") { output <- paste0(output, line, "\n") next } # match a parameter matches <- regmatches(line, regexec("^([^[:space:]]+)[[:space:]]+", line, perl=TRUE))[[1L]] if (length(matches) > 0L) { param_name <- matches[[2L]] cond <- parse_pcs_condition(conditions[[param_name]], param_types) output <- paste0(output, sprintf('%s "%s" %s %s%s%s\n', param_name, param_name, param_types[[param_name]], param_domains[[param_name]], cond, param_comments[[param_name]])) next } irace_error("unrecognized line: ", line) } if (length(forbidden) > 0L) { exp <- sapply(forbidden, function(x) { # FIXME: this will break if there are "," within the values. x <- strsplit(x, ",[[:space:]]*")[[1L]] paste0(collapse=" & ", sapply(regmatches(x, regexec("^([^=]+)=(.+)$", x, perl=TRUE)), function(matches) { rhs <- trim(matches[[3L]]) if (!any(startsWith(rhs, c("'", "\""))) && suppressWarnings(is.na(as.numeric(rhs)))) rhs <- paste0('"', rhs, '"') paste0("(", trim(matches[[2L]]), " == ", rhs, ")") }, USE.NAMES=FALSE)) }, USE.NAMES=FALSE) output <- paste0(output, "\n[forbidden]\n", paste0(collapse="\n", exp), "\n") } output } #' checkParameters #' #' FIXME: This is incomplete, for now we only repair inputs from previous irace #' versions. #' #' @inheritParams printParameters #' @export checkParameters <- function(parameters) { ## if (is.null(parameters$isDependent)) { ## parameters$isDependent <- sapply(parameters$domains, is.expression) ## names(parameters$isDependent) <- parameters$names ## } if (!inherits(parameters, "ParameterSpace")) { irace_error("parameters must be an object of class 'ParameterSpace'") } parameters } irace/R/configurations.R0000644000176200001440000001172214745735066014737 0ustar liggesusers.param_na_value_type <- list(i = NA_integer_, r = NA_real_, c = NA_character_, o = NA_character_) # Returns a data.table configurations_alloc <- function(colnames, nrow, parameters) { column_type <- function(x, n, types) { what <- switch(x, .ID. = NA_integer_, .PARENT. = NA_integer_, .WEIGHT. = NA_real_, .param_na_value_type[[ types[x] ]]) rep_len(what, n) } x <- sapply(colnames, column_type, n=nrow, types = parameters[["types"]], simplify=FALSE, USE.NAMES=TRUE) setDT(x) x } # FIXME: It may be faster to create a single expression that concatenates all # the elements of forbidden using '|' checkForbidden <- function(configurations, forbidden) { # We have to use a variable name that will never appear in # configurations, so .FORBIDDEN . for (.FORBIDDEN in forbidden) { #print(.FORBIDDEN) configurations <- subset(configurations, eval(.FORBIDDEN)) #print(configurations) #print(str(configurations)) ## FIXME: This is normally called with a single configuration. Thus, it ## would be faster to break as soon as nrow(configurations) < 1 } #print(nrow(configurations)) configurations } # FIXME: It may be faster to create a single expression that concatenates all # the elements of forbidden using '|' filter_forbidden <- function(configurations, forbidden) { # We have to use a variable name that will never appear in # configurations, so .FORBIDDEN for (.FORBIDDEN in forbidden) { configurations <- configurations[eval(.FORBIDDEN)] #print(configurations) #print(str(configurations)) if (nrow(configurations) == 0L) return(configurations) } #print(nrow(configurations)) configurations } which_satisfied <- function(configurations, condition) { # If there is no condition, do not waste time evaluating it. if (isTRUE(condition)) return(seq_nrow(configurations)) r <- eval(condition, configurations) # Return TRUE if TRUE, FALSE if FALSE or NA ## FIXME: If we byte-compile the condition, then we should incorporate the ## following into the condition directly. # r & !is.na(r) # Return indexes where r is TRUE. which(r) } #' removeConfigurationsMetaData #' #' Remove the columns with "metadata" of a data frame containing #' configurations. Currently, metadata corresponds to column names starting #' with a period. This function should be used before printing the #' configurations to output only the values for the parameters of the #' configuration without metadata possibly useless to the user. #' #' @param configurations `data.frame`\cr Parameter configurations of the #' target algorithm (one per row). #' #' @return The same data frame without "metadata". #' #' @seealso #' [configurations_print_command()] to print the configurations as command lines. #' [configurations_print()] to print the configurations as a data frame. #' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export removeConfigurationsMetaData <- function(configurations) configurations[, !startsWith(colnames(configurations), "."), drop = FALSE] #' Print configurations as a data frame #' #' @inheritParams removeConfigurationsMetaData #' #' @param metadata `logical(1)`\cr whether to print the metadata or #' not. The metadata are data for the configurations (additionally to the #' value of each parameter) used by \pkg{irace}. #' #' @return None. #' #' @seealso #' [configurations_print_command()] to print the configurations as command-line strings. #' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export configurations_print <- function(configurations, metadata = FALSE) { if (is.data.table(configurations) || !is.data.frame(configurations)) configurations <- as.data.frame(configurations, stringsAsFactors = FALSE) rownames(configurations) <- configurations[[".ID."]] if (!metadata) configurations <- removeConfigurationsMetaData(configurations) print.data.frame(configurations, digits = 15L) } #' Print configurations as command-line strings. #' #' Prints configurations after converting them into a representation for the #' command-line. #' #' @inheritParams removeConfigurationsMetaData #' @inheritParams printParameters #' #' @return None. #' #' @seealso #' [configurations_print()] to print the configurations as a data frame. #' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export configurations_print_command <- function(configurations, parameters) { if (nrow(configurations) <= 0L) return(invisible()) ids <- as.numeric(configurations$.ID.) configurations <- removeConfigurationsMetaData(configurations) # Re-sort the columns configurations <- configurations[, parameters$names, drop = FALSE] # A better way to do this? We cannot use apply() because that coerces # to a character matrix thus messing up numerical values. len <- nchar(max(ids)) for (i in seq_nrow(configurations)) { cat(sprintf("%-*d %s\n", len, ids[i], buildCommandLine(configurations[i, , drop=FALSE], parameters$switches))) } } irace/R/race-wrapper.R0000644000176200001440000006447314753644155014305 0ustar liggesusers#' Generate a command-line representation of a configuration #' #' @description `buildCommandLine` receives two vectors, one containing #' the values of the parameters, the other containing the switches of the #' parameters. It builds a string with the switches and the values that can #' be used as a command line to call the program to be tuned, thus generating #' one candidate configuration. #' #' #' @param values A vector containing the value of each parameter for the #' candidate configuration. #' @param switches A vector containing the switches of each paramter (in an #' order that corresponds to the values vector). #' #' @return A string concatenating each element of `switches` and #' `values` for all parameters with a space between each pair of #' parameters (but none between the switches and the corresponding values). #' #' @examples #' switches <- c("--switch1 ", "--switch2-", "--switch3=") #' values <- list("value_1", 1L, sqrt(2)) #' buildCommandLine (values, switches) #' ## Build a command-line from the results produced by a previous run of irace. #' # First, load the data produced by irace. #' logfile <- file.path(system.file(package="irace"), "exdata", "irace-acotsp.Rdata") #' iraceResults <- read_logfile(logfile) #' allConfigurations <- iraceResults$allConfigurations #' parameters <- iraceResults$scenario$parameters #' apply(allConfigurations[1:10, unlist(parameters$names)], 1, buildCommandLine, #' unlist(parameters$switches)) #' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export buildCommandLine <- function(values, switches) { irace_assert(length(values) == length(switches)) values <- as.list(values) sel <- !is.na(values) switches <- switches[sel] values <- format.default(values[sel], digits=15L, scientific=FALSE) paste0(switches, values, collapse=" ") } # This function tries to convert a, possibly empty, character vector into a # numeric vector. parse_output <- function(outputRaw, verbose) { if (verbose) cat(paste0(outputRaw, collapse = "\n")) # Initialize output as raw. If it is empty stays like this. output <- outputRaw # strsplit crashes if outputRaw == character(0) if (length(outputRaw)) { outputRaw <- paste0(outputRaw, collapse = "\n") output <- strsplit(trim(outputRaw), "[[:space:]]+")[[1L]] } # suppressWarnings to avoid messages about NAs introduced by coercion suppressWarnings(as.numeric(output)) } target_error <- function(err_msg, output, scenario, target_runner_call, target_evaluator_call = NULL) { if (!is.null(target_evaluator_call)) { err_msg <- paste0(err_msg, "\n", .irace_msg_prefix, "The call to targetEvaluator was:\n", target_evaluator_call) } if (!is.null(target_runner_call)) { err_msg <- paste0(err_msg, "\n", .irace_msg_prefix, "The call to targetRunner was:\n", target_runner_call) } if (is.null(output$outputRaw)) { # Message for a function call. output$outputRaw <- deparse1(output) advice_txt <- paste0( "This is not a bug in irace, but means that something failed in ", "a call to the targetRunner or targetEvaluator functions provided by the user.", " Please check those functions carefully.") } else { # Message for an external script. advice_txt <- paste0( "This is not a bug in irace, but means that something failed when", " running the command(s) above or they were terminated before completion.", " Try to run the command(s) above from the execution directory '", scenario$execDir, "' to investigate the issue. See also Appendix B (targetRunner troubleshooting checklist) of the User Guide (https://cran.r-project.org/package=irace/vignettes/irace-package.pdf).") } irace_error(err_msg, "\n", .irace_msg_prefix, "The output was:\n", paste(output$outputRaw, collapse = "\n"), "\n", .irace_msg_prefix, advice_txt) } check_output_target_evaluator <- function (output, scenario, target_runner_time, target_runner_call, bound) { if (!is.list(output)) { output <- list() target_error ("The output of targetEvaluator must be a list", output, scenario, target_runner_call = target_runner_call) return(output) } err_msg <- output$error if (is.null(err_msg)) { if (is.null(output$cost)) { err_msg <- "The output of targetEvaluator must contain 'cost'!" } else if (is_na_nowarn(output$cost)) { err_msg <- "The output of targetEvaluator is not numeric!" } if (scenario$maxTime > 0 || scenario$capping) { if (scenario$batchmode != 0) { if (is.null (output$time)) err_msg <- "When batchmode != 0 and maxTime > 0, the output of targetEvaluator must be two numbers 'cost time'!" # With scenario$capping == TRUE, we may have pre-executed targetRunner # (which_elite_exe) that already have recorded the time, so when we # reach this point, we may not have 'time'. } else if (!scenario$capping && is.null(target_runner_time) && is.null(output$time)) { err_msg <- "Either targetRunner or targetEvaluator must return 'time' !" } } if (is.null(output$time)) { output$time <- target_runner_time } else { if (!is.null(target_runner_time)) { err_msg <- "Both targetRunner and targetEvaluator cannot return 'time' !" } else if (is_na_nowarn(output$time)) { err_msg <- "The time returned by targetEvaluator is not numeric!" } else if (is.infinite(output$time)) { err_msg <- "The time returned by targetEvaluator is not finite!" } else if (output$time <= 0) { err_msg <- paste0("The value of time (", output$time, ") returned by targetEvaluator must be strictly positive!") } else { # Fix time. output$time <- max(output$time, scenario$minMeasurableTime) if (!is.null(bound) && !is.na(bound) && bound > 0 && bound + scenario$minMeasurableTime < output$time) { err_msg <- paste0("The time returned by targetEvaluator (", output$time, ") does not respect the given bound of ", bound, "!") } } } } if (is.null(err_msg)) { output$error <- NULL } else { target_error (err_msg, output, scenario, target_runner_call = target_runner_call, target_evaluator_call = output$call) } output } #' target_evaluator_default #' #' `target_evaluator_default` is the default `targetEvaluator` function that is #' invoked if `targetEvaluator` is a string (by default #' `targetEvaluator` is `NULL` and this function is not invoked). You can use it as #' an advanced example of how to create your own `targetEvaluator` function. #' #' @param experiment A list describing the experiment. It contains at least: #' \describe{ #' \item{`id_configuration`}{An alphanumeric string that uniquely identifies a configuration;} #' \item{`id_instance`}{An alphanumeric string that uniquely identifies an instance;} #' \item{`seed`}{Seed for the random number generator to be used for #' this evaluation, ignore the seed for deterministic algorithms;} #' \item{`instance`}{String giving the instance to be used for this evaluation;} #' \item{`bound`}{(only when `capping` is enabled) Time bound for the execution;} #' \item{`configuration`}{1-row data frame with a column per parameter #' name;} #' } #' @param num_configurations Number of configurations alive in the race. #' @param all_conf_id Vector of configuration IDs of the alive configurations. #' @inheritParams defaultScenario #' @param target_runner_call String describing the call to `targetRunner` that #' corresponds to this call to `targetEvaluator`. This is used for #' providing extra information to the user, for example, in case #' `targetEvaluator` fails. #' #' @return The function `targetEvaluator` must return a list with one element #' `"cost"`, the numerical value corresponding to the cost measure of the #' given configuration on the given instance. #' #' The return list may also contain the following optional elements that are used #' by \pkg{irace} for reporting errors in `targetEvaluator`: #' \describe{ #' \item{`error`}{is a string used to report an error;} #' \item{`outputRaw`}{is a string used to report the raw output of calls to #' an external program or function;} #' \item{`call`}{is a string used to report how `targetRunner` called #' an external program or function.} #' } #' #' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export target_evaluator_default <- function(experiment, num_configurations, all_conf_id, scenario, target_runner_call) { configuration_id <- experiment$id_configuration instance_id <- experiment$id_instance seed <- experiment$seed instance <- experiment$instance debugLevel <- scenario$debugLevel targetEvaluator <- scenario$targetEvaluator if (as.logical(file.access(targetEvaluator, mode = 1))) { irace_error ("targetEvaluator", shQuote(targetEvaluator), "cannot be found or is not executable!\n") } all_conf_id <- paste0(all_conf_id, collapse = " ") args <- c(configuration_id, instance_id, seed, instance, num_configurations, all_conf_id) withr::with_dir(scenario$execDir, { output <- runcommand(targetEvaluator, args, configuration_id, debugLevel, timeout = scenario$targetRunnerTimeout) }) cost <- time <- NULL err_msg <- output$error if (is.null(err_msg)) { v_output <- parse_output(output$output, verbose = (scenario$debugLevel >= 2L)) if (length(v_output) == 1L) { cost <- v_output[1L] } else if (length(v_output) == 2L) { cost <- v_output[1L] time <- v_output[2L] } else if (length(v_output) == 0L) { err_msg <- paste0("The output of targetEvaluator must be at least one number 'cost'!") } else { err_msg <- paste0("The output of targetEvaluator should not be more than two numbers!") } } list(cost = cost, time = time, error = err_msg, outputRaw = output$output, call = paste(targetEvaluator, args, collapse=" ")) } #' Check the output of the target runner and repair it if possible. If the #' output is incorrect, this function will throw an error. #' #' @param output The output from target runner. #' @inheritParams defaultScenario #' @param bound Optional time bound that the target runner should have respected. #' #' @return The output with its contents repaired. #' #' @export check_output_target_runner <- function(output, scenario, bound = NULL) { if (!is.list(output)) { output <- list() target_error ("The output of targetRunner must be a list", output, scenario, target_runner_call = NULL) return(output) } err_msg <- output$error if (is.null(err_msg)) { if (is.null(output$cost)) { output$cost <- NULL # make sure to delete it. } else if (is_na_or_empty(output$cost)) { err_msg <- "The cost returned by targetRunner is not numeric!" } if (is.null(output$time)) { output$time <- NULL # make sure to delete it. } else if (is.na(output$time)) { err_msg <- paste0("The time returned by targetRunner is not numeric!") } else if (is.infinite(output$time)) { err_msg <- paste0("The time returned by targetRunner is not finite!") } else if (output$time <= 0) { err_msg <- paste0("The value of time (", output$time, ") returned by targetRunner must be strictly positive!") } else { # Fix time. output$time <- max(output$time, scenario$minMeasurableTime) if (!is.null(bound) && !is.na(bound) && bound > 0 && bound + scenario$minMeasurableTime < output$time) { err_msg <- paste0("The time returned by targetRunner (", output$time, ") does not respect the given bound of ", bound, "!") } } if (is.null(err_msg)) { # When targetEvaluator is provided, targetRunner must return only the time. if (!is.null(scenario$targetEvaluator)) { # unless using batchmode, in that case targetRunner returns neither the # time nor the cost. if (scenario$batchmode != 0) { if (!is.null(output$time) || !is.null(output$cost)) { err_msg <- "When batchmode != 0, the output of targetRunner must not contain a cost nor a time!" } } else if (scenario$maxTime > 0 && is.null(output$time)) { err_msg <- "The output of targetRunner must be one number 'time'!" } else if (!is.null(output$cost)) { err_msg <- "The output of targetRunner must be empty or just one number 'time'!" } } else if (scenario$maxTime > 0 && (is.null(output$cost) || is.null(output$time))) { err_msg <- "The output of targetRunner must be two numbers 'cost time'!" } else if (scenario$maxExperiments > 0 && is.null(output$cost)) { err_msg <- "The output of targetRunner must be one number 'cost'!" } } } if (is.null(err_msg)) { output$error <- NULL } else { target_error (err_msg, output, scenario, target_runner_call = output$call) } output } # This function invokes target_runner. When used on a remote node by Rmpi, # environments do not seem to be shared and the default value is evaluated too # late, thus we have to pass race_state$target_runner explicitly. exec_target_runner <- function(experiment, scenario, target_runner) { doit <- function(experiment, scenario) { x <- target_runner(experiment, scenario) check_output_target_runner(x, scenario, bound = experiment$bound) } retries <- scenario$targetRunnerRetries while (retries > 0L) { output <- try (doit(experiment, scenario)) if (!inherits(output, "try-error") && is.null(output$error)) return (output) irace_note("Retrying (", retries, " left).\n") retries <- retries - 1L } doit(experiment, scenario) } parse.aclib.output <- function(outputRaw) { outputRaw <- paste0(outputRaw, collapse = "\n") text <- regmatches(outputRaw, regexec("Result of this algorithm run:\\s*\\{(.+)\\}\\s*\n", outputRaw))[[1]][2] aclib.match <- function(text, key, value) { pattern <- paste0('"', key, '":\\s*', value) return(regmatches(text, regexec(pattern, text))[[1]][2]) } cost <- runtime <- error <- NULL # AClib wrappers print: # Result of this algorithm run: {"status": "SUCCESS", "cost": cost, "runtime": time } # FIXME: This is not very robust. If we are going to be using jsonlite, then we can simply do: # jsonlite::fromJSON('{\"misc\": \"\", \"runtime\": 164.14, \"status\": \"SUCCESS\", \"cost\": \"0.121340\"}') status <- aclib.match(text, "status", '"([^"]+)"') if (!is.character(status)) { error <- paste0("Not valid AClib output") } else if (status %in% c("SUCCESS", "TIMEOUT")) { cost <- aclib.match(text, "cost", "([^[:space:],}]+)") cost <- suppressWarnings(as.numeric(cost)) runtime <- aclib.match(text, "runtime", "([^[:space:],}]+)") runtime <- suppressWarnings(as.numeric(runtime)) if (is.null.or.na(cost) && is.null.or.na(runtime)) error <- paste0("Not valid cost or runtime in AClib output") } else if (status %in% c("CRASHED", "ABORT")) { # FIXME: Implement ABORT semantics of fatal error error <- paste0("targetRunner returned status (", status, ")") } else { error <- paste0("Not valid AClib output status (", status, ")") } list(status = status, cost = cost, time = runtime, error = error) } target_runner_aclib <- function(experiment, scenario) { debugLevel <- scenario$debugLevel res <- run_target_runner(experiment, scenario) cmd <- res$cmd output <- res$output args <- res$args err_msg <- output$error if (is.null(err_msg)) { return(c(parse.aclib.output(output$output), list(outputRaw = output$output, call = paste(cmd, args)))) } list(cost = NULL, time = NULL, error = err_msg, outputRaw = output$output, call = paste(cmd, args)) } check_target_cmdline <- function(target_cmdline, launcher, capping) { required <- c("seed", "instance", "targetRunnerArgs") if (capping) required <- c(required, "bound") for (x in required) { if (!grepl(paste0("{", x, "}"), target_cmdline, fixed=TRUE)) irace_error("targetCmdline '", target_cmdline, "' must contain '{", x, "}'") } } expand_target_cmdline <- function(target_cmdline, experiment, targetRunner, targetRunnerArgs) { vars <- list(configurationID = experiment$id_configuration, instanceID = experiment$id_instance, seed = experiment$seed, instance = experiment$instance, bound = experiment$bound, targetRunner = targetRunner, targetRunnerArgs = targetRunnerArgs) for (x in names(vars)) { value <- vars[[x]] if (is.null(value)) value <- "" if (x == "targetRunner") value <- shQuote(value) target_cmdline <- gsub(paste0("{", x, "}"), value, target_cmdline, fixed=TRUE) } target_cmdline } run_target_runner <- function(experiment, scenario) { configuration_id <- experiment$id_configuration instance_id <- experiment$id_instance seed <- experiment$seed configuration <- experiment$configuration instance <- experiment$instance bound <- experiment$bound switches <- scenario$parameters$switches[names(configuration)] targetRunner <- scenario[["targetRunner"]] debugLevel <- scenario$debugLevel if (scenario$aclib) { # FIXME: Use targetCmdline for this has_value <- !is.na(configuration) # [] [] ... [--cutoff ] [--instance ] # [--seed ] --config [-param_name_1 value_1] [-param_name_2 value_2] ... args <- paste("--instance", instance, "--seed", seed, "--config", paste0("-", switches[has_value], " ", configuration[has_value], collapse = " ")) if (!is.null.or.na(bound)) args <- paste("--cutoff", bound, args) } else { args <- expand_target_cmdline(scenario$targetCmdline, experiment, targetRunner, targetRunnerArgs=buildCommandLine(configuration, switches)) } targetRunnerLauncher <- scenario$targetRunnerLauncher if (!is.null.or.empty(targetRunnerLauncher)) targetRunner <- targetRunnerLauncher output <- runcommand(targetRunner, args, configuration_id, debugLevel, timeout = scenario$targetRunnerTimeout) list(cmd=targetRunner, output=output, args=args) } #' Default `targetRunner` function. #' #' Use it as an advanced example of how to create your own `targetRunner` function. #' #' @param experiment A list describing the experiment. It contains at least: #' \describe{ #' \item{`id_configuration`}{An alphanumeric string that uniquely identifies a configuration;} #' \item{`id_instance`}{An alphanumeric string that uniquely identifies an instance;} #' \item{`seed`}{Seed for the random number generator to be used for #' this evaluation, ignore the seed for deterministic algorithms;} #' \item{`instance`}{String giving the instance to be used for this evaluation;} #' \item{`bound`}{(only when `capping` is enabled) Time bound for the execution;} #' \item{`configuration`}{1-row data frame with a column per parameter #' name;} #' } #' @inheritParams defaultScenario #' #' @return If `targetEvaluator` is `NULL`, then the `targetRunner` #' function must return a list with at least one element `"cost"`, #' the numerical value corresponding to the evaluation of the given #' configuration on the given instance. #' #' If the scenario option `maxTime` is non-zero or if `capping` is enabled #' then the list must contain at least another element `"time"` that reports the #' execution time for this call to `targetRunner`. #' The return list may also contain the following optional elements that are used #' by \pkg{irace} for reporting errors in `targetRunner`: #' \describe{ #' \item{`error`}{is a string used to report an error;} #' \item{`outputRaw`}{is a string used to report the raw output of calls to #' an external program or function;} #' \item{`call`}{is a string used to report how `targetRunner` called #' an external program or function.} #' } #' #' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export target_runner_default <- function(experiment, scenario) { res <- run_target_runner(experiment, scenario) cmd <- res$cmd output <- res$output args <- res$args debugLevel <- scenario$debugLevel cost <- time <- NULL err_msg <- output$error if (is.null(err_msg)) { v_output <- parse_output(output$output, verbose = (debugLevel >= 2L)) if (length(v_output) == 1L) { if (is.null(scenario$targetEvaluator)) { cost <- v_output[1L] } else { time <- v_output[1L] } } else if (length(v_output) == 2L) { cost <- v_output[1L] time <- v_output[2L] } else { err_msg <- "The output of targetRunner should not be more than two numbers!" } } list(cost = cost, time = time, error = err_msg, outputRaw = output$output, call = paste(cmd, args, collapse = " ")) } execute_experiments <- function(race_state, experiments, scenario) { parallel <- scenario$parallel mpi <- scenario$mpi target_runner <- race_state$target_runner execDir <- scenario$execDir if (!fs::dir_exists(execDir)) irace_error ("Execution directory '", execDir, "' is not found or not a directory\n") withr::local_dir(execDir) if (!is.null(scenario$targetRunnerParallel)) { # FIXME: We should remove the exec_target_runner parameter from # targetRunnerParallel and do lapply(target_output, # check_output_target_runner, scenario=scenario) after it to make sure the output is valid. # User-defined parallelization target_output <- scenario$targetRunnerParallel(experiments, exec_target_runner, scenario = scenario, target_runner = target_runner) if (length(target_output) != length(experiments)) { irace_error("Stopping because the output of targetRunnerParallel is missing elements. The output was:\n", paste0(utils::capture.output(utils::str(target_output)), collapse="\n")) } } else if (scenario$batchmode != 0L) { target_output <- cluster_lapply(experiments, scenario = scenario) } else if (parallel > 1L) { if (mpi) { if (scenario$loadBalancing) { target_output <- Rmpi::mpi.applyLB(experiments, exec_target_runner, scenario = scenario, target_runner = target_runner) } else { # Without load-balancing, we need to split the experiments into chunks # of size parallel. target_output <- unlist(use.names = FALSE, tapply(experiments, ceiling(seq_along(experiments) / parallel), Rmpi::mpi.apply, exec_target_runner, scenario = scenario, target_runner = target_runner)) } # FIXME: if stop() is called from mpi.applyLB, it does not # terminate the execution of the parent process, so it will # continue and give more errors later. We have to terminate # here, but is there a nicer way to detect this and terminate? if (any(sapply(target_output, inherits, "try-error"))) { # FIXME: mclapply has some bugs in case of error. In that # case, each element of the list does not keep the output of # each configuration and repetitions may occur. cat(unique(unlist(target_output[sapply( target_output, inherits, "try-error")])), file = stderr(), sep = "") irace_error("A slave process terminated with a fatal error") } } else { if (.Platform$OS.type == 'windows') { irace_assert(!is.null(race_state$cluster)) if (scenario$loadBalancing) { target_output <- parallel::parLapplyLB(race_state$cluster, experiments, exec_target_runner, scenario = scenario, target_runner = target_runner) } else { target_output <- parallel::parLapply(race_state$cluster, experiments, exec_target_runner, scenario = scenario, target_runner = target_runner) } # FIXME: if stop() is called from parLapply, then the parent # process also terminates, and we cannot give further errors. } else { target_output <- parallel::mclapply(experiments, exec_target_runner, # FALSE means load-balancing. mc.preschedule = !scenario$loadBalancing, mc.cores = parallel, scenario = scenario, target_runner = target_runner) # FIXME: if stop() is called from mclapply, it does not # terminate the execution of the parent process, so it will # continue and give more errors later. We have to terminate # here, but is there a nicer way to detect this and terminate? if (any(sapply(target_output, inherits, "try-error")) || any(sapply(target_output, is.null))) { # FIXME: mclapply has some bugs in case of error. In that # case, each element of the list does not keep the output of # each configuration and repetitions may occur. cat(unique(unlist( target_output[sapply( target_output, inherits, "try-error")])), file = stderr()) irace_error("A child process triggered a fatal error") } } } } else { # One process, all sequential target_output <- lapply(experiments, exec_target_runner, scenario = scenario, target_runner = target_runner) } target_output } execute_evaluator <- function(target_evaluator, experiments, scenario, target_output) { configurations_id <- unique(unlist_element(experiments, "id_configuration")) nconfs <- length(configurations_id) # Evaluate configurations sequentially. for (k in seq_along(experiments)) { experiment <- experiments[[k]] target_runner_call <- target_output[[k]]$call output <- target_evaluator(experiment = experiment, num_configurations = nconfs, all_conf_id = configurations_id, scenario = scenario, target_runner_call = target_runner_call) output <- check_output_target_evaluator(output, scenario, target_runner_time = target_output[[k]]$time, target_runner_call = target_runner_call, bound = experiment$bound) target_output[[k]]$cost <- output$cost # targetEvaluator may return time, for example for batchmode != 0. target_output[[k]]$time <- output$time } target_output } irace/R/main.R0000644000176200001440000004440614745735066012636 0ustar liggesusers# ========================================================================= # irace: An implementation in R of Iterated Race. # ------------------------------------------------------------------------- # # Copyright (C) 2010-2025 # Manuel López-Ibáñez # Jérémie Dubois-Lacoste # Leslie Perez Caceres # # ------------------------------------------------------------------------- # This program is free software (software libre); you can redistribute # it and/or modify it under the terms of the GNU General Public License # as published by the Free Software Foundation; either version 2 of the # License, or (at your option) any later version. # # This program is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, you can obtain a copy of the GNU # General Public License at: # http://www.gnu.org/copyleft/gpl.html # or by writing to the Free Software Foundation, Inc., 59 Temple Place, # Suite 330, Boston, MA 02111-1307 USA # ------------------------------------------------------------------------- # $Revision$ # ========================================================================= #' irace_license #' #' A character string containing the license information of \pkg{irace}. #' #' @export ## __VERSION__ below will be replaced by the version defined in R/version.R ## This avoids constant conflicts within this file. irace_license <- '#------------------------------------------------------------------------------ # irace: An implementation in R of (Elitist) Iterated Racing # Version: __VERSION__ # Copyright (C) 2010-2025 # Manuel Lopez-Ibanez # Jeremie Dubois-Lacoste # Leslie Perez Caceres # # This is free software, and you are welcome to redistribute it under certain # conditions. See the GNU General Public License for details. There is NO # WARRANTY; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # # irace builds upon previous code from the race package: # race: Racing methods for the selection of the best # Copyright (C) 2003 Mauro Birattari #------------------------------------------------------------------------------ ' cat_irace_license <- function() cat(sub("__VERSION__", irace_version, irace_license, fixed=TRUE)) #' Higher-level interface to launch irace. #' #' @inheritParams defaultScenario #' #' @param output.width `integer(1)`\cr The width used for the screen #' output. #' #' @details This function checks the correctness of the scenario, reads the #' parameter space from \code{scenario$parameterFile}, invokes [irace()], #' prints its results in various formatted ways, (optionally) calls #' [psRace()] and, finally, evaluates the best configurations on the test #' instances (if provided). If you want a lower-level interface that just #' runs irace, please see function [irace()]. #' #' @templateVar return_invisible TRUE #' @template return_irace #' @seealso #' \describe{ #' \item{[irace_cmdline()]}{a command-line interface to [irace()].} #' \item{[readScenario()]}{for reading a configuration scenario from a file.} #' \item{[readParameters()]}{read the target algorithm parameters from a file.} #' \item{[defaultScenario()]}{returns the default scenario settings of \pkg{irace}.} #' } #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @concept running #' @export irace_main <- function(scenario, output.width = 9999L) irace_common(scenario = scenario, simple=FALSE, output.width = output.width) #' Test configurations given in the logfile (typically `irace.Rdata`) produced by \pkg{irace}. #' #' `testing_fromlog` executes the testing of the target algorithm configurations #' found by an \pkg{irace} execution. #' #' @param logFile `character(1)`\cr Path to the logfile (typically `irace.Rdata`) produced by \pkg{irace}. #' #' @param testNbElites Number of (final) elite configurations to test. Overrides #' the value found in `logFile`. #' #' @param testIterationElites `logical(1)`\cr If `FALSE`, only the final #' `testNbElites` configurations are tested; otherwise, also test the best #' configurations of each iteration. Overrides the value found in `logFile`. #' #' @param testInstancesDir Directory where testing instances are located, either absolute or relative to current directory. #' #' @param testInstancesFile File containing a list of test instances and optionally additional parameters for them. #' #' @param testInstances Character vector of the instances to be used in the `targetRunner` when executing the testing. #' #' @return `logical(1)`\cr `TRUE` if the testing ended successfully otherwise, `FALSE`. #' #' @details The function `testing_fromlog` loads the `logFile` and obtains the #' testing setup and configurations to be tested. Within the `logFile`, the #' variable `scenario$testNbElites` specifies how many final elite #' configurations to test and `scenario$testIterationElites` indicates #' whether test the best configuration of each iteration. The values may be #' overridden by setting the corresponding arguments in this function. The #' set of testing instances must appear in `scenario[["testInstances"]]`. #' #' @seealso [defaultScenario()] to provide a default scenario for \pkg{irace}. #' [testing_fromfile()] provides a different interface for testing. #' #' @author Manuel López-Ibáñez and Leslie Pérez Cáceres #' @concept running #' @export testing_fromlog <- function(logFile, testNbElites, testIterationElites, testInstancesDir, testInstancesFile, testInstances) { if (is.null.or.empty(logFile)) { irace_note("No logFile provided to perform the testing of configurations. Skipping testing.\n") return(FALSE) } iraceResults <- read_logfile(logFile) scenario <- iraceResults[["scenario"]] instances_changed <- FALSE if (!missing(testNbElites)) scenario[["testNbElites"]] <- testNbElites if (!missing(testIterationElites)) scenario$testIterationElites <- testIterationElites if (!missing(testInstances)) scenario[["testInstances"]] <- testInstances if (!missing(testInstancesDir)) { scenario$testInstancesDir <- testInstancesDir instances_changed <- TRUE } if (!missing(testInstancesFile)) { scenario$testInstancesFile <- testInstancesFile instances_changed <- TRUE } cat("\n\n# Testing of elite configurations:", scenario$testNbElites, "\n# Testing iteration configurations:", scenario$testIterationElites,"\n") if (scenario$testNbElites <= 0) return (FALSE) # If they are already setup, don't change them. if (instances_changed || is.null.or.empty(scenario[["testInstances"]])) { scenario <- setup_test_instances(scenario) if (is.null.or.empty(scenario[["testInstances"]])) { irace_note("No test instances, skip testing\n") return(FALSE) } } # Get configurations that will be tested if (scenario$testIterationElites) testing_id <- sapply(iraceResults$allElites, function(x) x[seq_len(min(length(x), scenario$testNbElites))]) else { tmp <- iraceResults$allElites[[length(iraceResults$allElites)]] testing_id <- tmp[seq_len(min(length(tmp), scenario$testNbElites))] } testing_id <- unique.default(unlist(testing_id)) configurations <- iraceResults$allConfigurations[testing_id, , drop=FALSE] irace_note ("Testing configurations (in no particular order): ", paste(testing_id, collapse=" "), "\n") testing_common(configurations, scenario, iraceResults) return(TRUE) } #' Test configurations given an explicit table of configurations and a scenario file #' #' Executes the testing of an explicit list of configurations given in #' `filename` (same format as in [readConfigurationsFile()]). A `logFile` is #' created unless disabled in `scenario`. This may overwrite an existing one! #' #' @param filename `character(1)`\cr Path to a file containing configurations: one configuration #' per line, one parameter per column, parameter names in header. #' #' @inheritParams defaultScenario #' #' @return iraceResults #' #' @seealso [testing_fromlog()] provides a different interface for testing. #' #' @author Manuel López-Ibáñez #' @concept running #' @export testing_fromfile <- function(filename, scenario) { irace_note ("Checking scenario.\n") scenario <- checkScenario(scenario) if (!scenario$quiet) printScenario(scenario) configurations <- readConfigurationsFile(filename, scenario$parameters) configurations <- cbind(.ID. = seq_nrow(configurations), configurations, .PARENT. = NA_integer_) rownames(configurations) <- configurations[[".ID."]] num <- nrow(configurations) configurations <- checkForbidden(configurations, scenario$parameters$forbidden) if (nrow(configurations) < num) { irace_warning("Some of the configurations in the configurations file were forbidden", "and, thus, discarded.") } # To save the logs iraceResults <- list(scenario = scenario, irace_version = irace_version, allConfigurations = configurations) irace_note ("Testing configurations (in the order given as input): \n") testing_common(configurations, scenario, iraceResults) } testing_common <- function(configurations, scenario, iraceResults) { verbose <- !scenario$quiet if (verbose) configurations_print(configurations) iraceResults$testing <- testConfigurations(configurations, scenario) save_irace_logfile(iraceResults, logfile = scenario$logFile) irace_note ("Testing results (column number is configuration ID in no particular order):\n") if (verbose) print(cbind(seeds = iraceResults$testing$seeds, as.data.frame(iraceResults$testing$experiments))) irace_note ("Finished testing\n") iraceResults } #' Test that the given irace scenario can be run. #' #' Test that the given irace scenario can be run by checking the scenario #' settings provided and trying to run the target-algorithm. #' #' @inheritParams defaultScenario #' #' @return returns `TRUE` if successful and gives an error and returns `FALSE` #' otherwise. #' #' @details If the `parameters` argument is missing, then the parameters #' will be read from the file `parameterFile` given by `scenario`. If #' `parameters` is provided, then `parameterFile` will not be read. This function will #' try to execute the target-algorithm. #' #' @seealso #' \describe{ #' \item{\code{\link{readScenario}}}{for reading a configuration scenario from a file.} #' \item{\code{\link{printScenario}}}{prints the given scenario.} #' \item{\code{\link{defaultScenario}}}{returns the default scenario settings of \pkg{irace}.} #' \item{\code{\link{checkScenario}}}{to check that the scenario is valid.} #' } #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export checkIraceScenario <- function(scenario) { irace_note ("Checking scenario\n") scenario$debugLevel <- 2L scenario <- checkScenario(scenario) if (!scenario$quiet) printScenario(scenario) irace_note("Checking target runner.\n") if (checkTargetFiles(scenario = scenario)) { irace_note("Check successful.\n") return(TRUE) } irace_error("Check unsuccessful.\n") return(FALSE) } init <- function() { irace_note("Initializing working directory...\n") libPath <- system.file(package = "irace") tmplFiles <- list.files(file.path(libPath, "templates")) for (file in tmplFiles) { if (grepl(".tmpl", file) && (file != "target-evaluator.tmpl")) { newFile <- gsub(".tmpl", "", file) if (file == "target-runner.tmpl" && .Platform$OS.type == 'windows') { file.copy(file.path(libPath, "templates", "windows", "target-runner.bat"), file.path(getwd(), "target-runner.bat"), overwrite = FALSE) } else { file.copy(file.path(libPath, "templates", file), file.path(getwd(), newFile), overwrite = FALSE) } } } } #' Launch `irace` with command-line options. #' #' Calls [irace_main()] using command-line options, maybe parsed from the #' command line used to invoke R. #' #' @param argv `character()`\cr The arguments #' provided on the R command line as a character vector, e.g., #' `c("--scenario", "scenario.txt", "-p", "parameters.txt")`. #' Using the default value (not providing the parameter) is the #' easiest way to call [irace_cmdline()]. #' #' @details The function reads the parameters given on the command line #' used to invoke R, finds the name of the scenario file, #' initializes the scenario from the file (with the function #' [readScenario()]) and possibly from parameters passed in #' the command line. It finally starts \pkg{irace} by calling #' [irace_main()]. #' #' List of command-line options: #' ```{r echo=FALSE,comment=NA} #' cmdline_usage(.irace.params.def) #' ``` #' #' @templateVar return_invisible TRUE #' @template return_irace #' #' @seealso #' [irace_main()] to start \pkg{irace} with a given scenario. #' @examples #' irace_cmdline("--version") #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @concept running #' @export irace_cmdline <- function(argv = commandArgs(trailingOnly = TRUE)) { parser <- CommandArgsParser$new(argv = argv, argsdef = .irace.params.def) quiet <- !is.null(parser$readArg (short = "-q", long = "--quiet")) if (quiet) { op <- options(.irace.quiet = TRUE) on.exit(options(op)) } else { cat_irace_license() cat("# installed at: ", system.file(package="irace"), "\n", "# called with: ", paste(argv, collapse = " "), "\n", sep = "") } if (!is.null(parser$readArg(short = "-h", long = "--help"))) { parser$cmdline_usage() return(invisible(NULL)) } if (!is.null(parser$readArg(short = "-v", long = "--version"))) { print(utils::citation(package="irace")) return(invisible(NULL)) } if (!is.null(parser$readArg(short = "-i", long = "--init"))) { init() return(invisible(NULL)) } # Read the scenario file and the command line scenarioFile <- parser$readCmdLineParameter ("scenarioFile", default = "") scenario <- readScenario(scenarioFile) for (param in .irace.params.names) { scenario[[param]] <- parser$readCmdLineParameter(paramName = param, default = scenario[[param]]) } if (quiet) scenario$quiet <- TRUE # Check scenario if (!is.null(parser$readArg (short = "-c", long = "--check"))) { checkIraceScenario(scenario) return(invisible(NULL)) } # Only do testing testFile <- parser$readArg (long = "--only-test") if (!is.null(testFile)) { return(invisible(testing_fromfile(testFile, scenario))) } if (length(parser$argv)) irace_error ("Unknown command-line options: ", paste(parser$argv, collapse = " ")) irace_common(scenario = scenario, simple=FALSE) } #' @rdname irace_cmdline #' @export irace.cmdline <- function(argv = commandArgs(trailingOnly = TRUE)) { .Deprecated("irace.cmdline") irace_cmdline(argv = argv) } ## Check targetRunner execution checkTargetFiles <- function(scenario) { ## Create two random configurations configurations <- sampleUniform(scenario$parameters, 2L, repair = scenario$repairConfiguration) set(configurations, j = ".ID.", value = seq_nrow(configurations)) setcolorder(configurations, ".ID.", before = 1L) # Read initial configurations provided by the user. initConfigurations <- allConfigurationsInit(scenario) setDT(initConfigurations) if (nrow(initConfigurations) > 0L) { irace_assert(all(colnames(configurations) == colnames(initConfigurations))) configurations <- rbindlist(list(initConfigurations, configurations)) set(configurations, j = ".ID.", value = seq_nrow(configurations)) } bounds <- rep(scenario$boundMax, nrow(configurations)) instances_ID <- if (scenario$sampleInstances) sample.int(length(scenario$instances), 1L) else 1L setDF(configurations) experiments <- createExperimentList( configurations, scenario$parameters, instances = scenario$instances, instances_ID = instances_ID, seeds = 1234567L, bounds = bounds) race_state <- RaceState$new(scenario) race_state$start_parallel(scenario) on.exit(race_state$stop_parallel(), add = TRUE) # FIXME: Create a function try.call(err.msg,warn.msg, fun, ...) # Executing targetRunner irace_note("Executing targetRunner (", nrow(configurations), " times)...\n") result <- TRUE # We cannot let targetRunner or targetEvaluator modify our random seed, so we save it. withr::local_preserve_seed() output <- withCallingHandlers( tryCatch(execute_experiments(race_state, experiments, scenario), error = function(e) { cat(sep = "\n", "\n# Error occurred while executing targetRunner:", paste0(conditionMessage(e), collapse="\n")) result <<- FALSE NULL }), warning = function(w) { cat(sep = "\n", "\n# Warning occurred while executing targetRunner:", paste0(conditionMessage(w), collapse="\n")) invokeRestart("muffleWarning")}) if (scenario$debugLevel >= 1L) { cat("# targetRunner returned:\n") print(output, digits = 15L) } irace_assert(is.null(scenario$targetEvaluator) == is.null(race_state$target_evaluator)) if (!result) return(FALSE) if (!is.null(scenario$targetEvaluator)) { irace_note("Executing targetEvaluator...\n") output <- withCallingHandlers( tryCatch(execute_evaluator(race_state$target_evaluator, experiments, scenario, output), error = function(e) { cat(sep = "\n", "\n# Error ocurred while executing targetEvaluator:", paste0(conditionMessage(e), collapse="\n")) result <<- FALSE NULL }), warning = function(w) { cat(sep = "\n", "\n# Warning ocurred while executing targetEvaluator:", paste0(conditionMessage(w), collapse="\n")) invokeRestart("muffleWarning")}) if (scenario$debugLevel >= 1L) { cat("# targetEvaluator returned:\n") print(output, digits = 15L) } } result } irace/R/parameterAnalysis.R0000644000176200001440000001434514737241771015371 0ustar liggesusers#' Return the elite configurations of the final iteration. #' #' @inheritParams has_testing_data #' @param n `integer(1)`\cr Number of elite configurations to return, if \code{n} is larger than the #' number of configurations, then only the existing ones are returned. The default (\code{n=0}) returns all of them. #' @param drop.metadata `logical(1)`\cr Remove metadata, such as the #' configuration ID and the ID of the parent, from the returned #' configurations. See [removeConfigurationsMetaData()]. #' #' @return A data frame containing the elite configurations required. #' #' @examples #' log_file <- system.file("exdata/irace-acotsp.Rdata", package="irace", mustWork=TRUE) #' print(removeConfigurationsMetaData(getFinalElites(log_file, n=1))) #' #' @author Manuel López-Ibáñez and Leslie Pérez Cáceres #' @concept analysis #' @export getFinalElites <- function(iraceResults, n = 0L, drop.metadata = FALSE) { if (missing(iraceResults)) stop("argument 'iraceResults' is missing") iraceResults <- read_logfile(iraceResults) last_elites <- iraceResults$allElites[[length(iraceResults$allElites)]] if (n == 0L) n <- length(last_elites) if (length(last_elites) < n) { cat("Only", length(last_elites), "configurations available, reducing n,") n <- length(last_elites) } last_elites <- last_elites[seq_len(n)] configurations <- subset(iraceResults$allConfigurations, get(".ID.") %in% as.character(last_elites), drop = FALSE) if (drop.metadata) configurations <- removeConfigurationsMetaData(configurations) configurations } #' Returns the configurations selected by ID. #' #' @param ids `integer()`\cr The id or a vector of ids of the candidates configurations to obtain. #' @inheritParams getFinalElites #' #' @return A data frame containing the elite configurations required, in the #' order and with the repetitions given by `ids`. #' @examples #' log_file <- system.file("exdata/irace-acotsp.Rdata", package="irace", mustWork=TRUE) #' getConfigurationById(log_file, ids = c(2,1), drop.metadata = TRUE) #' #' @author Manuel López-Ibáñez and Leslie Pérez Cáceres #' @concept analysis #' @export getConfigurationById <- function(iraceResults, ids, drop.metadata = FALSE) { if (missing(iraceResults)) stop("argument 'iraceResults' is missing") iraceResults <- read_logfile(iraceResults) if (length(ids) < 1L) stop("You must provide at least one configuration id.") get_configuration_by_id_helper(iraceResults$allConfigurations, ids, drop_metadata = drop.metadata) } #' Returns the configurations by the iteration in which they were executed. #' #' @param iterations `integer()`\cr The iteration number or a vector of iteration numbers from where #' the configurations should be obtained. Negative values start counting from the last iteration. #' @inheritParams getFinalElites #' #' @return A data frame containing the elite configurations required. #' #' @examples #' log_file <- system.file("exdata/irace-acotsp.Rdata", package="irace", mustWork=TRUE) #' getConfigurationByIteration(log_file, iterations = c(-2, -1), drop.metadata = TRUE) #' #' @author Manuel López-Ibáñez and Leslie Pérez Cáceres #' @concept analysis #' @export getConfigurationByIteration <- function(iraceResults, iterations, drop.metadata = FALSE) { if (missing(iraceResults)) stop("argument 'iraceResults' is missing") iraceResults <- read_logfile(iraceResults) if (length(iterations) < 1L) stop("You must provide at least one iteration number.") n_iterations <- length(iraceResults$iterationElites) iterations <- as.integer(iterations) iterations <- ifelse(iterations >= 0L, iterations, n_iterations + 1L + iterations) # To silence warning. iteration <- NULL if (is.null(iraceResults$state$experiment_log) || nrow(iraceResults$state$experiment_log) == 0L) stop("'iraceResults' does not contain experiment_log, maybe the wrong file, an incomplete run or the wrong version of irace?") ids <- unique(subset(as.data.frame(iraceResults$state$experiment_log), iteration %in% iterations, select="configuration", drop=TRUE)) get_configuration_by_id_helper(iraceResults$allConfigurations, ids, drop_metadata = drop.metadata) } get_configuration_by_id_helper <- function(allConfigurations, ids, drop_metadata) { configurations <- allConfigurations[match(ids, allConfigurations[[".ID."]]), , drop=FALSE] if (nrow(configurations) == 0L) stop("No configuration found with ID:", ids, ".") if (drop_metadata) configurations <- removeConfigurationsMetaData(configurations) configurations } #' Returns the pairs of instance IDs and seeds used as instances in the race #' (and optionally the actual instances). #' #' @inheritParams getFinalElites #' @param index `integer()`\cr Indexes of the (instanceID,seed) pairs to be returned. The default returns everything. #' @param instances `logical(1)`\cr Whether to add the actual instances as an additional column (only if the instances are of atomic type). #' #' @return `data.table()`\cr With default arguments, a `data.table` containing two columns #' `"instanceID"` and `"seed"`. With `instances=TRUE` and if the instances #' are of atomic type (see [is.atomic()]) type, another column `instance` is #' added that contains the actual instance. #' #' @examples #' log_file <- system.file("exdata/irace-acotsp.Rdata", package="irace", mustWork=TRUE) #' head(get_instanceID_seed_pairs(log_file)) #' # Add the instance names #' get_instanceID_seed_pairs(log_file, index=1:10, instances=TRUE) #' @author Manuel López-Ibáñez #' @concept analysis #' @export get_instanceID_seed_pairs <- function(iraceResults, index, instances = FALSE) { if (missing(iraceResults)) stop("argument 'iraceResults' is missing") iraceResults <- read_logfile(iraceResults) instances_log <- iraceResults$state$instances_log if (!missing(index)) instances_log <- instances_log[index, , drop = FALSE] if (!instances) return(instances_log) instances <- iraceResults$scenario$instances if (!is.atomic(instances)) { warning("instances=TRUE requested, but instances are not of atomic type") return(instances_log) } instanceID <- instances_log[["instanceID"]] cbind(instances_log, instance = instances[instanceID]) } irace/R/utils.R0000644000176200001440000005426114745735066013052 0ustar liggesusers# Print a user-level warning message, when the calling context # cannot help the user to understand why the program failed. irace_warning <- function(...) { if (getOption(".irace.quiet", default=FALSE)) return() warning(paste0(.irace_msg_prefix, ..., collapse=""), call. = FALSE, immediate. = TRUE) } # Print a user-level fatal error message, when the calling context # cannot help the user to understand why the program failed. irace_error <- function(...) { # The default is only 1000, which is too small. 8170 is the maximum # value allowed up to R 3.0.2 withr::local_options(list(warning.length = 8170)) stop (.irace_msg_prefix, ..., call. = FALSE) } ## When irace crashes, it generates a file "iracedump.rda". To debug the crash use: ## R> load("iracedump.rda") ## R> debugger(iracedump) ## ## See help(dump.frames) for more details. irace_dump_frames <- function() { execDir <- getOption(".irace.execdir") if (!is.null(execDir)) cwd <- setwd(execDir) utils::dump.frames(dumpto = "iracedump", to.file = TRUE, include.GlobalEnv = TRUE) # FIXME: We want to use on.exit(setwd(cwd)) but q() does not run on.exit. if (!is.null(execDir)) setwd(cwd) # We need this to signal an error in R CMD check. See help(dump.frames) if (!interactive()) quit("no", status = 1) } # Print an internal fatal error message that signals a bug in irace. irace_internal_error <- function(...) { .irace.bug.report <- paste0(.irace_msg_prefix, "An unexpected condition occurred. ", "Please report this bug to the authors of the irace package ") op <- list(warning.length = 8170L) if (!base::interactive()) op <- c(op, list(error = irace_dump_frames)) withr::local_options(op) # 6 to not show anything below irace_assert() bt <- utils::capture.output(traceback(5)) warnings() stop (.irace_msg_prefix, paste0(..., collapse = "\n"), "\n", paste0(bt, collapse= "\n"), "\n", .irace.bug.report, call. = FALSE) invisible() } irace_assert <- function(exp, eval_after = NULL) { # FIXME: It would be great if we could save into a file the state of # the function that called this one. if (isTRUE(as.logical(exp))) return(invisible()) mc <- sys.call()[[2L]] msg <- paste0("'", deparse(mc), "' is not TRUE") if (!is.null(eval_after)) { msg_after <- eval.parent(utils::capture.output(eval_after)) msg <- paste0(msg, "\n", paste0(msg_after, collapse="\n")) } irace_internal_error(msg) invisible() } irace_note <- function(...) { # FIXME: If this was a function within an irace object, we could replace it # when using quiet. if (getOption(".irace.quiet", default=FALSE)) return() cat ("# ", format(Sys.time(), usetz=TRUE), ": ", paste0(..., collapse = ""), sep = "") } file.check <- function (file, executable = FALSE, readable = executable, writeable = FALSE, isdir = FALSE, notempty = FALSE, text = NULL) { EXEC <- 1 # See documentation of the function file.access() WRITE <- 2 READ <- 4 if (!is.character(file) || is.null.or.empty(file)) { irace_error (text, " ", shQuote(file), " is not a vaild filename") } file <- path_rel2abs(file) ## The above should remove the trailing separator if present for windows OS ## compatibility, except when we have just C:/, where the trailing separator ## must remain. if (!file.exists(file)) { if (writeable) { if (tryCatch({ suppressWarnings(file.create(file) && file.remove(file)) }, error=function(e) FALSE)) return(TRUE) irace_error("cannot create ", text, " ", shQuote(file)) return (FALSE) } irace_error (text, " '", file, "' does not exist") return(FALSE) } if (writeable && (file.access(file, mode = WRITE) != 0)) { irace_error(text, " '", file, "' cannot be written into") return(FALSE) } if (readable && (file.access(file, mode = READ) != 0)) { irace_error(text, " '", file, "' is not readable") return (FALSE) } if (executable && file.access(file, mode = EXEC) != 0) { irace_error(text, " '", file, "' is not executable") return (FALSE) } if (isdir) { if (!file.info(file)$isdir) { irace_error(text, " '", file, "' is not a directory") return (FALSE) } if (notempty && length(list.files (file, recursive=TRUE)) == 0) { irace_error(text, " '", file, "' does not contain any file") return (FALSE) } } else if (file.info(file)$isdir) { irace_error(text, " '", file, "' is a directory, not a file") return (FALSE) } return (TRUE) } # Returns the smallest multiple of d that is higher than or equal to x. round_to_next_multiple <- function(x, d) (x + d - 1L - (x - 1L) %% d) # This returns FALSE for Inf/-Inf/NA is.wholenumber <- function(x, tol = .Machine$double.eps^0.5) is.finite(x) & (abs(x - round(x)) < tol) is_na_nowarn <- function(x) length(x) == 1L && suppressWarnings(is.na(x)) is_na_or_empty <- function(x) (length(x) == 0L) || is_na_nowarn(x) is.null.or.na <- function(x) is.null(x) || is_na_nowarn(x) is.null.or.empty <- function(x) (length(x) == 0L) || (length(x) == 1L && !suppressWarnings(is.na(x)) && is.character(x) && x == "") is_null_or_empty_or_na <- function(x) (length(x) == 0L) || is_na_nowarn(x) || (length(x) == 1L && !suppressWarnings(is.na(x)) && is.character(x) && x == "") is.function.name <- function(FUN) { # FIXME: Is there a simpler way to do this check? is.function(FUN) || (!is.null(FUN) && !is.na(FUN) && as.character(FUN) != "" && !is.null(get.function(FUN))) } get.function <- function(FUN) { if (is.function(FUN)) return(FUN) FUN <- dynGet(as.character(FUN), ifnotfound = NULL, inherits = TRUE) if (is.function(FUN)) return(FUN) NULL } is.bytecode <- function(x) typeof(x) == "bytecode" bytecompile <- function(x) { if (is.bytecode(x)) return(x) compiler::cmpfun(x) } # This function is used to trim potentially large strings for printing, since # the maximum error/warning length is 8170 characters (R 3.0.2) strlimit <- function(s, limit = 5000L) { if (nchar(s) <= limit) return(s) paste0(substr(s, 1L, limit - 3L), "...") } #' Update filesystem paths of a scenario consistently. #' #' This function should be used to change the filesystem paths stored in a #' scenario object. Useful when moving a scenario from one computer to another. #' #' @inheritParams defaultScenario #' @param from `character(1)`\cr Character string containing a regular expression (or character #' string for `fixed = TRUE`) to be matched. #' @param to `character(1)`\cr The replacement string.character string. For `fixed = FALSE` #' this can include backreferences `"\1"` to `"\9"` to #' parenthesized subexpressions of `from`. #' @param fixed `logical(1)`\cr If `TRUE`, `from` is a string to be matched #' as is. #' @return The updated scenario #' @examples #' \dontrun{ #' scenario <- readScenario(filename = "scenario.txt") #' scenario <- scenario_update_paths(scenario, from = "/home/manuel/", to = "/home/leslie") #' } #' @seealso [base::grep()] #' @export scenario_update_paths <- function(scenario, from, to, fixed = TRUE) { pathParams <- .irace.params.def[.irace.params.def[, "type"] == "p", "name"] # Only consider the ones that actually appear in scenario. pathParams <- intersect(pathParams, names(scenario)) scenario[pathParams] <- lapply(scenario[pathParams], sub, pattern = from, replacement = to, fixed = fixed) scenario } test.type.order.str <- function(test.type) switch(test.type, friedman = "sum of ranks", t.none =, # Fall-throught t.holm =, # Fall-throught t.bonferroni = "mean value", irace_internal_error ("test.type.order.str() Invalid value '", test.type, "' of test.type")) trim_leading <- function(str) sub('^[[:space:]]+', '', str, perl = TRUE) ## white space, POSIX-style trim_trailing <- function(str) sub('[[:space:]]+$', '', str, perl = TRUE) ## white space, POSIX-style # remove leading and trailing white space characters trim <- function(str) trim_trailing(trim_leading(str)) ## This function takes two matrices x and y and merges them such that the ## resulting matrix z has: # rownames(z) <- setunion(rownames(x), rownames(y)) and # rownames(z) <- setunion(rownames(x), rownames(y)) and # z[rownames(x), colnames(x)] <- x and z[rownames(y), colnames(y)] <- y, and # z[i, j] <- NA for all i,j not in x nor y. merge_matrix <- function(x, y) { rownames_x <- rownames(x) colnames_x <- colnames(x) rownames_y <- rownames(y) colnames_y <- colnames(y) if (is.null(rownames_x) || is.null(colnames_x)) return(y) if (is.null(rownames_y) || is.null(colnames_y)) return(x) row_union <- union(rownames_x, rownames_y) col_union <- union(colnames_x, colnames_y) z <- matrix(NA_real_, nrow = length(row_union), ncol = length(col_union), dimnames = list(row_union, col_union)) # Map row and column names to indices for efficient assignment. row_idx_x <- chmatch(rownames_x, row_union) col_idx_x <- chmatch(colnames_x, col_union) row_idx_y <- chmatch(rownames_y, row_union) col_idx_y <- chmatch(colnames_y, col_union) z[row_idx_x, col_idx_x] <- x z[row_idx_y, col_idx_y] <- y # There must be a non-NA entry for each instance. irace_assert(all(rowAnyNotNAs(z))) return(z) } # FIXME: This may not work when working interactively. For example, # one cannot change the number of slaves. A more robust function # would try to close any open slaves, and then re-spawn a different # number. ## # FIXME2: Slaves will load the irace namespace independently of how the master # loaded it. This can be seen by doing in the slaves: # ## print(loadedNamespaces()) ## try(print(as.list(get(".__NAMESPACE__.", envir = asNamespace("irace", base.OK = FALSE), ## inherits = FALSE))$path)) ## try(print(path.package("irace"))) # # That is, neither R_LIBS, .libPaths or whether library was called with lib.loc # will affect the slaves. It also happens before we can set those variables on # the slaves. Thus, one may end up running a different version of irace on the # slaves than on the master. I wasted more than 12 hours trying to find a # work-around but nothing seems to work. mpiInit <- function(nslaves, debugLevel = 0) { # Load the Rmpi package if it is not already loaded. if ("Rmpi" %not_in% loadedNamespaces()) { if (!suppressPackageStartupMessages(requireNamespace("Rmpi", quietly = TRUE))) irace_error("The 'Rmpi' package is required for using MPI") # When R exits, finalize MPI. reg.finalizer(environment(Rmpi::mpi.exit), function(e) { # Rmpi already prints a message, so we don't need this. # cat("# Finalize MPI...\n") if (Rmpi::mpi.comm.size(1) > 0L) # FIXME: dellog == TRUE tries to delete log files, but it does # not take into account that we may have changed directory and # it does not fail gracefully but produces an annoying: # Warning message: running command 'ls *.30577+1.*.log 2>/dev/null' had status 2 Rmpi::mpi.close.Rslaves(dellog = FALSE) # We would like to use .Call("mpi_finalize", PACKAGE = "Rmpi"), which is # what mpi.finalize does, minus the annoying message: "Exiting Rmpi. Rmpi # cannot be used unless relaunching R", which we do not care about # because this finalizer should only be called when exiting R. utils::capture.output(Rmpi::mpi.finalize(), file = if (.Platform$OS.type == 'windows') 'NUL' else '/dev/null') }, onexit = TRUE) # Create slaves # needlog: a logical. If TRUE, R BATCH outputs will be saved in log # files. If FALSE, the outputs will send to /dev/null. # quiet: a logical. If TRUE, do not print anything unless an error occurs. # If FALSE, prints to stdio how many slaves are successfully # spawned and where they are running. Rmpi::mpi.spawn.Rslaves(nslaves = nslaves, quiet = (debugLevel < 2), needlog = (debugLevel > 0)) } } ## FIXME: Move this to the manual page. ## FIXME: Export this function. # Computes: # * Kendall's W (also known as Kendall's coefficient of concordance) # If 1, all configurations have ranked in the same order in all instances. # If 0, the ranking of each configuration on each instance is essentially random. # W = Friedman / (m * (k-1)) # # * Spearman's rho: average (Spearman) correlation coefficient computed on the # ranks of all pairs of raters. If there are no repeated data values, a # perfect Spearman correlation of +1 or −1 occurs when each of the variables # is a perfect monotone function of the other. # # data: matrix with the data, instances in rows (judges), configurations in # columns. concordance <- function(data) { irace_assert (is.matrix(data) && is.numeric(data)) n <- nrow(data) #judges k <- ncol(data) #objects if (n <= 1L || k <= 1L) return(list(kendall.w = NA_real_, spearman.rho = NA_real_)) # Get rankings by rows (per instance) r <- rowRanks(data, ties.method = "average", useNames=FALSE) TIES <- c(table(r,row(r))) # If everything is tied, then W=1, perfect homogeneity. if (all(TIES == k)) { W <- 1 } else { # FIXME: This formula seems slightly different from the one in # friedman.test. Why? TIES <- sum(TIES^3L - TIES) R <- colSums2(r, useNames=FALSE) W <- ((12 * sum((R - n * (k + 1) / 2)^2L)) / ((n^2L * (k^3L - k)) - (n * TIES))) } # Spearman's rho rho <- (n * W - 1) / (n - 1L) ## Same as in friedman test #STATISTIC <- n * (k - 1) * W #PARAMETER <- k - 1 #pvalue <- pchisq(PARAMETER, df = PARAMETER, lower.tail = FALSE) list(kendall.w = W, spearman.rho = rho) } ## FIXME: Move this to the manual page. ## FIXME: Reference! Explain a bit what is computed! # Calculates Performance similarity of instances # data: matrix with the data, instances in rows (judges), configurations # in columns. # Returns: variance value [0,1], where 0 is a homogeneous set of instances and # 1 is a heterogeneous set. # FIXME: How to handle missing values? dataVariance <- function(data) { irace_assert (is.matrix(data) && is.numeric(data)) # LESLIE: should we rank data?? # MANUEL: We should add the option. if (nrow(data) <= 1L || ncol(data) <= 1L) return(NA_real_) # Normalize #datamin <- apply(data,1,min,na.rm=TRUE) #datamax <- apply(data,1,max,na.rm=TRUE) #normdata <- (data - datamin) / (datamax-datamin) #standardize meandata <- rowMeans2(data, useNames=FALSE) stddata <- rowSds(data, useNames=FALSE) # If stddata == 0, then data is constant and it doesn't matter as long as it # is non-zero. stddata[stddata == 0] <- 1 zscoredata <- (data - meandata) / stddata # FIXME: We could log-tranform if needed # Variance of configurations mean(colVars(zscoredata)) } runcommand <- function(command, args, id, debugLevel, timeout = 0) { if (debugLevel >= 2L) { irace_note (command, " ", args, "\n") elapsed <- proc.time()["elapsed"] } err <- NULL output <- withCallingHandlers( tryCatch(system2(command, args, stdout = TRUE, stderr = TRUE, timeout = timeout), error = function(e) { err <<- c(err, paste(conditionMessage(e), collapse="\n")) NULL }), warning = function(w) { err <<- c(err, paste(conditionMessage(w), collapse="\n")) invokeRestart("muffleWarning") }) if (is.null(output)) output <- "" # If the command could not be run an R error is generated. If ‘command’ # runs but gives a non-zero exit status this will be reported with a # warning and in the attribute ‘"status"’ of the result: an attribute # ‘"errmsg"’ may also be available. if (!is.null(err)) { err <- paste(err, collapse = "\n") if (!is.null(attr(output, "errmsg"))) err <- paste(sep = "\n", err, attr(output, "errmsg")) if (debugLevel >= 2L) irace_note ("ERROR (", id, "): ", err, "\n") return(list(output = output, error = err)) } if (debugLevel >= 2L) { irace_note ("DONE (", id, ") Elapsed wall-clock seconds: ", formatC(proc.time()["elapsed"] - elapsed, format = "f", digits = 2), "\n") } # TODO: Return elapsed time so that we can report at the end the total # elapsed time taken by irace vs the time taken by the target-runner. list(output = output, error = NULL) } # Safe sampling of vector: resample <- function(x, ...) x[sample.int(length(x), ...)] # Rounds up the number x to the specified number of decimal places 'digits'. ceiling_digits <- function(x, digits) { multiple <- 10^-digits div <- x / multiple int_div <- trunc(div) int_div * multiple + ceiling(div - int_div) * multiple } # ceil.decimal <- function(x, d) { # # get the significant digits in the integer part. # ssd <- x * 10^(d) # # get the non significant digits # nsd <- ssd - floor(ssd) # ssd <- trunc(ssd) # sel <- nsd > 0 | ssd==0 # ssd[sel] <- ssd[sel] + 1 # x2 <- ssd/10^(d) # return(x2) # } is.file.extension <- function(filename, ext) substring(filename, nchar(filename) + 1L - nchar(ext)) == ext is.sub.path <- function(x, dir, n = nchar(dir)) substr(x, 1L, n) == dir # Same as !(x %in% table). Package data.table has %notin%. "%not_in%" <- function(x, table) is.na(match(x, table)) #' Save the log generated by \pkg{{irace}} to a file (by default `irace.Rdata`). #' #' This function may be useful if you are manually editing the log data generated by a run of \pkg{irace}. #' #' @param iraceResults `list()`\cr Object created by \pkg{irace} and typically saved in the log file `irace.Rdata`. #' #' @param logfile `character(1)`\cr Filename to save `iraceResults`. Usually, this is given by `scenario$logFile`. If `NULL` or `""`, no data is saved. #' #' @seealso [read_logfile()] #' @concept analysis save_irace_logfile <- function(iraceResults, logfile) { # FIXME: Raul Santa Maria proposed to only save if sufficient time (>= 1 # minute) has passed since the last save. We would need an option to force # saving the last one. if (is.null.or.empty(logfile)) return(invisible()) # Files produced by `saveRDS` (or `serialize` to a file connection) are not # suitable as an interchange format between machines, for example to download # from a website. The files produced by `save` have a header identifying the # file type and so are better protected against erroneous use. save(iraceResults, file = logfile, version = 3L) } valid_iracelog <- function(x) { is.list(x) && ("scenario" %in% names(x)) } #' Read the log file produced by irace (`irace.Rdata`). #' #' @param filename Filename that contains the log file saved by irace. Example: `irace.Rdata`. #' #' @param name Optional argument that allows overriding the default name of the object in the file. #' #' @return (`list()`) #' @examples #' irace_results <- read_logfile(system.file("exdata/irace-acotsp.Rdata", package="irace", #' mustWork=TRUE)) #' str(irace_results) #' @concept analysis #' @export read_logfile <- function(filename, name = "iraceResults") { if (is_na_or_empty(filename)) irace_error("read_logfile: 'filename' is NULL or NA.") # If filename is already the iraceResults object, just return it. if (valid_iracelog(filename)) return(filename) if (file.access(filename, mode = 4) != 0) irace_error("read_logfile: Cannot read file '", filename, "'.") load(filename) iraceResults <- get0(name, inherits=FALSE) if (!valid_iracelog(iraceResults)) irace_error("read_logfile: The file '", filename, "' does not contain the '", name, "' object.") # data.table recommends doing this after loading a data.table from a file. setDT(iraceResults$state$experiment_log) setDT(iraceResults$state$instances_log) iraceResults } get_log_clean_version <- function(iraceResults) { log_version <- iraceResults$irace_version if (is.null(log_version)) log_version <- iraceResults$irace.version if (is.null(log_version)) return(package_version("0.0.0")) if (log_version == "unknown") # This may happen during development. return(package_version("1000.0.0")) if (length(gregexpr("\\.", log_version)[[1L]]) > 3L || grepl("[a-z]", log_version)) log_version <- sub("\\.[^.]*$", "", log_version) package_version(log_version) } #' Check if the results object generated by irace has data about the testing phase. #' #' @param iraceResults `list()`|`character(1)`\cr Object created by \pkg{irace} and typically saved in the log file `irace.Rdata`. If a character string is given, then it is interpreted as the path to the log file from which the `iraceResults` object will be loaded. #' #' @return `logical(1)` #' @examples #' irace_results <- read_logfile(system.file("exdata/irace-acotsp.Rdata", package="irace", #' mustWork=TRUE)) #' print(has_testing_data(irace_results)) #' @export has_testing_data <- function(iraceResults) { ins <- iraceResults$scenario$testInstances exp <- iraceResults$testing$experiments !(length(ins) == 0L || (length(ins) == 1L && (is.na(ins) || ins == "")) || length(exp) == 0L || !(is.matrix(exp) || is.data.frame(exp))) } do_nothing <- function(...) invisible() seq_nrow <- function(x) seq_len(nrow(x)) clamp <- function(x, lower, upper) pmax.int(lower, pmin.int(x, upper)) truncate_rows <- function(x, n) { nx <- nrow(x) if (nx <= n) return(x) x[-seq.int(n + 1L, nx), ] } vlast <- function(x) { stopifnot(is.null(dim(x))) lx <- length(x) if (!lx) x else x[[lx]] } # 2147483647 is the maximum value for a 32-bit signed integer. # We use replace = TRUE, because replace = FALSE allocates memory for each possible number. # Do not use .Machine$integer.max unless it is smaller, to minimize differences # between machines. runif_integer <- function(size) sample.int(min(2147483647L, .Machine$integer.max), size = size, replace = TRUE) unlist_element <- function(x, element) unlist(lapply(x, "[[", element, exact=TRUE), recursive=FALSE, use.names=FALSE) # Extensions of matrixStats rowAnyNotNAs <- function(x, rows = NULL, cols = NULL, ..., useNames = FALSE) !rowAlls(x, rows = rows, cols = cols, value = NA, ..., useNames = useNames) colAnyNotNAs <- function(x, rows = NULL, cols = NULL, ..., useNames = FALSE) !colAlls(x, rows = rows, cols = cols, value = NA, ..., useNames = useNames) irace/R/parameters.R0000644000176200001440000006635115060315667014050 0ustar liggesusers# Checks that variables in the expressions are within the parameters names. check_domain_dependencies <- function (params, depends, types) { allnames <- names(params) allowed_fx <- c("+", "-", "*", "/", "%%", "min", "max", "round", "floor", "ceiling", "trunc") depends <- Filter(length, depends) for (p in names(depends)) { parameter <- params[[p]] domain <- parameter[["domain"]] vars <- depends[[p]] flag <- vars %in% allnames if (!all(flag)) { irace_error ("Domain (", paste0(domain, collapse=", "), ") of parameter '", p, "' is not valid: '", paste0(vars[!flag], collapse=", "), "' cannot be found in the scenario parameters: ", paste0(allnames, collapse=", ")," .") } flag <- types[vars] %in% c("i", "r") if (!all(flag)) { irace_error ("Domain of parameter '", p, "' depends on non-numerical", " parameters: ", paste0(vars[!flag], collapse=", "), " .") } # Supported operations for dependent domains fx <- setdiff(all.names(domain, unique=TRUE), all.vars(domain, unique=TRUE)) flag <- fx %in% allowed_fx if (!all(flag)) { irace_error ("Domain of parameter '", p, "' uses function(s) ", "not yet supported by irace: ", paste0(fx[!flag], collapse=", "), " .") } } invisible(TRUE) } check_forbidden_params <- function(x, pnames, filename = NULL) { if (length(x) == 0L) return(invisible()) if (any(unique(unlist(lapply(x, all.names))) %in% c("&&", "||"))) { for (ex in x) { if (any(all.names(ex) %in% c("&&", "||"))) irace_error("Please use '&' and '|' instead of '&&' and '|' in: ", deparse(ex), " .\n") } } if (all(all.vars(x) %in% pnames)) return(invisible()) for (ex in x) { v <- setdiff(all.vars(ex), pnames) if (length(v)) { v <- paste0(v, collapse=", ") if (is.null(filename)) { irace_error("Forbidden expression '", deparse(ex), "' contains unknown parameter(s): ", v) } else if (is.na(filename)) { irace_error("Expression '", deparse(ex), "' after [forbidden] contains unknown parameter(s): ", v) } else { irace_error("Expression '", deparse(ex), "' after [forbidden] in '", filename, "' contains unknown parameter(s): ", v) } } } } # --------------------------------------------------------------------------- # Ordering of the parameters according to conditions hierarchy # * The conditions hierarchy is an acyclic directed graph. # Function treeLevel() computes an order on vertex s.t: # level(A) > level(B) <=> There is an arc A ---> B # (A depends on B to be activated) # * If a cycle is detected, execution is stopped # * If a parameter depends on another one not defined, execution is stopped param_level <- function(paramName, varsTree, rootParam = paramName) { # The last parameter is used to record the root parameter of the # recursive call in order to detect the presence of cycles. vars <- varsTree[[paramName]] if (length(vars) == 0L) return(1L) # This parameter does not have conditions # This parameter has some conditions # Recursive call: level <- MAX( level(m) : m in children ) maxChildLevel <- 0L for (child in vars) { # The following line detects cycles if (child == rootParam) irace_error("Cycle detected in subordinate parameters! ", "Check definition of conditions and/or dependent domains.\n", "One parameter of this cycle is '", rootParam, "'") # The following line detects a missing definition if (child %not_in% names(varsTree)) irace_error("A parameter definition is missing! ", "Check definition of parameters.\n", "Parameter '", paramName, "' depends on '", child, "' which is not defined.") level <- param_level(child, varsTree, rootParam) if (level > maxChildLevel) maxChildLevel <- level } maxChildLevel + 1L } transform_domain <- function(transf, domain, type) { # We do not support transformation of dependent parameters, yet # TODO: think about dependent domain transformation if (is.expression(domain)) stop("Parameter domain transformations are not yet available for", " dependent parameter domains.") lower <- domain[1L] upper <- domain[2L] if (transf == "log") { # Reject log if domain contains zero or negative values if (any(domain <= 0)) stop("Domain (", lower, ", ", upper, ") of parameter of type 'log' contains non-positive values") trLower <- log(lower) # +1 to adjust before floor() trUpper <- if (type == "i") log(upper + 1) else log(upper) irace_assert(is.finite(trLower)) irace_assert(is.finite(trUpper)) attr(transf, "lower") <- trLower attr(transf, "upper") <- trUpper return(transf) } stop("unrecognized transformation type '", transf, "'") } named_stop <- function(name, ...) stop(errorCondition(paste0(..., collapse=""), class = name, call = sys.call(-1))) param_parse_condition <- function(x, type) { if (is.language(x)) x <- as.expression(x) else if (is.character(x)) { x <- tryCatch(str2expression(x), error = function(e) { msg <- sub(":[0-9]+:[0-9]+: ", "", conditionMessage(e), perl=TRUE) msg <- sub("\n1:", "\n ", msg, perl=TRUE) stop("Invalid condition '", x, "': ", msg, "\n") }) } if (!is.expression(x)) stop("Invalid condition '", x, "'") if (any(all.names(x) %in% c("&&", "||"))) stop("Please use '&' and '|' instead of '&&' and '|' in: ", deparse(x[[1L]]), " .\n") x } valid_real_bound <- function(x, digits) { if (is.na(x) || x == 0) return(TRUE) rx <- round(x, digits = digits) ((abs(rx - x) <= .irace_tolerance * max(1, abs(x))) && digits >= -log10(abs(x))) } parse_exp_domain <- function(domain) { # Convert to expression if string domain <- sapply(domain, USE.NAMES=FALSE, function(x) if (is.character(x)) str2expression(x) else x) # If is an expression or language without variables, then evaluate to convert it to numeric. domain <- sapply(domain, USE.NAMES=FALSE, function(x) if (is.numeric(x) || length(all.vars(x, unique=FALSE))) x else eval(x)) domain } Parameter <- function(name, type, domain, label, condition, transf, digits = if (type=='r') 15L else NULL) { cls <- switch(type, i="ParamInt", r="ParamReal", c="ParamCat", o="ParamOrd", stop("Unknown parameter type '", type, "'.")) if (!isTRUE(condition)) { condition <- param_parse_condition(condition) } if (type %in% c("i", "r")) { exp_domain <- parse_exp_domain(domain) domain <- sapply(exp_domain, USE.NAMES=FALSE, function(x) if (is.numeric(x)) x else NA) if (type == "r") { # digits >= 15 is almost infinite-precision so we do not complain. digits <- as.integer(digits) if (digits < 15L && (!valid_real_bound(domain[1L], digits) || !valid_real_bound(domain[2L], digits))) { for (i in seq.int(digits+1L, 15L)) { if (valid_real_bound(domain[1L], i) && valid_real_bound(domain[2L], i)) break } stop("Domain bounds (", domain[1L], ", ", domain[2L], ") of parameter '", name, "' of type 'real' must be representable within the given 'digits=", digits, "'; you would need at least 'digits=", i, "' or adjust the domain") } if (anyNA(domain)) domain <- as.expression(exp_domain) } else { # type == "i" if (any(!is.wholenumber(domain[!is.na(domain)]))) named_stop("invalid_domain", "for parameter '", name, "' of type 'integer', values must be integers") if (anyNA(domain)) domain <- as.expression(sapply(exp_domain, USE.NAMES=FALSE, function(x) if (is.numeric(x)) as.integer(x) else x)) else domain <- as.integer(domain) } if (!is.expression(domain) && domain[1L] >= domain[2L]) named_stop("invalid_range", "lower bound must be smaller than upper bound in numeric range") } else { # type %in% c("c", "o") if (anyDuplicated(domain)) { dups <- duplicated(domain) stop("duplicated values (", paste0('\"', domain[dups], "\"", collapse = ', '), ") for parameter '", name, "'") } if (type == "o") { tmp <- suppressWarnings(as.numeric(domain)) if (!anyNA(tmp) && !identical(order(tmp), seq_along(tmp))) stop("the domain of parameter '", name, "' appears to be a discretization of a numerical range, but the values are not in increasing order: ", paste0(domain, collapse = ', ')) } } if (transf != "") transf <- transform_domain(transf, domain, type) isFixed <- (type == "c" || type == "o") && (length(domain) == 1L) param <- list(name = name, type = type, domain = domain, label = label, is_dependent = is.expression(domain), isFixed = isFixed, transform = transf, condition = condition) if (type == "r") param$digits <- digits class(param) <- c(cls, "Parameter", class(param)) param } #' Create a parameter space to be tuned. #' #' @description #' - `param_cat()` creates a categorical parameter. #' - `param_ord()` creates an ordinal parameter. #' - `param_real()` creates a real-valued parameter. #' - `param_int()` creates an integer parameter. #' #' @param ... one or more parameters created by `param_int()`, `param_real()`, `param_ord()`, or `param_cat()`. #' @param forbidden `expression()|character(1)`\cr String or list of expressions that define forbidden parameter configurations. #' @inheritParams readParameters #' #' @return `ParameterSpace` #' @name parameters #' @examples #' digits <- 4L #' parametersNew( #' param_cat(name = "algorithm", values = c("as", "mmas", "eas", "ras", "acs"), label = "--"), #' param_ord(name = "localsearch", values = c("0", "1", "2", "3"), label = "--localsearch "), #' param_real(name = "alpha", lower = 0.0, upper=5.0, label = "--alpha ", digits = digits), #' param_real(name = "beta", lower = 0.0, upper = 10.0, label = "--beta ", digits = digits), #' param_real(name = "rho", lower = 0.01, upper = 1.00, label = "--rho ", digits = digits), #' param_int(name = "ants", lower = 5, upper = 100, transf = "log", label = "--ants "), #' param_real(name = "q0", label = "--q0 ", lower=0.0, upper=1.0, #' condition = expression(algorithm == "acs")), #' param_int(name = "rasrank", label = "--rasranks ", lower=1, upper=quote(min(ants, 10)), #' condition = 'algorithm == "ras"'), #' param_int(name = "elitistants", label = "--elitistants ", lower=1, upper=expression(ants), #' condition = 'algorithm == "eas"'), #' param_int(name = "nnls", label = "--nnls ", lower = 5, upper = 50, #' condition = expression(localsearch %in% c(1,2,3))), #' param_cat(name = "dlb", label = "--dlb ", values = c(0,1), #' condition = "localsearch %in% c(1,2,3)"), #' forbidden = "(alpha == 0) & (beta == 0)") #' #' @export parametersNew <- function(..., forbidden = NULL, debugLevel = 0L) ParameterSpace$new(..., forbidden = forbidden, verbose = debugLevel) ParameterSpace <- R6::R6Class("ParameterSpace", cloneable = TRUE, lock_class = TRUE, lock_objects = TRUE, public = list( .params = NULL, conditions = NULL, depends = NULL, domains = NULL, forbidden = NULL, hierarchy = NULL, isFixed = NULL, names = NULL, names_fixed = NULL, names_variable = NULL, names_numeric = NULL, nbParameters = NULL, nbFixed = NULL, nbVariable = NULL, switches = NULL, types = NULL, initialize = function(..., forbidden = NULL, verbose = 0L) { .params <- list(...) names(.params) <- sapply(.params, "[[", "name") self$names <- names(.params) self$types <- sapply(.params, "[[", "type") self$switches <- sapply(.params, "[[", "label") # This has to be a list because each element is a vector. self$domains <- lapply(.params, "[[", "domain") self$conditions <- lapply(.params, "[[", "condition") self$isFixed <- sapply(.params, "[[", "isFixed") self$names_fixed <- self$names[self$isFixed] self$names_variable <- self$names[!self$isFixed] self$names_numeric <- self$names[self$types %in% c("r", "i")] self$nbParameters <- length(.params) self$nbFixed <- sum(self$isFixed) self$nbVariable <- sum(!self$isFixed) # Obtain the variables in each condition self$depends <- lapply(self$domains, all.vars) # Check that domain dependencies are OK. check_domain_dependencies(.params, self$depends, self$types) # Merge dependencies and conditions self$depends <- Map(union, self$depends, lapply(self$conditions, all.vars)) # Sort parameters in 'conditions' in the proper order according to conditions hierarchyLevel <- sapply(names(.params), param_level, varsTree = self$depends) self$hierarchy <- hierarchyLevel self$conditions <- self$conditions[order(hierarchyLevel)] names(self$hierarchy) <- names(.params) stopifnot(length(self$conditions) == length(.params)) if (!is.null(forbidden)) { if (is.language(forbidden)) forbidden <- as.expression(forbidden) else if (is.character(forbidden)) forbidden <- str2expression(forbidden) if (!is.expression(forbidden)) stop ("Invalid forbidden expression.") check_forbidden_params(forbidden, names(.params)) # FIXME: Instead of a list, we should generate a single expression that is # the logical-OR of all elements of the list. # First we would need to handle the "is.na(x) | !(x)" case here. # Maybe: sapply(forbiddenExps, function(x) substitute(is.na(x) | !(x), list(x=x))) # x <- parse(text=paste0("(", paste0(forbiddenExps,collapse=")||("), ")")) self$forbidden <- sapply(forbidden, compile_forbidden) } # Optimize always TRUE conditions. cond_names <- names(which(!unlist(lapply(self$conditions, is.logical)))) for (p in cond_names) { deps <- self$depends[[p]] # p depends on deps and they are both fixed and always active. if (all(self$isFixed[deps]) && all(sapply(self$conditions[deps], isTRUE, USE.NAMES=FALSE))) { self$conditions[[p]] <- eval(self$conditions[[p]], envir = self$domains[deps], enclos = NULL) } } # Print the hierarchy vector: if (verbose >= 1L) { cat ("# --- Parameters Hierarchy ---\n") print(data.frame(Parameter = paste0(names(self$hierarchy)), Level = self$hierarchy, "Depends on" = sapply(names(self$hierarchy), function(x) paste0("", x, collapse=", ")), row.names=NULL)) cat("# ------------------------\n") } self$.params <- .params }, get = function(x) if (missing(x)) self$.params else self$.params[[x]], get_ordered = function() self$.params[names(self$conditions)], forbid_configurations = function(x) { # FIXME: This copies the whole list. Avoid the copy and just append. self$forbidden <- c(self$forbidden, buildForbiddenExp(configurations = x[, self$names, drop=FALSE])) }, as_character = function() { names_len <- max(nchar(names(self$.params))) label_len <- max(nchar(sapply(self$.params, "[[", "label"))) + 2L output <- "" digits <- c() for (p in self$.params) { type <- p[["type"]] domain <- p[["domain"]] if (is.expression(domain)) { domain <- sapply(domain, USE.NAMES=FALSE, function(x) { if (is.numeric(x)) { if (type == "r") x <- formatC(x, digits = p[["digits"]], format="f", drop0trailing=TRUE) } else { x <- paste0("\"", deparse(x, width.cutoff = 500L), "\"") } x }) } domain <- paste0('(', paste0(domain, collapse=","), ')') condition <- p[["condition"]] condition <- if (isTRUE(condition)) "" else paste0(" | ", condition) transf <- p[["transform"]] if (!is.null(transf) && transf != "") type <- paste0(type, ",", transf) label <- paste0('"', p[["label"]], '"') output <- paste0(output, sprintf('%*s %*s %s %-15s%s\n', -names_len, p[["name"]], -label_len, label, type, domain, condition)) if (type == "r") digits <- c(digits, p[["digits"]]) } if (!is.null(self$forbidden)) { output <- paste0(output, "\n[forbidden]\n", paste0(collapse="\n", sapply(self$forbidden, attr, "source", USE.NAMES=FALSE)), "\n") } digits <- unique(digits) n <- length(digits) if (n > 1L) stop("Different digits per parameter is not supported yet") if (n == 1L) output <- paste0(output, "\n[global]\ndigits = ", digits[[1L]], "\n") output }), ) #' @param name `character(1)`\cr Parameter name (must be alphanumeric). #' @param values `character()`\cr Domain as a vector of strings. #' @param label `character(1)`\cr Label associated to the parameter. Often used to encode a command-line switch that activates the parameter. #' @param condition `expression(1)|character(1)`\cr Expression that defines when the parameter is active according to the value of other parameters. #' @rdname parameters #' @export param_cat <- function(name = name, values, label = "", condition = TRUE) Parameter(name = name, type = "c", domain = as.character(values), label = label, condition = condition, transf = "") #' @rdname parameters #' @export param_ord <- function(name, values, label = "", condition = TRUE) Parameter(name = name, type = "o", domain = as.character(values), label = label, condition = condition, transf = "") #' @param lower,upper Lower and upper limits of the valid domain. #' @param transf `character(1)`\cr If `"log"`, then the parameter is sampled in a logarithmic scale. #' @inheritParams readParameters #' @rdname parameters #' @export param_real <- function(name, lower, upper, label = "", condition = TRUE, transf = "", digits = 15L) Parameter(name = name, type = "r", domain = c(lower, upper), label = label, condition = condition, transf = transf, digits = digits) #' @rdname parameters #' @export param_int <- function(name, lower, upper, label = "", condition = TRUE, transf = "") Parameter(name = name, type = "i", domain = c(lower, upper), label = label, condition = condition, transf = transf) transform_from_log <- function(x, transf, lower, upper) { trLower <- attr(transf, "lower") trUpper <- attr(transf, "upper") x <- exp(trLower + (trUpper - trLower) * x) clamp(x, lower, upper) } transform_to_log <- function(x, transf) { trLower <- attr(transf, "lower") trUpper <- attr(transf, "upper") (log(x) - trLower)/(trUpper - trLower) } ## How to sample integer values? # # The problem: If we have an integer with domain [1,3] and we sample a real value # and round, then there are more chances of getting 2 than 1 or 3: # [1, 1,5) -> 1 # [1.5, 2,5) -> 2 # [2.5, 3) -> 3 # # The solution: Sample in [lowerbound, upperbound + 1], that is, [1, 4], then floor(): # [1, 2) -> 1 # [2, 3) -> 2 # [3, 4) -> 3 # # Why floor() and not trunc()? # Because trunc(-1.5) -> -1, while floor(-1.5) -> -2, so for a domain [-3,-1]: # # [-3, -2) -> -3 # [-2, -1) -> -2 # [-1, 0) -> -1 # # Issue 1: We can sample 4 (upperbound + 1). In that case, we return 3. # # Issue 2: When sampling from a truncated normal distribution, the extremes are # not symmetric. # # nsamples <- 100000 # table(floor(rtnorm(nsamples, mean=1, sd=1, lower=1,upper=4)))/nsamples # table(floor(rtnorm(nsamples, mean=3, sd=1, lower=1,upper=4)))/nsamples # # To make them symmetric, we translate by 0.5, so that the mean is at the # actual center of the interval that will produce the same value after # truncation, e.g., given an integer value of 1, then mean=1.5, which is at the # center of [1,2). # # nsamples <- 100000 # table(floor(rtnorm(nsamples, mean=1.5, sd=1, lower=1,upper=4)))/nsamples # table(floor(rtnorm(nsamples, mean=3.5, sd=1, lower=1,upper=4)))/nsamples # # The above reasoning also works for log-transformed domains, because # floor() happens in the original domain, not in the log-transformed one, # except for the case of log-transformed negative domains, where we have to # translate by -0.5. # integer_round <- function(x, lower, upper) { x <- floor(x) # The probability of this happening is very small, but it happens. x <- pmin.int(upper, x) irace_assert(all(x >= lower)) as.integer(x) } sample_numeric_unif <- function(n, lower, upper, transf) { if (transf == "log") { value <- runif(n, min = 0, max = 1) return(transform_from_log(value, transf, lower, upper)) } runif(n, min = lower, max = upper) } sample_numeric_norm <- function(n, mean, sd, lower, upper, transf) { if (transf == "log") { x <- transform_to_log(mean, transf) x <- rtnorm(n, x, sd, lower = 0, upper = 1) return(transform_from_log(x, transf, lower, upper)) } rtnorm(n, mean, sd, lower, upper) } sample_model <- function(param, n, model, domain) UseMethod("sample_model") sample_unif <- function(param, n, domain) UseMethod("sample_unif") param_quantile <- function(param, probs, domain) UseMethod("param_quantile") #' @exportS3Method sample_model.ParamCat <- function(param, n, model, domain = param[["domain"]]) sample(domain, size = n, replace = TRUE, prob = model) #' @exportS3Method sample_unif.ParamCat <- function(param, n, domain = param[["domain"]]) sample(domain, n, replace=TRUE) #' @exportS3Method param_quantile.ParamCat <- function(param, probs, domain = param[["domain"]]) { n <- length(domain) z <- integer_round(as.numeric(probs) * n + 1L, 1L, n) domain[z] } #' @exportS3Method sample_model.ParamOrd <- function(param, n, model, domain = param[["domain"]]) domain[floor(sample_numeric_norm(n, mean = model[[2L]], sd = model[[1L]], lower = 1L, upper = length(domain), transf = ""))] #' @exportS3Method sample_unif.ParamOrd <- function(param, n, domain = param[["domain"]]) sample(domain, n, replace=TRUE) #' @exportS3Method param_quantile.ParamOrd <- param_quantile.ParamCat #' @exportS3Method sample_model.ParamInt <- function(param, n, model, domain = param[["domain"]]) { # Dependent domains could be not available because of inactivity of parameters # on which they are depedent. In this case, the dependent parameter becomes # not active and we return NA. if (anyNA(domain)) return(NA_integer_) lower <- domain[1L] upper <- domain[2L] transf <- param[["transform"]] mean <- model[[2L]] if (is.na(mean)) # +1 for correct rounding before floor() value <- sample_numeric_unif(n, lower, 1L + upper, transf) else { # + 0.5 because negative domains are log-transformed to positive domains. value <- sample_numeric_norm(n, mean + 0.5, sd = model[[1L]], lower = lower, # +1 for correct rounding before floor() upper = 1L + upper, transf = transf) } # We use original upper, not the +1L for 'i'. integer_round(value, lower, upper) } #' @exportS3Method sample_unif.ParamInt <- function(param, n, domain = param[["domain"]]) { # Dependent domains could be not available because of inactivity of parameters # on which they are depedent. In this case, the dependent parameter becomes # not active and we return NA. if (anyNA(domain)) return(NA_integer_) lower <- domain[1L] upper <- domain[2L] # +1 for correct rounding before floor() value <- sample_numeric_unif(n, lower, 1L + upper, transf = param[["transform"]]) # We use original upperBound, not the +1L for 'i'. integer_round(value, lower, upper) } #' @exportS3Method param_quantile.ParamInt <- function(param, probs, domain = param[["domain"]]) { # Dependent domains could be not available because of inactivity of parameters # on which they are depedent. In this case, the dependent parameter becomes # not active and we return NA. if (anyNA(domain)) return(NA_integer_) lower <- domain[1L] upper <- domain[2L] probs <- as.numeric(probs) transf <- param[["transform"]] if (transf == "log") { # +1 for correct rounding before floor() probs <- transform_from_log(probs, transf, lower, upper + 1L) } else { probs <- probs * (upper + 1L - lower) + lower } integer_round(probs, lower, upper) } #' @exportS3Method sample_model.ParamReal <- function(param, n, model, domain = param[["domain"]]) { # Dependent domains could be not available because of inactivity of parameters # on which they are depedent. In this case, the dependent parameter becomes # not active and we return NA. if (anyNA(domain)) return(NA_real_) lower <- domain[[1L]] upper <- domain[[2L]] transf <- param[["transform"]] mean <- model[[2L]] if (is.na(mean)) x <- sample_numeric_unif(n, lower, upper, transf) else x <- sample_numeric_norm(n, mean, sd = model[[1L]], lower, upper, transf) x <- round(x, digits = param[["digits"]]) irace_assert(all(x >= lower) && all(x <= upper)) x } #' @exportS3Method sample_unif.ParamReal <- function(param, n, domain = param[["domain"]]) { # Dependent domains could be not available because of inactivity of parameters # on which they are depedent. In this case, the dependent parameter becomes # not active and we return NA. if (anyNA(domain)) return(NA_real_) lower <- domain[[1L]] upper <- domain[[2L]] x <- sample_numeric_unif(n, lower, upper, transf = param[["transform"]]) x <- round(x, digits = param[["digits"]]) irace_assert(all(x >= lower) && all(x <= upper)) x } #' @exportS3Method param_quantile.ParamReal <- function(param, probs, domain = param[["domain"]]) { # Dependent domains could be not available because of inactivity of parameters # on which they are depedent. In this case, the dependent parameter becomes # not active and we return NA. if (anyNA(domain)) return(NA_real_) lower <- domain[1L] upper <- domain[2L] probs <- as.numeric(probs) digits <- param[["digits"]] transf <- param[["transform"]] if (transf == "log") return(round(transform_from_log(probs, transf, lower, upper), digits)) probs <- probs * (upper - lower) + lower clamp(round(probs, digits), lower, upper) } #' Print parameter space in the textual format accepted by irace. #' #' @param parameters `ParameterSpace`\cr Data structure containing the parameter #' space definition. The data structure has to similar to the one returned by the #' function [`readParameters`]. #' #' #' @return `character()` #' #' @seealso [readParameters()] #' @examples #' parameters_table <- ' #' # name switch type values [conditions (using R syntax)] #' algorithm "--" c (as,mmas,eas,ras,acs) #' localsearch "--localsearch " c (0, 1, 2, 3) #' alpha "--alpha " r (0.00, 5.00) #' beta "--beta " r (0.00, 10.00) #' rho "--rho " r (0.01, 1.00) #' ants "--ants " i,log (5, 100) #' q0 "--q0 " r (0.0, 1.0) | algorithm == "acs" #' q0dep "--q0 " r (0.0, q0) | algorithm != "acs" #' rasrank "--rasranks " i (1, "min(ants, 10)") | algorithm == "ras" #' elitistants "--elitistants " i (1, ants) | algorithm == "eas" #' nnls "--nnls " i (5, 50) | localsearch %in% c(1,2,3) #' dlb "--dlb " c (0, 1) | localsearch %in% c(1,2,3) #' #' [forbidden] #' (alpha == 0.0) & (beta == 0.0) #' ' #' parameters <- readParameters(text=parameters_table) #' printParameters(parameters) #' @export printParameters <- function(parameters) cat(parameters$as_character()) #' @export print.ParameterSpace <- function(x, digits = 15L, ...) { utils::str(x$.params, digits.d = digits) cat("Forbidden:\n") print(x$forbidden, digits = 15L) } irace/R/irace-package.R0000644000176200001440000000317614736526233014357 0ustar liggesusers#' The irace package: \packageTitle{irace} #' #' \packageDescription{irace} #' #' @name irace-package #' @import stats matrixStats withr data.table #' @importFrom utils str #' @importFrom R6 R6Class #' @importFrom graphics abline axis boxplot par plot points strwidth bxp grid #' @importFrom spacefillr generate_sobol_set #' #' @details License: GPL (>= 2) #' #' @author Maintainers: Manuel López-Ibáñez and Leslie Pérez Cáceres #' \email{irace-package@googlegroups.com} #' #' @keywords package optimize tuning automatic configuration #' #' @references #' Manuel López-Ibáñez, Jérémie Dubois-Lacoste, Leslie Pérez Cáceres, #' Thomas Stützle, and Mauro Birattari. The irace package: Iterated #' Racing for Automatic Algorithm Configuration. \emph{Operations Research #' Perspectives}, 2016. \doi{10.1016/j.orp.2016.09.002} #' #' Manuel López-Ibáñez, Jérémie Dubois-Lacoste, Thomas Stützle, and Mauro #' Birattari. \emph{The irace package, Iterated Race for Automatic #' Algorithm Configuration}. Technical Report TR/IRIDIA/2011-004, IRIDIA, #' Université Libre de Bruxelles, Belgium, 2011. #' #' Manuel López-Ibáñez and Thomas Stützle. The Automatic Design of #' Multi-Objective Ant Colony Optimization Algorithms. \emph{IEEE Transactions #' on Evolutionary Computation}, 2012. #' @seealso #' [irace()] for examples and `vignette(package = "irace")` for the user-guide. "_PACKAGE" # This silences CRAN NOTE: # Namespace in Imports field not imported from: 'codetools' # All declared Imports should be used. .ignore_unused_imports <- function() { # nocov start codetools::findGlobals } # nocov end irace/R/scenario.R0000644000176200001440000011237415060020356013471 0ustar liggesusers#' Reads from a file the scenario settings to be used by \pkg{irace}. #' #' The scenario argument is an initial scenario that is overwritten for every #' setting specified in the file to be read. #' #' @param filename `character(1)`\cr Filename from which the scenario will #' be read. If empty, the default `scenarioFile` is used. An example #' scenario file is provided in `system.file(package="irace", "templates/scenario.txt.tmpl")`. #' @inheritParams defaultScenario #' #' @return The scenario list read from the file. The scenario settings not #' present in the file are not present in the list, i.e., they are `NULL`. #' #' @seealso #' \describe{ #' \item{[printScenario()]}{prints the given scenario.} #' \item{[defaultScenario()]}{returns the default scenario settings of \pkg{irace}.} #' \item{[checkScenario()]}{to check that the scenario is valid.} #' } #' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export readScenario <- function(filename = "", scenario = list(), params_def = .irace.params.def) { # This function allows recursively including scenario files. scenario_env <- new.env() include.scenario <- function(rfilename, topfile = filename, envir. = scenario_env) { if (!file.exists (rfilename)) { irace_error ("The scenario file ", shQuote(rfilename), " included from ", shQuote(topfile), " does not exist.") } handle.source.error <- function(e) { irace_error("Reading scenario file ", shQuote(rfilename), " included from ", shQuote(topfile), " produced the following errors or warnings:\n", paste0(conditionMessage(e), collapse="\n")) return(NULL) } withCallingHandlers( tryCatch(source(rfilename, local = envir., chdir = TRUE), error = handle.source.error, warning = handle.source.error)) } # First find out which file... filename_given <- filename != "" if (!filename_given) { filename <- path_rel2abs(params_def["scenarioFile","default"]) if (file.exists(filename)) { irace_warning("A default scenario file ", shQuote(filename), " has been found and will be read.") } else { irace_warning ("No scenario file given (use ", params_def["scenarioFile", "short"], " or ", params_def["scenarioFile", "long"], ") and no default scenario file ", shQuote(filename), " has been found.") scenario[["scenarioFile"]] <- filename return(scenario) } } else { filename <- path_rel2abs(filename) } if (file.exists (filename)) { debug.level <- getOption(".irace.debug.level", default = 0) if (debug.level >= 1) cat("# Reading scenario file", shQuote(filename), ".......") # chdir = TRUE to allow recursive sourcing. handle.source.error <- function(e) { irace_error("Reading scenario file ", shQuote(filename), " produced the following errors or warnings:\n", paste0(conditionMessage(e), collapse="\n")) return(NULL) } withCallingHandlers( tryCatch(source(filename, local = scenario_env, chdir = TRUE), error = handle.source.error, warning = handle.source.error)) if (debug.level >= 1) cat (" done!\n") } else if (filename_given) { irace_error ("The scenario file ", shQuote(filename), " does not exist.") } ## Read scenario file variables. scenario[["scenarioFile"]] <- filename # If these are given and relative, they should be relative to the # scenario file (except logFile, which is relative to execDir). pathParams <- setdiff(params_def[params_def[, "type"] == "p", "name"], "logFile") params_names <- params_def[!startsWith(params_def[,"name"], "."), "name"] for (param in params_names) { if (exists (param, envir = scenario_env, inherits = FALSE)) { value <- get(param, envir = scenario_env, inherits = FALSE) if (filename_given && !is.null.or.empty(value) && is.character(value) && (param %in% pathParams)) { value <- path_rel2abs(value, cwd = dirname(filename)) } scenario[[param]] <- value } } unknown_scenario_vars <- setdiff(ls(scenario_env), params_names) if (length(unknown_scenario_vars)) { # We only accept variables that match irace.params.names and if the user # wants to define their own, they should use names starting with ".", which # are ignored by ls() irace_error("Scenario file ", shQuote(filename), " contains unknown variables: ", paste0(unknown_scenario_vars, collapse=", "), "\nMAKE SURE NO VARIABLE NAME IS MISSPELL (for example, 'parameterFile' is correct, while 'parametersFile' is not)", "\nIf you wish to use your own variables in the scenario file, use names beginning with a dot `.'") } scenario } setup_test_instances <- function(scenario) { testInstances <- scenario[["testInstances"]] if (is.null.or.empty(testInstances)) { if (!is.null.or.empty(scenario$testInstancesDir) || !is.null.or.empty(scenario$testInstancesFile)) { scenario$testInstancesDir <- path_rel2abs(scenario$testInstancesDir) if (!is.null.or.empty(scenario$testInstancesFile)) { scenario$testInstancesFile <- path_rel2abs(scenario$testInstancesFile) } testInstances <- readInstances(instancesDir = scenario$testInstancesDir, instancesFile = scenario$testInstancesFile) } else { testInstances <- NULL } } if (!is.null(testInstances)) { if (!is.null(dim(testInstances))) { if (length(dim(testInstances)) == 1L || (length(dim(testInstances)) == 2L && dim(testInstances)[1L] == 1L)) { # Remove useless dimensions testInstances <- c(testInstances) } else { irace_error("testInstances must be a one-dimensional vector or a list. If your instances are matrices or datasets in R, you can use scenario(testInstances=list(data1, data2, data3))") } } if (is.null(names(testInstances))) { # Create unique IDs for testInstances names(testInstances) <- paste0(seq_along(testInstances), "t") } } scenario[["testInstances"]] <- testInstances scenario } #' Check and correct the given scenario #' #' Checks for errors a (possibly incomplete) scenario setup of #' \pkg{irace} and transforms it into a valid scenario. #' #' @inheritParams defaultScenario #' #' @return The scenario received as a parameter, possibly corrected. Unset #' scenario settings are set to their default values. #' #' @details This function checks that the directories and the file names #' provided and required by the \pkg{irace} exist. It also checks that the #' settings are of the proper type, e.g. that settings expected to be integers #' are really integers. Finally, it also checks that there is no inconsistency #' between settings. If an error is found that prevents \pkg{irace} from #' running properly, it will stop with an error. #' #' @seealso #' \describe{ #' \item{[readScenario()]}{for reading a configuration scenario from a file.} #' \item{[printScenario()]}{prints the given scenario.} #' \item{[defaultScenario()]}{returns the default scenario settings of \pkg{irace}.} #' \item{[checkScenario()]}{to check that the scenario is valid.} #' } #' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export ## FIXME: This function should only do checks and return TRUE/FALSE. There ## should be other function that does the various transformations. checkScenario <- function(scenario = defaultScenario()) { quote.param <- function(name) { if (.irace.params.def[name, "long"] != "") { return(paste0("'", name, "' (", .irace.params.def[name, "long"], ")")) } paste0("'", name, "'") } as.boolean.param <- function(x, name) { if (is.logical(x)) return(x) if (is.na(x)) return(as.logical(NA)) # We handle "0" and "1" but not "TRUE" and "FALSE". x <- suppressWarnings(as.integer(x)) if (is.na(x) || (x != 0L && x != 1L)) { irace_error (quote.param(name), " must be either 0 or 1.") } as.logical(x) } check.valid.param <- function(x) { valid <- trimws(strsplit(.irace.params.def[x, "domain"],",",fixed=TRUE)[[1L]]) if (scenario[[x]] %not_in% valid) { irace_error ("Invalid value '", scenario[[x]], "' of ", quote.param(x), ", valid values are: ", paste0(valid, collapse = ", ")) } } # Fill possible unset (NULL) with default settings. scenario <- defaultScenario(scenario) # Duplicated entries will cause confusion. dups <- anyDuplicated(names(scenario)) if (dups) irace_error("scenario contains duplicated entries: ", names(scenario)[dups]) # We have characters everywhere, set to the right types to avoid problems # later. # Boolean control parameters. boolParams <- .irace.params.def[.irace.params.def[, "type"] == "b", "name"] for (p in boolParams) { scenario[[p]] <- as.boolean.param(scenario[[p]], p) } # Real [0, 1] control parameters. realParams <- .irace.params.def[.irace.params.def[, "type"] == "r", "name"] for (param in realParams) { if (is.null.or.empty(scenario[[param]])) scenario[[param]] <- NA_real_ if (is.na(scenario[[param]])) next # Allow NA default values scenario[[param]] <- suppressWarnings(as.numeric(scenario[[param]])) if (is.null(scenario[[param]]) || is.na (scenario[[param]]) || scenario[[param]] < 0.0 || scenario[[param]] > 1.0) irace_error (quote.param(param), " must be a real value within [0, 1].") } # Integer control parameters intParams <- .irace.params.def[.irace.params.def[, "type"] == "i", "name"] for (param in intParams) { p <- scenario[[param]] if (is.null.or.empty(p)) scenario[[param]] <- NA_integer_ if (is.na(scenario[[param]])) next # Allow NA default values p <- suppressWarnings(as.numeric(p)) if (is.null(p) || is.na (p) || !is.wholenumber(p) || p < 0) irace_error (quote.param (param), " must be a non-negative integer.") scenario[[param]] <- as.integer(p) } check_positive <- function(scenario, what) { if (any(scenario[what] <= 0L)) { for (op in what) { if (scenario[[op]] <= 0L) irace_error(quote.param (op), " = ", scenario[[op]], " must be larger than 0.") } } } check_positive(scenario, c("firstTest", "eachTest", "blockSize")) options(.irace.quiet = scenario$quiet) ## Check that everything is fine with external parameters # Check that the files exist and are readable. scenario$parameterFile <- path_rel2abs(scenario$parameterFile) if (is.null.or.empty(scenario$parameters)) { irace_note("Reading parameter file '", scenario$parameterFile, "'.\n") scenario$parameters <- readParameters(file = scenario$parameterFile, # AClib benchmarks use 15 digits digits = if (scenario$aclib) 15L else 4L, debugLevel = scenario$debugLevel) } scenario$parameters <- checkParameters(scenario$parameters) scenario$execDir <- path_rel2abs(scenario$execDir) file.check (scenario$execDir, isdir = TRUE, text = paste0("execution directory ", quote.param("execDir"))) options(.irace.execdir = scenario$execDir) if (is.null.or.empty(scenario$logFile)) { # We cannot use NULL because defaultScenario() would override it. scenario$logFile <- "" } else { scenario$logFile <- path_rel2abs(scenario$logFile, cwd = scenario$execDir) file.check(scenario$logFile, writeable = TRUE, text = quote.param('logFile')) } if (is.null.or.empty(scenario$recoveryFile)) { # We cannot use NULL because defaultScenario() would override it. scenario$recoveryFile <- "" } else { scenario$recoveryFile <- path_rel2abs(scenario$recoveryFile) file.check(scenario$recoveryFile, readable = TRUE, text = paste0("recovery file ", quote.param("recoveryFile"))) if (!is.null.or.empty(scenario$logFile) # Must have been set "" above. && scenario$recoveryFile == scenario$logFile) { irace_error("log file and recovery file should be different '", scenario$logFile, "'") } } if (is.null.or.empty(scenario$targetRunnerParallel)) { scenario$targetRunnerParallel <- NULL } else if (is.function.name(scenario$targetRunnerParallel)) { scenario$targetRunnerParallel <- get.function(scenario$targetRunnerParallel) } else { irace_error("'targetRunnerParallel' must be a function") } if (is.null.or.empty(scenario$repairConfiguration)) scenario$repairConfiguration <- NULL else if (is.function.name(scenario$repairConfiguration)) # Byte-compile it. scenario$repairConfiguration <- bytecompile(get.function(scenario$repairConfiguration)) else irace_error("'repairConfiguration' must be a function") if (is.na(scenario$capping)) scenario$capping <- (scenario$elitist && scenario$maxTime > 0 && !is.na(scenario$boundMax) && scenario$boundMax > 0) if (scenario$capping) { if (!scenario$elitist) irace_error("When capping == TRUE, elitist must be enabled.") if (scenario$boundMax <= 0) irace_error("When capping == TRUE, boundMax (", scenario$boundMax, ") must be > 0") check.valid.param("cappingType") check.valid.param("boundType") if (scenario$boundPar < 1) irace_error("Invalid value (", scenario$boundPar, ") ", quote.param("boundPar"), " must be >= 1") } else if (scenario$boundMax <= 0 || is.na(scenario$boundMax)) { # no capping scenario$boundMax <- NULL } if (is.function.name(scenario$targetRunner)) { scenario$targetRunner <- get.function(scenario$targetRunner) irace_assert(is.function(scenario$targetRunner)) } else if (is.null(scenario$targetRunnerParallel)) { if (is.character(scenario$targetRunner)) { scenario$targetRunner <- path_rel2abs(scenario$targetRunner) if (is.null.or.empty(scenario$targetRunnerLauncher)) { file.check (scenario$targetRunner, executable = TRUE, text = paste0("target runner ", quote.param("targetRunner"))) } else { scenario$targetRunnerLauncher <- path_rel2abs(scenario$targetRunnerLauncher) file.check (scenario$targetRunnerLauncher, executable = TRUE, text = paste0("target runner launcher ", quote.param("targetRunnerLauncher"))) file.check (scenario$targetRunner, readable = TRUE, text = paste0("target runner ", quote.param("targetRunner"))) if (!grepl("{targetRunner}", scenario$targetCmdline, fixed=TRUE)) { # If missing, we add it to the beginning for compatibility with irace 3.5. scenario$targetCmdline <- paste0("{targetRunner} ", scenario$targetCmdline) } } check_target_cmdline(scenario$targetCmdline, capping = scenario$capping) } else { irace_error(quote.param ('targetRunner'), " must be a function or an executable program") } } if (is.null.or.empty(scenario$targetEvaluator)) { scenario$targetEvaluator <- NULL } else if (is.function.name(scenario$targetEvaluator)) { scenario$targetEvaluator <- get.function(scenario$targetEvaluator) irace_assert(is.function(scenario$targetEvaluator)) } else if (is.character(scenario$targetEvaluator)) { scenario$targetEvaluator <- path_rel2abs(scenario$targetEvaluator) file.check (scenario$targetEvaluator, executable = TRUE, text = "target evaluator") } else { irace_error(quote.param('targetEvaluator'), " must be a function or an executable program") } # Training instances if (is.null.or.empty(scenario$instances)) { scenario$trainInstancesDir <- path_rel2abs(scenario$trainInstancesDir) if (!is.null.or.empty(scenario$trainInstancesFile)) { scenario$trainInstancesFile <- path_rel2abs(scenario$trainInstancesFile) } if (is.null.or.empty(scenario$trainInstancesDir) && is.null.or.empty(scenario$trainInstancesFile)) irace_error("Both ", quote.param ("trainInstancesDir"), " and ", quote.param ("trainInstancesFile"), " are empty: No instances provided") scenario$instances <- readInstances(instancesDir = scenario$trainInstancesDir, instancesFile = scenario$trainInstancesFile) } if (length(scenario$instances) == 0L) { irace_error("No instances found in `scenario$instances`") } else if (!is.null(dim(scenario$instances))) { if (length(dim(scenario$instances)) == 1L || (length(dim(scenario$instances)) == 2L && dim(scenario$instances)[1] == 1L)) { # Remove useless dimensions scenario$instances <- c(scenario$instances) } else { irace_error("Instances must be a one-dimensional vector or a list. If your instances are matrices or datasets in R, you can use scenario(instances=list(data1, data2, data3))") } } # Testing instances scenario <- setup_test_instances(scenario) # Configurations file if (!is.null.or.empty(scenario$configurationsFile)) { scenario$configurationsFile <- path_rel2abs(scenario$configurationsFile) file.check (scenario$configurationsFile, readable = TRUE, text = "configurations file") # We cannot read the configurations here because we need the parameters. # FIXME: We should have the parameters inside scenario. } if (is.null.or.empty(scenario$initConfigurations)) { scenario$initConfigurations <- NULL } else if (!is.data.frame(scenario$initConfigurations) && !is.matrix(scenario$initConfigurations)) { irace_error("if given, 'initConfigurations' must be a matrix or data.frame.") } if (length(scenario$instances) %% scenario$blockSize != 0) { irace_error("The number of instances (", length(scenario$instances), ") must be a multiple of ", quote.param("blockSize"), ".") } if (scenario$firstTest %% scenario$eachTest != 0) { irace_error(quote.param("firstTest"), " must be a multiple of ", quote.param("eachTest"), ".") } if (scenario$mu < scenario$firstTest * scenario$blockSize) { if (scenario$debugLevel >= 1) { irace_warning("Assuming 'mu = firstTest * blockSize' because 'mu' cannot be smaller.\n") } scenario$mu <- scenario$firstTest * scenario$blockSize } else if (scenario$mu %% scenario$blockSize != 0) { irace_error(quote.param("mu"), " must be a multiple of ", quote.param("blockSize"), ".") } if (!is.na(scenario$minExperiments)) scenario$maxExperiments <- max(scenario$maxExperiments, scenario$minExperiments) ## Only maxExperiments or maxTime should be set. Negative values are not ## allowed. if (scenario$maxExperiments == 0 && scenario$maxTime == 0) { irace_error("Tuning budget was not provided. Set ", quote.param("maxExperiments"), " or ", quote.param("maxTime"), ".") } else if (scenario$maxExperiments > 0 && scenario$maxTime > 0) { irace_error("Two different tuning budgets provided, please set only ", quote.param("maxExperiments"), " or only ", quote.param ("maxTime"), ".") } else if (scenario$maxExperiments < 0 ) { irace_error("Negative budget provided, ", quote.param("maxExperiments"), "must be >= 0." ) } else if (scenario$maxTime < 0) { irace_error("Negative budget provided, ", quote.param("maxTime"), " must be >= 0.") } if (scenario$maxTime > 0 && (scenario$budgetEstimation <= 0 || scenario$budgetEstimation >= 1)) irace_error(quote.param("budgetEstimation"), " must be within (0,1).") if (scenario$maxTime > 0 && !scenario$elitist) irace_error(quote.param("maxTime"), " requires using 'elitist=1'") if (scenario$deterministic && scenario$firstTest * scenario$blockSize > length(scenario$instances)) { irace_error("When deterministic == TRUE, the number of instances (", length(scenario$instances), ") cannot be smaller than firstTest (", scenario$firstTest, ") * blockSize (", scenario$blockSize, ")") } if (scenario$mpi && scenario$parallel < 2) irace_error (quote.param("parallel"), " must be larger than 1 when mpi is enabled.") if (is.null.or.empty(scenario$batchmode)) scenario$batchmode <- 0 if (scenario$batchmode != 0) { scenario$batchmode <- tolower(scenario$batchmode) check.valid.param("batchmode") } # Currently batchmode requires a targetEvaluator if (scenario$batchmode != 0 && is.null(scenario$targetEvaluator)) { irace_error(quote.param("batchmode"), " requires using ", quote.param("targetEvaluator"), ".") } if (scenario$batchmode != 0 && scenario$mpi) { irace_error(quote.param("mpi"), " and ", quote.param("batchmode"), " cannot be enabled at the same time.") } if (is_null_or_empty_or_na(scenario$testType)) { if (scenario$capping) scenario$testType <- "t-test" else scenario$testType <- "f-test" } scenario$testType <- switch(tolower(scenario$testType), "f-test" =, # Fall-through "friedman" = "friedman", "t-test" =, # Fall-through "t.none" = "t.none", "t-test-holm" =, # Fall-through, "t.holm" = "t.holm", "t-test-bonferroni" =, # Fall-through, "t.bonferroni" = "t.bonferroni", check.valid.param("testType")) scenario } #' Prints the given scenario #' #' @inheritParams defaultScenario #' #' @seealso #' \describe{ #' \item{[readScenario()]}{for reading a configuration scenario from a file.} #' \item{[printScenario()]}{prints the given scenario.} #' \item{[defaultScenario()]}{returns the default scenario settings of \pkg{irace}.} #' \item{[checkScenario()]}{to check that the scenario is valid.} #' } #' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export printScenario <- function(scenario) { scenario_names <- .irace.params.names cat("## irace scenario:\n") for (name in scenario_names) { cat(name, " = ", deparse(scenario[[name]]), "\n", sep = "") } cat("## end of irace scenario\n") } #' Default scenario settings #' #' Return scenario object with default values. #' #' @param scenario `list()`\cr Data structure containing \pkg{irace} #' settings. The data structure has to be the one returned by the function #' [defaultScenario()] or [readScenario()]. #' #' @param params_def `data.frame()`\cr Definition of the options accepted by #' the scenario. This should only be modified by packages that wish to extend #' \pkg{irace}. #' #' #' @return A list indexed by the \pkg{irace} parameter names, #' containing the default values for each parameter, except for those #' already present in the scenario passed as argument. #' The scenario list contains the following elements: # __IRACE_OPTIONS__BEGIN__ #' \itemize{ #' \item General options: #' \describe{ #' \item{`scenarioFile`}{Path of the file that describes the configuration scenario setup and other irace settings. (Default: `"./scenario.txt"`)} #' \item{`execDir`}{Directory where the programs will be run. (Default: `"./"`)} #' \item{`logFile`}{File to save tuning results as an R dataset, either absolute path or relative to execDir. (Default: `"./irace.Rdata"`)} #' \item{`quiet`}{Reduce the output generated by irace to a minimum. (Default: `0`)} #' \item{`debugLevel`}{Debug level of the output of `irace`. Set this to 0 to silence all debug messages. Higher values (1, 2 or 3) provide more verbose debug messages. (Default: `0`)} #' \item{`seed`}{Seed of the random number generator (by default, generate a random seed). (Default: `NA`)} #' \item{`repairConfiguration`}{User-defined R function that takes a configuration generated by irace and repairs it. (Default: `""`)} #' \item{`postselection`}{Perform a postselection race after the execution of irace to consume all remaining budget. Value 0 disables the postselection race. (Default: `1`)} #' \item{`aclib`}{Enable/disable AClib mode. This option enables compatibility with GenericWrapper4AC as targetRunner script. (Default: `0`)} #' } #' \item Elitist `irace`: #' \describe{ #' \item{`elitist`}{Enable/disable elitist irace. (Default: `1`)} #' \item{`elitistNewInstances`}{Number of instances added to the execution list before previous instances in elitist irace. (Default: `1`)} #' \item{`elitistLimit`}{In elitist irace, maximum number per race of elimination tests that do not eliminate a configuration. Use 0 for no limit. (Default: `2`)} #' } #' \item Internal `irace` options: #' \describe{ #' \item{`sampleInstances`}{Randomly sample the training instances or use them in the order given. (Default: `1`)} #' \item{`softRestart`}{Enable/disable the soft restart strategy that avoids premature convergence of the probabilistic model. (Default: `1`)} #' \item{`softRestartThreshold`}{Soft restart threshold value for numerical parameters. (Default: `1e-04`)} #' \item{`nbIterations`}{Maximum number of iterations. (Default: `0`)} #' \item{`nbExperimentsPerIteration`}{Number of runs of the target algorithm per iteration. (Default: `0`)} #' \item{`minNbSurvival`}{Minimum number of configurations needed to continue the execution of each race (iteration). (Default: `0`)} #' \item{`nbConfigurations`}{Number of configurations to be sampled and evaluated at each iteration. (Default: `0`)} #' \item{`mu`}{Parameter used to define the number of configurations sampled and evaluated at each iteration. (Default: `5`)} #' } #' \item Target algorithm parameters: #' \describe{ #' \item{`parameterFile`}{File that contains the description of the parameters of the target algorithm. (Default: `"./parameters.txt"`)} #' \item{`parameters`}{Parameters space object (usually read from a file using \code{readParameters}). (Default: `""`)} #' } #' \item Target algorithm execution: #' \describe{ #' \item{`targetRunner`}{Executable called for each configuration that executes the target algorithm to be tuned. See the templates and examples provided. (Default: `"./target-runner"`)} #' \item{`targetRunnerLauncher`}{Executable that will be used to launch the target runner, when `targetRunner` cannot be executed directly (e.g., a Python script in Windows). (Default: `""`)} #' \item{`targetCmdline`}{Command-line arguments provided to `targetRunner` (or `targetRunnerLauncher` if defined). The substrings `\{configurationID\}`, `\{instanceID\}`, `\{seed\}`, `\{instance\}`, and `\{bound\}` will be replaced by their corresponding values. The substring `\{targetRunnerArgs\}` will be replaced by the concatenation of the switch and value of all active parameters of the particular configuration being evaluated. The substring `\{targetRunner\}`, if present, will be replaced by the value of `targetRunner` (useful when using `targetRunnerLauncher`). (Default: `"{configurationID} {instanceID} {seed} {instance} {bound} {targetRunnerArgs}"`)} #' \item{`targetRunnerRetries`}{Number of times to retry a call to `targetRunner` if the call failed. (Default: `0`)} #' \item{`targetRunnerTimeout`}{Timeout in seconds of any `targetRunner` call (only applies to `target-runner` executables not to R functions), ignored if 0. (Default: `0`)} #' \item{`targetRunnerData`}{Optional data passed to `targetRunner`. This is ignored by the default `targetRunner` function, but it may be used by custom `targetRunner` functions to pass persistent data around. (Default: `""`)} #' \item{`targetRunnerParallel`}{Optional R function to provide custom parallelization of `targetRunner`. (Default: `""`)} #' \item{`targetEvaluator`}{Optional script or R function that provides a numeric value for each configuration. See templates/target-evaluator.tmpl (Default: `""`)} #' \item{`deterministic`}{If the target algorithm is deterministic, configurations will be evaluated only once per instance. (Default: `0`)} #' \item{`parallel`}{Number of calls to `targetRunner` to execute in parallel. Values `0` or `1` mean no parallelization. (Default: `0`)} #' \item{`loadBalancing`}{Enable/disable load-balancing when executing experiments in parallel. Load-balancing makes better use of computing resources, but increases communication overhead. If this overhead is large, disabling load-balancing may be faster. (Default: `1`)} #' \item{`mpi`}{Enable/disable MPI. Use `Rmpi` to execute `targetRunner` in parallel (parameter `parallel` is the number of slaves). (Default: `0`)} #' \item{`batchmode`}{Specify how irace waits for jobs to finish when `targetRunner` submits jobs to a batch cluster: sge, pbs, torque, slurm or htcondor. `targetRunner` must submit jobs to the cluster using, for example, `qsub`. (Default: `0`)} #' } #' \item Initial configurations: #' \describe{ #' \item{`initConfigurations`}{Data frame describing initial configurations (usually read from a file using \code{readConfigurations}). (Default: `""`)} #' \item{`configurationsFile`}{File that contains a table of initial configurations. If empty or `NULL`, all initial configurations are randomly generated. (Default: `""`)} #' } #' \item Training instances: #' \describe{ #' \item{`instances`}{Character vector of the instances to be used in the \code{targetRunner}. (Default: `""`)} #' \item{`trainInstancesDir`}{Directory where training instances are located; either absolute path or relative to current directory. If no `trainInstancesFiles` is provided, all the files in `trainInstancesDir` will be listed as instances. (Default: `""`)} #' \item{`trainInstancesFile`}{File that contains a list of training instances and optionally additional parameters for them. If `trainInstancesDir` is provided, `irace` will search for the files in this folder. (Default: `""`)} #' \item{`blockSize`}{Number of training instances, that make up a 'block' in `trainInstancesFile`. Elimination of configurations will only be performed after evaluating a complete block and never in the middle of a block. Each block typically contains one instance from each instance class (type or family) and the block size is the number of classes. The value of `blockSize` will multiply `firstTest`, `eachTest` and `elitistNewInstances`. (Default: `1`)} #' } #' \item Tuning budget: #' \describe{ #' \item{`maxExperiments`}{Maximum number of runs (invocations of `targetRunner`) that will be performed. It determines the maximum budget of experiments for the tuning. (Default: `0`)} #' \item{`minExperiments`}{Minimum number of runs (invocations of `targetRunner`) that will be performed. It determines the minimum budget of experiments for the tuning. The actual budget depends on the number of parameters and `minSurvival`. (Default: `NA`)} #' \item{`maxTime`}{Maximum total execution time for the executions of `targetRunner`. `targetRunner` must return two values: cost and time. This value and the one returned by `targetRunner` must use the same units (seconds, minutes, iterations, evaluations, ...). (Default: `0`)} #' \item{`budgetEstimation`}{Fraction (smaller than 1) of the budget used to estimate the mean computation time of a configuration. Only used when `maxTime` > 0 (Default: `0.05`)} #' \item{`minMeasurableTime`}{Minimum time unit that is still (significantly) measureable. (Default: `0.01`)} #' } #' \item Statistical test: #' \describe{ #' \item{`testType`}{Statistical test used for elimination. The default value selects `t-test` if `capping` is enabled or `F-test`, otherwise. Valid values are: F-test (Friedman test), t-test (pairwise t-tests with no correction), t-test-bonferroni (t-test with Bonferroni's correction for multiple comparisons), t-test-holm (t-test with Holm's correction for multiple comparisons). (Default: `""`)} #' \item{`firstTest`}{Number of instances evaluated before the first elimination test. It must be a multiple of `eachTest`. (Default: `5`)} #' \item{`eachTest`}{Number of instances evaluated between elimination tests. (Default: `1`)} #' \item{`confidence`}{Confidence level for the elimination test. (Default: `0.95`)} #' } #' \item Adaptive capping: #' \describe{ #' \item{`capping`}{Enable the use of adaptive capping, a technique designed for minimizing the computation time of configurations. Capping is enabled by default if `elitist` is active, `maxTime > 0` and `boundMax > 0`. (Default: `NA`)} #' \item{`cappingAfterFirstTest`}{If set to 1, elimination due to capping only happens after `firstTest` instances are seen. (Default: `0`)} #' \item{`cappingType`}{Measure used to obtain the execution bound from the performance of the elite configurations.\itemize{\item median: Median performance of the elite configurations.\item mean: Mean performance of the elite configurations.\item best: Best performance of the elite configurations.\item worst: Worst performance of the elite configurations.} (Default: `"median"`)} #' \item{`boundType`}{Method to calculate the mean performance of elite configurations.\itemize{\item candidate: Mean execution times across the executed instances and the current one.\item instance: Execution time of the current instance.} (Default: `"candidate"`)} #' \item{`boundMax`}{Maximum execution bound for `targetRunner`. It must be specified when capping is enabled. (Default: `0`)} #' \item{`boundDigits`}{Precision used for calculating the execution time. It must be specified when capping is enabled. (Default: `0`)} #' \item{`boundPar`}{Penalization multiplication constant (PARX) for timed out executions (executions that reach `boundMax` execution time). (Default: `1`)} #' \item{`boundAsTimeout`}{Replace the configuration cost of bounded executions with `boundMax`. (Default: `1`)} #' } #' \item Recovery: #' \describe{ #' \item{`recoveryFile`}{Previously saved log file to recover the execution of `irace`, either absolute path or relative to the current directory. If empty or `NULL`, recovery is not performed. (Default: `""`)} #' } #' \item Testing: #' \describe{ #' \item{`testInstancesDir`}{Directory where testing instances are located, either absolute or relative to current directory. (Default: `""`)} #' \item{`testInstancesFile`}{File containing a list of test instances and optionally additional parameters for them. (Default: `""`)} #' \item{`testInstances`}{Character vector of the instances to be used in the \code{targetRunner} when executing the testing. (Default: `""`)} #' \item{`testNbElites`}{Number of elite configurations returned by irace that will be tested if test instances are provided. (Default: `1`)} #' \item{`testIterationElites`}{Enable/disable testing the elite configurations found at each iteration. (Default: `0`)} #' } #' } # __IRACE_OPTIONS__END__ #' #' @seealso #' \describe{ #' \item{[readScenario()]}{for reading a configuration scenario from a file.} #' \item{[printScenario()]}{prints the given scenario.} #' \item{[defaultScenario()]}{returns the default scenario settings of \pkg{irace}.} #' \item{[checkScenario()]}{to check that the scenario is valid.} #' } #' #' @author Manuel López-Ibáñez and Jérémie Dubois-Lacoste #' @export defaultScenario <- function(scenario = list(), params_def = .irace.params.def) { params_names <- params_def[!startsWith(params_def[,"name"], "."), "name"] if (is.null(names(scenario))) { scenario <- setNames(as.list(params_def[params_names,"default"]), params_names) } else if (!all(names(scenario) %in% params_names)) { irace_error("Unknown scenario settings: ", paste(names(scenario)[!(names(scenario) %in% params_names)], collapse = ", ")) } else { for (k in params_names) { if (is.null.or.na(scenario[[k]])) { scenario[[k]] <- params_def[k, "default"] } } } scenario } readInstances <- function(instancesDir = NULL, instancesFile = NULL) { if (is.null.or.empty(instancesDir) && is.null.or.empty(instancesFile)) irace_error("Both instancesDir and instancesFile are empty: No instances provided") instances <- NULL if (is.null.or.empty(instancesFile)) { file.check (instancesDir, isdir = TRUE, notempty = TRUE, text = "instances directory") # The files are sorted in alphabetical order, on the full path if # 'full.names = TRUE'. instances <- list.files (path = instancesDir, full.names = TRUE, recursive = TRUE) if (length (instances) == 0) irace_error("No instances found in `", instancesDir, "'") } else { file.check (instancesFile, readable = TRUE, text = "instance file") # We do not warn if the last line does not finish with a newline. instances <- readLines (instancesFile, warn = FALSE) instances <- sub("#.*$", "", instances) # Remove comments instances <- sub("^[[:space:]]+", "", instances) # Remove leading whitespace instances <- instances[instances != ""] # Delete empty lines if (is.null.or.empty(instances)) irace_error("No instances found in '", instancesFile, "' (whitespace and comments starting with '#' are ignored)") if (!is.null.or.empty(instancesDir)) instances <- paste0 (instancesDir, "/", instances) } instances } update_scenario <- function(scenario, ...) { scenario_args <- list(...) if (length(scenario_args) == 0L) return(scenario) unknown_scenario_args <- setdiff(names(scenario_args), names(scenario)) if (length(unknown_scenario_args)) irace_error("Unknown scenario settings given: ", paste0(unknown_scenario_args, collapse=", ")) utils::modifyList(scenario, scenario_args) } irace/R/race.R0000644000176200001440000015741714754675041012627 0ustar liggesusers# ---------------------------------------- -*- mode: r; mode: font-lock -*- # # race.R Racing methods for the selection of the best # # ------------------------------------------------------------------------- # # ========================================================================= # # Racing methods for the selection of the best # # ------------------------------------------------------------------------- # # Copyright (C) 2003 Mauro Birattari # # ========================================================================= # # This program is free software; you can redistribute it and/or modify it # # under the terms of the GNU General Public License as published by the # # Free Software Foundation; either version 2 of the License, or (at your # # option) any later version. # # # # This program is distributed in the hope that it will be useful, but # # WITHOUT ANY WARRANTY; without even the implied warranty of # # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # # General Public License for more details. # # # # You should have received a copy of the GNU General Public License along # # with this program; if not, write to the Free Software Foundation, Inc., # # 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # # ========================================================================= # # ========================================================================= # # Mauro BIRATTARI # # IRIDIA - ULB, CP 194/6 # # Av. F. D. Roosevelt 50 mbiro@ulb.ac.be # # 1050 Brussels, Belgium http://iridia.ulb.ac.be/~mbiro # # ========================================================================= # # $Id: race.R,v 1.54 2005/03/30 12:40:42 mbiro Exp $ # createExperimentList <- function(configurations, parameters, instances, instances_ID, seeds, bounds) { n_configurations <- nrow(configurations) configurations_id <- configurations[[".ID."]] pnames <- parameters$names configurations <- as.list(configurations[, pnames, drop=FALSE]) # FIXME: How to do this faster? # - maybe purrr::transpose() ? # - mlr3misc::transpose_list() uses .mapply(list, configurations, list(), which is slower. configurations <- lapply(seq_len(n_configurations), function(i) lapply(configurations, `[`, i)) dots <- list(id_configuration = configurations_id, configuration = configurations) if (!is.null(bounds)) { # There must be a bound for each configuration. # FIXME: There must be a bound for each configuration AND each instance. irace_assert(length(bounds) == n_configurations) dots$bound <- bounds } configurations <- .mapply(list, dots = dots, MoreArgs = NULL) instances <- instances[instances_ID] n_instances <- length(instances_ID) instances <- .mapply(list, dots = list(id_instance = instances_ID, seed = seeds, instance = instances), MoreArgs = NULL) .mapply(c, dots = list( rep(instances, each = n_configurations), rep(configurations, times = n_instances)), MoreArgs = NULL) } race_wrapper_helper <- function(race_state, configurations, instance_idx, bounds, is_exe, scenario) { # Experiment list to execute experiments <- createExperimentList(configurations, parameters = scenario$parameters, instances = scenario$instances, instances_ID = race_state$instances_log[["instanceID"]][instance_idx], seeds = race_state$instances_log[["seed"]][instance_idx], bounds = bounds) irace_assert(length(is_exe) == length(experiments)) instance_idx <- rep(instance_idx, each = nrow(configurations)) if (race_state$recovery_mode) { # With targetEvaluator or if everything is executed, we recover everything. if (!is.null(scenario$targetEvaluator) || all(is_exe)) { configuration_id <- unlist_element(experiments, "id_configuration") target_output <- race_state$recover_output(instance_idx, configuration_id) } else { irace_assert(any(is_exe)) configuration_id <- unlist_element(experiments[is_exe], "id_configuration") target_output <- race_state$recover_output(instance_idx[is_exe], configuration_id) } } else { # !recovery_mode # We cannot let targetRunner or targetEvaluator modify our random seed, so we save it. withr::local_preserve_seed() target_output <- vector("list", length(experiments)) # Execute experiments for which is_exe is TRUE: if (any(is_exe)) target_output[is_exe] <- execute_experiments(race_state, experiments[is_exe], scenario) # If targetEvaluator is NULL, then target_output must contain the right # output already. Otherwise, targetEvaluator considers all experiments. if (!is.null(scenario$targetEvaluator)) { target_output <- execute_evaluator(race_state$target_evaluator, experiments, scenario, target_output) } else if (any(!is_exe)) { experiments <- experiments[is_exe] instance_idx <- instance_idx[is_exe] } target_output <- rbindlist(target_output, fill=TRUE, use.names=TRUE) set(target_output, j = setdiff(colnames(target_output), c("cost", "time")), value = NULL) if ("time" %notin% colnames(target_output)) set(target_output, j = "time", value = NA) set(target_output, j = "configuration", value = unlist_element(experiments, "id_configuration")) set(target_output, j = "instance", value = instance_idx) if (!is.null(bounds)) set(target_output, j = "bound", value = unlist_element(experiments, "bound")) } target_output } ## Executes a list of configurations in a particular instance ## configurations: description having the id of the configuration ## instance.idx: index of the instance,seed pair in race_state$instances_log ## bounds: execution bounds (NULL if not needed). ## is_exe: Boolean vector that determines which experiments need to executed. race_wrapper <- function(race_state, configurations, instance_idx, bounds, is_exe, scenario) { target_output <- race_wrapper_helper(race_state, configurations, instance_idx, bounds, is_exe, scenario) race_state$update_race_experiment_log(target_output, scenario) target_output } experiments_output_to_matrix <- function(output, scenario) { if (scenario$capping) output[["cost"]] <- applyPAR(output[["cost"]], boundMax = scenario$boundMax, boundPar = scenario$boundPar) as.matrix(dcast(output[, c("instance", "configuration", "cost")], instance ~ configuration, value.var = "cost"), rownames = "instance") } aux2.friedman <- function(y, I, alive, conf.level = 0.95) { dropped.any <- FALSE n <- nrow(y) k <- length(I) r <- rowRanks(y, cols = I, ties.method = "average", useNames = FALSE) R <- colSums2(r, useNames = FALSE) o <- order(R) best <- I[o[1L]] TIES <- tapply(c(r), row(r), table) STATISTIC <- ((12 * sum((R - n * (k + 1L) / 2)^2)) / (n * k * (k + 1L) - (sum(unlist(lapply(TIES, function (u) {u^3 - u}))) / (k - 1L)))) PARAMETER <- k - 1L PVAL <- pchisq(STATISTIC, PARAMETER, lower.tail = FALSE) #names(STATISTIC) <- "Friedman chi-squared" #names(PARAMETER) <- "df" alpha <- 1 - conf.level if (!is.nan(PVAL) && PVAL < alpha) { # This formula for multiple comparisons comes from Conover, "Practical # Nonparametric Statistics", 1999, pages 369-371. A <- sum(as.vector(r)^2) t <- qt(1 - alpha / 2, df = (n - 1L) * (k - 1L)) * (2 * (n * A - sum(R^2)) / ((n - 1L) * (k - 1L)))^(1 / 2) J <- best for (j in 2L:k) { if (abs(R[o[j]] - R[o[1L]]) > t) { break } else { J <- c(J, I[o[j]]) } } alive[-J] <- FALSE dropped.any <- TRUE } irace_assert(I[which.min(R)] == best) list(ranks = R, alive = alive, dropped.any = dropped.any, p.value = PVAL) } aux_friedman <- function(results, alive, which_alive, conf.level) { no.alive <- length(which_alive) if (no.alive > 2L) { # If more then 2 configurations are left, use Friedman return(aux2.friedman(results, which_alive, alive, conf.level = conf.level)) } ranks <- NULL dropped.any <- TRUE PVAL <- 0 # If only 2 configurations are left, switch to Wilcoxon V1 <- results[, which_alive[1L]] V2 <- results[, which_alive[2L]] diffs <- V1 - V2 # Avoid the test if the answer is obvious if (all(diffs <= 0)) { ranks <- c(1L,2L) } else if (all(diffs >= 0)) { ranks <- c(2L,1L) } else { ZEROES <- any(diffs == 0) if (ZEROES) diffs <- diffs[diffs != 0] r <- rank(abs(diffs)) TIES <- length(r) != length(unique(r)) diffs <- outer(diffs, diffs, "+") diffs <- sort(diffs[!lower.tri(diffs)])/2 PVAL <- wilcox.test(V1, V2, paired = TRUE, exact = if (ZEROES || TIES) FALSE else NULL)$p.value irace_assert(!is.nan(PVAL) & !is.na(PVAL)) if (PVAL >= 1 - conf.level) dropped.any <- FALSE # We use the pseudo median (see wilcox.test.default) ranks <- if (median(diffs) <= 0) c(1L,2L) else c(2L,1L) } if (dropped.any) alive[which_alive[ranks[2L]]] <- FALSE list(ranks = ranks, alive = alive, dropped.any = dropped.any, p.value = PVAL) } aux.one_ttest <- function(results, alive, which_alive, conf.level, adjust = c("none","bonferroni","holm")) { adjust <- match.arg(adjust) irace_assert(sum(alive) == length(which_alive)) results <- results[, which_alive] means <- colMeans2(results) best <- which.min(means) mean_best <- means[best] pvals <- sapply(means, function(x) as.numeric(isTRUE( all.equal.numeric(mean_best[[1L]], x[[1L]], check.attributes = FALSE)))) results_best <- results[, best] var_best <- var(results_best) which_test <- which(pvals < 1.0) for (j in which_test) { PVAL <- pvals[j] if (PVAL == 1.0) next results_j <- results[, j] # t.test may fail if the data in each group is almost constant. Hence, we # surround the call in a try() and we initialize p with 1 if the means are # equal or zero if they are different if (min(var(results_best), var(results_j)) < 10 * .Machine$double.eps) next # The t.test may fail if the data are not normal despite one configuration # clearly dominating the other. if (all(results_best <= results_j)) next try(PVAL <- t.test(results_best, results_j, alternative = "less", paired = TRUE)$p.value) irace_assert(!is.nan(PVAL) & !is.na(PVAL)) pvals[j] <- PVAL } pvals <- p.adjust(pvals, method = adjust) dropj <- which_alive[pvals < 1.0 - conf.level] dropped_any <- length(dropj) > 0 irace_assert(all(alive[dropj])) alive[dropj] <- FALSE list(ranks = means, alive = alive, dropped.any = dropped_any, p.value = min(pvals)) } aux.ttest <- function(results, alive, which_alive, conf.level, adjust = c("none","bonferroni","holm")) { adjust <- match.arg(adjust) irace_assert(sum(alive) == length(which_alive)) means <- colMeans2(results, cols = which_alive, useNames = FALSE) # FIXME: break ties using median or ranks? best <- which.min(means) mean_best <- means[best] pvals <- sapply(means, function(x) as.numeric(isTRUE( all.equal.numeric(mean_best[[1L]], x[[1L]], check.attributes = FALSE)))) results <- results[, which_alive] results_best <- results[, best] var_best <- var(results_best) which_test <- which(pvals < 1.0) for (j in which_test) { PVAL <- pvals[j] if (PVAL == 1.0) next results_j <- results[, j] # t.test may fail if the data in each group is almost constant. Hence, we # surround the call in a try() and we initialize p with 1 if the means are # equal or 0 if they are different. if (min(var(results_best), var(results_j)) < 10 * .Machine$double.eps) next # The t.test may fail if the data are not normal despite one configuration # clearly dominating the other. if (all(results_best <= results_j)) next try(PVAL <- t.test(results_best, results_j, paired = TRUE)$p.value) irace_assert(!is.nan(PVAL) & !is.na(PVAL)) pvals[j] <- PVAL } pvals <- p.adjust(pvals, method = adjust) dropj <- which_alive[pvals < 1.0 - conf.level] dropped_any <- length(dropj) > 0L irace_assert(all(alive[dropj])) alive[dropj] <- FALSE list(ranks = means, alive = alive, dropped.any = dropped_any, p.value = min(pvals)) } table_hline <- function(widths) paste0(collapse="+", c("",strrep("-", widths), "\n")) table_sprintf <- function(text, widths) paste0(collapse="|", c("", sprintf("%*s",widths, text), "\n")) .nocap_table_fields_width <- as.integer(c(1, 11, 11, 11, 16, 11, 8, 5, 4, 6)) .nocap_colum_names <- c(" ", "Instance", "Alive", "Best", "Mean best", "Exp so far", "W time", "rho", "KenW", "Qvar") .capping_table_fields_width <- as.integer(c(1, 11, 8, 11, 11, 16, 11, 8, 5, 4, 6)) .capping_colum_names <- c(" ", "Instance", "Bound", "Alive", "Best", "Mean best", "Exp so far", "W time", "rho", "KenW", "Qvar") .capping_hline <- table_hline(.capping_table_fields_width) .nocap_hline <- table_hline(.nocap_table_fields_width) .race_common_header <- "# Markers: x No test is performed. c Configurations are discarded only due to capping. - The test is performed and some configurations are discarded. = The test is performed but no configuration is discarded. ! The test is performed and configurations could be discarded but elite configurations are preserved. . All alive configurations are elite and nothing is discarded.\n\n" .nocap_header <- paste0(.race_common_header, .nocap_hline, table_sprintf(.nocap_colum_names, .nocap_table_fields_width), .nocap_hline) .capping_header <- paste0(.race_common_header, .capping_hline, table_sprintf(.capping_colum_names, .capping_table_fields_width), .capping_hline) race_print_header_nocap <- function() cat(sep = "", .nocap_header) race_print_header_cap <- function() cat(sep = "", .capping_header) elapsed_wctime_str <- function(now, start) { if (now <= start) return("00:00:00") elapsed <- difftime(now, start, units = "secs") # FIXME: Maybe better and faster if we only print seconds? format(.POSIXct(elapsed, tz="GMT"), "%H:%M:%S") } race_print_task <- function(res.symb, Results, instance, current_task, which_alive, id_best, best, experiments_used, start_time, bound, capping) { cat(sprintf("|%s|%11d|", res.symb, instance)) if (capping) { if (is.null(bound)) cat(" NA|") else cat(sprintf("%8.2f|", bound)) } # FIXME: This is the mean of the best, but perhaps it should # be the sum of ranks in the case of test == friedman? mean_best <- mean(Results[, best]) time_str <- elapsed_wctime_str(Sys.time(), start_time) cat(sprintf(paste0("%11d|%11d|", .irace.format.perf, "|%11d|%s"), length(which_alive), id_best, mean_best, experiments_used, time_str)) if (current_task > 1L && length(which_alive) > 1L) { res <- Results[, which_alive, drop = FALSE] conc <- concordance(res) qvar <- dataVariance(res) # FIXME: We would like to use %+#4.2f but this causes problems with # https://github.com/oracle/fastr/issues/191 cat(sprintf("|%+4.2f|%.2f|%.4f|\n", conc$spearman.rho, conc$kendall.w, qvar)) } else { cat("| NA| NA| NA|\n") } } race_print_footer <- function(bestconf, mean_best, break_msg, debug_level, capping = FALSE, old_best_id) { cat(sep = "", if (capping) .capping_hline else .nocap_hline, if (debug_level >= 1L) paste0("# Stopped because ", break_msg, "\n"), if (!is.null(old_best_id)) paste0("Best configuration for the instances in this race: ", old_best_id, "\n"), sprintf("Best-so-far configuration: %11d", bestconf[[".ID."]][1L]), " mean value: ", sprintf(.irace.format.perf, mean_best), "\n", "Description of the best-so-far configuration:\n") configurations_print(bestconf, metadata = TRUE) cat("\n") } ## This function calculates an execution bound ## data: matrix columns as configurations and rows instances ## type: ## median: bound based on the configuration's mean median ## mean: bound based on the configuration's mean mean ## worst: bound based on the worst configuration's mean ## best: bound based on the best configurations's mean executionBound <- function(data, type = "median") { irace_assert (ncol(data) >= 1L) if (ncol(data) == 1L) { return (mean(data[,1L], na.rm = TRUE)) } # This should never happen because the data used to obtain the execution # bound should be complete, that is, the bounding configurations should have # been executed on all previous instances. irace_assert (!anyNA(data)) colmeans <- colMeans2(data) bound <- switch (type, median = median(colmeans), mean = mean(colmeans), worst = max(colmeans), # default: min(colmeans)) bound } ## This function calculates an execution bound per instance ## data: array of the execution times on the current instance ## type: ## median: bound based on the median time ## mean: bound based on the mean time ## worst: bound based on the worst time ## best: bound based on the best time instanceBound <- function(data, type="median") { irace_assert (!anyNA(data)) switch (type, median = median(data), mean = mean(data), worst = max(data), # default: min(data)) } ## This function returns survivors obtained after applying the dominance elimination ## criterion. ## results: matrix of experiments results (all configurations) ## elites: index of elite configurations ## alive: bolean array of alive configurations ## eps: constant added to the bound to account for the measuring precision dom_elim <- function(results, elites, alive, scenario, minSurvival, eps = 1e-5) { which_alive <- which(alive) irace_assert(length(which_alive) >= minSurvival) cmeans <- colMeans2(results, cols = which_alive, useNames = FALSE) irace_assert(!all(is.na(cmeans))) # Only NA values when calculating mean for dominance. # When there are no protected elites left, select the best configurations to # calculate the bounds. This is quite aggressive and another alternative # would be to disable dom_elim when elites == 0. if (length(elites) == 0L) { # In the case we have only two alive configurations only one can be elite. if (length(which_alive) <= 2L) elites <- which_alive[which.min(cmeans)] else elites <- which_alive[order(cmeans, na.last = TRUE, decreasing = FALSE)[seq_len(minSurvival)]] } bound <- executionBound(results[, elites, drop = FALSE], type = scenario$cappingType) alive[which_alive] <- ((bound + eps) >= cmeans) alive } ## This function applies PARX (X=boundPar) to all experiments ## that exceed the maximum execution time (boundMax) applyPAR <- function(results, boundMax, boundPar) { # We do not want to change Inf or -Inf because those represent rejection. if (boundPar != 1) results[is.finite(results) & results >= boundMax] <- boundMax * boundPar results } ## This function calculates the execution time allowed for the executions ## of configurations based on previous execution times. ## It returns a list with two elements: ## * elite_bound : value (mean execution time or per instance execution time) ## used to calculate the maximum execution time (final_bounds) for each configuration. ## * final_bounds[i] : maximum execution time for candidate i on the current instance. final_execution_bound <- function(experimentsTime, elites, current_task, which_alive, which_exe, scenario) { boundMax <- scenario$boundMax final_bounds <- rep(boundMax, length(which_alive)) # Elite candidates can have NA values due to the rejection. if (length(elites)) elites <- elites[!is.na(experimentsTime[current_task,elites])] # Only apply bounds when there is previous data if (length(elites) == 0L || length(which_exe) == 0L) { # FIXME: should we use an adjusted boundMax ? return(list(final_bounds = final_bounds, elite_bound = boundMax)) } minMeasurableTime <- scenario$minMeasurableTime # The elite membership is updated before calling this function to know # the configurations used for calculating the bounds we need to do this. # Note that some of these configurations could be not elite anymore for the # elimination phase, given that all their evaluations have been used. if (scenario$boundType == "instance") { elite_bound <- instanceBound(experimentsTime[current_task, elites], type = scenario$cappingType) final_bounds_exe <- min(elite_bound + minMeasurableTime, boundMax) final_bounds_exe <- ceiling_digits(final_bounds_exe, scenario$boundDigits) } else { elite_bound <- executionBound(experimentsTime[seq_len(current_task), elites, drop = FALSE], type = scenario$cappingType) elite_bound <- min(elite_bound, boundMax) total_time <- (current_task * elite_bound) + minMeasurableTime final_bounds_exe <- total_time - colSums2(experimentsTime, rows = seq_len(current_task), cols = which_exe, na.rm = TRUE, useNames = FALSE) # There are cases in which a small negative budget is used. For example: # assuming candidates 1 and 2 are elite: # maxBound <- 80 # executionTime <- matrix(c(0.010,0.0170,0.010, 24,28,27, 0.010,0.017,NA), # byrow=TRUE, ncol=3, nrow=3, dimnames=list(c(1,2,3),c(1,2,3))) # current_task <- 3; boundDigits = 0 # elite_bound <- irace:::executionBound(executionTime[1:current_task,1:2], "median") # total_time <- elite_bound * current_task + 0.01 # final_bounds_exe <- total_time - colSums2(executionTime[1:current_task,3,drop=FALSE], na.rm=TRUE) # # We set the execution time to the elite_bound, this should be enough # to eliminate a bad candidate for the next task. final_bounds_exe[final_bounds_exe <= 0] <- elite_bound # We round up the bounds up to the specified number of digits. This may # be necessary if the target-algorithm does not support higher precision. final_bounds_exe <- ceiling_digits(final_bounds_exe, scenario$boundDigits) final_bounds_exe[final_bounds_exe > boundMax] <- boundMax irace_assert(all(final_bounds_exe > 0)) } final_bounds[which(which_alive %in% which_exe)] <- final_bounds_exe list(final_bounds = final_bounds, elite_bound = elite_bound) } get_ranks <- function(x, test) if (test == "friedman") colSums2(rowRanks(x, ties.method="average")) else rank(colMeans2(x)) # Recompute the best as follows. Given two configurations, the one evaluated # on more instances is ranked better. Otherwise, break ties according to the # criteria of the stat test. overall_ranks <- function(x, test) { if (ncol(x) == 1L) return(1L) ninstances <- colSums2(!is.na(x)) uniq_ninstances <- sort(unique(ninstances), decreasing = TRUE) last_r <- 0L ranks <- rep_len(Inf, ncol(x)) # Iterate from the largest to the lowest number of instances. for (k in uniq_ninstances) { confs <- which(ninstances == k) irace_assert(all(is.infinite(ranks[confs]))) r <- 1L if (length(confs) > 1L) { # Select only non-NA rows y <- x[, confs, drop = FALSE] y <- y[complete.cases(y), , drop = FALSE] r <- get_ranks(y, test = test) } r <- r + last_r last_r <- max(r) ranks[confs] <- r } ranks } # Remove one elite count from every configuration not executed. update_is_elite <- function(is_elite, which_exe) { which_notexecuted <- setdiff(which(is_elite > 0L), which_exe) is_elite[which_notexecuted] <- is_elite[which_notexecuted] - 1L irace_assert (all(is_elite >= 0L)) is_elite } update_elite_safe <- function(Results, is_elite) { elites <- is_elite > 0L if (!any(elites)) return(0L) # All elites rejected. max(which(rowAnyNotNAs(Results, cols = elites))) } generateTimeMatrix <- function(elite_ids, experiment_log) { # Silence CRAN warnings configuration <- bound <- NULL # Remove everything that we don't need. experiment_log <- experiment_log[(configuration %in% elite_ids) & !is.na(time), c("configuration", "instance", "time", "bound")] experiment_log[, time := pmin.int(time, bound)] experiment_log[, bound := NULL] time_matrix <- dcast(experiment_log, instance ~ configuration, value.var = "time") setcolorder(time_matrix, neworder = as.character(elite_ids)) as.matrix(time_matrix, rownames = "instance") } race <- function(race_state, maxExp, minSurvival = 1L, configurations, scenario) elitist_race(race_state = race_state, maxExp = maxExp, minSurvival = minSurvival, elite_data = NULL, configurations = configurations, scenario = scenario, elitist_new_instances = 0L) elitist_race <- function(race_state, maxExp, minSurvival = 1L, elite_data = NULL, configurations, scenario, elitist_new_instances, firstTest = scenario$firstTest) { blockSize <- scenario$blockSize conf.level <- scenario$confidence firstTest <- blockSize * firstTest eachTest <- blockSize * scenario$eachTest elitist <- scenario$elitist capping <- scenario$capping n_configurations <- nrow(configurations) alive <- rep_len(TRUE, n_configurations) is_rejected <- logical(n_configurations) ## FIXME: Remove argument checking. This must have been done by the caller. irace_assert(maxExp > 0L) if (n_configurations <= minSurvival) { irace_error("Not enough configurations (", n_configurations, ") for a race (minSurvival=", minSurvival, ")") } # Check argument: conf.level if (!missing(conf.level) && (!is.numeric(conf.level) || length(conf.level)!=1 || !is.finite(conf.level) || conf.level < 0 || conf.level > 1)) stop("conf.level must be a single number between 0 and 1") if (scenario$quiet) { print_header <- print_task <- print_footer <- do_nothing } else { print_header <- if (capping) race_print_header_cap else race_print_header_nocap print_task <- race_print_task print_footer <- race_print_footer } stat_test <- scenario$testType do_test <- switch(stat_test, friedman = function(results, alive, which_alive, conf.level = scenario$confidence) aux_friedman(results, alive, which_alive, conf.level = conf.level), t.none = function(results, alive, which_alive, conf.level = scenario$confidence) aux.ttest(results, alive, which_alive, conf.level = conf.level, adjust = "none"), t.holm = function(results, alive, which_alive, conf.level = scenario$confidence) aux.ttest(results, alive, which_alive, conf.level = conf.level, adjust = "holm"), t.bonferroni = function(results, alive, which_alive, conf.level = scenario$confidence) aux.ttest(results, alive, which_alive, conf.level = conf.level, adjust = "bonferroni")) do_test <- bytecompile(do_test) # Create the instance list according to the algorithm selected. if (elitist) { # FIXME: we should sample taking into account the block-size, so we sample blocks, not instances. irace_assert((race_state$next_instance - 1L) %% blockSize == 0, eval_after={cat("next_instance:", race_state$next_instance, ", block_size:", blockSize, "\n")}) race_instances <- elitist_init_instances(race_state, deterministic = scenario$deterministic, sampleInstances = scenario$sampleInstances, elitist_new_instances = elitist_new_instances) # It may be reduced further by elitist_init_instances() elitist_new_instances <- min(elitist_new_instances, race_state$elitist_new_instances) all_elite_instances_evaluated <- function() { if (race_state$next_instance == 1L) return(TRUE) evaluated <- !is.na(Results[, alive, drop=FALSE]) # All instances that have been previously seen have been evaluated by at # least one configuration all(rowAnys(evaluated)) && # and the number of instances evaluated per configuration is a multiple # of eachTest (which is scenario$blockSize * scenario$eachTest). all(colSums2(evaluated) %% eachTest == 0L) } } else { race_instances <- no_elitist_init_instances(race_state, deterministic = scenario$deterministic) all_elite_instances_evaluated <- function() TRUE } irace_assert(!anyDuplicated(race_instances)) irace_assert(identical(sort(race_instances), seq_along(race_instances))) nb_tasks <- length(race_instances) # Initialize some variables... experiments_used <- 0L ## FIXME: Probably, instead of this, we should keep elite_safe in the race_state. if (is.null(elite_data)) { elite_safe <- 0L elite_instances_ID <- NULL } else { irace_assert(race_state$next_instance - 1L == nrow(elite_data)) # There must be a non-NA entry for each instance. irace_assert(all(rowAnyNotNAs(elite_data)), eval_after = { print(elite_data)}) # There must be a non-NA entry for each configuration. irace_assert(all(colAnyNotNAs(elite_data)), eval_after = { cat("elite_data:\n") print(elite_data) cat("experiment_log:\n") print(race_state$experiment_log) }) # elite_safe: maximum instance number for which any configuration may be # considered elite. After evaluating this instance, no configuration is # elite. elite_safe <- elitist_new_instances + nrow(elite_data) elite_instances_ID <- as.character(race_instances[seq_len(elite_safe)]) } configurations_ID <- as.character(configurations[[".ID."]]) Results <- matrix(NA_real_, nrow = elite_safe, ncol = n_configurations, dimnames = list(elite_instances_ID, configurations_ID)) if (capping) experimentsTime <- matrix(NA_real_, nrow = elite_safe, ncol = n_configurations, dimnames = list(elite_instances_ID, configurations_ID)) if (is.null(elite_data)) { # is_elite[i] : number of instances to be seen in this race on which i has # been previously evaluated. is_elite <- integer(n_configurations) } else { Results[rownames(elite_data), colnames(elite_data)] <- elite_data irace_assert(all(colnames(elite_data) %chin% configurations_ID)) if (capping) { # Temporarily use only 0 or 1, we will calculate the real value below. is_elite <- as.integer(configurations_ID %chin% colnames(elite_data)) tmp <- generateTimeMatrix(elite_ids = colnames(elite_data), experiment_log = race_state$experiment_log) experimentsTime[rownames(tmp), colnames(tmp)] <- tmp # Preliminary execution of elite configurations to calculate # the execution bound of initial configurations (capping only). if (elitist_new_instances > 0L) { irace_assert(elitist_new_instances %% blockSize == 0L) # FIXME: This should go into its own function. n_elite <- ncol(elite_data) which_elites <- which(is_elite > 0L, useNames=FALSE) irace_assert(identical(which_elites, seq_len(n_elite))) irace_note("Preliminary execution of ", n_elite, " elite configuration(s) over ", elitist_new_instances, " instance(s).\n") # FIXME: Launch all seq_len(elitist_new_instances) experiments in parallel. for (k in seq_len(elitist_new_instances)) { output <- race_wrapper (race_state, configurations = configurations[which_elites, , drop = FALSE], instance_idx = race_instances[k], bounds = rep(scenario$boundMax, n_elite), # which_exe values are within 1:nbConfigurations, whereas experiments # indices are within 1:length(which_alive). The following line converts # from one to the other. is_exe = rep_len(TRUE, n_elite), scenario = scenario) # Extract results: irace_assert(length(output[["cost"]]) == n_elite) Results[k, which_elites] <- applyPAR(output[["cost"]], boundMax = scenario$boundMax, boundPar = scenario$boundPar) irace_assert(!anyNA(output[["time"]])) experimentsTime[k, which_elites] <- output[["time"]] # capping is enabled irace_assert(all.equal(configurations[[".ID."]][which_elites], output[["configuration"]])) irace_assert(all.equal(output[["bound"]], rep(scenario$boundMax, n_elite))) irace_assert(all.equal(unique(output[["instance"]]), race_instances[k])) experiments_used <- experiments_used + n_elite # We remove elite configurations that are rejected given that # is not possible to calculate the bounds. rejected <- is.infinite(output[["cost"]]) irace_assert(all.equal(as.vector(is.infinite(Results[k, which_elites])), rejected), eval_after={ cat("rejected:\n") print(output[["cost"]]) print(rejected) cat("Results:\n") print(Results[k, which_elites]) print(is.infinite(Results[k, which_elites])) }) is_rejected[which_elites] <- rejected is_elite[rejected] <- 0L which_elites <- which_elites[!rejected] n_elite <- length(which_elites) # If all elite are eliminated we stop execution of initial instances. if (n_elite == 0L) { irace_note ("All elite configurations are rejected. Execution of non-elites will be not bounded.\n") break } } if (any(is_rejected)) { irace_note ("Immediately rejected configurations: ", paste0(configurations[[".ID."]][is_rejected], collapse = ", ") , "\n") alive[is_rejected] <- FALSE # Calculate the maximum instance that has any non-NA value. # FIXME: Use update_elite_safe() if (n_elite > 0L) elite_safe <- max(which(rowAnys(!is.na(Results[, which_elites, drop=FALSE])))) else elite_safe <- 0L irace_assert(identical(update_elite_safe(Results, is_elite), elite_safe)) # FIXME: If sum(alive) <= minSurvival, we stop later but we will have # elites that have not been evaluated in any instance, which is # bad. Ideally we would sample new configurations here but that is # too hard to do in iterated-racing. } } } # end if(capping) # Compute the elite membership. is_elite <- colSums2(!is.na(Results)) # Remove rejected configurations. is_elite[is_rejected] <- 0L } no_elimination <- 0L # number of tasks without elimination. print_header() # Start main loop. break_msg <- NULL best <- NA_integer_ which_alive <- which(alive) nb_alive <- length(which_alive) for (current_task in seq_len(nb_tasks)) { which_exe <- which_alive if (elitist && any(is_elite > 0L)) { # Filter configurations that do not need to be executed (elites). # This is valid only for previous iteration instances. irace_assert(current_task <= elite_safe) # Execute everything that is alive and not yet executed. which_exe <- which(alive & is.na(Results[current_task, ])) if (length(which_exe) == 0L) { is_elite <- update_is_elite(is_elite, which_exe) # LESLIE: This is the case in which there are only elite configurations alive # and we are still in the previous instances execution, but we can still # continue with the race. (This is only possible because the early termination # criterion is disabled) ## MANUEL: So what is the reason to not immediately terminate here? Is ## there a reason to continue? if (current_task == 1L) { # We may reach this point in the first iteration so we need to calculate best. if (nb_alive == 1L) { best <- which_alive } else { tmpResults <- Results[1L, which_alive, drop = FALSE] irace_assert(!anyNA(tmpResults)) # which.min returns only the first minimum. best <- which_alive[which.min(get_ranks(tmpResults, test = stat_test))] } } if (is.na(best)) { utils::dump.frames(dumpto = "best_crash", to.file = TRUE, include.GlobalEnv = TRUE) irace_assert(!is.na(best)) } id_best <- configurations[[".ID."]][best] print_task(".", Results[seq_len(current_task), , drop = FALSE], race_instances[current_task], current_task, which_alive = which_alive, id_best = id_best, best = best, experiments_used, start_time = Sys.time(), # FIXME: Why do we pass NA as bound? Why not pass the actual bound if any? bound = NA_real_, capping = capping) next } } if (all_elite_instances_evaluated()) { # We stop when we have less configurations than required. if (nb_alive <= minSurvival) { # Stop race if we have less or equal than the minimum number of # configurations. break_msg <- paste0("number of alive configurations (", nb_alive, ") <= minimum number of configurations (", minSurvival, ")") break } ## We continue running if (1) we have not reached the firstTest or (2) ## there are instances previously seen that have not been evaluated on any ## alive configuration. If we just did a test, check that we have enough ## budget to reach the next test. # FIXME: In post-selection racing, we want to consume all budget, so we # should discard configurations until we have 2. if (current_task > firstTest && ( (current_task - 1L) %% eachTest) == 0L && experiments_used + length(which_exe) * eachTest > maxExp) { break_msg <- paste0("experiments for next test (", experiments_used + length(which_exe) * eachTest, ") > max experiments (", maxExp, ")") break } if (elitist && scenario$elitistLimit != 0L && no_elimination >= scenario$elitistLimit) { break_msg <- paste0("tests without elimination (", no_elimination, ") >= elitistLimit (", scenario$elitistLimit, ")") break } } if (nrow(Results) < current_task) { Results <- rbind(Results, rep_len(NA_real_, ncol(Results))) rownames(Results) <- race_instances[seq_nrow(Results)] if (capping) { experimentsTime <- rbind(experimentsTime, rep_len(NA_real_, ncol(experimentsTime))) rownames(experimentsTime) <- race_instances[seq_nrow(experimentsTime)] } } start_time <- Sys.time() # Calculate bounds for executing if needed. which_elite_exe <- intersect(which_exe, which(is_elite > 0L)) irace_assert(setequal(which_elite_exe, which(is_elite & is.na(Results[current_task,])))) if (capping) { # Pre-execute elite configurations that are not yet executed in the current instance. if (length(which_elite_exe)) { # FIXME: This should go into its own function output <- race_wrapper(race_state, configurations = configurations[which_elite_exe, , drop = FALSE], instance_idx = race_instances[current_task], # FIXME: What if we already have a bound for this instance? bounds = rep(scenario$boundMax, length(which_elite_exe)), is_exe = rep_len(TRUE, length(which_elite_exe)), scenario = scenario) # Extract results irace_assert(length(output[["cost"]]) == length(which_elite_exe)) Results[current_task, which_elite_exe] <- applyPAR(output[["cost"]], boundMax = scenario$boundMax, boundPar = scenario$boundPar) irace_assert(!anyNA(output[["time"]])) irace_assert(all.equal(configurations[which_elite_exe, ".ID."], output[["configuration"]])) experimentsTime[current_task, which_elite_exe] <- output[["time"]] irace_assert(all.equal(unique(output[["instance"]]), race_instances[current_task])) experiments_used <- experiments_used + length(which_elite_exe) # We remove elite configurations that are rejected given that # is not possible to calculate the bounds rejected <- is.infinite(output[["cost"]]) if (any(rejected)) { irace_note("Immediately rejected configurations: ", paste0(configurations[[".ID."]][which_elite_exe[rejected]], collapse = ", ") , "\n") is_rejected[which_elite_exe] <- rejected is_elite[is_rejected] <- 0L alive[which_elite_exe] <- !rejected if (!any(alive)) { # FIXME: Only report this error if (all(is_rejected)); otherwise # restore non-rejected non-alive ones. Restoring a non-alive # configuration is difficult. We need to evaluate it in all the # instances that it has missed. irace_error("All configurations have been immediately rejected (all of them returned Inf) !") } which_alive <- which(alive) nb_alive <- length(which_alive) elite_safe <- update_elite_safe(Results, is_elite) } which_exe <- setdiff(which_exe, which_elite_exe) # FIXME: There is similar code above. Can we merge these code paths? if (length(which_exe) == 0L) { is_elite <- update_is_elite(is_elite, which_elite_exe) if (current_task == 1L) { # We may reach this point in the first iteration so we need to calculate best. if (nb_alive == 1L) { best <- which_alive } else { tmpResults <- Results[1L, which_alive, drop = FALSE] irace_assert(!anyNA(tmpResults)) # which.min returns only the first minimum. best <- which_alive[which.min(get_ranks(tmpResults, test = stat_test))] } } if (is.na(best)) { utils::dump.frames(dumpto = "best_crash", to.file = TRUE, include.GlobalEnv = TRUE) irace_assert(!is.na(best)) } id_best <- configurations[[".ID."]][best] print_task(".", Results[seq_len(current_task), , drop = FALSE], race_instances[current_task], current_task, which_alive = which_alive, id_best = id_best, best = best, experiments_used, start_time = start_time, # FIXME: Why do we pass NA as bound? Why not pass the actual bound if any? bound = if (is.null(scenario$boundMax)) NA_real_ else scenario$boundMax, capping) next } } all_bounds <- final_execution_bound(experimentsTime, elites = which(is_elite > 0L), current_task, which_alive, which_exe, scenario) final_bounds <- all_bounds$final_bounds elite_bound <- all_bounds$elite_bound } else { final_bounds <- rep(scenario$boundMax, length(which_alive)) elite_bound <- NULL } # Execute experiments. output <- race_wrapper(race_state, configurations = configurations[which_alive, , drop = FALSE], instance_idx = race_instances[current_task], bounds = final_bounds, is_exe = which_alive %in% which_exe, scenario = scenario) # Extract results # Set max execution bound to timed out executions which have execution # times smaller than boundMax and implement parX if required. vcost <- output[["cost"]] # Output is not indexed in the same way as configurations. which_has_time <- which(which_alive %in% which_exe) # With !is.null(scenario$targetEvaluator) we will have duplicated (instance, configuration) in output. irace_assert(all.equal(output[["bound"]], if (is.null(scenario$targetEvaluator)) final_bounds[which_has_time] else final_bounds)) if (capping) { vcost <- applyPAR(vcost, boundMax = scenario$boundMax, boundPar = scenario$boundPar) if (scenario$boundAsTimeout) { timeout_bounds <- if (is.null(scenario$targetEvaluator)) final_bounds[which_has_time] else final_bounds irace_assert(all.equal(output[["bound"]], timeout_bounds)) # We do not want to change Inf or -Inf because those represent rejection. vcost[is.finite(vcost) & (vcost >= timeout_bounds) & (vcost < scenario$boundMax)] <- scenario$boundMax } # If targetEvaluator was used, we do not update the times because no # evaluation actually happened, only the cost values possibly changed. vtimes <- if (is.null(scenario$targetEvaluator)) output[["time"]] else output[["time"]][which_has_time] # Correct higher execution times. irace_assert(all.equal(if (is.null(scenario$targetEvaluator)) output[["bound"]] else output[["bound"]][which_has_time], final_bounds[which_has_time])) experimentsTime[current_task, which_has_time] <- pmin(vtimes, final_bounds[which_has_time]) } ## Currently, targetEvaluator always re-evaluates, which implies that the ## value may change without counting as an evaluation. We do this to allow online normalization. which_has_cost <- if (is.null(scenario$targetEvaluator)) which_exe else which_alive irace_assert(length(output[["cost"]]) == length(which_has_cost)) Results[current_task, which_has_cost] <- vcost experiments_used <- experiments_used + length(which_exe) # We update the elites that have been executed. is_elite <- update_is_elite(is_elite, which_elite_exe) ## Drop bad configurations. ## Infinite values denote immediate rejection of a configuration. # FIXME: Should this be which_has_cost? rejected <- is.infinite(Results[current_task, which_exe]) if (any(rejected)) { irace_note ("Immediately rejected configurations: ", paste0(configurations[which_exe[rejected], ".ID."], collapse = ", ") , "\n") is_rejected[which_exe] <- rejected is_elite[is_rejected] <- 0L alive[is_rejected] <- FALSE if (!any(alive)) { # FIXME: Only report this error if (all(is_rejected)); otherwise # restore non-rejected non-alive ones. Restoring a non-alive # configuration is difficult. We need to evaluate it in all the # instances that it has missed. irace_error("All configurations have been immediately rejected (all of them returned Inf) !") } which_alive <- which(alive) nb_alive <- length(which_alive) # FIXME: Should we stop if (nbAlive <= minSurvival) ??? elite_safe <- update_elite_safe(Results, is_elite) } irace_assert(!anyNA(Results[seq_len(current_task), alive, drop=FALSE])) irace_assert(!any(is.infinite(Results[, alive, drop=FALSE]))) # Variables required to produce output of elimination test. cap_done <- FALSE # if dominance elimination was performed test.done <- FALSE # if statistical test elimination was performed cap_dropped <- FALSE # if capping has drop any configuration test.dropped <- FALSE # if any candidates has been eliminated by testing cap_alive <- test.alive <- alive ## Dominance elimination (Capping only). # The second condition can be false if we eliminated via immediate # rejection. The third condition ensures that we see the block before capping. if (capping && nb_alive > minSurvival && (current_task %% blockSize) == 0L && (!scenario$cappingAfterFirstTest || current_task >= firstTest)) { irace_assert(!any(is_elite > 0L) == (current_task >= elite_safe)) cap_alive <- dom_elim(Results[seq_len(current_task), , drop = FALSE], # Get current elite configurations. elites = which(is_elite > 0L), alive, scenario, minSurvival) cap_dropped <- nb_alive > sum(cap_alive) cap_done <- TRUE } # We assume that firstTest is a multiple of eachTest. In any # case, this will only do the first test after the first multiple # of eachTest that is larger than firstTest. if (current_task >= firstTest && (current_task %% eachTest) == 0L && nb_alive > 1L) { irace_assert(sum(alive) == nb_alive) test_res <- do_test(Results[seq_len(current_task), ], alive, which_alive) # FIXME: This race_ranks is unused. We should check if it matches the one computed below. race_ranks <- test_res$ranks test.alive <- test_res$alive test.dropped <- nb_alive > sum(test.alive) test.done <- TRUE } # Merge the result of both eliminations. prev_nb_alive <- nb_alive prev_which_alive <- which_alive alive <- cap_alive & test.alive # Handle elites when elimination is performed. The elite configurations # can be removed only when they have no more previously-executed instances. irace_assert(!any(is_elite > 0L) == (current_task >= elite_safe)) if (!is.null(elite_data) && any(is_elite > 0L)) { irace_assert (length(alive) == length(is_elite)) alive <- alive | (is_elite > 0L) } # It may happen that the capping and the test eliminate together all # configurations. In that case, we only trust the capping elimination. if (capping && !any(alive)) { if (scenario$debugLevel >= 2L) { irace_warning("Elimination tests have eliminated all configurations, keeping the capping results.\n") irace_note("Alive according to capping:", which(cap_alive), "\n") irace_note("Alive according to test:", which(test.alive), "\n") } alive <- cap_alive } which_alive <- which(alive) nb_alive <- length(which_alive) # Output the result of the elimination test. res.symb <- if (cap_dropped && !test.dropped && prev_nb_alive != nb_alive) { "c" # Removed just by capping. } else if (cap_dropped || test.dropped) { if (prev_nb_alive != nb_alive) "-" else "!" } else if (cap_done || test.done) "=" else "x" # Rank alive configurations: order all configurations (eliminated or not) # LESLIE: we have to make the ranking outside: we can have configurations eliminated by capping # that are not eliminated by the test. # MANUEL: I don't understand the above comment. if (length(prev_which_alive) == 1L) { race_ranks <- 1L best <- prev_which_alive } else { tmpResults <- Results[seq_len(current_task), prev_which_alive, drop = FALSE] irace_assert(!anyNA(tmpResults)) race_ranks <- get_ranks(tmpResults, test = stat_test) # which.min() returns only the first minimum. best <- prev_which_alive[which.min(race_ranks)] } irace_assert(best == prev_which_alive[order(race_ranks)][1L]) irace_assert(length(race_ranks) == length(prev_which_alive)) # Remove the ranks of those that are not alive anymore race_ranks <- race_ranks[which_alive] irace_assert(length(race_ranks) == nb_alive) id_best <- configurations[[".ID."]][best] print_task(res.symb, Results[seq_len(current_task), , drop = FALSE], race_instances[current_task], current_task, which_alive = which_alive, id_best = id_best, best = best, experiments_used, start_time = start_time, bound = elite_bound, capping) if (elitist) { # Compute number of statistical tests without eliminations. irace_assert(!any(is_elite > 0L) == (current_task >= elite_safe)) if (!any(is_elite > 0L) && current_task > firstTest && (current_task %% eachTest) == 0L) { no_elimination <- if (nb_alive == prev_nb_alive) no_elimination + 1L else 0L } } } if (is.null(break_msg)) break_msg <- paste0("all instances (", nb_tasks, ") evaluated") # Adding this given that when ncandidates = minsurvival+1 # and there one elite configuration that gets discarded in the new instances # execution the race is finished with no executions. # FIXME: we should handle this better, maybe allowing irace to handle no elite # in irace() # MANUEL: Leslie, how can we reach this error in normal circumstances? # Can we handle this better? if (current_task == 1L && all(is_elite == 0L)) irace_error ("Maximum number configurations immediately rejected reached!") # All instances that are not new in this race must have been evaluated by at # least one configuration. irace_assert(all_elite_instances_evaluated(), eval_after = { print(Results[,alive, drop=FALSE])}) # If we stop the loop before we see all new instances, there may be new # instances that have not been executed by any configuration. Results <- Results[rowAnyNotNAs(Results), , drop = FALSE] # If we reject configurations so that sum(alive) <= minSurvival, we may stop # before we evaluate some configurations in any instance. if (any(is_rejected)) { alive <- alive & colAnyNotNAs(Results) which_alive <- which(alive) if (!any(alive)) { # FIXME: Only report this error if (all(is_rejected)); otherwise # restore non-rejected non-alive ones. Restoring a non-alive # configuration is difficult. We need to evaluate it in all the # instances that it has missed. irace_error("All configurations have been immediately rejected (all of them returned Inf) !") } } race_ranks <- overall_ranks(Results[, alive, drop = FALSE], test = stat_test) if (!scenario$quiet) { old_best <- best # old_best could be NA. best <- which_alive[which.min(race_ranks)] mean_best <- mean(Results[, best]) print_footer(bestconf = configurations[best, , drop = FALSE], # FIXME: This is the mean of the best, but perhaps it # should be the sum of ranks in the case of test == friedman? mean_best = mean_best, break_msg = break_msg, debug_level = scenario$debugLevel, capping = capping, old_best_id = if (old_best == best || is.na(old_best)) NULL else id_best) } rejected_ids <- configurations[is_rejected, ".ID."] scenario$parameters$forbid_configurations( race_state$update_rejected(rejected_ids, configurations) ) # Only return alive ones. configurations <- configurations[alive, , drop=FALSE] irace_assert(all(configurations[[".ID."]] %not_in% rejected_ids)) # No rejected is alive. # Assign the proper ranks in the configurations data.frame. configurations[[".RANK."]] <- race_ranks # Now we can sort the data.frame by the rank. configurations <- configurations[order(configurations[[".RANK."]]), , drop=FALSE] if (scenario$debugLevel >= 3L) { irace_note ("Memory used in race():\n") race_state$print_mem_used() } local_experiment_log <- race_state$reset_race_experiment_log() # nrow(Results) may be smaller, equal or larger than current_task. if (is.null(scenario$targetEvaluator)) { # With targetEvaluator, we may have the recorded a new cost value but not # counted it as an experiment used if targetRunner was not called. irace_assert(anyDuplicated(local_experiment_log[, c("instance", "configuration")]) == 0L, eval_after = { print(local_experiment_log) print(mget(ls())) }) irace_assert(nrow(local_experiment_log) == experiments_used) } list(experiments = Results, experiment_log = local_experiment_log, experimentsUsed = experiments_used, configurations = configurations) } irace/R/multi_irace.R0000644000176200001440000001266714736573012014202 0ustar liggesusers#' Execute [irace()] multiple times with the same or different scenarios and parameter space definitions. #' #' There are three modes of operation: #' \itemize{ #' \item One `scenarios` and `k` `parameters`: `k` runs with the same scenario and each parameter space definition. #' \item One `parameters` and `k` `scenarios`: `k` runs with the same parameter space definition and each scenario. #' \item `k` `parameters` and `k` scenarios: `k` runs with each scenario and parameter space definition. #' } #' Each of the `k` runs can be repeated `n` times by supplying a value for `n`. #' #' @param scenarios `list()`\cr A list of scenarios. #' If only a single scenario is supplied, it is used for all parameters. #' @param parameters `list()`\cr A list of parameter space definitions. #' If only a single definition is supplied, it is used for all scenarios. #' @param n `integer(1)`\cr The number of repetitions. #' @param parallel `integer(1)`\cr The number of workers to use. #' A value of `1` means sequential execution. Note that `parallel > 1` is not supported on Windows. #' @param split_output `logical(1)`\cr If `TRUE`, the output of [irace()] is written to `{execDir}/run_{i}/irace.out` #' instead of the standard output. #' @param global_seed `integer(1)`\cr The global seed used to seed the individual runs. #' #' @return A list of the outputs of [irace()]. #' #' @seealso #' \describe{ #' \item{[irace()]}{the main interface for single irace runs.} #' } #' #' @concept running #' @export multi_irace <- function(scenarios, parameters, n = 1L, parallel = 1L, split_output = parallel > 1L, global_seed = NULL) { # Parallel execution is not available on Windows. if (.Platform$OS.type == 'windows' && parallel > 1L) { irace_error("multi_irace() does not yet support parallel > 1 on Windows") } # Allow either the same number of scenarios and parameters, or a single scenario or parameter space definition. if (length(scenarios) != length(parameters)) { if (length(scenarios) == 1L) { scenarios <- rep(scenarios, each = length(parameters)) } else if (length(parameters) == 1L) { parameters <- rep(parameters, each = length(scenarios)) } else { irace_error("Invalid arguments: ", "Cannot execute 'irace' with", length(scenarios), "scenarios and", length(parameters), "parameters.", "Either supply the same number of scenarios and parameters, or one scenario or one parameter space definition.") } } # Repeat the existing scenarios and parameters 'n' times. if (n > 1L) { scenarios <- rep(scenarios, each = n) parameters <- rep(parameters, each = n) } # Overwrite scenario seeds. seeds <- gen_random_seeds(length(scenarios), global_seed = global_seed) for (i in seq_along(scenarios)) { # FIXME: We should store this seed in state not in scenario. We should not modify scenario. scenarios[[i]]$seed <- seeds[[i]] scenarios[[i]]$parameters <- parameters[[i]] # Check each scenario. scenarios[[i]] <- checkScenario(scenarios[[i]]) # Modify 'logFile' and 'execDir' with the index of the run. # Paths are guaranteed to be absolute because of 'checkScenario'. logFile_old <- scenarios[[i]]$logFile execDir_old <- scenarios[[i]]$execDir # 'path/to/execDir' -> 'path/to/execDir/run_{i}'. execDir <- file.path(execDir_old, sprintf("run_%02d", i)) scenarios[[i]]$execDir <- execDir fs::dir_create(execDir) if (logFile_old != "") { logFile <- if (is.sub.path(logFile_old, execDir_old)) { # 'logFile' is located in the old 'execDir', so move it into the new 'execDir'. # 'path/to/execDir/logFile.rdata' -> 'path/to/execDir/run_{i}/logFile.rdata'. sub(execDir_old, execDir, logFile_old) } else { # 'logFile' is located outside, so adapt the file name. # 'path/to/logFile.rdata' -> 'path/to/logFile_{i}.rdata'. # pathWithoutExt <- tools::file_path_sans_ext(logFile_old) # pathExt <- tools::file_ext(logFile_old) # pathWithoutExtWithIndex <- sprintf("%s_%02d", pathWithoutExt, i) # paste(pathWithoutExtWithIndex, pathExt, sep = ".") irace_error("Invalid 'logFile' path (", logFile_old, "): ", "The 'logFile' must be located inside the 'execDir' (", execDir_old, ").") } scenarios[[i]]$logFile <- logFile } } irace_run <- function(scenario) { if (scenario$quiet || !split_output) { irace(scenario) } else { withr::with_output_sink(file.path(scenario$execDir, "irace.out"), { irace(scenario) }) } } if (parallel > 1L) { runs <- parallel::mcmapply(irace_run, scenarios, mc.cores = parallel, SIMPLIFY = FALSE) # FIXME: if stop() is called from mcmapply, it does not # terminate the execution of the parent process, so it will # continue and give more errors later. We have to terminate # here, but is there a nicer way to detect this and terminate? if (any(sapply(runs, inherits, "try-error")) || any(sapply(runs, is.null))) { # FIXME: mcmapply has some bugs in case of error. In that # case, each element of the list does not keep the output of # each configuration and repetitions may occur. errors <- unique(unlist(runs[sapply(runs, inherits, "try-error")])) cat(errors, file = stderr()) irace_error("A child process triggered a fatal error") } } else { runs <- mapply(irace_run, scenarios, SIMPLIFY = FALSE) } runs } irace/R/irace-options.R0000644000176200001440000003472515060020356014445 0ustar liggesusers## This file was generated by scripts/generate-options.R # Non-variable options (such as --help and --version) have names starting with "." # Variables that do not have a command-line option have description == "" # Types are b(oolean), i(nteger), s(tring), r(eal), p(ath), x (R object or no value) # FIXME: Add special type for R functions. # FIXME: For i and r add their domain. .irace.params.def <- structure(list(name = c(".help", ".version", ".check", ".init", ".onlytest", "scenarioFile", "execDir", "parameterFile", "parameters", "initConfigurations", "configurationsFile", "logFile", "recoveryFile", "instances", "trainInstancesDir", "trainInstancesFile", "sampleInstances", "testInstancesDir", "testInstancesFile", "testInstances", "testNbElites", "testIterationElites", "testType", "firstTest", "blockSize", "eachTest", "targetRunner", "targetRunnerLauncher", "targetCmdline", "targetRunnerRetries", "targetRunnerTimeout", "targetRunnerData", "targetRunnerParallel", "targetEvaluator", "deterministic", "maxExperiments", "minExperiments", "maxTime", "budgetEstimation", "minMeasurableTime", "parallel", "loadBalancing", "mpi", "batchmode", "quiet", "debugLevel", "seed", "softRestart", "softRestartThreshold", "elitist", "elitistNewInstances", "elitistLimit", "repairConfiguration", "capping", "cappingAfterFirstTest", "cappingType", "boundType", "boundMax", "boundDigits", "boundPar", "boundAsTimeout", "postselection", "aclib", "nbIterations", "nbExperimentsPerIteration", "minNbSurvival", "nbConfigurations", "mu", "confidence"), type = c("x", "x", "x", "x", "p", "p", "p", "p", "x", "x", "p", "p", "p", "s", "p", "p", "b", "p", "p", "x", "i", "b", "s", "i", "i", "i", "p", "p", "s", "i", "i", "x", "x", "p", "b", "i", "i", "i", "r", "r", "i", "b", "b", "s", "b", "i", "i", "b", "r", "b", "i", "i", "x", "b", "b", "s", "s", "i", "i", "i", "b", "b", "b", "i", "i", "i", "i", "i", "r"), short = c("-h", "-v", "-c", "-i", "", "-s", "", "-p", "", "", "", "-l", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "-q", "", "", "", "", "-e", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "", "" ), long = c("--help", "--version", "--check", "--init", "--only-test", "--scenario", "--exec-dir", "--parameter-file", "", "", "--configurations-file", "--log-file", "--recovery-file", "", "--train-instances-dir", "--train-instances-file", "--sample-instances", "--test-instances-dir", "--test-instances-file", "", "--test-num-elites", "--test-iteration-elites", "--test-type", "--first-test", "--block-size", "--each-test", "--target-runner", "--target-runner-launcher", "--target-cmdline", "--target-runner-retries", "--target-runner-timeout", "", "", "--target-evaluator", "--deterministic", "--max-experiments", "--min-experiments", "--max-time", "--budget-estimation", "--min-measurable-time", "--parallel", "--load-balancing", "--mpi", "--batchmode", "--quiet", "--debug-level", "--seed", "--soft-restart", "--soft-restart-threshold", "--elitist", "--elitist-new-instances", "--elitist-limit", "", "--capping", "--capping-after-first-test", "--capping-type", "--bound-type", "--bound-max", "--bound-digits", "--bound-par", "--bound-as-timeout", "--postselection", "--aclib", "--iterations", "--experiments-per-iteration", "--min-survival", "--num-configurations", "--mu", "--confidence"), default = c(NA, NA, NA, "", "", "./scenario.txt", "./", "./parameters.txt", "", "", "", "./irace.Rdata", "", "", "", "", "1", "", "", "", "1", "0", "", "5", "1", "1", "./target-runner", "", "{configurationID} {instanceID} {seed} {instance} {bound} {targetRunnerArgs}", "0", "0", "", "", "", "0", "0", NA, "0", "0.05", "0.01", "0", "1", "0", "0", "0", "0", NA, "1", "1e-04", "1", "1", "2", "", NA, "0", "median", "candidate", "0", "0", "1", "1", "1", "0", "0", "0", "0", "0", "5", "0.95"), domain = c(NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, "F-test,t-test,t-test-holm,t-test-bonferroni", NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, "sge,pbs,torque,slurm,htcondor", NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, "median,mean,worst,best", "instance,candidate", NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA, NA), description = c("Show this help.", "Show irace package version.", "Check scenario.", "Initialize the working directory with template config files.", "Only test the configurations given in the file passed as argument.", "File that describes the configuration scenario setup and other irace settings.", "Directory where the programs will be run.", "File that contains the description of the parameters of the target algorithm.", "", "", "File that contains a table of initial configurations. If empty or `NULL`, all initial configurations are randomly generated.", "File to save tuning results as an R dataset, either absolute path or relative to execDir.", "Previously saved log file to recover the execution of `irace`, either absolute path or relative to the current directory. If empty or `NULL`, recovery is not performed.", "", "Directory where training instances are located; either absolute path or relative to current directory. If no `trainInstancesFiles` is provided, all the files in `trainInstancesDir` will be listed as instances.", "File that contains a list of training instances and optionally additional parameters for them. If `trainInstancesDir` is provided, `irace` will search for the files in this folder.", "Randomly sample the training instances or use them in the order given.", "Directory where testing instances are located, either absolute or relative to current directory.", "File containing a list of test instances and optionally additional parameters for them.", "", "Number of elite configurations returned by irace that will be tested if test instances are provided.", "Enable/disable testing the elite configurations found at each iteration.", "Statistical test used for elimination. The default value selects `t-test` if `capping` is enabled or `F-test`, otherwise. Valid values are: F-test (Friedman test), t-test (pairwise t-tests with no correction), t-test-bonferroni (t-test with Bonferroni's correction for multiple comparisons), t-test-holm (t-test with Holm's correction for multiple comparisons).", "Number of instances evaluated before the first elimination test. It must be a multiple of `eachTest`.", "Number of training instances, that make up a 'block' in `trainInstancesFile`. Elimination of configurations will only be performed after evaluating a complete block and never in the middle of a block. Each block typically contains one instance from each instance class (type or family) and the block size is the number of classes. The value of `blockSize` will multiply `firstTest`, `eachTest` and `elitistNewInstances`.", "Number of instances evaluated between elimination tests.", "Executable called for each configuration that executes the target algorithm to be tuned. See the templates and examples provided.", "Executable that will be used to launch the target runner, when `targetRunner` cannot be executed directly (e.g., a Python script in Windows).", "Command-line arguments provided to `targetRunner` (or `targetRunnerLauncher` if defined). The substrings `\\{configurationID\\}`, `\\{instanceID\\}`, `\\{seed\\}`, `\\{instance\\}`, and `\\{bound\\}` will be replaced by their corresponding values. The substring `\\{targetRunnerArgs\\}` will be replaced by the concatenation of the switch and value of all active parameters of the particular configuration being evaluated. The substring `\\{targetRunner\\}`, if present, will be replaced by the value of `targetRunner` (useful when using `targetRunnerLauncher`).", "Number of times to retry a call to `targetRunner` if the call failed.", "Timeout in seconds of any `targetRunner` call (only applies to `target-runner` executables not to R functions), ignored if 0.", "Optional data passed to `targetRunner`. This is ignored by the default `targetRunner` function, but it may be used by custom `targetRunner` functions to pass persistent data around.", "Optional R function to provide custom parallelization of `targetRunner`.", "Optional script or R function that provides a numeric value for each configuration. See templates/target-evaluator.tmpl", "If the target algorithm is deterministic, configurations will be evaluated only once per instance.", "Maximum number of runs (invocations of `targetRunner`) that will be performed. It determines the maximum budget of experiments for the tuning.", "Minimum number of runs (invocations of `targetRunner`) that will be performed. It determines the minimum budget of experiments for the tuning. The actual budget depends on the number of parameters and `minSurvival`.", "Maximum total execution time for the executions of `targetRunner`. `targetRunner` must return two values: cost and time. This value and the one returned by `targetRunner` must use the same units (seconds, minutes, iterations, evaluations, ...).", "Fraction (smaller than 1) of the budget used to estimate the mean computation time of a configuration. Only used when `maxTime` > 0", "Minimum time unit that is still (significantly) measureable.", "Number of calls to `targetRunner` to execute in parallel. Values `0` or `1` mean no parallelization.", "Enable/disable load-balancing when executing experiments in parallel. Load-balancing makes better use of computing resources, but increases communication overhead. If this overhead is large, disabling load-balancing may be faster.", "Enable/disable MPI. Use `Rmpi` to execute `targetRunner` in parallel (parameter `parallel` is the number of slaves).", "Specify how irace waits for jobs to finish when `targetRunner` submits jobs to a batch cluster: sge, pbs, torque, slurm or htcondor. `targetRunner` must submit jobs to the cluster using, for example, `qsub`.", "Reduce the output generated by irace to a minimum.", "Debug level of the output of `irace`. Set this to 0 to silence all debug messages. Higher values (1, 2 or 3) provide more verbose debug messages.", "Seed of the random number generator (by default, generate a random seed).", "Enable/disable the soft restart strategy that avoids premature convergence of the probabilistic model.", "Soft restart threshold value for numerical parameters.", "Enable/disable elitist irace.", "Number of instances added to the execution list before previous instances in elitist irace.", "In elitist irace, maximum number per race of elimination tests that do not eliminate a configuration. Use 0 for no limit.", "User-defined R function that takes a configuration generated by irace and repairs it.", "Enable the use of adaptive capping, a technique designed for minimizing the computation time of configurations. Capping is enabled by default if `elitist` is active, `maxTime > 0` and `boundMax > 0`.", "If set to 1, elimination due to capping only happens after `firstTest` instances are seen.", "Measure used to obtain the execution bound from the performance of the elite configurations: median, mean, worst, best.", "Method to calculate the mean performance of elite configurations: candidate or instance.", "Maximum execution bound for `targetRunner`. It must be specified when capping is enabled.", "Precision used for calculating the execution time. It must be specified when capping is enabled.", "Penalization multiplication constant (PARX) for timed out executions (executions that reach `boundMax` execution time).", "Replace the configuration cost of bounded executions with `boundMax`.", "Perform a postselection race after the execution of irace to consume all remaining budget. Value 0 disables the postselection race.", "Enable/disable AClib mode. This option enables compatibility with GenericWrapper4AC as targetRunner script.", "Maximum number of iterations.", "Number of runs of the target algorithm per iteration.", "Minimum number of configurations needed to continue the execution of each race (iteration).", "Number of configurations to be sampled and evaluated at each iteration.", "Parameter used to define the number of configurations sampled and evaluated at each iteration.", "Confidence level for the elimination test.")), row.names = c(".help", ".version", ".check", ".init", ".onlytest", "scenarioFile", "execDir", "parameterFile", "parameters", "initConfigurations", "configurationsFile", "logFile", "recoveryFile", "instances", "trainInstancesDir", "trainInstancesFile", "sampleInstances", "testInstancesDir", "testInstancesFile", "testInstances", "testNbElites", "testIterationElites", "testType", "firstTest", "blockSize", "eachTest", "targetRunner", "targetRunnerLauncher", "targetCmdline", "targetRunnerRetries", "targetRunnerTimeout", "targetRunnerData", "targetRunnerParallel", "targetEvaluator", "deterministic", "maxExperiments", "minExperiments", "maxTime", "budgetEstimation", "minMeasurableTime", "parallel", "loadBalancing", "mpi", "batchmode", "quiet", "debugLevel", "seed", "softRestart", "softRestartThreshold", "elitist", "elitistNewInstances", "elitistLimit", "repairConfiguration", "capping", "cappingAfterFirstTest", "cappingType", "boundType", "boundMax", "boundDigits", "boundPar", "boundAsTimeout", "postselection", "aclib", "nbIterations", "nbExperimentsPerIteration", "minNbSurvival", "nbConfigurations", "mu", "confidence"), class = "data.frame") .irace.params.names <- c("scenarioFile", "execDir", "parameterFile", "parameters", "initConfigurations", "configurationsFile", "logFile", "recoveryFile", "instances", "trainInstancesDir", "trainInstancesFile", "sampleInstances", "testInstancesDir", "testInstancesFile", "testInstances", "testNbElites", "testIterationElites", "testType", "firstTest", "blockSize", "eachTest", "targetRunner", "targetRunnerLauncher", "targetCmdline", "targetRunnerRetries", "targetRunnerTimeout", "targetRunnerData", "targetRunnerParallel", "targetEvaluator", "deterministic", "maxExperiments", "minExperiments", "maxTime", "budgetEstimation", "minMeasurableTime", "parallel", "loadBalancing", "mpi", "batchmode", "quiet", "debugLevel", "seed", "softRestart", "softRestartThreshold", "elitist", "elitistNewInstances", "elitistLimit", "repairConfiguration", "capping", "cappingAfterFirstTest", "cappingType", "boundType", "boundMax", "boundDigits", "boundPar", "boundAsTimeout", "postselection", "aclib", "nbIterations", "nbExperimentsPerIteration", "minNbSurvival", "nbConfigurations", "mu", "confidence") ## FIXME: If these values are special perhaps they should be saved in $state ? .irace.params.recover <- c("instances", "seed", "testInstances", # We need this because this data may mutate "targetRunnerData", "elitist", "deterministic") irace/R/race_state.R0000644000176200001440000002557315060047643014014 0ustar liggesusersRaceState <- R6Class("RaceState", lock_class = TRUE, public = list( # This may be dynamically adjusted cluster = NULL, completed = "Incomplete", elapsed = 0L, elapsed_recovered = 0L, elitist_new_instances = 0L, experiment_log = NULL, instances_log = NULL, minSurvival = NULL, next_instance = -1L, race_experiment_log = NULL, recovery_info = NULL, recovery_mode = FALSE, rejected_ids = NULL, rng = NULL, seed = NULL, session_info = NULL, target_evaluator = NULL, target_runner = NULL, timeUsed = 0, time_next_save = 0, timer = NULL, # Methods. initialize = function(scenario, new = TRUE, recover = FALSE) { self$timer <- Timer$new() self$target_runner <- if (is.function(scenario$targetRunner)) bytecompile(scenario$targetRunner) else if (scenario$aclib) target_runner_aclib else target_runner_default if (!is.null(scenario$targetEvaluator)) { self$target_evaluator <- if (is.function(scenario$targetEvaluator)) bytecompile(scenario$targetEvaluator) else target_evaluator_default } irace_assert(new || !recover) if (new) { # elitist_new_instances must be a multiple of blockSize. self$elitist_new_instances <- scenario$elitistNewInstances * scenario$blockSize # We cannot recover if we did not get to initialize self$rng. if (recover && !is.null(self$rng)) { restore_random_seed(self$rng) self$recovery_mode <- TRUE set(self$experiment_log, j = "iteration", value = NULL) self$recovery_info <- rbindlist(c(list(self$experiment_log), self$race_experiment_log), use.names = TRUE) # Reinitialize some state. self$completed = "Incomplete" self$elapsed = 0L self$elapsed_recovered = 0L self$experiment_log = NULL self$next_instance = -1L self$race_experiment_log = NULL self$rejected_ids = NULL self$timeUsed = 0 self$time_next_save = 0 # Just in case anything is still running. self$stop_parallel() } else { seed <- scenario$seed if (is.na(seed)) seed <- trunc(runif(1, 1, .Machine$integer.max)) set_random_seed(seed) self$seed <- seed self$rng <- get_random_seed() } } else { # !new self$elapsed_recovered <- self$elapsed restore_random_seed(self$rng) } if (is.null(self$experiment_log)) { self$experiment_log <- data.table(iteration=integer(0), instance=integer(0), configuration=integer(0), cost = numeric(0), time = numeric(0), bound = if (is.null(scenario$boundMax)) NULL else numeric(0)) } if (scenario$debugLevel >= 3L) { irace_note("RNGkind: ", paste0(self$rng$rng_kind, collapse = " "), "\n", "# .Random.seed: ", paste0(self$rng$random_seed, collapse = ", "), "\n") } # We do this here, so it is available even if we crash. self$session_info <- utils::sessionInfo() invisible(self) }, update_experiment_log = function(output, instances, scenario) { # FIXME: The instances parameter is not needed. irace_assert(all.equal(rep(instances, each = length(unique(output[["configuration"]]))), output$instance)) # Extract results self$experiment_log <- rbindlist(list(self$experiment_log, output), use.names=TRUE) experiments_output_to_matrix(output, scenario) }, save_recovery = function(iraceResults, logfile) { now <- self$timer$wallclock() # Do not save to disk too frequently. if (now >= self$time_next_save) { # irace_note("Saving recovery info.\n") iraceResults$state <- self save_irace_logfile(iraceResults, logfile) self$time_next_save <- .get_time_next_save(now) } }, update_race_experiment_log = function(experiment_log, scenario) { self$race_experiment_log <- c(self$race_experiment_log, list(experiment_log)) now <- self$timer$wallclock() # Do not save to disk too frequently. if (now >= self$time_next_save) { # irace_note("Saving recovery info.\n") iraceResults <- list( scenario = scenario, irace_version = irace_version, state = self) save_irace_logfile(iraceResults, logfile = scenario$logFile) self$time_next_save <- .get_time_next_save(now) } invisible() }, reset_race_experiment_log = function() { res <- rbindlist(self$race_experiment_log, use.names=TRUE) self$race_experiment_log <- NULL res }, recover_output = function(instance_idx, configuration_id) { search <- data.table(instance = instance_idx, configuration = configuration_id) res <- self$recovery_info[search, on = .(instance,configuration), mult="first", nomatch=NULL, which=TRUE] irace_assert(length(res) == 0L || length(res) == nrow(search)) if (length(res) == 0L) { irace_note("Cannot find the following in recovery info:") print(search[!self$recovery_info, on = .(instance,configuration)]) irace_error("Recovery terminated.") } # Get the rows. output <- self$recovery_info[res] # Delete those rows. self$recovery_info <- self$recovery_info[-res] if (nrow(self$recovery_info) == 0L) { irace_note("Recovery completed.\n") self$recovery_mode <- FALSE self$recovery_info <- NULL } output }, update_rejected = function(rejected_ids, configurations) { if (length(rejected_ids) == 0L) return(NULL) self$rejected_ids <- c(self$rejected_ids, rejected_ids) configurations[configurations[[".ID."]] %in% rejected_ids, , drop = FALSE] }, time_elapsed = function() { self$elapsed <- self$timer$elapsed() + self$elapsed_recovered self$elapsed }, start_parallel = function(scenario) { parallel <- scenario$parallel data.table::setDTthreads(if (parallel <= 1L) 1L else min(4L, parallel)) if (!is.null(scenario$targetRunnerParallel) || parallel <= 1L) return(invisible(self)) # Starting the parallel environment may set some logs to the current # directory, so switch to execDir momentarily. withr::local_dir(scenario$execDir) # setwd() if (scenario$mpi) { mpiInit(parallel, scenario$debugLevel) } else { requireNamespace("parallel", quietly = TRUE) if (.Platform$OS.type == 'windows' && is.null(self$cluster)) { # FIXME: makeCluster does not print the output generated by the workers # on Windows. We need to use the future package for that: # https://stackoverflow.com/questions/56501937/how-to-print-from-clusterapply self$cluster <- parallel::makeCluster(parallel) if (scenario$debugLevel >= 1L) irace_note("makeCluster initialized for ", parallel, " jobs.\n") # We export the global environment because the user may have defined # stuff there. There must be a better way to do this, but I cannot # figure it out. R sucks sometimes. parallel::clusterExport(self$cluster, ls(envir=.GlobalEnv)) # In Windows, this needs to be exported, or we get: ## Error in checkForRemoteErrors(val) : ## 2 nodes produced errors; first error: could not find function "target_runner" parallel::clusterExport(self$cluster, list("target_runner"), envir=self) if (is.function(scenario$targetRunner) && !identical(environment(scenario$targetRunner), globalenv())) { env_target_runner <- environment(scenario$targetRunner) funglobs <- codetools::findGlobals(self$target_runner, merge=TRUE) common <- intersect(funglobs, ls(envir=env_target_runner)) if (length(common)) parallel::clusterExport(self$cluster, common, envir=env_target_runner) } } } invisible(self) }, stop_parallel = function() { if (!is.null(self$cluster)) { try(parallel::stopCluster(self$cluster), silent=TRUE) self$cluster <- NULL } invisible(self) }, print_mem_used = function(objects) { object_size_kb <- function (name, envir) utils::object.size(get(name, envir = envir)) / 1024 envir <- parent.frame() if (missing(objects)) objects <- ls(envir = envir, all.names = TRUE) x <- sapply(objects, object_size_kb, envir = envir) y <- sapply(names(get(class(self)[1L])$public_fields), object_size_kb, envir = self) names(y) <- paste0("RaceState$", names(y)) x <- c(x, y) # Do not print anything that is smaller than 32 Kb x <- x[x > 32] cat(sep="", sprintf("%30s : %17.1f Kb\n", names(x), x), sprintf("%30s : %17.1f Mb\n", "Total", sum(x) / 1024), # This does garbage collection and also prints memory used by R. sprintf("%30s : %17.1f Mb\n", "gc", sum(gc()[,2L]))) invisible(self) } )) no_elitist_init_instances <- function(self, deterministic) { max_instances <- nrow(self$instances_log) # if next.instance == 1 then this is the first iteration. # If deterministic consider all (do not resample). if (self$next_instance == 1L || deterministic) return(seq_len(max_instances)) irace_assert(self$next_instance < max_instances) self$next_instance : max_instances } elitist_init_instances <- function(self, deterministic, sampleInstances, elitist_new_instances) { max_instances <- nrow(self$instances_log) # if next_instance == 1 then this is the first iteration. next_instance <- self$next_instance if (next_instance == 1L) return(seq_len(max_instances)) # Consider all new_instances <- NULL last_new <- next_instance - 1L + elitist_new_instances # Do we need to add new instances? if (elitist_new_instances > 0L) { if (last_new > max_instances) { # This may happen if the scenario is deterministic and we would need # more instances than what we have. irace_assert(deterministic) if (next_instance <= max_instances) { # Add all instances that we have not seen yet as new ones. last_new <- max_instances new_instances <- next_instance : last_new } # else new_instances remains NULL and last_new remains > number of instances. # We need to update this because the value is used below and now there # may be fewer than expected, even zero. self$elitist_new_instances <- length(new_instances) } else { new_instances <- next_instance : last_new } } past_instances <- if (sampleInstances) sample.int(next_instance - 1L) else seq_len(next_instance - 1L) # new_instances + past_instances + future_instances if (last_new + 1L <= max_instances) { future_instances <- (last_new + 1L) : max_instances return(c(new_instances, past_instances, future_instances)) } c(new_instances, past_instances) } irace/R/argparser.R0000644000176200001440000000741314736526233013667 0ustar liggesusers# R6 Class for parsing command-line arguments CommandArgsParser <- R6::R6Class("CommandArgsParser", cloneable = FALSE, lock_class = TRUE, list( argv = NULL, argsdef = NULL, initialize = function(argv, argsdef) { # Handle the case where we are given a single character string like a # command-line. if (!missing(argv) && length(argv) == 1) { # strsplit does not respect quoted strings. argv <- scan(text=argv, what='character', quiet=TRUE) } self$argv <- argv required_colnames <- c("name", "short", "long", "type", "default") if (any(required_colnames %not_in% colnames(argsdef))) { stop("argsdef must contain the column names: ", paste0(required_colnames, collapse=", ")) } self$argsdef <- argsdef rownames(self$argsdef) <- argsdef$name self }, readCmdLineParameter = function (paramName, default = NULL) { short <- self$argsdef[paramName, "short"] long <- self$argsdef[paramName,"long"] value <- self$readArg(short = short, long = long) if (is.null(value)) { value <- if (is.null(default)) self$argsdef[paramName, "default"] else default } else if (is.na(value) && self$argsdef[paramName,"type"] != 'x') { stop("option '", long, "' requires an argument", call. = FALSE) } return(value) }, # Function to read command-line arguments. ## FIXME: This function always consumes two arguments. This is problematic ## for flags that have no arguments, like --check. readArg = function(short = "", long = "") { if (length(short) == 0) short <- "" if (length(long) == 0) long <- "" if (short == "" && long == "") return(NULL) argv <- self$argv pos <- c() pattern <- "" if (short != "") { pattern_equal <- paste0("^", short, "=") pattern <- paste0("^", short, "$|", pattern_equal) } if (long != "") { pattern_long_equal <- paste0("^", long, "=") pattern_long <- paste0("^", long, "$|", pattern_long_equal) if (short != "") { pattern <- paste0(pattern, "|", pattern_long) pattern_equal <- paste0(pattern_equal, "|", pattern_long_equal) } else { pattern <- pattern_long pattern_equal <- pattern_long_equal } } pos <- grep(pattern, argv) if (length(pos) == 0) { return(NULL) # Not found } else if (length(pos) > 0) { # Allow repeated parameters pos <- max(pos) } if (grepl(pattern_equal, argv[pos])) { value <- unlist(strsplit(argv[pos], '=', fixed = TRUE))[2] if (is.null (value) || is.na(value)) value <- "" } else { value <- argv[pos + 1] self$argv <- argv[-(pos + 1)] } self$argv <- self$argv[-pos] return (value) }, readAll = function() { params <- list() for (param in self$argsdef$name[self$argsdef$type != 'x']) { value <- self$readCmdLineParameter(paramName = param) if (is.na(value) || (length(value) > 0 && value == "")) value <- NULL params[[param]] <- value } params }, cmdline_usage = function() { cmdline_usage(self$argsdef) }) ) # `cmdline_usage()` prints the output of `--help` # # @param cmdline_args Definition of the command-line arguments. # # @export cmdline_usage <- function(cmdline_args) { for (i in seq_len(nrow(cmdline_args))) { short <- cmdline_args[i,"short"] long <- cmdline_args[i,"long"] desc <- cmdline_args[i,"description"] if (desc == "" || (short == "" && long == "")) next if (short != "") short <- paste0(short,",") default <- cmdline_args[i,"default"] if (!is_null_or_empty_or_na(default)) { desc <- paste0(desc, " Default: ", default, ".") } cat(sep = "\n", strwrap(desc, width = 80, initial = sprintf("%3s%-20s ", short, long), exdent = 25)) } } irace/cleanup0000755000176200001440000000154515060500572012717 0ustar liggesusers#!/bin/sh rm -f irace.Rdata irace-Ex.R config.* confdefs.h \ src/*.so src/*.o src/*/*.o src/*.gcno src/*/*.gcno src/*.gcov src/config.h src/symbols.rds \ inst/doc/*.blg inst/doc/*.bbl \ tests/testthat/*.log tests/testthat/*.Rout tests/testthat/Rplots.pdf tests/testthat/iracedump.rda tests/testthat/irace.Rdata \ examples/vignette-example/Rplots.pdf \ devel-examples/vignette-example/*.stderr \ devel-examples/vignette-example/*.stdout \ devel-examples/vignette-example/irace-acotsp.Rdata \ devel-examples/vignette-example/examples.Rdata \ devel-examples/vignette-example/irace-acotsp-stdout.txt \ devel-examples/vignette-example/mean.pdf \ devel-examples/vignette-example/sann.rda rm -fr tests/testthat/run_* tests/testthat/multi_irace* \ autom4te.cache find . -name '*.orig' -o -name '.Rhistory' -o -name '*.rej' | xargs rm -f irace/vignettes/0000755000176200001440000000000015060500572013345 5ustar liggesusersirace/vignettes/irace-acotsp-stdout.txt0000644000176200001440000010651614746202625020021 0ustar liggesusers#------------------------------------------------------------------------------ # irace: An implementation in R of (Elitist) Iterated Racing # Version: 4.2.0.ee928b9 # Copyright (C) 2010-2025 # Manuel Lopez-Ibanez # Jeremie Dubois-Lacoste # Leslie Perez Caceres # # This is free software, and you are welcome to redistribute it under certain # conditions. See the GNU General Public License for details. There is NO # WARRANTY; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. # # irace builds upon previous code from the race package: # race: Racing methods for the selection of the best # Copyright (C) 2003 Mauro Birattari #------------------------------------------------------------------------------ # installed at: /home/manu/R/x86_64-pc-linux-gnu-library/4.1/irace # called with: --parallel 2 # 2025-01-28 13:47:31 GMT: Reading parameter file '/home/manu/work/irace/git/devel-examples/vignette-example/parameters.txt'. # 2025-01-28 13:47:31 GMT: 1 expression(s) specifying forbidden configurations read. # 2025-01-28 13:47:31 GMT: Read 1 configuration(s) from file '/home/manu/work/irace/git/devel-examples/vignette-example/default.txt' # 2025-01-28 13:47:31 GMT: Initialization # Elitist race # Elitist new instances: 1 # Elitist limit: 2 # nbIterations: 5 # minNbSurvival: 5 # nbParameters: 11 # seed: 687542627 # confidence level: 0.95 # budget: 1000 # mu: 5 # deterministic: FALSE # 2025-01-28 13:47:31 GMT: Iteration 1 of 5 # experimentsUsed: 0 # remainingBudget: 1000 # currentBudget: 200 # nbConfigurations: 33 # Markers: x No test is performed. c Configurations are discarded only due to capping. - The test is performed and some configurations are discarded. = The test is performed but no configuration is discarded. ! The test is performed and configurations could be discarded but elite configurations are preserved. . All alive configurations are elite and nothing is discarded. +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ | | Instance| Alive| Best| Mean best| Exp so far| W time| rho|KenW| Qvar| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ |x| 1| 33| 22| 33693816.00| 33|00:01:41| NA| NA| NA| |x| 2| 33| 31| 33233161.00| 66|00:01:41|+0.96|0.98|0.0060| |x| 3| 33| 31| 33285969.67| 99|00:01:35|+0.97|0.98|0.0052| |x| 4| 33| 31| 33264133.00| 132|00:01:37|+0.97|0.98|0.0046| |-| 5| 3| 31| 33251469.40| 165|00:01:36|-0.05|0.16|0.4985| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ Best-so-far configuration: 31 mean value: 33251469.40 Description of the best-so-far configuration: .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. 31 31 acs 3 4.2717 0.2871 0.9362 6 17 0.918 1 NA NA 5 NA # 2025-01-28 13:55:43 GMT: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time 31 acs 3 4.2717 0.2871 0.9362 6 17 0.918 1 NA NA 5 9 mmas 3 3.4904 4.9746 0.5959 13 33 NA 0 NA NA 5 22 as 3 2.0842 2.1621 0.7506 25 26 NA 1 NA NA 5 # 2025-01-28 13:55:43 GMT: Iteration 2 of 5 # experimentsUsed: 165 # remainingBudget: 835 # currentBudget: 208 # nbConfigurations: 31 # Markers: x No test is performed. c Configurations are discarded only due to capping. - The test is performed and some configurations are discarded. = The test is performed but no configuration is discarded. ! The test is performed and configurations could be discarded but elite configurations are preserved. . All alive configurations are elite and nothing is discarded. +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ | | Instance| Alive| Best| Mean best| Exp so far| W time| rho|KenW| Qvar| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ |x| 6| 31| 31| 32987639.00| 31|00:01:31| NA| NA| NA| |x| 1| 31| 61| 33317599.00| 59|00:01:19|+0.95|0.97|0.0029| |x| 4| 31| 61| 33270043.67| 87|00:01:19|+0.94|0.96|0.0026| |x| 3| 31| 61| 33282925.75| 115|00:01:20|+0.94|0.96|0.0025| |-| 2| 7| 61| 33187955.60| 143|00:01:19|+0.20|0.36|0.7082| |=| 5| 7| 31| 33207497.67| 147|00:00:12|+0.09|0.24|0.7586| |=| 7| 7| 31| 33197039.29| 154|00:00:21|+0.16|0.28|0.7095| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ Best-so-far configuration: 31 mean value: 33197039.29 Description of the best-so-far configuration: .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. 31 31 acs 3 4.2717 0.2871 0.9362 6 17 0.918 1 NA NA 5 NA # 2025-01-28 14:03:09 GMT: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time 31 acs 3 4.2717 0.2871 0.9362 6 17 0.9180 1 NA NA 5 61 ras 3 3.6680 2.4269 0.8717 96 14 NA 1 89 NA 5 37 acs 3 3.8927 1.6181 0.6188 6 35 0.8587 1 NA NA 5 41 ras 3 3.3674 5.9837 0.7399 12 40 NA 1 95 NA 5 9 mmas 3 3.4904 4.9746 0.5959 13 33 NA 0 NA NA 5 # 2025-01-28 14:03:09 GMT: Iteration 3 of 5 # experimentsUsed: 319 # remainingBudget: 681 # currentBudget: 227 # nbConfigurations: 32 # Markers: x No test is performed. c Configurations are discarded only due to capping. - The test is performed and some configurations are discarded. = The test is performed but no configuration is discarded. ! The test is performed and configurations could be discarded but elite configurations are preserved. . All alive configurations are elite and nothing is discarded. +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ | | Instance| Alive| Best| Mean best| Exp so far| W time| rho|KenW| Qvar| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ |x| 8| 32| 65| 32811562.00| 32|00:01:27| NA| NA| NA| |x| 2| 32| 75| 32779249.00| 59|00:01:16|+0.92|0.96|0.0031| |x| 6| 32| 75| 32799039.00| 86|00:01:16|+0.92|0.95|0.0036| |x| 7| 32| 75| 32842164.00| 113|00:01:16|+0.89|0.92|0.0044| |-| 5| 8| 75| 32875613.60| 140|00:01:16|+0.64|0.71|0.2166| |-| 1| 7| 75| 32974384.00| 143|00:00:10|+0.66|0.71|0.2031| |-| 4| 6| 75| 32972595.86| 145|00:00:05|+0.53|0.60|0.2424| |-| 3| 1| 75| 32995307.50| 146|00:00:05| NA| NA| NA| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ Best-so-far configuration: 75 mean value: 32995307.50 Description of the best-so-far configuration: .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. 75 75 ras 3 4.5827 1.6918 0.818 13 10 NA 1 15 NA 5 37 # 2025-01-28 14:10:04 GMT: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time 75 ras 3 4.5827 1.6918 0.818 13 10 NA 1 15 NA 5 # 2025-01-28 14:10:04 GMT: Iteration 4 of 5 # experimentsUsed: 465 # remainingBudget: 535 # currentBudget: 267 # nbConfigurations: 30 # Markers: x No test is performed. c Configurations are discarded only due to capping. - The test is performed and some configurations are discarded. = The test is performed but no configuration is discarded. ! The test is performed and configurations could be discarded but elite configurations are preserved. . All alive configurations are elite and nothing is discarded. +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ | | Instance| Alive| Best| Mean best| Exp so far| W time| rho|KenW| Qvar| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ |x| 9| 30| 113| 32734055.00| 30|00:01:21| NA| NA| NA| |x| 6| 30| 91| 32806654.50| 59|00:01:21|+0.83|0.91|0.0624| |x| 2| 30| 91| 32738738.33| 88|00:01:21|+0.75|0.83|0.1103| |x| 5| 30| 117| 32780373.75| 117|00:01:21|+0.72|0.79|0.1597| |-| 1| 12| 117| 32893747.40| 146|00:01:21|-0.01|0.19|0.9196| |=| 4| 12| 113| 32874137.17| 157|00:00:32|+0.05|0.20|0.8612| |=| 3| 12| 113| 32910758.71| 168|00:00:32|+0.12|0.25|0.8001| |-| 8| 6| 113| 32892247.62| 179|00:00:32|+0.08|0.19|0.7116| |=| 7| 6| 113| 32894494.00| 184|00:00:16|+0.09|0.19|0.6930| |=| 10| 6| 113| 32912596.60| 190|00:00:16|+0.01|0.11|0.7806| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ Best-so-far configuration: 113 mean value: 32912596.60 Description of the best-so-far configuration: .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. 113 113 ras 3 4.1042 1.531 0.8104 16 12 NA 1 24 NA 5 75 # 2025-01-28 14:19:01 GMT: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time 113 ras 3 4.1042 1.5310 0.8104 16 12 NA 1 24 NA 5 117 ras 3 4.1817 0.5324 0.8893 13 18 NA 1 23 NA 5 92 ras 3 4.9632 4.1134 0.8790 7 9 NA 1 26 NA 5 102 ras 3 4.3582 2.8458 0.7508 9 19 NA 1 86 NA 5 93 ras 3 4.1500 2.0082 0.8067 10 14 NA 1 32 NA 5 # 2025-01-28 14:19:01 GMT: Iteration 5 of 5 # experimentsUsed: 655 # remainingBudget: 345 # currentBudget: 345 # nbConfigurations: 35 # Markers: x No test is performed. c Configurations are discarded only due to capping. - The test is performed and some configurations are discarded. = The test is performed but no configuration is discarded. ! The test is performed and configurations could be discarded but elite configurations are preserved. . All alive configurations are elite and nothing is discarded. +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ | | Instance| Alive| Best| Mean best| Exp so far| W time| rho|KenW| Qvar| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ |x| 11| 35| 124| 32571228.00| 35|00:01:37| NA| NA| NA| |x| 2| 35| 147| 32589096.50| 65|00:01:21|+0.28|0.64|0.1066| |x| 1| 35| 102| 32841930.67| 95|00:01:21|+0.42|0.61|0.0852| |x| 3| 35| 102| 32917142.50| 125|00:01:21|+0.48|0.61|0.0727| |-| 4| 19| 102| 32911207.20| 155|00:01:23|-0.06|0.15|0.9874| |=| 7| 19| 113| 32914603.83| 169|00:00:38|-0.07|0.10|0.9949| |=| 9| 19| 113| 32888811.14| 183|00:00:38|-0.02|0.12|0.9456| |=| 8| 19| 113| 32873043.50| 197|00:00:37|+0.02|0.14|0.9046| |=| 5| 19| 113| 32870705.56| 211|00:00:38|+0.05|0.15|0.8924| |=| 10| 19| 113| 32891187.00| 225|00:00:37|+0.03|0.13|0.9091| |=| 6| 19| 118| 32883264.36| 239|00:00:37|+0.03|0.12|0.9045| |=| 12| 19| 118| 32868008.67| 258|00:00:54|+0.05|0.13|0.8860| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ Best-so-far configuration: 118 mean value: 32868008.67 Description of the best-so-far configuration: .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. 118 118 ras 3 4.0526 2.5427 0.7522 17 20 NA 1 9 NA 5 113 # 2025-01-28 14:30:48 GMT: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time 118 ras 3 4.0526 2.5427 0.7522 17 20 NA 1 9 NA 5 124 ras 3 3.1663 0.6971 0.8396 24 26 NA 1 19 NA 5 113 ras 3 4.1042 1.5310 0.8104 16 12 NA 1 24 NA 5 117 ras 3 4.1817 0.5324 0.8893 13 18 NA 1 23 NA 5 133 ras 3 3.1718 0.9297 0.8386 9 24 NA 1 23 NA 5 # 2025-01-28 14:30:48 GMT: Iteration 6 of 6 # experimentsUsed: 913 # remainingBudget: 87 # currentBudget: 87 # nbConfigurations: 11 # Markers: x No test is performed. c Configurations are discarded only due to capping. - The test is performed and some configurations are discarded. = The test is performed but no configuration is discarded. ! The test is performed and configurations could be discarded but elite configurations are preserved. . All alive configurations are elite and nothing is discarded. +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ | | Instance| Alive| Best| Mean best| Exp so far| W time| rho|KenW| Qvar| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ |x| 13| 11| 113| 32697505.00| 11|00:00:32| NA| NA| NA| |x| 4| 11| 113| 32789761.00| 17|00:00:16|+0.55|0.77|0.4924| |x| 2| 11| 113| 32743042.67| 23|00:00:16|+0.62|0.75|0.4061| |x| 5| 11| 113| 32770282.50| 29|00:00:16|+0.62|0.72|0.4564| |-| 1| 6| 113| 32872017.00| 35|00:00:16|+0.24|0.39|0.6408| |!| 9| 6| 113| 32849023.33| 36|00:00:05|+0.31|0.42|0.6091| |=| 3| 6| 113| 32889232.57| 37|00:00:05|+0.18|0.29|0.6964| |=| 12| 6| 113| 32872605.38| 38|00:00:05|+0.04|0.16|0.7812| |=| 7| 6| 113| 32877034.22| 39|00:00:05|-0.05|0.07|0.8518| |=| 8| 6| 113| 32865597.80| 40|00:00:05|-0.06|0.05|0.8491| |=| 11| 6| 113| 32844551.82| 41|00:00:05|-0.04|0.06|0.8388| |=| 6| 6| 113| 32844854.83| 42|00:00:05|-0.06|0.03|0.8736| |=| 10| 6| 118| 32867735.77| 43|00:00:05|-0.05|0.03|0.8810| |=| 14| 6| 118| 32851875.71| 49|00:00:16|-0.05|0.03|0.8818| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ Best-so-far configuration: 118 mean value: 32851875.71 Description of the best-so-far configuration: .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. 118 118 ras 3 4.0526 2.5427 0.7522 17 20 NA 1 9 NA 5 113 # 2025-01-28 14:33:27 GMT: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time 118 ras 3 4.0526 2.5427 0.7522 17 20 NA 1 9 NA 5 113 ras 3 4.1042 1.5310 0.8104 16 12 NA 1 24 NA 5 149 ras 3 3.7080 0.4873 0.9452 8 11 NA 1 50 NA 5 117 ras 3 4.1817 0.5324 0.8893 13 18 NA 1 23 NA 5 124 ras 3 3.1663 0.6971 0.8396 24 26 NA 1 19 NA 5 # 2025-01-28 14:33:27 GMT: Iteration 7 of 7 # experimentsUsed: 962 # remainingBudget: 38 # currentBudget: 38 # nbConfigurations: 7 # Markers: x No test is performed. c Configurations are discarded only due to capping. - The test is performed and some configurations are discarded. = The test is performed but no configuration is discarded. ! The test is performed and configurations could be discarded but elite configurations are preserved. . All alive configurations are elite and nothing is discarded. +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ | | Instance| Alive| Best| Mean best| Exp so far| W time| rho|KenW| Qvar| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ |x| 15| 7| 149| 32886640.00| 7|00:00:21| NA| NA| NA| |x| 10| 7| 149| 32935718.00| 9|00:00:05|+0.21|0.61|0.8607| |x| 14| 7| 118| 32827495.67| 11|00:00:05|+0.11|0.40|0.7721| |x| 13| 7| 149| 32825055.00| 13|00:00:05|+0.26|0.44|0.7264| |=| 11| 7| 113| 32781621.00| 15|00:00:05|+0.20|0.36|0.6939| |=| 2| 7| 149| 32759707.67| 17|00:00:05|+0.06|0.21|0.8239| |=| 12| 7| 149| 32751285.86| 19|00:00:05|+0.10|0.23|0.7552| |=| 7| 7| 149| 32771017.12| 21|00:00:05|+0.02|0.15|0.8085| |=| 5| 7| 149| 32787355.11| 23|00:00:05|+0.06|0.17|0.7630| |=| 3| 7| 149| 32819324.50| 25|00:00:05|+0.04|0.14|0.8019| |=| 4| 7| 149| 32831015.73| 27|00:00:05|+0.05|0.13|0.7921| |=| 1| 7| 149| 32874709.83| 29|00:00:05|+0.05|0.13|0.7786| |=| 9| 7| 113| 32854146.77| 31|00:00:05|+0.03|0.11|0.8026| |=| 6| 7| 149| 32859769.21| 33|00:00:05|+0.05|0.12|0.7833| |=| 8| 7| 149| 32855221.33| 35|00:00:05|+0.06|0.13|0.7587| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ Best-so-far configuration: 149 mean value: 32855221.33 Description of the best-so-far configuration: .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. 149 149 ras 3 3.708 0.4873 0.9452 8 11 NA 1 50 NA 5 117 # 2025-01-28 14:35:06 GMT: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time 149 ras 3 3.7080 0.4873 0.9452 8 11 NA 1 50 NA 5 113 ras 3 4.1042 1.5310 0.8104 16 12 NA 1 24 NA 5 118 ras 3 4.0526 2.5427 0.7522 17 20 NA 1 9 NA 5 117 ras 3 4.1817 0.5324 0.8893 13 18 NA 1 23 NA 5 124 ras 3 3.1663 0.6971 0.8396 24 26 NA 1 19 NA 5 # 2025-01-28 14:35:06 GMT: Stopped because there is not enough budget left to race more than the minimum (5). # You may either increase the budget or set 'minNbSurvival' to a lower value. # Iteration: 8 # nbIterations: 8 # experimentsUsed: 997 # timeUsed: 0 # remainingBudget: 3 # currentBudget: 3 # number of elites: 5 # nbConfigurations: 5 # Total CPU user time: 5397.903, CPU sys time: 78.97, Wall-clock time: 2855.561 # 2025-01-28 14:35:06 GMT: Starting post-selection: # Configurations selected: 149, 113, 118. # Pending instances: 0, 0, 0. # 2025-01-28 14:35:07 GMT: seed: 687542627 # Configurations: 3 # Available experiments: 3 # minSurvival: 1 # Markers: x No test is performed. c Configurations are discarded only due to capping. - The test is performed and some configurations are discarded. = The test is performed but no configuration is discarded. ! The test is performed and configurations could be discarded but elite configurations are preserved. . All alive configurations are elite and nothing is discarded. +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ | | Instance| Alive| Best| Mean best| Exp so far| W time| rho|KenW| Qvar| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ |.| 8| 3| 118| 32752987.00| 0|00:00:00| NA| NA| NA| |.| 7| 3| 118| 32831948.00| 0|00:00:00|-0.50|0.25|1.1636| |.| 12| 3| 118| 32788030.67| 0|00:00:00|+0.17|0.44|0.7047| |.| 15| 3| 118| 32818929.25| 0|00:00:00|-0.08|0.19|0.7650| |.| 14| 3| 118| 32784282.40| 0|00:00:00|-0.20|0.04|0.8230| |.| 2| 3| 118| 32753987.33| 0|00:00:00|-0.10|0.08|0.7476| |.| 6| 3| 118| 32749294.86| 0|00:00:00|+0.00|0.14|0.6861| |.| 3| 3| 118| 32795767.12| 0|00:00:00|+0.07|0.19|0.6370| |.| 9| 3| 118| 32798232.11| 0|00:00:00|-0.03|0.09|0.6834| |.| 10| 3| 118| 32810925.60| 0|00:00:00|+0.02|0.12|0.6534| |.| 13| 3| 118| 32815792.45| 0|00:00:00|-0.04|0.06|0.6765| |.| 1| 3| 118| 32856770.08| 0|00:00:00|-0.07|0.02|0.7054| |.| 11| 3| 118| 32843955.00| 0|00:00:00|-0.08|0.01|0.7140| |.| 5| 3| 118| 32849549.21| 0|00:00:00|-0.07|0.01|0.7135| |.| 4| 3| 118| 32855859.00| 0|00:00:00|-0.07|0.00|0.7072| |=| 16| 3| 113| 32847622.06| 3|00:00:10|-0.06|0.00|0.7069| +-+-----------+-----------+-----------+----------------+-----------+--------+-----+----+------+ Best-so-far configuration: 113 mean value: 32847622.06 Description of the best-so-far configuration: .ID. algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time .PARENT. 113 113 ras 3 4.1042 1.531 0.8104 16 12 NA 1 24 NA 5 75 # 2025-01-28 14:35:17 GMT: Elite configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time 113 ras 3 4.1042 1.5310 0.8104 16 12 NA 1 24 NA 5 118 ras 3 4.0526 2.5427 0.7522 17 20 NA 1 9 NA 5 149 ras 3 3.7080 0.4873 0.9452 8 11 NA 1 50 NA 5 # Total CPU user time: 5414.035, CPU sys time: 79.259, Wall-clock time: 2866.547 # Best configurations (first number is the configuration ID; listed from best to worst according to the sum of ranks): algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time 113 ras 3 4.1042 1.5310 0.8104 16 12 NA 1 24 NA 5 118 ras 3 4.0526 2.5427 0.7522 17 20 NA 1 9 NA 5 149 ras 3 3.7080 0.4873 0.9452 8 11 NA 1 50 NA 5 # Best configurations as commandlines (first number is the configuration ID; listed from best to worst according to the sum of ranks): 113 --ras --localsearch 3 --alpha 4.1042 --beta 1.531 --rho 0.8104 --ants 16 --nnls 12 --dlb 1 --rasranks 24 --time 5 118 --ras --localsearch 3 --alpha 4.0526 --beta 2.5427 --rho 0.7522 --ants 17 --nnls 20 --dlb 1 --rasranks 9 --time 5 149 --ras --localsearch 3 --alpha 3.708 --beta 0.4873 --rho 0.9452 --ants 8 --nnls 11 --dlb 1 --rasranks 50 --time 5 # Testing of elite configurations: 5 # Testing iteration configurations: TRUE # 2025-01-28 14:35:18 GMT: Testing configurations (in no particular order): 31 9 22 61 37 41 75 113 117 92 102 93 118 124 133 149 algorithm localsearch alpha beta rho ants nnls q0 dlb rasrank elitistants time 31 acs 3 4.2717 0.2871 0.9362 6 17 0.9180 1 NA NA 5 9 mmas 3 3.4904 4.9746 0.5959 13 33 NA 0 NA NA 5 22 as 3 2.0842 2.1621 0.7506 25 26 NA 1 NA NA 5 61 ras 3 3.6680 2.4269 0.8717 96 14 NA 1 89 NA 5 37 acs 3 3.8927 1.6181 0.6188 6 35 0.8587 1 NA NA 5 41 ras 3 3.3674 5.9837 0.7399 12 40 NA 1 95 NA 5 75 ras 3 4.5827 1.6918 0.8180 13 10 NA 1 15 NA 5 113 ras 3 4.1042 1.5310 0.8104 16 12 NA 1 24 NA 5 117 ras 3 4.1817 0.5324 0.8893 13 18 NA 1 23 NA 5 92 ras 3 4.9632 4.1134 0.8790 7 9 NA 1 26 NA 5 102 ras 3 4.3582 2.8458 0.7508 9 19 NA 1 86 NA 5 93 ras 3 4.1500 2.0082 0.8067 10 14 NA 1 32 NA 5 118 ras 3 4.0526 2.5427 0.7522 17 20 NA 1 9 NA 5 124 ras 3 3.1663 0.6971 0.8396 24 26 NA 1 19 NA 5 133 ras 3 3.1718 0.9297 0.8386 9 24 NA 1 23 NA 5 149 ras 3 3.7080 0.4873 0.9452 8 11 NA 1 50 NA 5 # 2025-01-28 15:14:38 GMT: Testing results (column number is configuration ID in no particular order): seeds 31 9 22 61 37 41 75 113 117 92 102 93 118 124 133 149 1t 2046302398 33098140 33239149 33129540 33109131 33201943 32976999 32852645 32833572 32784419 32875049 32777346 32820444 32780508 32923537 32871318 32805333 2t 827626108 32954263 32987443 33039177 33056915 32887007 32780271 32567740 32607386 32661338 32605363 32590997 32514163 32595732 32744276 32605256 32534045 3t 978077451 33313400 33446626 33376846 33415690 33387989 33266730 33183722 33097013 33077258 33090479 33197150 33032552 33062375 33147446 33042611 33083261 4t 1348269770 33021447 32897210 33165710 33045317 32904477 33068992 32790813 32824418 32867503 32755126 32926804 32786122 32782807 32923497 32806207 32739172 5t 243391689 32944849 33053706 33204377 33003552 32978293 33010655 32819909 32919999 32726458 32790603 32821804 32783965 32760191 32934499 32771715 32921237 6t 1588668262 32690232 32914113 32953951 32875620 32928834 32963678 32717001 32651566 32595506 32617190 32749972 32626934 32573375 32602356 32606513 32692338 7t 423372130 33026281 33211159 33194496 33148754 33213814 33170418 32939764 32989898 32895022 32943900 32836359 32892448 32884365 33044123 32877345 32974480 8t 652122407 33071578 33255488 33244533 33055455 33040474 33118029 32937849 32891239 32898150 32855158 32925102 32884699 33028371 32911267 32946823 33040194 9t 317806051 33033130 33463764 33244304 33208355 33143082 33190011 32874390 32849625 32913245 32933978 32994331 32925936 32887713 33015638 32747034 32812335 10t 747706567 32823698 32971836 32989937 32984342 32843959 32913020 32706959 32756032 32756123 32765789 32614143 32719902 32728343 32857316 32700242 32868859 11t 50933476 33002325 33221790 33224313 33062546 32955083 33021896 32967532 32857920 32912572 32883301 32937237 32842348 33001929 32989392 32982455 32855097 12t 1739405996 32929209 32941102 32856706 33027699 32825183 32820039 32587056 32598397 32561786 32506831 32581913 32643712 32519887 32636143 32595651 32504224 13t 1957203093 32788760 32959449 33065012 32989738 32823546 33016071 32661707 32638211 32607545 32684776 32619844 32841206 32575148 32759115 32739988 32687250 14t 2032826826 32831117 33062008 32923055 32885600 32905397 32973393 32562020 32627456 32606172 32803159 32630108 32648439 32832893 32796466 32593158 32662798 15t 1206236755 32893521 33041201 33083686 33068523 33049312 32984492 32699498 32633428 32733892 32745855 32724253 32731723 32677060 32928235 32739696 32665586 16t 763851782 33159005 33378520 33316334 33228824 33111096 33182851 32900293 33104319 32979011 32964066 32928362 33018843 32927669 33144143 32968303 32984175 17t 349953428 33536691 33491757 33602689 33536763 33359766 33523909 33238122 33127750 33073259 33175404 33148729 33110551 33272981 33264666 33129590 33190112 18t 897599183 32616720 32867845 32930559 32859355 32788206 32793763 32566177 32556947 32550547 32516828 32705214 32629259 32429610 32573661 32525739 32593153 19t 1400970198 33203949 33268680 33373564 33183534 33177561 33272717 33039458 32908001 32889287 32964453 33028715 32959378 32917060 32981568 33004692 33082049 20t 1782793253 32920981 33086354 33144045 32971140 33023459 33035694 32692608 32735331 32742981 32743665 32755353 32721542 32768699 32951370 32864674 32829495 21t 1660845956 33060600 32970450 33071257 32906030 33161795 33015612 32682340 32780886 32764194 32784629 32666204 32614331 32811076 32868182 32716060 32684395 22t 290410408 32969734 33240958 33158577 33077389 33089633 33099623 32791908 32898456 32846216 32885236 32964063 32795078 32902746 32869519 32718033 32851606 23t 1495337927 33058071 33043020 33172331 33129424 32900129 33131786 32825696 32716477 32857494 32876402 32902149 32739830 32773697 32750528 32780460 32807391 24t 1364123825 32936519 33175700 33216248 33127283 33119860 33094549 32862215 32855988 32975309 32893142 32914054 32846935 32862968 33047339 32841995 33002341 25t 915618954 33221437 33340356 33324125 33267849 33350663 33061580 32938672 32895342 32896788 32923194 33025517 32940073 32916386 33044500 33003287 32811036 26t 1066725944 33309234 33413626 33378515 33377976 33100601 33284699 33064354 32973237 32962348 33040082 32977859 32956226 33102636 33196297 33077908 33060265 27t 1012167492 33116237 33212972 33144225 33187194 33123043 33089640 32775498 32747432 32728200 32870284 32843344 32744397 32722944 33026740 32873759 32812007 28t 405072419 32944552 33183445 33260768 33190870 33030890 33070083 32809618 32824751 32825054 32804746 32709328 32709832 32788312 32880987 32854797 32763291 29t 343646063 33287500 33435506 33446599 33374038 33291798 33367811 33077554 33075585 33179998 33127231 33178973 33074167 33081719 33222527 32937251 33268605 30t 2062824562 32892958 33144194 33079207 33019337 33016877 32994210 32758852 32730821 32705369 32776245 32768151 32684416 32675816 32801298 32757259 32647672 31t 1315404608 33362159 33420071 33375785 33466081 33364889 33308878 33035136 33050152 32977172 33184455 33100022 33086417 33007821 33060061 33201336 33111088 32t 718135261 33080581 33148836 33178597 33209213 33075259 33090111 32808179 33073806 32824862 32862792 32910569 32809320 32832661 32916787 33013878 32868270 33t 1756600186 32912578 33140256 33172058 33122171 33082595 32980922 32945569 32778835 32836996 32805414 32804116 32776818 32789844 32877954 32783432 32825277 34t 1084471193 33058926 33089505 33039589 33036017 32921606 32947257 32721148 32709831 32685964 32693570 32687319 32690564 32707820 32743104 32705370 32621660 35t 1163321042 32483594 32752995 32803194 32828186 32771137 32690789 32506559 32396349 32366269 32479742 32416846 32376150 32458495 32567191 32361580 32418296 36t 1318942249 32670682 32971766 32936095 32913150 32717950 32698178 32527927 32575185 32547154 32572291 32474010 32532734 32627421 32750209 32653173 32612411 37t 424612740 32701106 33002602 32819010 32800764 32830202 32743558 32373724 32380917 32391978 32440573 32431067 32539230 32508876 32564875 32388837 32421787 38t 507331628 33045967 33268562 33103741 33098088 32963127 32987509 32829137 32679588 32726309 32747132 32804395 32698469 32753361 32922700 32763021 32692158 39t 1640400685 33006811 33199652 33170436 33106016 33188579 33118545 32864846 32896914 32870078 32916441 32911873 32885323 32959799 33071726 32922589 32905922 40t 1364449617 32623835 32874869 32761751 32828312 32583441 32723534 32457904 32452281 32404861 32344336 32366792 32410717 32465983 32524448 32533048 32335468 41t 752897771 33218038 33453910 33489170 33450105 33482242 33400984 33084008 33033849 33008163 33011304 32990212 32965457 33100836 33246586 33109029 33052769 42t 1025182031 33308181 33416507 33348551 33353589 33078035 33194525 32937731 33096404 32873261 32856868 32900612 32859590 32917712 33107581 32950684 32954981 43t 696942227 32794732 33077276 32835350 33004223 32905747 32813582 32543692 32458725 32599183 32662034 32659345 32519506 32665713 32802829 32620304 32619233 44t 1699723808 32859196 33014447 33005264 33098533 32851130 32863953 32647020 32620048 32544882 32629933 32675907 32701896 32691564 32808682 32804464 32680084 45t 410855221 32973357 33262500 33202569 33189160 33055825 33044003 32646518 32844211 32823867 32790803 32658075 32812383 32899920 32935307 32971815 32763179 46t 1502438571 32936173 33260372 33144182 33085343 33088220 33037123 32752761 32735861 32749653 32834556 32824232 32788860 32760354 32856958 32786384 32726020 47t 1580656301 32814152 33179790 32882078 32970050 32863158 32909574 32521834 32706354 32582668 32607947 32648027 32492769 32721354 32846821 32790095 32548698 48t 1099243655 32864590 33060995 33010305 32994239 32896624 32941142 32602521 32629683 32612903 32561942 32651405 32725952 32587778 32799409 32722572 32540867 49t 1611639012 33389112 33558167 33487694 33554983 33454508 33472490 33183976 33256869 33202243 33186646 33124595 33205308 33311120 33235674 33293977 33179711 50t 1429022466 32683888 32708498 32842868 32898058 32851506 32778780 32610345 32425351 32506621 32515430 32592616 32513821 32588071 32672271 32437732 32531407 # 2025-01-28 15:14:38 GMT: Finished testing irace/vignettes/section/0000755000176200001440000000000015055371305015015 5ustar liggesusersirace/vignettes/section/irace-options.tex0000644000176200001440000000000015060020356020272 0ustar liggesusersirace/vignettes/section/irace-options.Rnw0000644000176200001440000005114615060020356020261 0ustar liggesusers \subsection[General options]{General options} \begin{description} \defparameter[h]{-{}-help}{help}{}% Show the list of command-line options of \irace. \defparameter[v]{-{}-version}{version}{}% Show the version of \irace. \defparameter[c]{-{}-check}{check}{}% Check that the scenario and parameter definitions are correct and test the execution of the target algorithm. See \autoref{sec:execution}. \defparameter[i]{-{}-init}{init}{}% Initialize the working directory with the template config files. This copies the files in \code{\$IRACE_HOME/templates} to the working directory without overwriting the files with the same names as those of the template files. \defparameter[s]{scenarioFile}{scenario}{./scenario.txt}% File that contains the scenario setup and other irace options. All options listed in this section can be included in this file. See \IRACEHOME{/templates/} for an example. Relative file-system paths specified in the scenario file are relative to the scenario file itself. \defparameter{execDir}{exec-dir}{./}% Directory where the target runner will be run. The default execution directory is the current directory. \begin{xwarningbox} The execution directory must exist before executing \irace, it will not be created automatically. \end{xwarningbox} \defparameter[l]{logFile}{log-file}{./irace.Rdata}% File to save tuning results as an \aR dataset. The provided path must be either an absolute path or relative to \parameter{execDir}. See \autoref{sec:output r} for details on the format of the \aR dataset. \defparameter[q]{quiet}{quiet}{0}% Reduce the output generated by irace to a minimum. \defparameter{debugLevel}{debug-level}{0}% Level of information to display in the text output of \irace. A value of 0 silences all debug messages. Higher values (1, 2 or 3) provide more verbose debug messages. Details about the text output of \irace are given in \autoref{sec:output text}. \defparameter{seed}{seed}{}% Seed to initiallize the random number generator. The seed must be a positive integer. If the seed is \code{""} or \code{NULL}, a random seed will be generated. \defparameter{repairConfiguration}{}{}% User-defined \aR function that takes a configuration generated by \irace and repairs it. See \autoref{sec:repairconf} for details. \defparameter{postselection}{postselection}{1}% Perform a postselection race after the execution of \irace to consume all remaining budget. Value 0 disables the postselection race. See \autoref{sec:postselection}. \defparameter{aclib}{aclib}{0}% Enable/disable AClib mode. This option enables compatibility with \texttt{GenericWrapper4AC} (\url{https://github.com/automl/GenericWrapper4AC/}) as \parameter{targetRunner} script. \end{description} \subsection[Elitist irace]{Elitist \irace} \begin{description} \defparameter[e]{elitist}{elitist}{1}% Enable/disable elitist \irace. In the \textbf{elitist} version of \code{irace}~\citep{LopDubPerStuBir2016irace}, elite configurations are not discarded from the race until non-elite configurations have been executed on the same instances as the elite configurations. Each race begins by evaluating all configurations on a number of new instances. This number is defined by the option \parameter{elitistNewInstances}. After the new instances have been evaluated, configurations are evaluated on instances seen in the previous race. Elite configurations already have results for most of these previous instances and, therefore, do not need to be re-evaluated. Finally, after configurations have been evaluated on all these instances, the race continues by evaluating additional new instances. The statistical tests can be performed at any moment during the race according to the setting of the options \parameter{firstTest} and \parameter{eachTest}. The elitist rule forbids discarding elite configurations, even if the show poor performance, until the last of the previous instances is seen in the race. The \textbf{non-elitist} version of \irace can discard elite configurations at any point of the race, instances are not re-used from one race to the next, and new instances are sampled for each race. \defparameter{elitistNewInstances}{elitist-new-instances}{1}% Number of new instances added to each race before evaluating instances from previous races (only for elitist \irace). \begin{xwarningbox} If \parameter{deterministic} is \code{TRUE} then the number of \parameter{elitistNewInstances} will be reduced or set to \code{0} once all instances have been evaluated. \end{xwarningbox} \defparameter{elitistLimit}{elitist-limit}{2}% Maximum number of statistical tests performed without successful elimination after all instances from the previous race have been evaluated. If the limit is reached, the current race is stopped. Only valid for elitist \irace. Use \code{0} to disable the limit. \end{description} \subsection[Internal irace options]{Internal \irace options} \begin{description} \defparameter{sampleInstances}{sample-instances}{1}% Enable/disable the sampling of the training instances. If the option \parameter{sampleInstances} is disabled, the instances are used in the order provided in the \parameter{trainInstancesFile} or in the order they are read from the \parameter{trainInstancesDir} when\parameter{trainInstancesFile} is not provided. For more information about training instances see \autoref{sec:training}. \defparameter{softRestart}{soft-restart}{1}% Enable/disable the soft-restart strategy that avoids premature convergence of the probabilistic model. When a sampled configuration is \emph{similar} to its parent configuration, the probabilistic model of these configurations is soft restarted. The soft-restart mechanism is explained in the \irace paper~\citep{LopDubPerStuBir2016irace}. The similarity of categorical and ordinal parameters is given by the hamming distance, and the option \parameter{softRestartThreshold} defines the similarity of numerical parameters. \defparameter{softRestartThreshold}{soft-restart-threshold}{1e-04}% Soft restart threshold value for numerical parameters. \defparameter{nbIterations}{iterations}{0}% Maximum number of iterations to be executed. Each iteration involves the generation of new configurations and the use of racing to select the best configurations. By default (with 0), \irace calculates a \emph{minimum} number of iterations as $\Niter = \lfloor 2 + \log_{2}\Nparam \rfloor$, where $\Nparam$ is the number of non-fixed parameters to be tuned. Setting this parameter may make \irace stop sooner than it should without using all the available budget. We recommend to use the default value. \defparameter{nbExperimentsPerIteration}{experiments-per-iteration}{0}% Number of runs of the target algorithm per iteration. By default (when equal to 0), this value changes for each iteration and depends on the iteration index and the remaining budget. Further details are provided in the \irace paper~\citep{LopDubPerStuBir2016irace}. We recommend to use the default value. \defparameter{minNbSurvival}{min-survival}{0}% Minimum number of configurations needed to continue the execution of each race (iteration). If the number of configurations alive in the race is not larger than this value, the current iteration will stop and a new iteration will start, even if there is budget left to continue the current race. By default (when equal to 0), the value is calculated automatically as $\lfloor 2 + \log_{2}\Nparam \rfloor$, where $\Nparam$ is the number of non-fixed parameters to be tuned. \defparameter{nbConfigurations}{num-configurations}{0}% The number of configurations that will be raced at each iteration. By default (when equal to 0), this value changes for each iteration and depends on \parameter{nbExperimentsPerIteration}, the iteration index and \parameter{mu}. The precise details are given in the \irace paper~\citep{LopDubPerStuBir2016irace}. We recommend to use the default value. \defparameter{mu}{mu}{5}% Parameter used to define the number of configurations to be sampled and evaluated at each iteration. The number of configurations will be calculated such that there is enough budget in each race to evaluate all configurations on at least $\mu + \min(5,j)$ training instances, where $j$ is the index of the current iteration. The value of $\mu$ will be adjusted to never be lower than the value of \parameter{firstTest}. We recommend to use the default value and, if needed, adjust \parameter{firstTest}and \parameter{eachTest}, instead. \end{description} \subsection[Target algorithm parameters]{Target algorithm parameters} \begin{description} \defparameter[p]{parameterFile}{parameter-file}{./parameters.txt}% File that contains the description of the parameters of the target algorithm. See \autoref{sec:target parameters}. \end{description} \subsection[Target algorithm execution]{Target algorithm execution} \begin{description} \defparameter{targetRunner}{target-runner}{./target-runner}% Executable or \aR function that evaluates a configuration of the target algorithm on a particular instance. See \autoref{sec:runner} for details. \defparameter{targetRunnerLauncher}{target-runner-launcher}{}% Executable that will be used to launch the target runner, when \parameter{targetRunner} cannot be executed directly (e.g., a Python script in Windows). \defparameter{targetCmdline}{target-cmdline}{\{configurationID\} \{instanceID\} \{seed\} \{instance\} \{bound\} \{targetRunnerArgs\}}% Command-line arguments provided to \parameter{targetRunner} (or \parameter{targetRunnerLauncher} if defined). The substrings \code{\{configurationID\}}, \code{\{instanceID\}}, \code{\{seed\}}, \code{\{instance\}}, and \code{\{bound\}} will be replaced by their corresponding values. The substring \code{\{targetRunnerArgs\}} will be replaced by the concatenation of the switch and value of all active parameters of the particular configuration being evaluated. The substring \code{\{targetRunner\}}, if present, will be replaced by the value of \parameter{targetRunner} (useful when using \parameter{targetRunnerLauncher}). Example: <>= targetRunner="./real_target_runner.py" targetRunnerLauncher="python" targetCmdLine="-m {targetRunner} {configurationID} {instanceID}\ --seed {seed} -i {instance} --cutoff {bound} {targetRunnerArgs}" @ \defparameter{targetRunnerRetries}{target-runner-retries}{0}% Number of times to retry a call to \parameter{targetRunner} if the call failed. \defparameter{targetRunnerTimeout}{target-runner-timeout}{0}% Timeout in seconds of any \parameter{targetRunner} call (only applies to \code{target-runner} executables not to R functions), ignored if 0. \defparameter{targetRunnerData}{}{}% Optional data passed to \parameter{targetRunner}. This is ignored by the default \parameter{targetRunner} function, but it may be used by custom \parameter{targetRunner} functions to pass persistent data around. \defparameter{targetRunnerParallel}{}{}% Optional \aR function to provide custom parallelization of \parameter{targetRunner}. See \autoref{sec:parallel} for more information. \defparameter{targetEvaluator}{target-evaluator}{}% Optional script or \aR function that returns a numerical value for an experiment after all configurations have been executed on a given instance using \parameter{targetRunner}. See \autoref{sec:evaluator} for details. \defparameter{deterministic}{deterministic}{0}% Enable/disable deterministic target algorithm mode. If the target algorithm is deterministic, configurations will be evaluated only once per instance. See \autoref{sec:training} for more information. \begin{xwarningbox} If the number of instances provided is less than the value specified for the option \parameter{firstTest}, no statistical test will be performed. \end{xwarningbox} \defparameter{parallel}{parallel}{0}% Number of calls of the \parameter{targetRunner} to execute in parallel. Values 0 or 1 mean no parallelization. For more information on parallelization, see \autoref{sec:parallel}. \defparameter{loadBalancing}{load-balancing}{1}% Enable/disable load-balancing when executing experiments in parallel. Load-balancing makes better use of computing resources, but increases communication overhead. If this overhead is large, disabling load-balancing may be faster. See \autoref{sec:parallel}. \defparameter{mpi}{mpi}{0}% Enable/disable use of \pkg{Rmpi} to execute the \parameter{targetRunner} in parallel using MPI protocol. When \parameter{mpi} is enabled, the option \parameter{parallel} is the number of slave nodes. See \autoref{sec:parallel}. \defparameter{batchmode}{batchmode}{0}% Specify how irace waits for jobs to finish when \parameter{targetRunner} submits jobs to a batch cluster: \code{sge}, \code{pbs}, \code{torque}, \code{slurm} or \code{htcondor} (\parameter{targetRunner} must submit jobs to the cluster using. for example, \code{qsub}). See \autoref{sec:parallel}. \end{description} \subsection[Initial configurations]{Initial configurations} \begin{description} \defparameter{configurationsFile}{configurations-file}{}% File containing a table of initial configurations. If empty or \code{NULL}, \irace will not use initial configurations. See \autoref{sec:initial}. \begin{xwarningbox} The provided configurations must not violate the constraints described in \parameter{parameterFile} and \parameter{forbiddenFile}. \end{xwarningbox} \end{description} \subsection[Training instances]{Training instances} \begin{description} \defparameter{trainInstancesDir}{train-instances-dir}{}% Directory where training instances are located; either absolute path or relative to current directory. See \autoref{sec:training}. \defparameter{trainInstancesFile}{train-instances-file}{}% File that contains a list of instances and optionally additional parameters for them. See \autoref{sec:training}. \begin{xwarningbox} The list of instances in \parameter{trainInstancesFile} is interpreted as file-system paths relative to \parameter{trainInstancesDir}. When using an absolute path or instances that are not files, set \code{trainInstancesDir=""}. \end{xwarningbox} \defparameter{blockSize}{block-size}{1}% Number of training instances, that make up a 'block' in \parameter{trainInstancesFile}. Elimination of configurations will only be performed after evaluating a complete block and never in the middle of a block. Each block typically contains one instance from each instance class (type or family) and the block size is the number of classes. The value of \parameter{blockSize} will multiply \parameter{firstTest}, \parameter{eachTest} and \parameter{elitistNewInstances}. \end{description} \subsection[Tuning budget]{Tuning budget} \begin{description} \defparameter{maxExperiments}{max-experiments}{0}% The maximum number of runs (invocations of \parameter{targetRunner}) that will be performed. It determines the maximum budget of experiments for the tuning. See \autoref{sec:budget}. \defparameter{minExperiments}{min-experiments}{}% The minimum number of runs (invocations of \parameter{targetRunner}) that will be performed. If this option is set, then \parameter{maxExperiments} is ignored and the actual budget will depend on the number of parameters and \parameter{minSurvival}, but it will not be smaller than this value. See \autoref{sec:budget}. \defparameter{maxTime}{max-time}{0}% The maximum total time for the runs of \parameter{targetRunner} that will be performed. The mean execution time of each run is estimated in order to calculate the maximum number of experiments (see option \parameter{budgetEstimation}). When \parameter{maxTime} is positive, then \parameter{targetRunner} \textbf{must} return the execution time as its second output. This value and the one returned by \parameter{targetRunner} must use the same units (seconds, minutes, iterations, evaluations, \ldots). See \autoref{sec:budget}. \defparameter{budgetEstimation}{budget-estimation}{0.05}% Fraction (smaller than 1) of the budget used to estimate the mean execution time of a configuration. Only used when \parameter{maxTime} $> 0$. See \autoref{sec:budget}. \defparameter{minMeasurableTime}{min-measurable-time}{0.01}% Minimum time unit that is still (significantly) measureable. \end{description} \subsection[Statistical test]{Statistical test} \begin{description} \defparameter{testType}{test-type}{}% Specifies the statistical test used for elimination: \begin{itemize} \item[] \code{F-test} (Friedman test) \item[] \code{t-test} (pairwise t-tests with no correction) \item[] \code{t-test-bonferroni} (t-test with Bonferroni's correction for multiple comparisons) \item[] \code{t-test-holm} (t-test with Holm's correction for multiple comparisons). \end{itemize} We recommend to not use corrections for multiple comparisons because the test typically becomes too strict and the search stagnates. See \autoref{sec:stat test} for details about choosing the statistical test most appropriate for your scenario. \begin{xwarningbox} The default setting of \parameter{testType} is \code{F-test} unless the \parameter{capping} option is enabled in which case, the default setting is \code{t-test}. \end{xwarningbox} \defparameter{firstTest}{first-test}{5}% Specifies how many instances are evaluated before the first elimination test. \begin{xwarningbox} The value of \parameter{firstTest} must be a multiple of \parameter{eachTest}. \end{xwarningbox} \defparameter{eachTest}{each-test}{1}% Specifies how many instances are evaluated between elimination tests. \defparameter{confidence}{confidence}{0.95}% Confidence level for the elimination test. \end{description} \subsection[Adaptive capping]{Adaptive capping} \begin{description} \defparameter{capping}{capping}{}% Enable the use of adaptive capping. Capping is enabled by default if \parameter{elitist} is active, $\parameter{maxTime} > 0$ and $\parameter{boundMax} > 0$. When using this option, \irace provides an execution bound to each target algorithm execution (See \autoref{sec:runner}). For more details about this option See \autoref{sec:capping}. \defparameter{cappingAfterFirstTest}{capping-after-first-test}{0}% If set to 1, elimination due to capping only happens after \parameter{firstTest} instances are seen. \defparameter{cappingType}{capping-type}{median}% Specifies the measure used to define the execution bound: \begin{itemize} \item[] \code{median} (the median of the performance of the elite configurations) \item[] \code{mean} (the mean of the performance of the elite configurations) \item[] \code{best} (the best performance of the elite configurations) \item[] \code{worst} (the worst performance of the elite configurations). \end{itemize} \defparameter{boundType}{bound-type}{candidate}% Specifies how to calculate the performance of elite configurations for the execution bound: \begin{itemize} \item[] \code{candidate} (performance of candidates is aggregated across the instances already executed) \item[] \code{instance} (performance of candidates on each instance). \end{itemize} \defparameter{boundMax}{bound-max}{0}% Maximum execution bound for \parameter{targetRunner}. It must be specified when capping is enabled. \defparameter{boundDigits}{bound-digits}{0}% Precision used for calculating the execution time. It must be specified when capping is enabled. \defparameter{boundPar}{bound-par}{1}% Penalty used for PARX. This value is used to penalize timed out executions by multiplying \parameter{boundMax}, see \autoref{sec:capping}. \defparameter{boundAsTimeout}{bound-as-timeout}{1}% Replace the configuration cost of bounded executions with \parameter{boundMax}. See \autoref{sec:capping}. \end{description} \subsection[Recovery]{Recovery} \begin{description} \defparameter{recoveryFile}{recovery-file}{}% Previously saved \irace log file that should be used to recover the execution of \irace; either absolute path or relative to the current directory. If empty or \code{NULL}, recovery is not performed. For more details about recovery, see \autoref{sec:recovery}. \end{description} \subsection[Testing]{Testing} \begin{description} \defparameter{-{}-only-test}{only-test}{}% Run the configurations contained in the file provided as argument on the test instances. See \autoref{sec:testing}. \defparameter{testInstancesDir}{test-instances-dir}{}% Directory where testing instances are located, either absolute or relative to current directory. \defparameter{testInstancesFile}{test-instances-file}{}% File containing a list of test instances and optionally additional parameters for them. \defparameter{testNbElites}{test-num-elites}{1}% Number of elite configurations returned by irace that will be tested if test instances are provided. For more information about the testing, see \autoref{sec:testing}. \defparameter{testIterationElites}{test-iteration-elites}{0}% Enable/disable testing the elite configurations found at each iteration. \end{description} irace/vignettes/fig1u-acotsp-instances.pdf0000644000176200001440000010420514736526233020344 0ustar liggesusers%PDF-1.4 % 1 0 obj << /Pages 2 0 R /Type /Catalog >> endobj 8 0 obj << /ExtGState 4 0 R /Font 3 0 R /Pattern 5 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ] /Shading 6 0 R /XObject 7 0 R >> endobj 10 0 obj << /Annots [ ] /Contents 9 0 R /Group << /CS /DeviceRGB /S /Transparency /Type /Group >> /MediaBox [ 0 0 720 468 ] /Parent 2 0 R /Resources 8 0 R /Type /Page >> endobj 9 0 obj << /Filter /FlateDecode /Length 11 0 R >> stream xKslǑ;ǯa~ K]ݲѕv{P^HMIDE;O!fՒ(ȌɝO?|pLJߩ4_ܚ~Wb99GmI}5kq_/Iov_2=;TVcn%>ߒJLώ%7m\O-2ow\\j/;JGĘ"_r߱ͅ{hn o~-п/gWW7W &Og#sO~߿wNr}ϟo?9}.z۱4u_fӫ~{?/V_$Ϛڿ",ϙ<_N_/ cz9~:}O/F-k˟OߚwKw7ҫ{-*|˯߉)Ur;Q<-'Bv"y/%N"/+/_QV}^'u/%cǁne箫ۤ$)ԴZ~eL˼/ I$AŧCtsjZ6Z|lsk[kx΅|mQJ3=m1iܹcԣ 0w;%E]6WeW?ä{ٝnTỪ?_ͻ}t75-gM}w' O?u:D9r$O.3S|dߊhAWNI1WɆ8^ASY<j>x7םnZzR([2ăf}s%X?3rl#8C??}_?w jK>apBnI:"[Vϥ=/"D[Pd[-KEEH g_$E]yvO.B75_yK;-5`rB()w'cOթ\]i}RFa'a˰0s~nE4_] vopysu> :rv}r~7'Q'a=\]$ݙvOV.Ovô=a'a2eGwOvê;t}ݰ7뛎uk/>Ϸp9GAg'h gCm,ECD@Ϳ:}'j!iFvN?ìꋥ~Aؿ'+폧n|(cȧteHvtj.R.Q F{Rt:>:kѲDeh3/?Zgl2|Y*MBOqE]@-,TF4\$YNzr9̜]_RzXwZG|.Э7QvEpiztyX[^%}=;]zkt;#>ΒΑFZ.!ߪ`gIίW5g.wEb ]sԕrVOUy,v׋Betqmι>JWK}ΚnHp%$eGVüepIR!绹^X6ӟR,km9k /}^ ":S=tr{e+l/1v"MKZD ãzpջ`p6E5't$yeAdj +z5N']GIJ.m w5e#t1hf kmn B-rt$%C^Gi}Wu(Joy|ꓽl]ʛĘ[o+W1N.`Xzǁ J@?GX(1rpAMRO7K yՃ>L}=YW$^˂۽G 5R>Zh FA% BVC_}|邮Z5 ij,# PK m*{l}$;֦X5]H6vM4 h:H~vk5BsxRjZoDKk:9hרeՒ$i^\Mr9Y$Pl(8[׳ppz2s6MYp"KvYo®Ye]#q "HS;@&zy:>),S]ta}5/^"ܤ%#@JK灾s4zHRIF\/hz\Qïo&җ's 5>W2"lm(},Hžˣ'aSrO2S 2rYGD7m[ NFvI /벇52AeVT7A7i$.? <0a93NL%ϗJ H a ;(QZu)ȸHŰuO X0cZ,CɲR%a V$8~ϸɘ絗`|zY!A*e$@45`ٲ<V7k|}_"%a^ugm҄dȭP: &k5-DHBHɴXaYWbAu $V:0B2$׺lŗĭBеn?ⲵn9И,o:< QCD-,:A'#:ȇ֚2Fu|5tҕ]m)uQ1;NZN%87k2j~0:7X!cdNvX{"Idw =9)EzxdkCKNHtK yTZ!ޙ8|:N7}y>_o !? Z з<6p.xݮ`!p{ܭĐdrT q*Ro9 A6d.Z64 LqK_ۍ`{|s[]J;?/q=V&@nAk-(ItQh, ^2!Ă! D(BrG#(6`¤K^'qݲ`[c\ϰ^uhkAى_K?|8$ē[ JS@A+p bkA9,-5&.aYeUc_&"C4 Yo9.hSrI&1ޕe= lIC%cۨH /DgS;uRB7B XĮo`B ) Qtc6xIY"GlRu,I!v1< k!FF\g4I6fAg =  v?K I,ņdњkZ1LMs7O8hd 0t-u(  w؈BD * q#%&M:OlGg_zk]},B78CT7 E2 |QZΥ N!ƍaTqFP!VT:s)K'Hdx}qϸPhQ݄A5TS(Wjx< *  :QgL̆Ò.iZG G4!8$g hzɸLeŔugo"Nsg`CU5 (0PE$,ڄ*%j:UŲAJe`2#$ϵlHNZDJt8(Hk GIr;8l2$3Z pޘ/89F":aj$2`&d+&dEpĔ -0+kvkiFɼpѓNUH5`%Ĕ R. | Hq 57#~I܁0<[9:rM#uPbq_GA3 am(u!rB+h{ņJ}@Mgmv,DY/:)Tц2:AAض-o^xZjd?Jޏa `snhСÛAK$cDO"C(!RWStT1m)7 .un!8ґH[ dI$ShHtilI]]ƞc@JP&ALdh(m}29O xI6(Lj&ԸipxynSx^kt1 fXxse꺘 vgk#%Yai3D yMߏk>"7?уށ 1? N1Ξ{Hyj4ҥ5D TS,V\_LcS?Bh?3T%K/`S@?b}QMH,NHpxNI诟o龡UFP juZ-I 8zhr)^L$g4X=A^ PlH4Ɔsb! Ap{ڣ * z{TZwA%PF6# P<%a-/=Qǂ! !NV+h+ԉߵOg*m 5{I4rCT"\:x:7'Z6IH F$Ee$ ~rI= 0lxѰza^@ ng]Ѥ?V  EJQ+d1W6jLBuqdpt~Q="pGse7tg GG,POEpw`!ɧᲶxCܜp]=Rt7#/ ^bAԝHhHIBƝK!\Bp,4 b,$=#fו<0dKGC@hVY8Sj =- -DJPn5`Pu!`ܗ *6LBL/[R*ꛁFG:YzӒ!57dE  jvf:Fw=dYOv~l\~tܴ{8ױJWhBf.AqdxÊ?lwYdd]J{fH 6<־ |L)֜c*iPp2YHJs#кgG|Z:b-ljs_؅cn#P$&^ k` #mX5Lsb5%Y׬I>746u54MC $g(hŒ."1e12cTde7&lBgЂ2)7 u>5H p% H:9ZHmx:+(Iz 63/VА5F^,'>I[t!kTLZ-OLR*؟GJ%):] lk _YV†"r!$a*XU`@Rβ0jJC07dIJG;$$oiaDb(V#`md;8/(R ϣsS4'ɾA@yHy`֎JO䵻L"&' #hG,)Q[N@¦ dž!-Q,JkACS:!$cޠ:6=V'E Y7Mυ:a Qu`>E 6K }M )jբ]r{@UhC|F"qۼ_2ibI*Q3!  yC_F*Cд*Ep_O3 #RcxY:CӠ@|&+hʼnZsBU4JG@40_YZZZG0`:"bg_ Vɚ70}$|<"dEh) ݩ))1XCX7 a];浡Ux)f} K{q[nKBe4S4!t<auc^Ew`0Qpu ]FCǍJz.t qmOX[ 76 /oKT7Ǵy jQhNL@ْEd'ENLgg dLB[]hJ+?:bQy96D (tDH́n!ĢiuψװtZGneK8W+l4>)>X fNwib"*o/ђ,A"`ʮ7kMGZX26]fA&F;jdj?;MR- ZsU3hH rGkRC[}i]" Gg mA .uu~$壁u|d .eEhѰoI ;Bמ֜$Xb;-;gyȖ0;G ^%w;e>M/\pPQwݐŖ'Wj =!9Rߢ:6tDПb$)u!eF,xZ&= ;:j)2 +;?ժt{a!` KLA- Sހf( n%TO[vMR'9HiZ`Ѓ`p7Adt{EKV U[Ueܕ6"φ LS"i2c ;~Agn\{&M^9&>\HtJ4傁:p܉ Yu$KbuCo͂s.?%KUg5Vt=X/dæ Cuk0TxB{]9& HQV!7BHA]Be\}neYY2CT.}7B r+2,E7aDmL#/\4 *㊥k4qҀQi`GPf(!!י2\X2nh($j5C@48VFz2$;DC΍OEܷ]Z V/"LUײ)8 >_3]xa,;t66?uR_iC IdÆiSΓR:jI o AM@*ѻ\'Jv.lM2RY?eׯmxzRr% ־6~Z ; ›#ʊFe4hCgRfzY&tD)IB֮7b*f+M Nl]^ MRJ3DgzHڮ *HDÍܼ`ХL5zB2mJd` 0.::Lfs;dh+Ƙ!=YL&B]c#O#a>%T3U(6Xd^(R^k $5'[vs%Ԙ\4(L{Jqʈ4D^mMbζNIY,y<҆A8ۨǣIy 5Y)> x=~}Q uXOu \@c/C4Gja6I^R!8H]s _ `H*%DM0N#ݒ j+&#Bi)*:|:L3"TՆt0Z%֩u1>z(`~zdjX,M]TI<10Y, M[9HH:(Grm(~PwҚ wq3pC * ;d9Y>)OE( D5΋ޛdRa dWOYAOӿq6HB߆{ 4Ap &}o|::Q҃$VMvw\y⷏ŢNƹ!R u8s5M rjAʵb('G_Tѓ76꺔b`iR(Ӱݕ0s_Bp̖lEw#1cB6KM}P=Q'RS8 %=eg J! KO@ 2mKSZFK] ,DDǒ g$33g+880"Uz@6\g6((fN%x8Eh=b2!{)= ( [1M"Z2-imܿ%Z AÐʏ/Ov:*AY!w\9vAI2tR,w`d ͏#!paB!O7Ёvb]ap P#C9tZҬ[܊TK_#ߦ&dWEGbiS46̙0ɔ[&Ĺ tX Q0QJh\ sn ? jmft/ M#&b#[Z-jgY5#XMzm-Jk[Av/z>c'O6Y8Iښ:Qe"WVG}`r >UOi6`FiRhXeh&@{JB " H [?gnz V+yIk$<ѥJ7Ln42R1"krZmj0凗R0,G=a78bՂJ lP:z:baCnS}Ig|2B%"6"ԖYrKؤsvQi3 Mڐ$s&zYp YsuzM<?j(d !*E{Rg*gi_lh#-pa6CFU hioy9y1i]ILL{ ~l7#˹LrL Slel8՚B3TLdgB?FGdBjzΓL/ Rg&+,']f D-}H._K,@͒eG6M(dZ) p A w R!F9RՑIDYRƉY@YbQ)5*7/v𱐨WWQƔu|iNT1y)犷ɣ h|;son~2 GnQA7rS_d<Չr st}eF8 7sMiIx.ăo̍90S5~>ցyNsӠb liR<]= !r A\z=DRWIBX-ՙFhPYO˗-8[K+׹j0jzvZZ|Tuq㫠La1Cѻn%#p??yɫb؞w(ެ598gURRE`%~ޫۚ~3+9b'iꨱ}U TQzgn?9~$W;ϙT(l@ fE~DJ˒7|F$F޳u>G>L#gޕ\A jWrڃX 6/oBr:0<3/3 iYg"p#RvmPtǺj : QRw!O.$˰ Y#ة,w]llxꮃ4h/.T n+DC8+g؍BN ؊Nk4 :AR;ǀkDֵRB׶9-V J%gGDţ4<#$[Hj :ß<2xF,E_2 pKRkaxG< WCa,̢G92q&#;դCR WK {P0l+չxaCRa3˭#pIp#[fFq4%V=X'(#(!(2!Sq9ҽsL]Z썲oh<}+aKE‡9wȏe.ft7-iU1Ajp`^Q2\N?1MoBSBޑoaί)Ȼ |_yuԘAlsfᛁ%y&S+7qS9e֟t8W>+$rx.Q'S!%Ipc kF‚S'ݖsxC?MtGJ|2WԶ_s'IU|*›*V)H \YJp@*yQTZՖ>W=/#`Z=vwW*bkaL}t{%aLCro,i@+hY3[&xrEb;:XwcGBmq^HFS O1X e OK:UT)IQ>eHf_Cemz.szN)={rޜ.E< >̹N+!i GYOYny-! \Zgl鞻AfKdzܞ]=:Yv9Iy'e3zH$Ȩ>,MrՋ.tf S;d Dɥ`Od̽9sMP^f#m͖P]/c#| PL5Q)|=9 ]PyM Nߤ1feDqZsz#zT\C0`a$DPH~b1 CIHej -̥t76OtZ3Zzϓ,|S8ZzrVG,Kf/B /Ya1ܿcG],#Ѳr7@9 [Cky1rx{d^z^b}3:kKvm}I?.,Nk6M⛓P&@ FFPʗ8l`f9awmi2g`} MDZn}lontFD{ތ=cVnoo Pwo6#OA+eWyK 3:]/69 F屋4HMWɧdhJẀntwΙ'ϙKS@; |ҽ~TTt_7kn4I lۥYH8Nǻ+F ,!rd(+m 5{tNtVwmE h0YKKN)ڕ <8%.v-8a%{7_6_>rP`d~{]CU iꩪ_-à}AS誋kw9 N9@T`ak="lLbh{Cӥ.VL,x{k|<68%g$/m>_$Rk9o[2RWWuLpί֯BudDq[gLCK2RUbu'0mTS %*Tb3`R 1%v &̟[Z>ؗQB{  fu;t%>y%^r~N'<y}|-nQR)K?] l5 P 6R Y#qB׋/'/S];Fy PR2؞宽[)(9~7ݵܧ_`9 si]r,x;~ ywNjMƩVl-z+-V !3%I /!<:Ez`9W( ;h!ܓ )<#W,>x纭2ZJTڿ=e(g/PK 9  Fb[S;mK/tz7h}c*xs.YDFK+@U~Z-.r5OUVˡh@'WJٙ"bK3Ch7{%CwW;nV'}f$IO&XALu:{' =o9*)q7BHnw/pfnS0{76fHE(']zp +`&@)]%tUr2'5*t+d-i8Yk_Jw} #hvȋ)K.Ěd^e}9Zdƽ %} U?܍Htfrdn2nY|-Y;~{DjPmݼT(3{fe'^ ?2t,{&Y };ڦ-(Mb0- ޅiaɞ_틝6z-D~.#QF4p:2\*دB{*ok9!OLu}nFfx*k)Ppikgjo]e4o6 4n>(֙Cjz-0] .ա[fs5ZNCfUHN]WfcAJ.*=" M+6)Cɉ|mW'5T{Ox١bæ?e Z2e(Q%%117 *ܚt;#\xk;wQ+}ϓ};A8M+ddG;<>io\klݲ6.t6/@Kʠ\ 0%!a׼.f[BUn|ūW&eU <̩fe\iBF&qqaw|<^~%|ky5YsEPʍzKx=A0tR)tȥ7g5"VrS[NQ:݁Ht,n\$,%TmpO4_7&>hnAF`fFnƸ;`B>W¦y֞RuQP)il"G |*!OK1.ޡw =* G&-LR Pl5:lع mKrҽ}# po8* ivt!iQP Uro |%~i7` efUnp?ӆSPxqA[znC̈P9XJ\ArW`[&Awlh?4(\NLT&9ׇu# 6`9f'P7V;2Ub֎VR!_$3!.c$꩞>kHz-KW;)0ұgN֣$=qSUXBX7/c$xaeɾW1 > [!5wiMFnSpz,9r;IvSD>(S75[ܾi_\/Q-m͡y-GB mof@CC?񭍖8xu ^ޖBnS)\7Qcz&Q 7r6L31߼7\ypW7˶r&4m~Ɇ з 9,F7$%?5ɮJZX{IķF7暯/w?\YwIk yG fG8*-m"{h{ߝrSo:IBAN ?| B{85!{?~os 9N?>6g*ǃ Q5\eޖ\t~|ւPGcVf7+_Y]3hª?>.%퟿ : ^/fuy 툴R~^zz>EŚBg?E.)z37^i1\Cei~1tb*D>7>EʼdpMov*rܶvzvKjTVlteF_S+ ?_N姇[7xfY䍊{J7]i$(%%oov#O?'O `]?M;\ǒ4iLױO_{xطaz|yݗ]~~tm~!fvhj*^>.Rg{cߞ>3˜hI 1nu%l_> stream x5Q;r1} ] 3og3JFfa =؈ėoYf~'Y)QTEX!Yjs#Sr&>'b8Ύf01h9f=!#n4U i[ZSEȺ)Z[=- c _ĜE'~3尒4#15όO}>hw/̈́LHœƘ1T$?г>0TG endstream endobj 21 0 obj << /Filter /FlateDecode /Length 49 >> stream x34U0P F )\@> 4XiLEWD endstream endobj 22 0 obj << /Filter /FlateDecode /Length 227 >> stream xEK!CGp;.:l%(3{Lnd;0sJlLN\#T޼頉{DΣY jg.7ɭFiwyHT9λi Yմlk?*MlVWw'K~0%?Q|]7g'W&kyI_ʣq)$_SS{ginV9 endstream endobj 23 0 obj << /Filter /FlateDecode /Length 304 >> stream x=;0 C{Ȍd'>2VI(/u< i& b;w؞D/)ϡ+E:Ū0[M*K õ}74uK hY pu;Gw5<TQ!OJ|<(!\{0FS@\^BAjI'> stream xEQIr! ++Nͩk,3IN`k1i-x!́/|M4|72:pO.ܗ˂g–WTw/]H}w~fd{H͐:B4&!=#2V,s)R([.ꕶN;#o#JvMl:˶.:$vaqj!"uc5N"v0Q$gq&M]9yV*9>^Ƅtc׶i_>}?\" endstream endobj 25 0 obj << /Filter /FlateDecode /Length 230 >> stream x5QIn0 .AO ``hKxA[֌]< 'Q^{ .o8|^Z9O2 D`@ai'ώH5YN_KK(O~ J.kO8'O [WLcD.A|!]'@;綟Wuu)O645I"%Ҹ[{TS endstream endobj 26 0 obj << /Filter /FlateDecode /Length 227 >> stream x5O;! 9.m`ϳT/od HDG&^S=Ä=Ba$5븛ſ]3 b0řvX'qNcDlc]L(!0%)}9:Nb.a}1WOdP=&ՍI4^2`a$YNkGQ"1'2Ҝb :; *s>h][M endstream endobj 27 0 obj << /Filter /FlateDecode /Length 245 >> stream xEPC1 =`,{wHۿ=JFp!Z?ZK oGFA= 3A΄@xFnvpμ39Zpә\'mBITqTqLύׁlӑ!KI%&~S*)[*EH䁓M4,?Cb̠Q0qGuٜ9-L|X&Q)2>'\N}䢥Uޑ"ۡW%Qէ<Y> endstream endobj 28 0 obj << /Filter /FlateDecode /Length 392 >> stream x=RKn1)@Mr[T /1 %?ꒈ3L~r]Qljg!.6Xr_rњbO/ȴTXVݣC(-װr{d`Jn@CHYAaPl( WԬtb ) ٠[]aP[[xfޑ3qYk?=Q2QMg|2RCgB'`$Ip#A 1qOl)V;ޒ{,\L'ib?lK\+E(~Aq|XdDw#h% 0xyDhDԎ=(ͱ&{ǫvzcw. endstream endobj 29 0 obj << /Filter /FlateDecode /Length 133 >> stream xMA0~tzJ-6팀03 endstream endobj 30 0 obj << /Filter /FlateDecode /Length 247 >> stream xMQmD1 \ky R]oC /)%K [UC?13,=?TPbht/"+ߏe s`&4`oI&ռ3d‰ATwM,3V7: lx%D`r Z`Q+ tĺv7C/਺x} K{,|BL;wI#fR:=b}@e+ (\* endstream endobj 31 0 obj << /Filter /FlateDecode /Length 90 >> stream xMA "OPDtz_NE5jK02kP)U0\ 2IL{qIqzz"X endstream endobj 32 0 obj << /Filter /FlateDecode /Length 338 >> stream xERKr0\ 3gNWp:<  2=eH6dWdՐFD)򹼖\nJ?72ͮЪG6F5+# CzVQdv!:Sp,Cu)mA#o<rLn[ :[m@ s` )(UI­\';PЪt79`Òho>F, f1H'N=q:őpI8@/ u:eMžBRq"n]ElO ?*3b Ԓ枾?9 endstream endobj 33 0 obj << /Filter /FlateDecode /Length 68 >> stream x32P0P4& f )\@B.H  %[B4AXf&fI8"ɴ endstream endobj 34 0 obj << /Filter /FlateDecode /Length 45 >> stream x32P0P4& f )\V.L,іp "} endstream endobj 35 0 obj << /Filter /FlateDecode /Length 255 >> stream xEK D#> stream x240S065U276r,#s# $`Ad_ endstream endobj 37 0 obj << /Filter /FlateDecode /Length 161 >> stream xEK CBGG|tJ￭!M@w'/mK >[ x6n5uVhR}ith6s+ fz:rGp_Gdf)|Q]dcnk]3s: endstream endobj 38 0 obj << /Filter /FlateDecode /Length 320 >> stream x5Qq0 54sۿ @;a@dJ\UGM>`!S֖{&UF!}W2j]* UYFp&I8d Rӿccz endstream endobj 39 0 obj << /Filter /FlateDecode /Length 214 >> stream x=PC1= |7˥m$B6BLɔ:ʒ)O>Kbnd6%*E/% }ՖC4h9~ 3*K6p*3 mtV[ Ф`׶ r " JMrR=ot-N=Dkq: DpFjtaŲC5=kz7hGt4CָR endstream endobj 40 0 obj << /Filter /FlateDecode /Length 80 >> stream xE 0D{`~&f( JpO{:2Sa ,S`5FR죰n_uzS*Ovvq= endstream endobj 41 0 obj << /Filter /FlateDecode /Length 49 >> stream x36P0P040F@B!H Y@8&+ & endstream endobj 42 0 obj << /Filter /FlateDecode /Length 157 >> stream xEC1DsUA wJo-%S'"h0yM%V,&rAJ1xN1븨ufihW3=5'M<[ }@8IP1}bv">G)#qbn fW7y endstream endobj 43 0 obj << /Filter /FlateDecode /Length 332 >> stream x-R9$1 ~`LtIUls#h/#xE=f۴[iGiK,W ;BjW0wy.2meDkag؏]e8*Jl !2J'Qw\I2[E™w2;yNE{ kF9+%|6vzrYɩHHӺ NKؖߗ3| endstream endobj 44 0 obj << /Filter /FlateDecode /Length 68 >> stream x336S0P0 F )\@>,́,# .C c0mbl`fbdY 1 r endstream endobj 45 0 obj << /Filter /FlateDecode /Length 317 >> stream x5RKrC1ۿSpΘ}tj'+-@B./YK~%ۥW%B>R-G- Q=2'":xa>N)x_xN;2$KMH=I+4t~&+s{rj X+)$=Hr7VސWg%&&MܕBXtLX㰄*aՃM5fcdxLP} #GMv²[6!D3,($Nc$ Ұ9 9e, mh%zМaמE[{ endstream endobj 46 0 obj << /Filter /FlateDecode /Length 17 >> stream x36P0C. endstream endobj 47 0 obj << /Filter /FlateDecode /Length 131 >> stream xE ! CT>՞0ABA";06Ѣ76իc,zRV鐇Pi0QąYLCaΘȖ2MlTv<e~ma, U^ ?KwUBS0 endstream endobj 48 0 obj << /Filter /FlateDecode /Length 338 >> stream x5R9@ } ] v͜~߆_ CVie!U-.Im W%ڥ Pt,6˯JH+kLwIi"Eo7o}=@.^ AS(i|Ъc(ew 4<3}(~_K&(? _osџa`Ś}@*z`yT endstream endobj 49 0 obj << /Filter /FlateDecode /Length 248 >> stream x-Q9AzBsˑ C :-qPO+Uwu9HTM]vf5,?c 7zqxLu5{kOfP2+qSușO \ ȹeƌ#M!RH&3AQ~#aU#j \Ks4;<9GW +ET<pC7ҹ^s0XM7/=[ endstream endobj 50 0 obj << /Filter /FlateDecode /Length 171 >> stream xMMB!0h\vK!CGFGx1 2&^$ m;,1',#`kܛG ADko5u~~]ԥu# ȎP p=&T)8T bSUhV=^; endstream endobj 51 0 obj << /Filter /FlateDecode /Length 72 >> stream x50{6X`KEoC|N/Z #pu#])Ec΂q_H1F=#O_p} endstream endobj 52 0 obj << /Filter /FlateDecode /Length 210 >> stream x5P C1g dVukm;aBXȔy)K>:L." u%ʚ +`p&^7`i5tႦ.B%|u{OxjrvC` jMX> /FirstChar 0 /FontBBox [ -1021 -463 1794 1233 ] /FontDescriptor 17 0 R /FontMatrix [ 0.001 0 0 0.001 0 0 ] /LastChar 255 /Name /DejaVuSans /Subtype /Type3 /Type /Font /Widths 16 0 R >> endobj 17 0 obj << /Ascent 929 /CapHeight 0 /Descent -236 /Flags 32 /FontBBox [ -1021 -463 1794 1233 ] /FontName /DejaVuSans /ItalicAngle 0 /MaxWidth 1342 /StemV 0 /Type /FontDescriptor /XHeight 0 >> endobj 16 0 obj [ 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 600 318 401 460 838 636 950 780 275 390 390 500 838 318 361 318 337 636 636 636 636 636 636 636 636 636 636 337 337 838 838 838 531 1000 684 686 698 770 632 575 775 752 295 295 656 557 863 748 787 603 787 695 635 611 732 684 989 685 611 685 390 337 390 838 500 500 613 635 550 635 615 352 635 634 278 278 579 278 974 634 612 635 635 411 521 392 634 592 818 592 592 525 636 337 636 838 600 636 600 318 352 518 1000 500 500 500 1342 635 400 1070 600 685 600 600 318 318 518 518 590 500 1000 500 1000 521 400 1023 600 525 611 318 401 636 636 636 636 337 500 500 1000 471 612 838 361 1000 500 500 838 401 401 500 636 636 318 500 401 471 612 969 969 969 531 684 684 684 684 684 684 974 698 632 632 632 632 295 295 295 295 775 748 787 787 787 787 787 838 787 732 732 732 732 611 605 630 613 613 613 613 613 613 982 550 615 615 615 615 278 278 278 278 612 634 612 612 612 612 612 838 612 634 634 634 634 592 635 592 ] endobj 19 0 obj << /C 20 0 R /I 21 0 R /R 22 0 R /a 23 0 R /b 24 0 R /c 25 0 R /d 26 0 R /e 27 0 R /eight 28 0 R /f 29 0 R /five 30 0 R /four 31 0 R /g 32 0 R /i 33 0 R /l 34 0 R /m 35 0 R /n 37 0 R /nine 38 0 R /o 39 0 R /one 40 0 R /period 41 0 R /r 42 0 R /s 43 0 R /seven 44 0 R /six 45 0 R /space 46 0 R /t 47 0 R /three 48 0 R /two 49 0 R /u 50 0 R /v 51 0 R /zero 52 0 R >> endobj 3 0 obj << /F1 18 0 R >> endobj 4 0 obj << /A1 << /CA 0 /Type /ExtGState /ca 1 >> /A2 << /CA 1 /Type /ExtGState /ca 1 >> /A3 << /CA 0.8 /Type /ExtGState /ca 0.8 >> >> endobj 5 0 obj << >> endobj 6 0 obj << >> endobj 7 0 obj << /F1-DejaVuSans-minus 36 0 R /P0 12 0 R /P1 13 0 R /P2 14 0 R /P3 15 0 R >> endobj 12 0 obj << /BBox [ -2.5 -2.5 2.5 2.5 ] /Filter /FlateDecode /Length 36 /Subtype /Form /Type /XObject >> stream x3T2P5R\.##񸜸 endstream endobj 13 0 obj << /BBox [ -3.1494897428 -3.1494897428 3.1494897428 3.1494897428 ] /Filter /FlateDecode /Length 138 /Subtype /Form /Type /XObject >> stream xmA!E |հטͤIQ4!y7qzZjVIYKSDvK(N.lb"`ԩ`ԫi0>7<>p;cOY`ʡV endstream endobj 14 0 obj << /BBox [ -4.0166247904 -4.0166247904 4.0166247904 4.0166247904 ] /Filter /FlateDecode /Length 49 /Subtype /Form /Type /XObject >> stream x3T2P5P5364332U3 rpn.T4 endstream endobj 15 0 obj << /BBox [ -4.6785548508 -4.0843608999 4.6785548508 4.8833001327 ] /Filter /FlateDecode /Length 105 /Subtype /Form /Type /XObject >> stream xm10w^kǾ%*eua%8TQjJKdg60d+SeB"BB]6xxԹduՇ'/R() endstream endobj 2 0 obj << /Count 1 /Kids [ 10 0 R ] /Type /Pages >> endobj 53 0 obj << /CreationDate (D:20210416181907-03'00') /Creator (Matplotlib v3.3.3, https://matplotlib.org) /Producer (Matplotlib pdf backend v3.3.3) >> endobj xref 0 54 0000000000 65535 f 0000000016 00000 n 0000033572 00000 n 0000032305 00000 n 0000032337 00000 n 0000032479 00000 n 0000032500 00000 n 0000032521 00000 n 0000000065 00000 n 0000000385 00000 n 0000000208 00000 n 0000021487 00000 n 0000032614 00000 n 0000032780 00000 n 0000033085 00000 n 0000033300 00000 n 0000030871 00000 n 0000030671 00000 n 0000030210 00000 n 0000031924 00000 n 0000021509 00000 n 0000021814 00000 n 0000021935 00000 n 0000022235 00000 n 0000022612 00000 n 0000022922 00000 n 0000023225 00000 n 0000023525 00000 n 0000023843 00000 n 0000024308 00000 n 0000024514 00000 n 0000024834 00000 n 0000024996 00000 n 0000025407 00000 n 0000025547 00000 n 0000025664 00000 n 0000025992 00000 n 0000026162 00000 n 0000026396 00000 n 0000026789 00000 n 0000027076 00000 n 0000027228 00000 n 0000027349 00000 n 0000027579 00000 n 0000027984 00000 n 0000028124 00000 n 0000028514 00000 n 0000028603 00000 n 0000028807 00000 n 0000029218 00000 n 0000029539 00000 n 0000029783 00000 n 0000029927 00000 n 0000033632 00000 n trailer << /Info 53 0 R /Root 1 0 R /Size 54 >> startxref 33789 %%EOF irace/vignettes/irace-scheme.pdf0000644000176200001440000005432714157104332016400 0ustar liggesusers%PDF-1.4 % 4 0 obj << /Length 39 /Filter /FlateDecode >> stream x+2T0BC]C#S]SC\.}\C|@.l( endstream endobj 3 0 obj << /Type /Page /Contents 4 0 R /Resources 2 0 R /MediaBox [0 0 380 239] /Parent 6 0 R /Group 5 0 R >> endobj 1 0 obj << /Type /XObject /Subtype /Form /FormType 1 /PTEX.FileName (./irace-scheme-simple.pdf) /PTEX.PageNumber 1 /PTEX.InfoDict 7 0 R /BBox [0 0 595 842] /Group 5 0 R /Resources << /Font << /F1 8 0 R/F2 9 0 R>> /ProcSet [ /PDF /Text ] >> /Length 4301 /Filter /FlateDecode >> stream x[ˎ% W:t  ĉ;'dd5y!o!aս31>XEQɛÿt$C:C8~q!{Hgr|C|:}qFu ͝Թ0vF/K˅;υsw΅*(^qD}O}.5ޥ;bzn1ljsG2o٬ͫ3Gݴ |? m_Um3K%di>.ӛ쎚ӛn> +tkK{ޔvr_-fn1 +odYcoMtZ((7KRo :P^\U30"^)66q6~^'N2v2˅{L_׌1߈afTΨDz8>\׌R4/#PQ! F-V'jwB%Top dy7Ob;Ur7 XX4b ZilmVnA#}$[aaS$` LK%G&%L)|ӎm9+d:XB!M5RF)(ӘH*ׄȭ`7 =o&fST,|K11P:1 6iރشa0jսlTePB*MFqүbr4||xW]'#X2؅%#v#Dyuf'D;QB}@'u%8S9l`o!;r $ewG1-e,H4)a=eƦ^ F7PX;Ɯ!'8~>֝+vs= e=MjГs(0_ NQs-V PxE*=|e!'-櫖M`4`FOFCJXN &w\p7pw>_5nlpg[ V<5\^3r `wʣyBr N7 ݱ{L}]F P*n\qmn`&@0M =Rᣖ92(ȓcȈh]KS8%#P:@p:32ox^2QԣCAdo24q'8K653`ub酉g{Κn߿Bt$t^4d+i qhVW桛1(G,.RӠ3cV܅ƗB;wwqL1{NyS;C_pRi8ə OQg G$.t_*&׽}zy3>)X9^M}bGKVDJ( N }nPwca߿~~w7A¤0 ؝`'O$c+B]㑌Se"08J둎x#QG)'ɳc:S<[1쎠7;ocNh #!iR n?hDs×ry̶/ 6=jA\oϗ$ X;OwI{iyq=_25nKSNIɽ*t7Hjf9Ӝc&𬖫%Y`']^2b씆C*yP|l-Kr Bpc#ǖrz ))t P߹R(`ڕ⹿RH9V'b༿z'+!^ t0hjPgӢ A=.K6\`쨊uq3.+㞊 D#i+gM6E7f"4iu.vx]!XEf'5̮I8͠Шe7LMF3RdB{Ǻ Qsnj8꧔Wa`*#4vzZӸ48S&1b[ F T@AǺȭEV+' F#q&>҈3:>#^zj!i<ϋX0=xDltF_Ŋ!((r<" +((R^n=hFy46`HCX ciup݁:}X7k#/H2/e_5(~&+W1 Ò ̪00',̀yA(;М0l),1H͆ 0*t+wL8&V'ذwϠ&Q&Ԝ8*SrρA YFN&d`uӝ6hG}AAS8qݴd f,>nH?136΄ACYZ:șj"F"*,*jܘ8{z3 L ;rTT{G{diFG< xE>6n!7({;R;8$j2Nް`{ FxmX2Jh0 &*0w[h "iㆥ> $#Vcn&l0Qatm083dX׊ESޑ2n2d̢M t( } >8-ae0%t\.IiSK!ARqf\6ZѩcVSǚ-"]?%P#TQQv8s 5tnQ./JK-eSMQG;Gɜinjٟ9.=r_Fl4(!&Q\(ͅDIc]Po|Z)K)q(tTP8=n0Jjt;id\Дb;2O^&YL[+.dVlʈoA)%Ju#07q (QȠp/P (A&ٕ.K+F;< V:Y댫mxGPi.F+rj;,Qdy`Dệ>/r=Gj{o&܈&D?Z0-Z7ODSr\p*IɎ?)Tsl)F` endstream endobj 7 0 obj << /Creator /Producer /CreationDate (D:20160720164806+01'00') >> endobj 8 0 obj << /Type /Font /Subtype /TrueType /BaseFont /BAAAAA+LiberationSans /FirstChar 0 /LastChar 18 /Widths [ 365 610 333 556 222 556 556 500 277 500 556 666 833 556 722 556 277 556 722] /FontDescriptor 10 0 R /ToUnicode 11 0 R >> endobj 9 0 obj << /Type /Font /Subtype /TrueType /BaseFont /CAAAAA+LiberationSans-Italic /FirstChar 0 /LastChar 17 /Widths [ 365 500 556 222 500 277 722 222 277 556 542 277 333 556 556 556 333 333] /FontDescriptor 12 0 R /ToUnicode 13 0 R >> endobj 10 0 obj << /Type /FontDescriptor /FontName /BAAAAA+LiberationSans /Flags 4 /FontBBox [ -203 -303 1049 910] /ItalicAngle 0 /Ascent 905 /Descent -211 /CapHeight 910 /StemV 80 /FontFile2 14 0 R >> endobj 11 0 obj << /Length 305 /Filter /FlateDecode >> stream x]n0~ CMBH$f8tQiCj8޾OJ=Y,١96քՏ{ciyWc̹6*ܭWCX<ۏUŲb > endobj 13 0 obj << /Length 302 /Filter /FlateDecode >> stream x]Mk0 >v; !Ц-8!~mC#}%eusl ٫U jx 2smTG鯆α,zy 04˒eo6?^x`ًGƸ9*}: %ײѱl¼?)5j\w \1_MdQ*TKqN\$&q./ )FQy:$>PԿ&}|$Dgʟ"KA^#. { ƻqxi=cntJ7nL endstream endobj 14 0 obj << /Length 16 0 R /Filter /FlateDecode /Length1 11508 >> stream xzkxS׵#%ɖ:?u,ǀe6l [,$$D^GـcAqG ߍN-h+P#VC;Ip } 9h/c_Eo( L$or[<3Qǧ;Z7nhq7_FWvjQ#VWUVZ|Yͺ ?/לmJ3uڄ^Vq,C0*7;9[f.R@B 9onEL]RmwHJIiQjTD7 uf2ㅶLc"=}r^PMi%7#w;:?Cp:74I-[e@Ü*3[߱ [lwOP XK{POY$,2馜Nr-pwa66l^c||+@< ҩ0?gfBC> shukd\ Um-7l`/ ԙӒys_D]w¸ܴq*7G"QF4ULelvzaÊV$ѵ<@oܼZ rW"jMk@2[3eZ$|`cgTZLjtL{B%J(B5mDg^SBP)4 $B-MRhYJC10AcGelk0Q1f9ּU)xiɞq]t$n}\TjdV}Vjf˦T/2@4P2Gg*곘l}^>suP@V^K.<~+\q#y<Γ?Yg;o~z="ZD HdgrNt q^JKԩP;l-mevr+)g1^l_ $K"064f &aMejjUMfKQͼqzxVV祲QHFRǻb Xx[e9%+Xo*fU혡!zېĕh01)%'jq-eYfWvKaJE gػ f0AYn`Z0a}bm1alXf3te̖ٖz, x5ātzClȜdC ߝlyZRjykM1 _Y 9Fk2߽˼i[?gvke֮\[X&dlYJx.` L-GBR<.cKYFR'XVG e~'yxbR5Sq}NYYSMF!ph^ܑBV}lYiR=_uNx/g?ZG0yh3?#c]yNɍq43t\gR*s:x]|iLxNrtIxeD-el}t֒aY@b*R9#Ldltv>Z@^m\3sjK\IEXr}7lin3smBP=HrY6V?&#$YD\ԃʘbm|6TS%\I c*'G98&Ch 5 rҀpsk8@ rH\]NN** xf%PJꉦ `Y)ZVpa+] rvteФ`J;ޠJZ!Pa&%OsCU[-=ai+%+{WLRO݊ esz3ZrjQS~:7fe![ |hmw)}7w3oܬDoImaVHޗ4fĊ ( hi{<ȃ oH [f&+S|u O7Ⱥ㩓c^gu.~w_pKڬ{–u;V[:ۚVec{Z,FɿִEWtؔ/߶ȱ+ڻJ* tmM{ķk+W\ x?b1+@J>Ǿ̲Ź)RHJ\| s #N ıjn, kU8G In&k4`]vljݖi3Σ\o.'ۓѴLV.'?;25_=?>tmC"]Kh2ã72XpL ZSN23L>aB"4A͘fF!Nӭ@Pk4;!T=(B-0oN2t^"Y2uq=45*,Nʔkx nD!aĿ_$9kv޷b߻S6K?StM.,]vͤ3k01>2y>A2$wXZJVfT[+ 84KQ} [>Zv[;HJ"F\̍#]W/n{vǵk ~|liok%WDVҥ[m8 g-ΊlD&ȵ~bpT ܔ\~KOͿiN [0m;~>ry߾i߾z}󁾊㎿c.W˶8Z#RըpP %/!SM!Ѹ6GbLra5DפksJsqr؅J.No0ieEK#eX:^,em|{WO8k[M;JkI's;)uN ==ߣq` {KLLR=6 dPp.]"86#'Ǎj%U 1FiS(cȜQɈ$0NWi1N3aeF@N3VÅJ #'c-+Ws QʑJJ.V@ p0z[)IqHp툒Wjj$c֮Xr8hz ØW~ya!YWy( L9 6d`.#F*A>tP'D s ~ 3c c;y3f}cP71>YĬ;ccҗ%}~RBK¤pB`ҙN2jVGl]t6=uq V㉜y$V[wgMO6^В d⠅sO5^"O{5n)iw\~ýHsG>cV_\=|?}ԥ";x7:۬2s uG]5sQ͟y6| +y `Ch31:O ob R%Ly&*"0c4@eo#\%d0wos˹*Mc\1UHBΨ 1PMv/ڍ!0Aj @0u49ȓǣ WApyQn±`]Xq( ǣ~> '16MGa(LPS, Ea-aGa/Fa5?Ga *dGa-agp,Z~( >Pm 8: EBYiJaOpy%BPUXc_7g6߰y ^!@Շ@6_ zHv-?ojk 9%h ?G(TzQ)K7`A8u{z $Oۅae?YȌ[;!F=4;Ю[ߊ2򌬘3K۔~"^ =~)޶*~e~C?@Gg?JӚ[n˾H0_xm4+㘕71ױp.4Yhb5uuU|3WO?2NX3郙+3Wgiƾ9L3J?L̴7iaQnһi5]mO|.4_cZϟO,s95 w3'gY<7F㵫1cLdyZ,333++d˯ˎx/STԥSp(tuMHiߑ#G''ؓ8816A&' |p0stCLMCrSNkKL .\f2u3,eL[Z--Dj)_Z HnL.3N|yIƜ8,Mm2m #l2i.>-ڴڀvB{E;U;vU@89|ONnXΫ74j6һ)ȨsKOt$ 3N5Px^8 QS) bp.8zo8 Y"A( @>Ci[!7Bǟ endstream endobj 15 0 obj << /Length 17 0 R /Filter /FlateDecode /Length1 11236 >> stream xyxSו=~Zz- ۲-Y-Ho [&e,ABh&BB ]%M@"C8@'o4v2[N~|{}5Igfgog9s9sKL 2EX" E&B!97!vcaJ&v|#!\!J~辡5FOeF+O KOQ*)f%̇#8ƓhQ%؊أ$!c}$Jb(yy?ODʭ'}GFbh;!ɴGAv"N#3@^N> 5}-HuF%S/=,T3EKV,a|F3~vTX%"cOo|īT#cU=6߷?.[{yEҺ-K{{e]{斮Zrl|Wспe ׹RD:10U8*XfZ2=}k=_tqM _@ <(-kM&;hjlX6˄ x]r ҚmUV5fk|";w.&у}Yu}?u\=SUˋd8XlBʎU6-e|ԱI@amXqO>_)y宾&|Vў jp~6uVڲKJ6?JP7њ]UK3u䖙nMK66X7;0  /Ybjq6Cφ0'&^a3W0+2]mk"րm,_'dE\Q)C|) 2pBnaW$SfvpCUNO >. AR\QON辭cu [4f5lƭݏֻpKHyAE,(0ukl5n73^4*I%+5djk9`O9,MNtzNkcͩ͋F \e&n10H.A\jтZ!x7^9^/ފr!^C|J¥pywD?Pށgq*wy^iĖ\W?7VeJl2Bl7;yIP3j#ՠ){3C!ބFQü:h՝]q:]a.)# s(bϷ,h)'zuy6#} cI#5;L^kU饭o_?߽ z Bo-K폭\m psHqJfҢd'Uh@C2\ 5͎ c<2w.~W9(6ɧzzBhJY#|yϾ~7I*~$OZ_36~" +M}#;kX,/;b0([ĆQQfxtrotL\hji(ň F"$Fbbgd<ޒ 4IT]L1|1uSt2NIeeoP_P 9OD'82.vumDt=!C[F[;G-Hv#ۀr#L"W/%»I@]//(_1c)KIȲS#n蔹_ yq Fld}wpȲ-)1Ӟ߅R`Phf_0O4F'1JcxjWsLO1Gۆel]Kʐnie24&d?F9߉/e|O{h\^C{1/ǀNѻЙowQxZ!y&CGeoԝ#8#ݩiڽ,r=/ƚd7x;"i%n^y8kp&\ H\O.'7؍7ݸ|' ץm'O]WֿE*_ֿ^vktͷ"p-cokv| }a | Տk~Vdk⭩XivbGclfe@xuf2Ͽj΂ 5Vi._qU''3$;30/|9]xl,=;gZϜ>s }*VM<pY%5nÛ@.}ov!*:<~?»>ar 85~}$`-?"a@-FKt4c#c )=Wx5N2 z&UxKE%q|!6wtr>["3c0P?vߎwA *h ~ @DP]XgA%U}[*֭-ΪqzJ;Ƃֳ̉Hv\AgBGtvFjZ ]m n,T nՆ?r;J9 /gBjkK@јx 3sevnCVP/l xV!& i7+)30 tuz> endobj 2 0 obj << /XObject << /Im1 1 0 R >> /ProcSet [ /PDF ] >> endobj 6 0 obj << /Type /Pages /Count 1 /Kids [3 0 R] >> endobj 18 0 obj << /Type /Catalog /Pages 6 0 R >> endobj 19 0 obj << /Producer (pdfTeX-1.40.16) /Creator (TeX) /CreationDate (D:20160720164946+01'00') /ModDate (D:20160720164946+01'00') /Trapped /False /PTEX.Fullbanner (This is pdfTeX, Version 3.14159265-2.6-1.40.16 (TeX Live 2015/Debian) kpathsea version 6.2.1) >> endobj xref 0 20 0000000000 65535 f 0000000250 00000 n 0000021747 00000 n 0000000133 00000 n 0000000015 00000 n 0000021686 00000 n 0000021812 00000 n 0000004853 00000 n 0000005023 00000 n 0000005262 00000 n 0000005504 00000 n 0000005706 00000 n 0000006084 00000 n 0000006298 00000 n 0000006673 00000 n 0000014287 00000 n 0000021644 00000 n 0000021665 00000 n 0000021869 00000 n 0000021919 00000 n trailer << /Size 20 /Root 18 0 R /Info 19 0 R /ID [ ] >> startxref 22186 %%EOF irace/vignettes/examples.Rdata0000644000176200001440000000073014746200372016145 0ustar liggesusersTN18!H\z R8l Pc@'d6k.#zOCqP ^<`LovyǛԡQ#PoZG?6@XPt\)ZOp,W+^.o `$LGZr1~T<)vxwOp+R)ٔc}PdSXʋ3fePJۡ qRU\zTCؤ"AТjh<m $ (']ҫ[@OBNL1# T֏mD*ˋt})LU'fV13̾'pf˥({)6o+~\0]'*irace/vignettes/irace-package.bib0000644000176200001440000004253714745735066016533 0ustar liggesusers @preamble{{\providecommand{\MaxMinAntSystem}{{$\cal MAX$--$\cal MIN$} {Ant} {System}} } # {\providecommand{\rpackage}[1]{{#1}} } # {\providecommand{\softwarepackage}[1]{{#1}} } # {\providecommand{\proglang}[1]{{#1}} }} @string{and = { and }} @string{lncs = {Lecture Notes in Computer Science}} @string{add-berlin = { Berlin, Germany }} @string{add-cham = { Cham, Switzerland }} @string{add-heidelberg = { Heidelberg }} @string{add-ny = { New York, NY }} @string{aaaip-pub = {{AAAI} Press}} @string{acm-pub = {ACM Press}} @string{ieeep = {IEEE Press}} @string{ieeep-ad = {Piscataway, NJ}} @string{springer = {Springer}} @string{iridia = {IRIDIA, Universit{\'e} Libre de Bruxelles, Belgium}} @string{proc_of = {Proceedings of }} @string{proc_of_the = proc_of # { the }} @string{aaai = proc_of_the # {{AAAI} Conference on Artificial Intelligence}} @string{cec = { Congress on Evolutionary Computation (CEC}} @string{cec2006 = proc_of_the # {2006} # cec # { 2006)}} @string{gecco = proc_of_the # {Genetic and Evolutionary Computation Conference, GECCO }} @string{gecco2013 = gecco # {2013}} @string{icml = { International Conference on Machine Learning, {ICML} }} @string{icml2014 = proc_of_the # {31st} # icml # {2014}} @string{acm-cs = {{ACM} Computing Surveys}} @string{cor = {Computers \& Operations Research}} @string{jair = {Journal of Artificial Intelligence Research}} @string{joh = {Journal of Heuristics}} @string{orp = {Operations Research Perspectives}} @string{telo = {ACM Transactions on Evolutionary Learning and Optimization}} @string{alba_e = { Alba, Enrique }} @string{balaprakash = { Prasanna Balaprakash }} @string{bartz-beielstein = { Thomas Bartz-Beielstein }} @string{battiti = { Roberto Battiti }} @string{biedenkapp = { Biedenkapp, Andr{\'e} }} @string{birattari = { Mauro Birattari }} @string{blum = { Christian Blum }} @string{branke = { J{\"u}rgen Branke }} @string{chiarandini = { Marco Chiarandini }} @string{desouza = { Marcelo {De Souza} }} @string{dubois-lacoste = { J{\'e}r{\'e}mie Dubois-Lacoste }} @string{fawcett = { Chris Fawcett }} @string{fonseca = { Carlos M. Fonseca }} @string{hamadi = { Youssef Hamadi }} @string{hoos = { Holger H. Hoos }} @string{hutter = { Frank Hutter }} @string{lau_hc = { Hoong Chuin Lau }} @string{leyton-brown = { Kevin Leyton-Brown }} @string{lindauer_m = { Marius Thomas Lindauer }} @string{lopez-ibanez = { Manuel L{\'o}pez-Ib{\'a}{\~n}ez }} @string{mcgeoch_cc = { Catherine C. McGeoch }} @string{montesdeoca = { Marco A. {Montes de Oca} }} @string{paquete = { Lu{\'i}s Paquete }} @string{pardalos = { Panos M. Pardalos }} @string{perez_l = { P{\'e}rez C{\'a}ceres, Leslie}} @string{preuss_m = { Mike Preuss }} @string{ritt = { Marcus Ritt}} @string{schneider_m = { Marius Schneider }} @string{schoenauer = { Marc Schoenauer }} @string{stuetzle = { Thomas St{\"u}tzle }} @string{yuan_z = { Zhi Yuan }} @article{DesRitLopPer2021acviz, author = desouza # and # ritt # and # lopez-ibanez # and # perez_l, title = {{\softwarepackage{ACVIZ}}: A Tool for the Visual Analysis of the Configuration of Algorithms with {\rpackage{irace}}}, journal = orp, year = 2021, doi = {10.1016/j.orp.2021.100186}, supplement = {https://zenodo.org/record/4714582}, abstract = {This paper introduces acviz, a tool that helps to analyze the automatic configuration of algorithms with irace. It provides a visual representation of the configuration process, allowing users to extract useful information, e.g. how the configurations evolve over time. When test data is available, acviz also shows the performance of each configuration on the test instances. Using this visualization, users can analyze and compare the quality of the resulting configurations and observe the performance differences on training and test instances.}, volume = 8, pages = 100186 } @article{FawHoos2015ablation, title = {Analysing Differences Between Algorithm Configurations through Ablation}, author = fawcett # and # hoos, journal = joh, pages = {431--458}, volume = 22, number = 4, year = 2016 } @article{HutHooLeyStu2009jair, author = hutter # and # hoos # and # leyton-brown # and # stuetzle, title = {{\softwarepackage{ParamILS}:} An Automatic Algorithm Configuration Framework}, journal = jair, year = 2009, volume = 36, pages = {267--306}, month = oct, doi = {10.1613/jair.2861} } @article{LopBraPaq2021telo, author = lopez-ibanez # and # branke # and # paquete, title = {Reproducibility in Evolutionary Computation}, journal = telo, year = 2021, volume = 1, number = 4, pages = {1--21}, doi = {10.1145/3466624}, epub = {https://arxiv.org/abs/2102.03380}, abstract = {Experimental studies are prevalent in Evolutionary Computation (EC), and concerns about the reproducibility and replicability of such studies have increased in recent times, reflecting similar concerns in other scientific fields. In this article, we suggest a classification of different types of reproducibility that refines the badge system of the Association of Computing Machinery (ACM) adopted by TELO. We discuss, within the context of EC, the different types of reproducibility as well as the concepts of artifact and measurement, which are crucial for claiming reproducibility. We identify cultural and technical obstacles to reproducibility in the EC field. Finally, we provide guidelines and suggest tools that may help to overcome some of these reproducibility obstacles.}, keywords = {Evolutionary Computation, Reproducibility, Empirical study, Benchmarking} } @article{LopDubPerStuBir2016irace, author = lopez-ibanez # and # dubois-lacoste # and # perez_l # and # stuetzle # and # birattari, title = {The {\rpackage{irace}} Package: Iterated Racing for Automatic Algorithm Configuration}, journal = orp, year = 2016, supplement = {http://iridia.ulb.ac.be/supp/IridiaSupp2016-003/}, doi = {10.1016/j.orp.2016.09.002}, volume = 3, pages = {43--58} } @article{McG1992vrt, author = mcgeoch_cc, title = {Analyzing Algorithms by Simulation: Variance Reduction Techniques and Simulation Speedups}, abstract = {Although experimental studies have been widely applied to the investigation of algorithm performance, very little attention has been given to experimental method in this area. This is unfortunate, since much can be done to improve the quality of the data obtained; often, much improvement may be needed for the data to be useful. This paper gives a tutorial discussion of two aspects of good experimental technique: the use of variance reduction techniques and simulation speedups in algorithm studies. In an illustrative study, application of variance reduction techniques produces a decrease in variance by a factor 1000 in one case, giving a dramatic improvement in the precision of experimental results. Furthermore, the complexity of the simulation program is improved from $\Theta(m n/H_n)$ to $\Theta(m + n \log n)$ (where $m$ is typically much larger than $n$), giving a much faster simulation program and therefore more data per unit of computation time. The general application of variance reduction techniques is also discussed for a variety of algorithm problem domains.}, volume = 24, doi = {10.1145/130844.130853}, number = 2, journal = acm-cs, year = 1992, keywords = {experimental analysis of algorithms, move-to-front rule, self-organizing sequential search, statistical analysis of algorithms, transpose rule, variance reduction techniques}, pages = {195--212} } @article{SouRitLop2021cap, author = desouza # and # ritt # and # lopez-ibanez, title = {Capping Methods for the Automatic Configuration of Optimization Algorithms}, journal = cor, doi = {10.1016/j.cor.2021.105615}, year = 2022, volume = 139, pages = 105615, supplement = {https://github.com/souzamarcelo/supp-cor-capopt}, abstract = {Automatic configuration techniques are widely and successfully used to find good parameter settings for optimization algorithms. Configuration is costly, because it is necessary to evaluate many configurations on different instances. For decision problems, when the objective is to minimize the running time of the algorithm, many configurators implement capping methods to discard poor configurations early. Such methods are not directly applicable to optimization problems, when the objective is to optimize the cost of the best solution found, given a predefined running time limit. We propose new capping methods for the automatic configuration of optimization algorithms. They use the previous executions to determine a performance envelope, which is used to evaluate new executions and cap those that do not satisfy the envelope conditions. We integrate the capping methods into the irace configurator and evaluate them on different optimization scenarios. Our results show that the proposed methods can save from about 5\% to 78\% of the configuration effort, while finding configurations of the same quality. Based on the computational analysis, we identify two conservative and two aggressive methods, that save an average of about 20\% and 45\% of the configuration effort, respectively. We also provide evidence that capping can help to better use the available budget in scenarios with a configuration time limit.} } @inproceedings{BieLinEggFraFawHoo2017, author = biedenkapp # and # lindauer_m # and # {Eggensperger, Katharina} # and # hutter # and # fawcett # and # hoos, title = {Efficient Parameter Importance Analysis via Ablation with Surrogates}, crossref = {AAAI2017}, doi = {10.1609/aaai.v31i1.10657} } @incollection{BirYuaBal2010:emaoa, author = birattari # and # yuan_z # and # balaprakash # and # stuetzle, title = {{F}-Race and Iterated {F}-Race: An Overview}, pages = {311--336}, crossref = {BarChiPaqPre2010emaoa}, keywords = {F-race, iterated F-race, irace, tuning}, doi = {10.1007/978-3-642-02538-9_13} } @incollection{FonPaqLop06:hypervolume, author = fonseca # and # paquete # and # lopez-ibanez, title = {An improved dimension-\hspace{0pt}sweep algorithm for the hypervolume indicator}, crossref = {CEC2006}, pages = {1157--1163}, doi = {10.1109/CEC.2006.1688440}, pdf = {FonPaqLop06-hypervolume.pdf}, abstract = {This paper presents a recursive, dimension-sweep algorithm for computing the hypervolume indicator of the quality of a set of $n$ non-dominated points in $d>2$ dimensions. It improves upon the existing HSO (Hypervolume by Slicing Objectives) algorithm by pruning the recursion tree to avoid repeated dominance checks and the recalculation of partial hypervolumes. Additionally, it incorporates a recent result for the three-dimensional special case. The proposed algorithm achieves $O(n^{d-2} \log n)$ time and linear space complexity in the worst-case, but experimental results show that the pruning techniques used may reduce the time complexity exponent even further.} } @incollection{HutHooLey2013lion, author = hutter # and # hoos # and # leyton-brown, title = {Identifying Key Algorithm Parameters and Instance Features using Forward Selection}, crossref = {LION2013}, pages = {364--381}, doi = {10.1007/978-3-642-44973-4_40}, keywords = {parameter importance} } @inproceedings{HutHooLey2014icml, author = hutter # and # hoos # and # leyton-brown, title = {An Efficient Approach for Assessing Hyperparameter Importance}, crossref = {ICML2014}, pages = {754--762}, url = {https://proceedings.mlr.press/v32/hutter14.html}, keywords = {fANOVA, parameter importance} } @techreport{IRIDIA-2004-001, author = birattari, title = {On the Estimation of the Expected Performance of a Metaheuristic on a Class of Instances. How Many Instances, How Many Runs?}, institution = iridia, year = 2004, number = {TR/IRIDIA/2004-001} } @techreport{LopDubStu2011irace, author = lopez-ibanez # and # dubois-lacoste # and # stuetzle # and # birattari, title = {The {\rpackage{irace}} package, Iterated Race for Automatic Algorithm Configuration}, institution = iridia, year = 2011, number = {TR/IRIDIA/2011-004}, url = {http://iridia.ulb.ac.be/IridiaTrSeries/link/IridiaTr2011-004.pdf}, note = {Published in } # orp # {~\cite{LopDubPerStuBir2016irace}} } @incollection{PerLopHooStu2017:lion, author = perez_l # and # lopez-ibanez # and # hoos # and # stuetzle, title = {An Experimental Study of Adaptive Capping in {\rpackage{irace}}}, crossref = {LION2017}, pages = {235--250}, pdf = {PerLopHooStu2017lion.pdf}, doi = {10.1007/978-3-319-69404-7_17}, supplement = {http://iridia.ulb.ac.be/supp/IridiaSupp2016-007/} } @incollection{SchHoo2012quanti, title = {Quantifying Homogeneity of Instance Sets for Algorithm Configuration}, crossref = {LION2012}, keywords = {Quantifying Homogeneity; Empirical Analysis; Parameter Optimization; Algorithm Configuration}, author = schneider_m # and # hoos, pages = {190--204}, doi = {10.1007/978-3-642-34413-8_14} } @incollection{YuaStuMonLauBir13, author = yuan_z # and # montesdeoca # and # stuetzle # and # lau_hc # and # birattari, title = {An Analysis of Post-selection in Automatic Configuration}, pages = {1557--1564}, crossref = {GECCO2013} } @book{AAAI2017, booktitle = aaai, editor = {Satinder P. Singh and Shaul Markovitch}, title = {Proceedings of the Thirty-First {AAAI} Conference on Artificial Intelligence, February 4-9, 2017, San Francisco, California, {USA}}, year = 2017, month = feb, publisher = aaaip-pub } @book{BarChiPaqPre2010emaoa, title = {Experimental Methods for the Analysis of Optimization Algorithms}, booktitle = {Experimental Methods for the Analysis of Optimization Algorithms}, publisher = springer, address = add-berlin, year = 2010, editor = bartz-beielstein # and # chiarandini # and # paquete # and # preuss_m } @proceedings{CEC2006, key = {IEEE CEC}, title = {Proceedings of the 2006 Congress on Evolutionary Computation (CEC 2006)}, booktitle = cec2006, year = 2006, month = jul, publisher = ieeep, address = ieeep-ad } @book{GECCO2013, title = {Genetic and Evolutionary Computation Conference, GECCO 2013, Proceedings, Amsterdam, The Netherlands, July 6-10, 2013}, booktitle = gecco2013, editor = blum # and # alba_e, year = 2013, publisher = acm-pub, address = add-ny, isbn = {978-1-4503-1963-8} } @proceedings{ICML2014, editor = {Xing, Eric P. and Jebara, Tony}, title = {Proceedings of the 31st International Conference on Machine Learning, {ICML} 2014, Beijing, China, 21-26 June 2014}, booktitle = icml2014, volume = 32, year = 2014, url = {http://jmlr.org/proceedings/papers/v32/} } @book{LION2012, title = {6th International Conference, LION 6, Paris, France, January 16-20, 2012. Selected Papers}, booktitle = {Learning and Intelligent Optimization, 6th International Conference, LION 6}, year = 2012, series = lncs, publisher = springer, address = add-heidelberg, editor = hamadi # and # schoenauer, volume = 7219 } @book{LION2013, title = {7th International Conference, LION 7, Catania, Italy, January 7-11, 2013. Selected Papers}, booktitle = {Learning and Intelligent Optimization, 7th International Conference, LION 7}, year = 2013, series = lncs, editor = pardalos # and # {G. Nicosia}, volume = 7997, publisher = springer, address = add-heidelberg } @book{LION2017, title = {11th International Conference, LION 11, Nizhny Novgorod, Russia, June 19-21, 2017, Revised Selected Papers}, booktitle = {Learning and Intelligent Optimization, 11th International Conference, LION 11}, year = 2017, editor = battiti # and # {Dmitri E. Kvasov} # and # {Yaroslav D. Sergeyev}, volume = 10556, series = lncs, publisher = springer, address = add-cham } irace/vignettes/Warning-icon.png0000644000176200001440000012750114157104332016414 0ustar liggesusersPNG  IHDRXPStIME""NbKGDIDATxx4z/Rދ ҋޥ "vD+Ҥ(* (H]AEQHoIH=g7ݙI{}J ٙ7!H,n$ʶ&[J -LW2DH5L4z+rS0?N?6^Ex^DbMAX5BZ^HQ|GQ "yUfRī A,ED jZ-hf*"x~Nϼ1DLa9.':*;7= =DiE|aSc=QέD}, o[BL"bh\@T-2@ZH 3b+Q+ïrq dG> 0 o F46^dgVWҍD-Ƌ,c5 7'@#?Ǖ__%-&L25Dq)DYYH2D!=U*t1uc>߻#>ݓ/̧:ߪWo3gTWRU}@d$  ARBKڰ #j:3Od+[qe1Ѭ0y$o(3K&>ףǰ{"&{dG&Uy$ "^rdي(["A[f.#eXl8g WW;ҤOuLp\ICyVWqXJ#2MՑ+s:`%q%]Kt$Q *#⪂Uqùĕo5*%86>ψ>%,8"cE}*򙮭V3kf%q0DmMK޺U#hʭ_*VWi-A3!*oY@ACDUqMt|y`D , )U,~5*+L#LիM%Z\I/[SxVF\mp _@T$-QPW?$QpjJpW{c[vo%:l=(ڥWvIнvȦ$B$1ψ(rkPB>6U|k6aY:=oΕD'`Yyq-ۈ( SWf輍8ss#-|g,>?.$ #t8r`/U^h9W1w-^wiR+VQaD UվBmm F|4 ,G^h b!"#j[HmanjbO8|]RWq:F1 IT^UP{7<սy-,-yS`-uc9*DV~Ip;jW,:5k29mǸ2|)%qe8h2ofzVɶM̱؟x+xktJJ(%Qc*NxR|\VW?/lo}ٗ0+tHo>ҨzP\.v~O*C8t"NSž-ՍqZl7UZ9)\yVWo 1 2T8 *W|իrw&<ޱ]W֙nmxcP-A3WGT$ R^\UW u+C9$i`Jjm&L8(ڮa|ӓ3j(;~UY`!+/<1v㭙2(DOV a 8 ʭel\\Y\XKY h &~I*Vnq ,*vM ո w+c_W̧$ &z$ ST^1[1{ÇFCW`=`%q%]ItQ *#ДmDDŨ=䲵5D+%5{իFE%q&ιDVaf! QW`{` `+ϱ5Xvlk0~`]ԝgɬ$E%joX%A8J4H/Zڰ6C+WN[6JPXW_BT$hz"o2˃[Kby_~_W{PW 8" Q4Oձ?8cЧ<{^ycp%ѵjDuL"H1CU\?; %y\,>xoc\zyDK|=LOA,eAA|G;quA qc>KxϑMI\ Q7*MxD#Tn Vի[7%[\FX1mWU08B?cQET̙b5nM_3O?q؄*.?,$ C[p~T^Xl1c'JvqqzexwGޘ&uBJVQaVLXUXߋ,L\NW׫$uG,'J-/Uq5$?r\x륄?9c޽|g,VwΩ$t K!8r^*תb}>v+?Q+WvZ}qt)DL"+?\GJTHQڕ=ngMٸ~p6-ӭ%_2F4̷ 3|GTŕdfGf/}:۷q}=ƕBN)}:%qRN &&U0"ڪ + /~X%q\޽w>1 hWaeĕ|4LW‘ ;2H:*Nu>>5{Wg+-V7ɭ$ _'jXepDz5QU\IWs}9Wq]/6xN-yMP-A3&:Id $3[JP*{(m9nϝwqŻw3|o`XFI\"C[pdPFU\m ӝWF=ÎCe\Icn+rC\]^ۙ7e̠$ޣ:=iX%8d@DQaes#׬۸]wO渍+#6yD{ܬb$1;`*Mq~WN+]*WI\MI`Ƒ$!"fMqiT=㘽{/6+Z&W3]Z`[+L\JtQM*#H"DU*v?`giW7;v1X]Z^I\!O[Yp@M?́|[ c#Jز 1{JJLr/&UJ#HDD=զxTUƵŕ9rG[KzE}j ~At Id$rn%*vdHոxߏ+V +޶G]r"ĕD|=-@^}`|6{`8vZo1΁n:ۭ MJI\-w _JT$(f+QPUq<|qt=yիK\RZDK|= ": >׳帺 єJذ 1[rdSWq>M$J  "jZի6iG~oՈWxW\DGz &Kv!u-Aod=-խKqqexQĕ$ dU D0-UWYl1c"7neK]Jz{[^&uBW\7 b;ʸH[[KZ+#4+ްyz?ρշCu)+ֹYʁw`DiDLvpKHr\x%WDฒ:֮+z+mr*83 2SzSU\I]ճ}8)>[Qrb`h۔W$(+N lYN,o wfʡCzY۳"׮ub?-$ M% x^U\X7>`vrPm\ɰ5k8vtOӸXzu&dK.݂I`' Q^e\/Y`{%Wvzg$Wtrĕ%DQZVIa Q9{+eKl_qe1xre8b$quNlTGI\~J*xGH "b겷VҥKYcW6ҥ|!q/8)VUŕhۇ}? /^1s?KrC\IOwjS(+"UDVz\CU\}_ ;_|rJU/Z7?uWzWp6d><ڲ(DojcPkqu}HWHy+V+[q%u̟ϡr)q%gUWq:&iXxuj zE[se*⊿yBǸ2<ڢ`{KD'`Y" oyjW;d=-֭iC\%r\9]|R`]՞w/$_ %H[N2Uq%=Ѻq}$+iGs\Jzs+^&Wae՗uV1aލt* ,Z`{W WIW<r4 =/(+9Dݬb;߳(cjs>ߧ帺KB\%Y\3)_+WrPWÈVaޙ}-[=V`{>sVԔI|wGq%=޲hkŮU1GqUTT&G6r`ݞ5%W7F>g).l{JSWVaFC%"6+[C!WW>wGq؋<^l{˕TW>&xY(@.Uq1 Owne}g! qq%꾸9s8v ۸rֹxu jkn&URw0d3SJlZ̿JqexghӭA#voˇRWfa"*K˗vWaA\x\2ϚWWm(+7^&4x')D\}*G׵0a8滝+_ٳ=mB umS^uX' -AT$ V cUzwys tOJeqN%ZD[Y,! gEj\]u;d<+#W>Wc7/ʁw u}3^*DI\I/5AH'BͼU|gkk΂Oĕ +Uĕi֋=ƕ8? ZANd' !uXU\m s:XlWL!>xTvL̗uWs][YUDL0xITl Z9ݸqI\)N#njXE`i8/V+BT$ B_mUq=kfr`ݚnƕ ++V>WN'OkC~[2O?2*Lw>^Z*'4>>bqa\s̄wwxWiR+;߮G$$b\v*{5x>JӸ2@J^ !ngAF\}NXZ'-Ur\?a2x$/f)ʡ$ !hXe!8P&Uŕ:l Goل={i=eW维͟E pDUL"Q (CQ1WjGl:=s2*ĕ\*.JQW Œ8#P[ZXk#CNz0#IJ;~_c\МK-Ώ3D3f&U0[6uVWJ5dƕVJ*?cb#׮*qw9|pqexJpN7Xpf`Vet-AiR]-Uظ\ak\ɏMṄV4+0yxK\Jad08CH"&+^=חco3;} M;8k\V|EC^hkj?%:_Idm6*vikk0|t=JY)r5K=#Jr:~<}\tOJDVaf)XD@}>-._z2t+s8Qu}DZcnV\[%xnO$J q` J}Uŕl^E-_bW2uENTu kWN~0C\KzlCʛD8cV"B4ܒ>_`oW;!K\Q\9}-U|7{DI\L [DHQW#Z~c~q%J*k q%JE\4՗X\|e#^l+>w۽帺0vȈ-Ǔvq%u{W:=x_%uۆDM0g^=*zuw2JHT(&G<܃^qYmĕVIW[y`b#_;J(>ߥǸݪ1/Ng&#Qq*Jx4DO\JދJ>ƒ)(:s`utcJp* 7XqfT"vjm냃\_)J~;^}/1ulg;4ٲ(\~DM0gHzEU\IYճ}UW2+髯rLux~@WaWNIGT$ )G vKp[|yp/ˁukʻŕ?'q5Q\9}&=.)$ G[pUJzM#q쮝ŕ}C}W" cFzNI\4էzD MaRՓ*bl G,TZeiw5.XU\/3O*؝2UO#f+'8XCND o 7 };[oOXS\ɰrW<]\Q/ȗW.ٕĕ3DOVa<9"ƫX2nu[WDD :UQ5WRWS9D֣8l *.*RU\mϑCX?_\IeB>G~7Fq%WxHާǸ:7 ++D3|0@VUq. Oոv|K~fq%}~8_tk0~`jӄJ6A[KS&U0Z^uv*?.fk=rǕVWR?D~x(c\YYI\O'f+7DU&9Uq)Mп {Jk*?svq#F8ܭǸ2\C^W=oXeqF}k:Eq%=\իrŕQQEC[M %q%|&Ug$MѪվӼ+VWK27k|/FqÇ;Twqu]3JSW#,83D|K+/=Pŕ8tJ|SKX4%ۚr⋚DmL0g(իJJFSҶV./~:! RkWǪ)+7Gp?d9QDWTզŕA֒  ĕV>WNCW>$>%W|= TWsUŕ&[~usǕ +emfq%~z kR`k>/rJ68S!(7q&=bkkܙŕi⊟yyPc\Iϴmߕ-$ G%g D"AA|G;quavJ~)C'dX%S\I [z+̙% v#bXA8s#"j-AUZ|~q%Y>/pVqC8)qexJpo!DL"+/\ì"zDTڒ)_~/,q(&,,+>&'O'KXoppsphVqTVdJzZ^6X"~\HjK,ɗ彮_ŕӧtX ٔĕDM0tĈzGU\Iխno}jJ<>dw3dWqeXzu;/_#_+#W}8+gw q ?ywN3N&jYp]*!)JE\n˙CX_\!pJcXY_ƕa6m̍W:zeXA%qe,2*̄3$"Xq& Ov|r\]{q;١_\?Fl|~WU,,woكϵm1r}a ?$lNI`:nJC#}_\\w~X7%ݷo wxa!W/=v[UcZE%qe8;7XqDd=Q&QTib*l+)?hkg ʰ\r|]/,xo>[5Us&hKFVYaj 5ynիr̺5WFX%e\ɟ+gE*m _S{]}uc-swzW -YDI\#̷ L C("{o 򩮭z~_\ɏ&C^.Xދ!Y\qnۭ+oc\IO>ـȐ>AqMݖI`15PU\IW(mk*/+lٲVFqB|U\9ڕ;X[65+++éD?7GpFH!ꊪژ>_r\|goĕ +Uq%gq%wXiӦ(hNΜgKrڥ _nc\lkNЖ#^#83Wx;o?ֳ0a8vzJz69yqeG9;wY\XGOlLkʼnjDV ֋*ڙ?;s>/)Ngap|FWFXq\qNNomqpWJ|=ΐv: >ۣ帺0vȠ򗸒ae%ǜ@٣}`LkWRGǎ|uq%=d^)c3+QW*Mx#z^U\IT-ok/+7o&8 34cVq"CӮ:dCvU( @E\9J[z &gJ<"T9tpO_\-\/ v>ʐI(.f4+6<ƕyĕ+D|0Θ`}KaexT\,7WC` oGwq%۷mY\+uy^j?7=aXE8kj2+gWs[RB`iƻ)2y۵sֺի]¶*V9 RUq6$h9n>VF\aq%Q?ռys_+nۖ">Ϸll5?_fͬ$tWZ6W SW\X2*̀3)wj[ymS#Cݸ2WJyyy8{, ?j1-OzSMOӤ*fsQDML8aWmT՞ ~q`K0ܦ]l<ƕbi%qe8hUpF&"NuS񅾝~aK\͉5LDnjWN|/X ,ٔĕ@&UF gV@Gһ[kl_L?/""Q+SLZ/E#[,թx_ 5oDsU58R4kJ*=CXNLaq5>#XɓGzwԿhׯQ\9mޜh>KH%q7ţ>tU gX@D\DLvKpu@帺6r0;l/!H(QBյQC!CJg1x`k׮]",Q#mؐZ+ɦ 鼎1 Q{*) -"+Jel^E-qe,oDDX?S⿈<ǎ*Xĕ4qqes {D?7Gphz"z;oe!Cz4帺9uJ?'%Ǐ:~WplkKrڠ_jj!s\ Do{Z:^}*9DZkVWN>}ցuԩ{1Pn|B\Iz VWiDJ1"8b c"n ~?;W}2)c ğs`]IpFq'z',REĕXd> gd Zg{W_~2%ǟ X2z.z\Sq%=q^)a.DL0gfW#TŕTYx~q%֭d u99>noǕVE\Iԯ1 ֨3Tĕ'`Yyqf{\uy+Í3qֻcO?e>x^۸ʘ1cgϺ4+~qurzd=$ _$f0%JzU#p캵ŕs풉~Mʙ3gcB:u8|ʕWҿR؊nJtQ}*& Yw%jbKh~q%=w]vVp c\au_\կk!wLq%qe8y*2]ʁ35AF_jMH{帺WPWLXXUL}qAc|=ƕY2+68z%~ѪJK*6܇,/Op3!00PVZ'jW\ӈn XjVq{W>DL"+?`QaT[nΞ/e}}Ҹĉ+##͓O+d˖M/!x݄ŕZJ:WN6vѼJen 3 իuJh`AXNnlVB 2L|y,W"֬1kgHU:֠6M:![t ѝ:DML8aˈpjbKЈ% lCO/]b_RJZVw|ݻÆiWNkujy\:ٸ=J*qC*C8|%DDXVWkR Xl+t]א[i:}E>p@՝oPc\.ΞUI\'jXep&k&+z5 $+q%}XFz$⊫UZ5\1e!~]gŕtDUM"BUiQw2Pr`ݞ٩帺6r0;6/&S烌1B2e,J?eWDI4 |S4 @zSŖ޲% Ǖ +;q%\QƎe`͝;^lXõ+[{\:٨*VWq%Yʍ3; YAE%Uq6]Z8 {eJL>]R~.}w`iW\vTP1qeJp(ы&UVgx@!"i'Qz/ǬX_\C~3_]t UMWFXx\IVX3-'/4B8SU1[w{ܶç_\}1co!w+Vtz.-ժ&f0 $e QUZIWnmz2t+1;w2{O2,BT4|4x>3QG*% WUŕt_2V/俻~/СCZ1__=U:bq%Gi|ZeW\VU\dDBʛDVIz"A-qWc̙3ZEe_Q6Fq1-tW*2F4HիW)+?=a}}X?]Bwt֭[ ۷o'`,J]۶iW\4;ďg6X',c[VD$J q%$jm^]4_M "EhXOx( T1 T-Sĕ D'`Yp%(eQ^OTՆ,8tpOquk›G1mPE\Hu %K'An-Y,KbK{+yeJ8'f0=ex۰2JWFۇU+ƕ +gm`5pDWIWFX%b\IK j{N[g$8޿nW"oXń*FQ3qH[[Ws؟СVU_F?rsZ(\8ߨZc\9_-YTI\!Zf+' AFDQUqJ|yO'quVĕPsU`W#~4+Cx?ϒ)A[f oXe!BYU|>VUŕy~X#G*3U\F+c)VS;LT$ HPWg!,_\͙#3:0n8k [ WFX)+КULm-^%q5)ΑDlfĕ`RͬyvJ~<5`ƌZ(y?!MJ#~Ω'j]q%=V:J*a?&#51 ℁w͸2,jkk0ŕqBXoVTsϞZŕ?e++ñqTU\b,(3jM|_otEƕVvJv8Nu`XSL?n\ɰJ@\9?\nWd\IOȖEI\ŭbq_gaЩpXYzyWճ}8EŕZݻ q%eǛoWqF--s\Iը +#slnj&UW[DQuWU\mɕC>;ՇaׅOߵ /qI`^z|FqexzeKu≚@^%qe8h̸햠ኀ>ٹ帺6r0;V2ʈ+9/zֹs x%,%Kz5=ƕha4 |#jD$J q%UDUŕtoۣ>P?^e`kX:5 (vU\ުX?q:!Bl8+ñD{ݬb=+ _%*벪Z.-_r\yI=cE;&&Ʃ*U*m?x%9w2K2P+WsfWW^"4^RVյ~ϫr̒/+D߹u`=CֱcǴ+itt4OhWҨ2^2O 5oeEtbQM*+ j"bUնmm O_\͚1G8/p:VbŴ /jXѡMjWתVXҍUW-tUWR8KD0t ƍkW HvU\._c\9cRJm!DM"+0PVuU+#eLϗW7Ǿ_\ɭ={qeX:EVN̙3kXƱxW>d\`iWkUqX" uX1d>WR "櫊+ϱ_/.s'|zw,FEEq̄ ŕn ( G<58XI\M+H-IT$J@ C"jGL]\Is^RJ`KZVI) \a{7^ǸrZo-RPI\<gR{;oe"8`9+4u\X'N"~,a|iWRXGG&UZ+)ckpT9+#)ϑ)*>o~)&"#"8N2W\V*I*D#hYp@s1u-AY2q̙θJiD|>t֭[;F#vbiWjTWeJЖÉ&*L+#Bim`tvqfDxx -[hX={LQѓ'kWR%( g*<-$DI\D L0W!4d%QsqhA[3jW.F\XRPҘ+Èe<^*ĕt,9D֣" "vK87ơCzYﾥ]\9wJ%?qa)?)4 ŕ=ƕʼ+ q f0#LhMJz}ss1˾.b~vKʚ5y ,1>)e!ԪSJpXŏ+;DV ;~WTREmm F@9yyaUB }`Ǩq ~c5* *WNE,YTI\&*V.\Sety+UiR]+t>໛7۷XqT\kǎ)vJ*۵+ó5~5sM? :T!"WWCOԲu+u׮%J`kd5lKn"4紋+idRq!ez*ZO TWqDsU3Rg *'*-A͹sr]\I#w1Xbu/9q2n?0Jq ,yU,xVaf\#@.Uq, Ovie}}``2ʏ*F|lnbmB' x񢶫WvKߑY\IcK]2Ve6WqοېI`l QW˕55gvq%ӉXYFÔ=h֭78' ,e=ƕUywbhݬb=+>l".{;oe:}Z8 =J>o7o.@"+*ֻq% ߵKU,L%hf|= `ՇJzq]s~/,Z&dXH\Ŋֵk+%m?VlٴY&c[һ#GjXQJ:J[<)(PI\I& -LT$ Xo }`+_wѹ?M(?K: (b͓'QuRyqe@>%q%/|h¬{[A"~RWWmW_SUX&͛7u`w0Ka`Ŗ(佱<V>V2J֫2j -:V)ah+QW+7>CJO;/4,6}:{1l!~`]1kY˖W{˔PWoBT$^Auy+õ2{Xc_/Mo7nX,ۄYǎj޼ۃ 2yyG5 *z c\.~(2|htZbJTgۇ_hWq劥JۄW^ҥKX^*ŋ eZU,ŕ`[V|zYM(a$/"aPW 泵5(s0XHSoݻl>1s` 80EoZ gpl͚z ?HG5? WWXq C"갪Z&:;ӫrڪ_xT~WQ˗םYV+mf͚o^J̻?(|/UDBk-syX^rY dHu\GG^&UZ+ɳ5VKSJj@gZ* _-F=eƕCgMX`A z/W=hwʈ+鵫W9[7x#KླྀeX+W`I+Tĕ1D#hYp YB_DTZ53i?/|/ []sU\)|g.vJmBJ*m`Mwk{^`E?3.7quGxwT-KJ#{j>VǭA#ϕ.ɝ3A[LB@Z`|ib?5o\jt|ghwQ "bğ{ʕVRo*V 6>Ǔ^ݞ$|XssU? ?qk0~\IWCB8D MaAT qmBjh/Z!i\ЪG +#+i3.$IM*V6=>ֵ9n] "]sVFXX[d@Ji[Kh%qeF7X9q ӊ:*V }:ͫX^x'*rjg\%v`:޿ 7p{RlJMy;_z`|ᬼ#kn~p&˫W.+9)a3 *# Dd98o{N5W:qZe%7|&_;wr`%6?~X{n*ʕ+էOUL|"O6ko?,ĕX҃ Z%q%Mt2QzW@/TWrdС^~^\X7ZTI$k\Io `_&zw6:䗫W=h5+;ʕK ,V{ +wqB\IgῪWzuRWpUWBgj[=ޡHI\ݳh{c&OL?C/_v^0GVro*ևsᯁu)-ۓj{xH'MJ:?r/WYŕW#*ԩln+M$JPWUwܓdkm\yIZ4'Y\I9¡ &ꫯ6Uի,yi0ꕝ~X`oRWN_!f+(b!Q&VgU4BWVHܕ۸ V58zۉW'k wۄbm޼o^W;Wɵ=hV]Lm|RXNeɔ{ 7 Ը2fj2aekW+ClU*HxG3Q*Vгg#Ke`麊?e\e̘oVs]`P!="vUbĕ[`۲yzuuBJa>Cxs!*%W+OQG㗫{o7Tʫ\q, ψ^g?}'YQv7k׮ϴ+vAE_g} |:5v*r_rELؗ&WNuUSFDfx!2ULh~}+gl/WS]ȕk~Kv3KkLh)M'Xsέ+$X?>Թe-\m(Wv\|!rU-X^Pdh0E1Qԑե7Du+;??(~-l0di_ʗ`%ޝ%oKFlO #ԔY$>MN"WBTwR8pڊ+vQE_g{Ӵ+g6U;5MߊP k?V !p8W+ʱv-j|(2+JQ^IDĄ"(4kΝ,:+t, LG$ʋWKgD݋G|Bd$+pYPi%F V\C/2_rE2˟u }H^}^ܻ J 12QlMOXJbi$Mb%ɕT^V  N"LkE½w-ջrWc|wLYv52BTHvf,}6b4:ңܮ`QN̞iRgBv4Lf% !rfuw8u-Rv7ᗫͲxS.u+IrlMk.E^Pd]HQ!l6~- j^*k ]&6rFaЅree-'S\IFh+p"Wߊ+ʪ!ճuJgwULh)$+%%4rmJxYk' LۻF;S"r(OV32UJ;.:u>G9QrE)؆_& 7 Ct]]u,_8-XbB#LF4h`v|2>ا){Wr1L$W%»zұR"Bd$+ pR4XH?jcrAwX VLk,[//re䲻) ]t1`5ip+Orez%ܮ5t»R#Lٻr+ʖ |Ec镝MUr zK!*;/Rl&B_OA /WYwE^-K\KPm]QM#X:u2U4JxP,ʉ'fv(\Ik%Wfx~H~B=ew_8V^ %W%jH|IezWWjRb }52d}21ZnWJ|w[kFPkE|fؑ%D$~^)V*܁X|P F\ ?M/XݛZc43.W,eIJ)KT+ɚ?ik̘1}UnRjO g5+g؞n(:ߢ1~?6o03Y2՘ wb8V^&J(痫q͂`k,JU,51ewQS,w_ ֲeL#X>AQv+_ăkÇ JH^:\QK oJGd2"R?-i)rEiSDb7j ]02KTxO?mXb-53t;ӣMٻ$XclLr X7ƫS%Vr%1 QaܑX搫yZVO;/W:. ܂勲;oS,Q$~m"XrbjA5K9oMٻzžx+ʙfaB䪚vE`lpgchp((+ᗫ#(k)w{3 Pm])oT+ɢNpp)룏>2\)rzG@\Q*w◫1}F6!XUYsqGL)OHEKAvmޕ+=jA{E|h.+;N+`l\g "WO8692 wh8=Dn#*J6ު%L|h SXKN,ew5(TEFe :uJD+r;s<$X;2gR\Q6D&E^AI4+2҄ȕ}-U('NTk[m6rկuzW]#|` 9UTKzM!X^rg4h镨xPz-uLۻr/;ScAI(Jar<܍0*;3N%WíՔafq%ڣH|qYv7WxbUhhcdYWzEă`}xLٻr+,Ѡ`QjSSx(WvAt0Be2wl89!Tߗ 1Qե%,sa'b Uv{5*KN8ax*++ӥЮ\izWGũ\٧XJx!r%1gs13QrE9۫\iR;R<3kdi 22dя)((ȒQ+WzF^Mke… %*&Yv:#*dM2Űr]p+GF^8}xU*@R'U=%K(g P >_"4NFJAp'cCqQrpr5i0YjeT/w,_ƄFbE> V˖-.WA +]y1lY1xqrfrfB(DQpGc4B\Q._meT.I.{挋`2&˗Q+Z뜚7Q3\QlkzW<( #v(\I :Sٸ?,D("%#X@u48\D$(_7Xb= 6gKiQvc~g!dN prG4h長x"]/Z`ޕ\Ilg8U`Qމ[=a=,#X p+zC\}F8uӫq KӫVi񣲂-bSx׳"Y"EK-]BCCʕ+u+_ȕh+Qv%нzwI(yrctHVތ5$W+:jfR;»/E'&u]Kt]/bfi-5K;J/ٻҺAJm'Ug725ZU>5X+K1˕U}Ɓ

iJiVZ 60KX+ޕhPJjA ^>u ooQdR`Q2,]9BJbBkd\%q)JVEFkSF Ŗ]w^͝nX: KTLȻKk],=D]v܉333*88>[nP+HP\}1R[nJE9t1hR\Q6ƆTJ,q1\rF!B=Ci(Tݙ_F7,LԾ]C\Y~b):&4KT睅"%KIxdB?ΥKqᲲ2"S񸰰Ϟ=^*UZV+bW5cȁxZѠ3_5Ha^I9 "QG/9.#Yy`pDU{¯ZVlL痫cV,ӻ[' `i݉e)Q/$KhʖHԈHr-T3RA+D8S< ֚@|8Y$ԯ'DXx=-'|v"D%Wl6|yd~%~g ژWewXF,ih-[,V\"1ҳ"XC?cޕ\IlMf*-/jG\nV መ^ Jn]/W~]( (|fvJ=Fb%B}劵Ԯg4h镚xӧve]˕r%.@:Ss1feσE.Wdž.["5&4KDT+Y-Qg?JD4Slu7*[SHv9Ta.JT!rU-Xx:BXI` pFB0%W˃񕱃WZ[cTlЗ"6&4KKT(RLo)WF^,{Mۻr+;aoI+ _$V\UkO+ރFO%֒z5i0QjX|f^拤jX y%KmdE|-[,XL WzYW~)˰K[SVɕġr%?7 ? B<75L08^("!_>_:XbOr] b EOލ(YjDUDKϟZʕ޽+3NƃΜظoL12Q •%'BJbB)DQ`pM֋+ʅ=jXw˔)W:Z}1ZIxEKo}S+HP\ {z%>sajvsY"dEy sWsV(i[I,!,嗫8M]}Ϯ]髲X zKhU}LZŊej#WV^:8}(ðDYYV,Ѡ`Q6e +mWbIu>F(%S+*W+BC{4L^l1L\]v7_U.?[$W^]nđw2|ez%1.yzj!#X0 8 R޹-\Mo[wu, W>i ަX" ZB032\Y`(tvOr%q 3K(܀!rE,0 8`1%WSc] /jb .YDKlZxVzʕAQ+#=qrV!1̰Dy䊲*†+L%I "92G]0کo%)^ݫwXcTl|0嫲)?BQOp~ be bzr'9{Wnf{r,\QNgl^2UHÀhp(--䗫i#&іX&(3 8e.2*dy-R+^j~_\+4J`>qo.62QZx^h`ٚY_\ٙ~)V*F>#"uUk$(2< 4i84xW˔)_ϙ(TxZXz }%Y-OH@{{{yJJJxЙwƒ` )oӫ|ތR%Vr%12՘ QwWo+ʩnWA,[Uǎ8Qv7r]; J*\c޶m~'p޽qII 4AAA~M6xxѢE1Py+rG4镞OɕľѠ`sldžk0YloAgQr~=~ӫi)7G].Fb)YY%A+V~Z2C&MycQbe ^],ĩݻ(.呫 ӫ3E$dt$Ɓqԡ4BDi[I,%? |7/`hy5nFֲѦXzE,-M駟ɚxȑ#^JXIDF^B(kR\IE2)MHCCU b ̣DSEeR~2(LrA8>}FewXQ?$'2T-^٢3<ㅋ;䇟Cڿ"JRSju4(zzjSxsB.+9^AIٙnGVz:N^sMe$>G݈TEk$(**_2?Zdev'h)»RTOSv؁ u+wbbbo(U"VzɕU]^I[hr)6:SO1Ѫa CI{8R٣ jto|#fg>0i%"*Qz,!hz!DrL =V"ʗѠW,{}@S؝5tyq[z%2bC_E՗ j|E$X>ygEֲS,EZ$KihѯMn*V ;MĊ%4\,ky栿 Ǜw$WdlfXU/IU$(V(D,xv"Dm66DȕQwh)P; "YjE~ #WʌrG4hā?ce,̉M \rEBU뗻!#X% 0Mf+er5y(QnR;}sQڽŻ_S,#Jhђd~ 'W鸢BXDJޕk2imaȕġ9+Ξ^Xre~* 0kӉ+RŽѠ\QNe'l6!rEy UFNL~u)WTG-L>ʱ+>EQ)BJ7ђdk٦+ +O]HUq^Uϔ+w, y O \Y"D$ ^ab0"PF\- W -ӻ|3/fbMX#C5k.l#?M#zhU*W< EFV^I|e9{W!!6)t0?CK V K$+2J*+{v5Bj劲44d+;yYxUr D!*+#j ~7^U]bΦ+S,B,d,-!h+, ~ѠWket H+ʱ»Qxk6Be$+Ƿ`G"Qj%deՌ1zzW"']rCOS,Gf,ڹs%+==]XYEZ]SkLUjOˉ0Ven^JJb>8"LD(ZbGL^lrB= "XzJOdFf9jٲ(rW4Slk+~;C/e+/b"R?"DFV ! Hhp(lӌz5y(^f\gg,)/B4KN:wl95jp\ ֕U O aޕ'_OV< V%]i»3 HV0 }w^5 2}+Uq1ƴQӫNyX&*qYB»K; IȐGx K/UL|!WzDui-7YWFLٻrO‚i"N,+*WDk+CzA!*ozR\Qz5eRZ*NYb(5*%Y"#CEٱckBʗrw օ LMٻr+q\rEGm6!ruV#Bؐ""A/ }4,26|f:q!4rTh$R+ZeeedM4y4\A]5 LG(aUNN"Wڨ0J#x"FkQr4$_7?D%2\yXBHH$[ekرJMߊG*C4W^62Q9$X5Qa\rE9Mx>,[S;Be f$n-ӢrVӫ.5IU%bU%Yk ZjX +=zW0̦N &{+'{i= @\~12H\bEYoL?jee5s~:y!T[xU]d-ٲcsk>W^\+ӫ xETJJY "2r% .F\I Gu0 I{}(\p\ :+ZloSg (YZD~,999 &Vf+(u`Sʎ M1 r*X'ǂljf53ws(k[0p@U N7L>z:}!4bT(ZX#CѢh"UII rѠ'o+Mٻr+'# {]SȕDv)L)8ZQr,,8a(fX&*u#_Q ,4EeW^ eU li{Wr%yx(>*WՂEy9!wn?_olR];t !ˌ,-%B(soذkΜ9LR%'VF+}Þ~c +%k4,W 4P@C"Be$+)jLZJ/%3OfXb$W N B_IPOڶm 1\i5X+w B4;,DYWYOdex%8>V\ID}0̉+^OE_҇_vLYν\E޵H^&[o& ?#UQQ{FDNDʕ/A,WNc8>6)tDʞR%VrUA`GcED{GFQ~+ ~Y>z.~ }-Y"YZDa%+;;*Vf+ W9M>w%B< Pz*r:T/YʝRbA)U<(ZN?zW.-j,>%Khy{pմiS|I!bW$\A]G;fޕ';DA$.D$"B= LJ^ :SqWG~_ϷYw"X|.> )YzlQ~mnݻk*Ube4hЇvDʝّbzg!di l9KCB'Z[oe})VF+}ñ5 g5+w|+n,,QBJbB 0BDŽʞ]Wzk vmT\y\M.YYZEKlQ<Νucbb*==s=_ħN$TJRX\Y=[Ԯ$X ٚ<8Q?D~!W)52WHQg"V-W3k͓-LTI4 a_XZ%Kddhi-g۷/Y3gδ3~xhV)\P0JQri _=]Zw&WosٕQ|!Y"Ci7-s!V+v+vNI[4,Xw"j!BTYFΜ{merUCa>KpѬhKx?6oH(rѕ+Fٖ\&#W⢙W5(D9 ,B`5"ԅb{Or5?[E}4^`8ȕ`IBCDPdFz%_ZLϋW W0Rɽ{eFk-I,W4#t6)L--W  #(Z Ӳd䪆,po{|qաpQթ+5l k2x |?/+bBmj7auze'L` ³!jw!4YFJ Vȏj@| DUɕDo׷ChrFUޘJ N I>4YʇpڍSk-fO#2Usע93TrRRS!)frp㳦pNGOŇd8xL$3j;mG0nbdvv0m0_/f!CLG$v%%0˕nZx"WS޿Ed5(G\!G VлR,IdZ \9/9g2\9+TPN}]p48)ÉTW8.Qϕc Hq = IUbmɰW"= 얈we;݉sC히uM |6KDfS-QlP`;P`;ʰFᮬRs0WVʰB"6eXEZC¢@|\)VE5GG=107h+ ֗Z#Ag. >e F\"Wʣ`NNYdrIxjj*F\m\\) \Y!μ+wX4pلe2BiP*T`r8_֍ejAruA\U!WU"AN ^q?!Uvg&@+5ӫ*KN ~""DyJPZhBJb BX,!W"MD(Z8? J,+?9?iUv:g: @i r&(W_ +F ~+/+v镝4;/;Xy+{z@a!V^oLyWck2QJU*߽]%]MK4D"䪮>s./IJM) ?&"ZFM-WK'A꿢j]j2\I3qUmR_zW"JлޕRzWKP]( m&oѠ$W cUEr iQa) #@jHPba꬟KPj7m6]E]ADYY(k"AgNǂdmP_*"1'J({ZI"M[jޕvij o=ҿv]w.WvȽ{JhНI X-i*#YiE%y5,p|uh[ ,һBϥv]i/oRiKлLG((W^S%WeУ 03M%WSwu^`(,,eL\j$W [bI& jVLDF}n\S6D~ Jr|ۯJeл,5E}KлeZKjQ:V"='BJbwxCrlDOl6݈~rua(,eL2QX&R'X%WSABj_"4XF 6#G3E嫖MNGCwD-Rjrq*W{VXN#T&#Y醔־Ċ(|mHb{MѠL4ſv]A BT媢"…ȕĽ=F{X\Qz/W#{:л8CwD!л[ʝ&WNu85?(D(kBd+P^P"Sj%6e7/O5Ov w%Bw+.ozWuw.Wȵ j)V!](%WˣOzeL2QX& Da*ɕ`QV2-:ImHXkGFV !)B+v58q0VD!g(CwD}ܻ+] ^AINU=)Aq!Wv"t.f2`C"Uڷ<>_6z!Aqeuqe~-sF|dNZ=fdjW #9Z#J(/W8C CwD!]ɱ>*k4,W@!r5QxgBd_ Dz%6ךC zWL`}~.eLڍػ:}z&bgE\HPmQb%A(Z7zգ,eLzWL2Q,e+BBAg:IG"z@h`}YQrE9ҡ \5 ,`(<zWY& 2O$vUEGR,B-e$S@?"Ċ|0qETe2QX& Da(,eL`DyYH)reP-Vr%17h_L6EӫA!g]2Qx3weR"ýFrE9^/ ?d "W|P;EG%SGk1Pj +]AʯD[j`ǁf:lgM|( CZ0Ū`*WU$, 8q4xWSX& Da(a(,e_&`hY(υh+IVcBrI\Qwm?{/lR;`_K-C zW~]&R;Ա56yz%Dȕ-Qbe!Xh%֑/\UL2QX& Da(,@f2\!Q"AOE5W8ar0"S!W _l zWL +x3<zW]ɱ:<]9(OI³m6r%qB3(jz_\Q4-^MGA ^R;`h].CDհ/1Y$6 +P*%M`o%42_ ol :s^~1,oq1]"X&b>rEхz57geл,5d}Kлef.+9:yzEr"% O&W Ew"D_4ղH|}hL2QX& Da(,`]@ɉ^YNCXX8:%*W}-G}C zWLzWлe:лʰP+XrE+G;˴ĩ58y(Zs8CL4ſv]2QX& q6\I!*W:A$#`5Q"E\Q.5rH0|}L& ve'9@c\a-ƸhFS^όOpwz;P??3<vxbݮJ?g頯d*O/.zVsKtv-ɕe[::ˍ;;x\C+]%nMv.|Y-mtr-:p{KWKsޙ$N+m9Ɲ2izV4s -ӼT;g$yA;EzAw 4vPNiڔ8,LQ<..39vJ3Y24LNf-NIdK4p%K"6 %]8Df}Wx"Avݩ_O4WR%w )IvXAgjq?/bTUE䨧VZHqP.ahWIreSGf;#Lvg-&9V0`ㇸpUb`yre;4ڝN#`-~tg!->v~pgDoW;˕A=\=(пGwUWrr镜\ݫBLjɕ镌\)NJ+9^W䪕 0!W\+r镜\+镜\^JNLO+߻`Rf܋9+AM+]M$NbhA!hA!+V%VrE3-XO#1B"WODT hA!hA)Ok+JGsVwn"W ײ D B4 D B4Ѡh9.V\~IA`-F-'7;!hA!hAͦo(.rT6ZJbqX>7D B4 D B4Ѡh@R<~i5XѰS(GHdo$(G0 (D B4 D B4ѠhpkB,~z8r%v"JC::I.׈A4 D B4 D }+xR@POqUc$\P3"V+g> _=ſ\jh@_3)[CSC!6nnr1B2Eyx3+6Ր렀rc5FDG-rYWCkYȖhP5rdIdPՄU.;dX)~5PA},H4~G걐?s&H,HJSJbI!dzs8$b刖'ƕjegD4 vHDEF?tDX3?P1*FZZCF>`} .bV-d F yYȋ x #yx232WyG #sd <,dFg>Ff0 Lcd S$&22A#V1+ N*&[ [(WqFzՋ:.WOk'\=rI$W-Wtz 4@ZFVw4[u\^$Wρ\i'\(WqQqQ W*+I`I ruE\}rIިrFz4ռ:.Ws5գu\fi\M(W\q.f+Jq>B"OMԷ2\oeF2RʌreHHHX,Pf2;١ev(Cf[I@|"Bm4V:y05"Sev(CPf2;(WPW rEDrltu١ev(CPf2Nru5U#!7P:ɂ2;١ev(CPf7z2KQ(1HC$7D~2;١ev(CPf7b2 ) WWHVw"So}+o}+[A VF[s/BoxΠܳ#/("ZI,or}+[A Vзgj8B V_S3J/ Wзo}+[Aבޅ+ <{Ԟ* зo}+[A V:/:ŊRDAF:EI.S t-?PX Cay(,,塂"Pr"X7!M"t}?XIn+Mx 'qP  + A /32_yA 3{cY<#VFNR%QBG?6B_BI-Շo&z0dC77QJHB&?Ӭ d}cz57_Ґ,|; S? @/hM)L%vSjZwY:}M8SslhB!Z%xxC]zUp8p8peIENDB`irace/vignettes/light-bulb-icon.png0000644000176200001440000011267314157104332017044 0ustar liggesusersPNG  IHDR\rfIDATx$gu/z8qg6gII$"/߻~|f{g 8 clYHͻ:wW}gvfΑzCuOuu XUY VR_b7nFG?SyJ=ޯUCKW*,SAE߉z(?6/eF@qo AaRUY2++{A);)Uv?yJ=O?矿8Q@!Uyc۫>g2XPӖ*ں /~l^#bu|#_>-26B?,; OZs*@Pw@[zVnT@+>Q (W 3 a:{XRvڗl@*V~M 1}*,ܫ=)qSJOOBLccHzz@0%_{Rxّ;0V| (E :m7liPc7%UX\~n`E7Uzvw]CI=J7d+2r\@ o(v:N$%'P0E?F$O_BUxrdoR\Aon +&$B[@ .~RnO e,ϟÀ@Jތ l)@.(e z] rK'kIII]'Yo-}9O"Pby1np 6Y* υnDi1+:Աb_B xhI @VEsZ'kmThPe t=@^p?7@0P,̠FB aK H~ A`R+QVȥ8`Mx{ *E% ?;!y[w~~ޔ2szAI:=G3~- ! 4Mv(l(f@@j*7M~-gVKσyo:Y`xblX %_wUю1N͐TEj@xtx OI} @@قv:3 (64 3h598lU6hYE*NPOwc>r2oGz-UzR]2:hkbP:S4gkV@[h+@lKWU)b/*g@LyE.hA B<%GQ4d:~Rsmk8n1UH1 JU`A oYKZ *G@P4Կ|.  A/ @&W:Z9Eg :'>@=rTfB6 Vf {/*pr/j>o$edB~@Yz ~2OgޮhH^n•KEwi*"ȳ@E@%o l:NШG3T,'Yy *?G:u9UC|ң@ ]gOXch\^$Lv _@*4brlTqV~ :]Iۖ4,YyVˊқ 0v{K}*' dshFUhU/0UbVY݆@CV`cr)%%gʏ@PΡGEAl5ΝK;OPF$-h`z sualz N7=﭂@Y>͏]zn>A}󖬽f@L'R~1^/хb/.H tΠ[w ?.Q*t@\[f g ׭@OYV~ (|~a(@4!Zkm n% J!7ԧ`hBu 2R4NB7݆M]5ب\\+> F+\›E&iD-0x/нfOb@bAOrg 12[yL]iVAd9]U/|/h/P+r/Q~Q* b1,x;|~v_/ qi>/r*$ 5v1@C)UpeRAX@XohymS{ԥx涂M~'f5nq1t+hxwl7U@Zt:SP2ׯUТ? >*BMl!X"1TR{axSZʧK^tQA]h)GrH9}ٞPFQ NS"lԡ>9@Y 䌁J"E:INVX)LYHYGKe}=P vOi&Tҁע4GBVBu\ W泟L܏AE)IN" wB Dl|AE .R~*Z~<X7Shѳo6D!Ev1?'y,cb޻w -L)5Om*feVb&_[3 g%CbusHQŅn)>/HYTfDxA4y|ɹZ^2DN<}k_H.݋.ej5TL>s g. E.+R.h@1-l.)TyT"|cV)?BS|Z@Z#sF|'-os| W ,Q%TeI pS'ZMhIit jPzU SP0$uF8Xr*?G)O%To{#`W_G<G,gCE=6;;x~^:@T<,2(8 D9Jev'e-$;eնK߀A~ ű1*a7DiE]8Ԅ][YwΟH'zYGPfHvW9ZԸz;ix@;#B @@|A^ g=eT?(`p<~?]k~B4=}z&3x>zs:Fȿr Is:dZrN3_JAnj6z4A'(x-`<.ߧA.9ݗtTߍ?" XW\"_wf> z(tf0磡]M`BT*Z*YE0{[| *3И9@FMI4ht-U$tA*?ꏔB0X>'шx55|Xs=~|2Y A,s v"q{X@#fL)1 &C^s V+pA\`)+ AqF|M6X ~XE o^[k~̧(}/[{Qz*Nf39@!@g_FP9y50.AS_ynbYw%s~$?b=/b,@n^R^SH[RWU .#w`\$!r^3w{Cj #iGVQÝs+r|BtBs@^*2." Viq^s`  F+QB B+@QxȍO/'}˧O(u\1yȜ9,JnRukk<_6ׁ8vt{9R UH'U,@&rye-%{q󡒶@iV{_ b=.0"7S&Nz+dO/={hj9)3ɲޓЯUL)t"K9GaAimO{@"ބ DU 9xe4 n×'Kfz ڀZ-,\O.ه _y@-Q( _ q("ou@5~_FC2Yg4NER>.~{*n|F2L#a-h@|ڙݨ2  0=aeW4WRx APmWbpR1eI(׷dUo6~Ԭz} jX[ ][">» ÍwJOSJޥe$עy V8_|n>A}M+0*t! Zf2n;F X]xVYX)̠ҧ2 ?=k}5H+n?@ u\ E`#:s0 rv+О*OfE3Za;kVTJ<#o(a@o$eâ2)AVO󽨼ZR.." YDWҏ3|~`elj ^pib[@kנpJtn%|-~:TN>)T`ɊT~g}70aF  bg^RB}7jfV<[{>T~H1A$s? |ieN+_!2 $ -b7 l#"} `:ԗuktZD @FXQW| 7hQ`70_oO I[ FeS3~_?{YxCS܅AWe`,p—Ou)$z^݁73:h0 >4!j\q0PtGT%`ejwtU`-R}?hJ5v`0:QnFܹ(hԯDhKQ|zن|!~9JmӮc^$XA[6;.ؑ~X@iAo`U{8{W yq;h}ch qo+fjЊE7KRL%6 c0@=1CA~]@N$%Ck!h`Xf\ry!Rp$xcvQaoCJ/ܼP[i>$,R1N fI?m&A8H+3<$  +]OT` K1 `'`7 xQVƱEFgYD3At&?Hu) yN$2sA3s,7Mے4(ys@n@Nyo8ip'G5ah!=b+7඗:1dp|X7k.r5ț;<ݤ筲! x{J[)KUYk s>tJ'.LF2k DBM? v}TD!CᏂ?nI:'|gA\ûq! O| jOBs(ԧOpCP[-(7\ګ_`^`?Y/=~Z %wcgYS` wm|100 m^3-i'‚M2:;)40Xew^ :&;]tg`m]`92V\6q v ЮMCԳPwVdҁku`%m``ax# ︅~72ZyEdQ~cܘ#Ho!TZc:*wRn5'{ agMI*x_B2oGשAS[ CtsNyjFuN;Pg!&sЃ ֿDqg8HNEl ɖM }}Vju^S~sy7|x'ue}Ry4\Rk ?@ ::}}-^*/ d,tN}1E`N=@=NŰ )qiˆk3+xL!G` _'K`^0K߃_|O'`{(]> <B:QCLm}}2ٖd д ^a$PF1T hƕ%G{S',uJQGXAvx#cVD0(oo:ȭ}'tEU }aK8}P0WКd:H(~ @2j/9>"9])_wn*΋Kn$1t:'{ی0aWv&6-h 𢰉Yßa|޸"J@jR}օʲ/~xC/ fgP/&]`3@`K h+b@&:& Q>Yd<4LAϟ (H 7f݁E(v%r? [Q24d{t^;=P.o H:ͨ 0`b41->DrX8LWXS1v4;PW`I@x\CK87AqR(nx=_*)4Q~iThat- _{ߣ!(Q#%XA'HBn1P?ٺ=r9 ?7>}9 Gu1HD[!P~.2Q 0,ȡ| o@)"~fSOA>- *,M.[@ 8]8_!dJe봶&N*>Ad)1 89/dzTIk=n[_Y<V6W<~8ԣڳkQvYTI J\n 1p3q vnF@ 4&Fl5 `RBe~5d4Cq}:mm.5ʯ@J]}kگ΢=eMv-=/~]@2>D{@jŎgJd~&lz3$jRMA17$}/V:[[^ _AW!? [~\믢m~hn/e\ t<(;Y/W Rd{qjP$: {DWWy(8esƊy3 !cs:쬵MT)0)r4_?O7qcׁ(dud\ 4 p>yBYVKUR6eG{oҁvPXv|&5ps*V fp@kQG4-OS95ʋ{م笈ӕ24K-Pt3;8VWyÖd9n*0U z@`0~:R)&myxS A_B䷠z6 -! Jҫ>iumW?"%?7dcwiPAꘀ. i7 Ewxs@s k2dJslp@@nAb?P8ӀjZ$ ڀWu>DM[5;mUzXy($l])s>G*g/Rϔ*ې3~R`eG ؽS vV~ x"*и QG `!1w,@7 mNp4p҂'\'CM̀F6@M@}VwP T@_ A2"cـTwT=>}\CIE0S2-L]Ǐ^.Mo X7OK5Y'W{wb+l[ݱBivJ#@EMՈ,^&YAop޶BRS or:J11Ų̌]-X9718.@۸i^rέ%x0(I(_peB£L*}GY0[@̾ p>.-uXRxjH)s]bJ+Jkz9HwITCP(!Q엳ŷqyԿES&$cxT( u[ Vq<A!ʡډ@an W=ut:x'6.h7. 6VXZkr8 (,2^Ed$Lj*9K|ӖgaQvO)H+1 Un@ + P&u'@ :^Mu9@7tAA7@Q@j@8AhQS*,ycQi@ӁW`)$5jXw9SL16DHp~ O6/vAA k'S"7h-*gҬA -LB>_"v_GaG0ߏW% _v?*0ՊKݭO#@Dte (p4u~7HtP}ܘ kA;B0u.TRN~``{l|FĴj1Ha꩔ϭ,0QI^tJix4LEƋY B/% {>Q"sbwQ8D3Th7@Fc XB Jf$W`o9&t}T?O $J- 4XRT]PZ@7@BaWYi@4[žևr5Xm.0z 7>8 hۉ~{XwY L$z5 ]p8h}ArS}̂C9ڟW3L04 ȣT£VhV4~ᨐ|?JZ}G&.+COĖMrIu.tٰHbW4_:+ P62#ٳFiOBXtԝ `~* T~oXT!1_V}*ՎV`A@ ) 0v 6,NDA@kFNǭ*tx hDLP>T1U5#ܘReJ PAs^*˭0M NiƝ3YS d0PngBUDR}Tm@Gd9 P3 ӳ ~Tvd Dmp3qf}ph "JL?l7@ ~iua`  fbCG7ȥ[ ^eil n:im pjLY~Z;Tee܎2mP9[(AC?oWT(*:}(@ⱆ"Qs4oU@[Sa+2TZZ5RxIؔ_ClRMex͐f'!{;:5?p)68AT2A}uy/{ӑ_ wua~g/ ^.ǭ ]=*(eTPo'omK ] D:?7s9=hR_if(G)=X`zT*F)_7]7%5֤I;ݟȹGmd,YO1(e8JWRBi+-0~m[°ή] h@ ]}HDW}'{LΟb 3)Xe_rzǬo[P/[j]X,oc۝y Z}^\r ]06M@&4>u)l JԸ~NLj.p|t1t"_G Jxqui<5 WjU3@J[r4[1ǭc 6vdjPeQ+bA '!5h2 -LB˰ 46CQՂ7bc-*0hˆ!;`dK pNA(lS"ff`TO@ZPS/.,T;͇he`=T]o8 ԟ[/8I@ZI-A.`ՙ +JTFNeEQVv%($Z#abnE!@ϋf+?]r]3v]Yx)XHt,?cABd~ ocv%r?N[7]@7>@T>t|x?*jq +@nbm. Akn$Ko[`Hθ2k h<| L#Pr |ШNlGht`1swlUA&ĩ-%N'@T?.DAPܽE1- {MTx:DQyLS*a4݋[QywV3u:VZvA2C9 V<>L b7EDFd(`@Gy.(%/|{D_0LUį1v] ̄qڌg{xcqٯ^S͘4cJa5B 6N? Kh+@Af˔S:g`1s{JRzZ'F7ؕqQv1%oZ4]Zl|B)H4??|V6T *(nB|Q2Gk z}ЖE4!a g@ Q )ca˼e9\$EYb͛EȍBPŏ|.0n~Uw&1d۴MӈDQT<7VpL~"(HT*~ oRN{e+-+A* lrSPp#c=48$L_74*\jҁ il# \nJTn*ikk 7\C{uR"^-G׾]Tюӂ(T׮kS{F"ct|m:)Uk֬:}>T<*վ*QF;3zކ H.CȦjqPf\00>?,SMZ:5>L=?`*ߎ8oҁ(#[jXP :FSV`Kދbx!uТp 06e违 Y@_gre:`Pԅ@y<Ux.Cu*~=fզ grz20-=рy?q1L#ܣ\x$ 0: uK3n*hL66\T[9%7p,u ;6V~}ta6&"xOXHxw?el0`ZZe_;|lJ}@ Y{ejG㠟tY3زgY,g˨Qibd (q'P?FS tc`QĪB A?z!$I*xV@^ak\ Sթ94fhW%{y.w쇶EB 0p޿6"QV[2:玣;?Rgp=@[^Vzg`g+d Sֆ++fln49q:%f[= (i )W w [` B L@O ,E sPmృ<2 RsGO6y&ݟV͋XW +3 ` :Jq"؄V_0| TMB۵`o`V~C\30"ke]S1SCyp@aM9 @{s!+ n'Hp0 \ 3HM!3PCO7o>āA՛Y 1[x6B"QƋU׎[%i=@|>L 09Eվzxw\Pus>_..v1p4B O<;> cW7pğo} |^/&"bw(d6T47p,'TX@CP}W 05lg ?/4U~<h%OA@S*{8{r+`tMfEƒ+L^cD@)zD <=Ձ6w=3؀*:x_yQ=)~dW'W~CZAZLO^3;ȯS;X {&ޯ'D ʠBAl@ ``C[S1LCXn@)@T .U}1 ا2q }Jֆee3BLAH<߀֍ij 8Yαf x`txR͌DfZpf+ xG!G!/L( @jQ/迗w85PCIVV+_uȝi7z>aÍm,ڄNy[| :w27;3|-=EMj88AfNLpp™ZGILmBx*tP5JK\?U'@ 8>VjPkzMPQ1D* ݭ uڕ3p컟Q~[c߻? ;Ӗ)#T)A ln(KwɸM~;W:ݣ˺/-+?mVAee$Xnq|z Ϭbv(̢c= ۂ 3n6M7<ւG|`3qx7C`dЇhl<9݀V2`tg a ?3$RVf !iwEJnR2͐l P=;'SJM5 CK|WYto@;[n\  9S,b||+Pt3s v~ Gozai:=1цgFzϓ]S=ofʮ}<}nw%dSq$ȏ,N9ppoYO5w9JV  mƯ}/0hq& px$X[P-R I$?~ϴaۆ<\%ӟo7zTSg>?LӛǞEMW; ݚ}M&PovB"c \w@ɔ`Җ.Ohkwƺ^w&B$k/ƩP]ȃ8yW־{v $[b z% io=< x]gf[™ڪXO0p81/)_4%Yp[OQ15ޱ}턾d%mp #G3@AX=Zg*p=0@;x7`-/+gZ5t7`+9 x>gL;)UsYR73g^]Qf3 M=ЖKfܨZ3]} ~XNԟ++H (Xmy ;a균! /®@{$l ~xYmSB5MZNt@p.k2 go@kweY$|p-MD3;{md-zDz/!^}YB'T sjT'w ='e)+n' kw37 rku@0v'( E2}Q3 e ;Ѐc[p RK ȺKB'7@ϟg$^8~űw~4Rp?7^U5A6[ѴS_8Ӏ^F5 Ľd1g-?,L<{z';DX %+H?nzhp%)= 0 P| lXQ, rs.OA(gK5b `}O֠pݟfOwO^5 ]Tn~ŠlRC]=;݆{rg^}]w@EADWd}Pׂ} Z}5|ߢN\.؃b<:\Wusu ,@X੿(r ` Rv[7|Y+`l8^iv߯R= *]2gp YÕ;cc纨c 1OEÉa >=%lMϿF߹ oȊJwt>MxްF y~P{ h!fX?ߌ=}}L1}zn_ARRYv} N>M\U͸}+. x 159YCe7R YAQsY@^ێY@O߿? 8o~0= Rؼ>$Zy\30tȩ<\5d||@hL3P7xu2ɳ&gC}?_`AZ9?j15{݂6w@DzqJ Dt08Ў/v9Uu @FMgK0Y+A;aX3.aSm8<|'iy`oe \7G ^i\-ŮG)'h3A}zu9EyN!ش_ƩA5 ы.Mxj5 Fup 6 tk ϛU~PW,;!ȣ{n)$}OVdW_yT[ӫ+WQiVIM<_|>qT7WOP@2Qu93/ BQ'ڦz-Z_ꤶѪS>z.NM4[f L~<Sjղ(9p9i=0KQ>mج ?gYpOV~mN+4 (0P޸lؒ/GL8:J'%HApʭŋsɧB~l-˶GkpBUX6 +6|$ӌ0-;W!z?_G߯mo7*"W.h ܸ70 @Pٺ7_HȐк 0H7Ehsa޿7#Cܐ|, Zp\uhEp˵Ĺ$$ ̛GЬסvL?GJO>sR_R=?' HTf z.  ahfխ.O>9z؂rTtSc}+zua  ~=Ǟ?ي 8i:+\7vI_LˁW\é>> MhLO\ǑF}:r;*U0S+h޺ A`>EyhHmPaQ,\,W >لW_1WpEDjsa~o< +>ڡQ4f΢bK7UO*hp@@/,bF 6ByA@\(D@+ *jt[" {oY/>0 C>\!x ~㰮\ Zf|O[H4q?'Vl*8ظ7K0i-inL L/6 tـp"+t\wga @/@\#'pS54CưC6Pz%h|Ѷb?{,V?#xⶅʓUHg~cCF)Eذ l6 ܃%Cͽ]sRtU{*!.D ށpkJC 6`Q-e }s}\ݏV``ז ^{U!ʧ3e7/)}}PinAZLj3uYʿYy1kʆs<@* (o@, tKXGtK,/4z#pXo a| JSNYC^7.V^ O1Zhn` !(bӢDQP?<8޻5lBxf8y ?R}PR=UiЪWQ[Hj[ӯ/LVS@o#&@D<_bpx SX&#@W^Jy1' @;oIV}iVkm9rzgvۏWVS@zv(P d!nK9(V(N%Y@l=z H yΡˡ[B`|I*jTچl}B |>c|_q [6LA3e8p r"9 /+pUΣ^Ao?Ya.41wd{M dP/@W)WЋy.jm -TC>;7L@+NV` 7A,ZVEˁ4JSKW` @FS`=܀^M+9R+t_4$DϹ'x|6B%ŏB+ ZMeOYUXկ߃Ն ;u ny]9hP7]}AjJ MO!O5Kt+_=6KwZ["A U f%1^H>x57A89֯h|2iSa:бY96ڗjW(4tQ~V|A?~4aFuk GCQRʽ-O"d΃ @(U0kHr6lhCo#hgQx{58|UnCf;k8hY.d'6]w_b r q^gWK|Qb*+=YPjv*_ŖU8@AC(b:K@7# daD+Q|{)W P aT<,\hlOCT?P 5i@Zm)lTyL߱U78~QW{σy OtL*""^oc쾮~޾Ͼr!E)dQR-K1R-ɊvYq?Q@ĖEĖbIEoEm#q3Eq'Erַמﹿ;3p޳{o|9sm`O @P-CaF$Ыs4<ߺ}Dm O[q[EsCza מ~-8j _g7{nZ-y|./v3Ky3 zq-U߾0GU}gY6y0 `ҭGx}~j~xJ췓Í$`2kw"lW`x@/)ґq>t4lBRb$ e 5hd9w`w#ȳ?ۯ}3Z@+8?er  =tI VY0 9)P1X[R" 9=ŕS[\YM/%x= c1fZJ<ٜH%^/3(ƥPUւ (s~~P:ʼ෬+)XK fg~(j*+TNBSժ)<z?nLJo}+9*J4.ٿ`EWC\G,+ Yp GɃX2] n`uY@`vf>=)} &t6*i@P %3zඈX9BOS.BHy/Uj6߿|w0H9do $Onc3 y|Xb8.MLQcw-(ylf_lP/#áUk=,OoEt=yH薭aekV \:J|SqUfZCV`!!%.RGX6I]*Țī*@՜TI:uf >y(9!TNn`8w2^ߡ5%i :6꣝[X *igc\Dpy! g}փS;:1Jr7FSVZR @p~8Cu)yx.fN% qOO1:JetcCG\U!4ߺeqrXp i 3|I2ْ-'V@iZ/bFuB}RPG$O8@^Y@+ʱVf )Ў-EiH!)gZ^gX+2CAnc?u@7m_yJM?{R,9_E P{jP¦A!ax]ԿMV ـ0 V*P1_\GC nIr7f s@\V@v>=rW޷LT:s+xTR}UҙjVӏ}-jZ}Lvߣ$k,~_v<AaN÷YT=`H;E(P @%X J }fD;6ٽmΊՇVBllto3t^܎ꢟ-y l/ebT.'1XcaH0(N ̏DGo wXK@eR>A3%Vhm VٲGi| =E?[BJ[oRrvJd7b7 `lYtR܂k>[f~m0=3+饲eiz]nH߿Ȏ@0"z9DJ;*% Y}j/}֛Au 65п#z'I('%)rcIDBnѻT)._#W!#IZdn HB^-hR TE)y5@}~lNtz6wrɯ\7=NAo xd,@Nӡ4B:oIqrmTQL3NOeː<(b2[ 0I& T?,?Hʨ?Mp JrSFN+8beBj (˒05y\a؜4ZUv fyI[%Сyog2 hB7mpb# vʦ쎔$ο˃ }o֛AOel$#+. ![iX X[H6rjs,D4Qeϧ;}'M.pu:G2wy2eI!&3'~B< $ M ,?@ IQ'"XkU Xjt*MC]N||>~װvߗ(m + H@ Q潲f ` AJ/^| uů=r:uj4.@^\h(R{[y,'V$Ff=Հ$MSPٲT`z6@Vg`X VV"b'`lv PzsO> :xQ!/G1h=v9TpJ@ȯ܁c!wdCj{pX9Љ)~\^tybT$X-nj xU ] ?)IXY' 2`O4s@C˃3hIO34.V.rwm"1%/X5f.i 7Z^ɝ+fOTbF<@/CV"dY.y=յu_b,xL[ynb6Dm6#ƛ4޸s*Z Wj5E:d2*! `Z|#vߋ*m@ 𪀠{l"ds # VgT/;ݻ)IHǟfrD\[W̻\'m_9 TpY T@xb@対x  !Gl zkTՀ,E:t:&A+rqffK{fr|?W7Xqk F Kp+j^JKJK6KU ( D/w#)u |fzFj*tl xˇݺ'`*(UhU,3؅dž3K x6 {l^8¦a3O/a#.Pz%Ȇs>g1h0vns+ v; ٖ`0 @ kۮ\M񍬋bGgw PͪmK {R~'miO5m'mÓ|U*I z#U_2Z~'qER~ejC>/:VBt @" v@((zji!φܥJF@"\.Wi WKp1␓5RcS>Lb]uA cX(Z~3lF :J{ί3|E-+"n y*;9k`g'E0\S C2פب.mC C5?̈́I10ywN|֡ ˲}=^)P|vZ]N\ֹ"% 3,X,(Hos(Q砍 Pjȹ7m\Bɜ ,`FG1rT\2?0EabnHɾ} E5Hgf>w100Mp98[N:zD/7wCY\.&}w'Yѽn^e Bg@uS0`$H"=!m΀T\)RD[n;>> /\iوk`ƍrYL^)_L,xISX^)U)/R"b6IŹ85*QeʖAg͕o**+5>飍t48}}]y `M|ҽ`zer` ~8p_mccY??2AaB6oތ mڴo߾&FfuJcpm@@w?<, J799)@Bb@t /sVHR2JHY ;\I+>pWpxЦ??яwtٳ痬%9 S {7cڇJ #XxDGs8b1J*g{(<@k7(@EuwX1?200 > fAڱc(B~€4;ǀ2Y~ Y>a@qx|P~XQ ˗/K> ˛oi:L aVyi .0ܹSnWtm#2u8 81 0_ϟgJN {017C䡇廬;1œyY-"̺P> 7mk@h Y aa"@qD+?iV?l:L 0?%V?oPv,2@@N= AznL}(}v RڙQx)p:& vڽ8p{017d޽ PjXw#sϛ ,,W H`YE@P~">a/Šg:L :L+O5` <ڴ Ŷ+Vz:PbJtk+@[,$oQt1Љ'ɓ2Bb] 5 ^ەU+25V6vhAelllY@ǎrxU(ף?L1PLj@ d*Lm{1 ˁ:Ctl@[zv݂++-Œb d\ (nb'xl?EA ,)@~|7 +$h$~zmci/kPD(@kY @qgŸ׹E55|›bf@b (4CfL}VXײpDRNY>@@+}@y7C :@O1N@u6 x805mY p-@(H7K a 5V")L@Pl(:8<ׁch ^`~ܐ{>3?b {޽.B-(ʮv8h`h7߯ktHw3r ܐ~wXu+-XKiiѹPJ(;@;Y^ZA5gHA:M 0yG+YKapv4(?Z@Z@8Lb <~[!&wuX*\(8ogs`kVX_ &Jo/°@-v%pn:O 0ٻwه-U WR sb ]2$!(µb m؋0p N>]>uꔷݿ_#{ܹ7Pб=g]{B@`:W @:.x"@_x:qС[5X :PS̏Vbi?q j{b/ih A?5+;4v B gzկ~%{ FUW/5X :K0eͮ|>8o׮]2h3^ s:?U K >W1cǞC _~|9Նab>Wv۰?v۷?>pr`,aMCL}qESM>|Wls}FVC Vvo|| 6m4{/}/Ϥn }`Qya.˵ۛǘ Ph  aA`-?77>.7<՗k5^VrOy#@0 ?.AoثK` hW?,f`^Mgk=u@w遰PfƳC8rVk_94ԖR<)#=6Sk ̜Kʍ }AH쯻-ue<&xĬZ:a?J[vd1XUVG|R o?2#Ha,}t#`],IxYZ5-'ȩ_G1XJLV^Ga򸋔;ХХ:OuEixV5۾Or"t[pl1پ$بK-+~2ઈ7SVA񁥔QQMalGNj,E2&EJٗR5Nl4vʿ5KTV.RKZ+df"k':=_Jeչ7ƍD}-h6n4>/Q5VZhvE?nf7SYܮ̭}+kѝh8V 2`FP :Gg~/5 LpL; h6cv5F,lt6 ,{NB[\ #",i Z֊[mx 1D Y{H"IENDB`irace/vignettes/irace-package.Rnw0000644000176200001440000052506515060315677016540 0ustar liggesusers% !Rnw weave = knitr %%% DO NOT EDIT the .tex file directly since it is generated from the .Rnw %%% sources. %\VignetteEngine{knitr::knitr} %\VignetteIndexEntry{irace package: User Guide} %\VignetteDepends{knitr} %\VignetteCompiler{knitr} \synctex=1 \RequirePackage{xparse} \RequirePackage[dvipsnames]{xcolor} \documentclass[a4paper,english]{article} \usepackage[utf8]{inputenc} \usepackage[T1]{fontenc} \usepackage{lmodern} \usepackage[a4paper]{geometry} % It saves some pages \usepackage[english]{babel} \usepackage{ifthen} \newboolean{Release} \setboolean{Release}{true} \usepackage{calc} \usepackage{afterpage} \usepackage{algorithm,algorithmic} \usepackage{booktabs} \usepackage{tabularx} \usepackage{xspace} \usepackage{amsmath,amssymb} \usepackage{relsize} \usepackage{fancyvrb} \usepackage{underscore} \usepackage{microtype} % \texttt{test -- test} keeps the "--" as "--" (and does not convert it to an en dash) \DisableLigatures{encoding = T1, family = tt* } \usepackage[hyphens]{url} \usepackage{hyperref} \usepackage[numbers]{natbib} \usepackage[nottoc]{tocbibind} %% For autoref \hypersetup{ colorlinks, linkcolor={red!50!black}, citecolor={blue!50!black}, urlcolor={blue!70!black} } \addto\extrasenglish{% \def\sectionautorefname{Section} \let\subsectionautorefname\sectionautorefname \let\subsubsectionautorefname\sectionautorefname } \usepackage[titletoc, title]{appendix} % Fix use with \autoref \newcommand*{\Appendixautorefname}{Appendix} \usepackage{tocloft} \setlength{\cftsubsecnumwidth}{3em}% Set length of number width in ToC for \subsection \usepackage[inline]{enumitem} \setlist[enumerate]{leftmargin=*,widest=00} \setlist[itemize]{leftmargin=1.5em} %% FIXME: listing is very limited, we should use 'minted' \usepackage{listings} \lstdefinestyle{BashInputStyle}{ language=bash,% basicstyle=\ttfamily,% numbers=none,% frame=tb,% rulecolor=\color{lightgray}, % framesep=1ex, framexleftmargin=1ex, columns=fullflexible,% backgroundcolor=\color{yellow!05},% linewidth=\linewidth,% % xleftmargin=1\linewidth,% identifierstyle=\color{darkgray},% keywordstyle=\color{darkgray},% keywordstyle={[2]\color{Cyan}},% keywordstyle={[3]\color{olive}},% stringstyle=\color{MidnightBlue},% commentstyle=\color{RedOrange},% morestring=[b]',% showstringspaces=false } \DefineVerbatimEnvironment{Code}{Verbatim}{} \DefineVerbatimEnvironment{CodeInput}{Verbatim}{fontshape=rm} \DefineVerbatimEnvironment{CodeOutput}{Verbatim}{} \newenvironment{CodeChunk}{}{} \newcommand{\IRACEHOME}[1]{\hyperlink{irace_home}{\path{$IRACE_HOME}}\path{#1}} \providecommand{\keywords}[1]{\textbf{\textit{Index terms---}} #1} % Simple font selection is not good enough. For example, |\texttt{--}| % gives `\texttt{--}', i.e., an endash in typewriter font. Hence, we % need to turn off ligatures, which currently only happens for commands % |\code| and |\samp| and the ones derived from them. Hyphenation is % another issue; it should really be turned off inside |\samp|. And % most importantly, \LaTeX{} special characters are a nightmare. E.g., % one needs |\~{}| to produce a tilde in a file name marked by |\file|. % Perhaps a few years ago, most users would have agreed that this may be % unfortunate but should not be changed to ensure consistency. But with % the advent of the WWW and the need for getting `|~|' and `|#|' into % URLs, commands which only treat the escape and grouping characters % specially have gained acceptance \makeatletter \DeclareRobustCommand\code{\bgroup\@makeother\_\@makeother\~\@makeother\$\@noligs\@codex} \def\@codex#1{\texorpdfstring% {{\text{\normalfont\ttfamily\hyphenchar\font=-1 #1}}}% {#1}\egroup} \makeatother \let\proglang=\textsf \newcommand{\pkg}[1]{{\fontseries{b}\selectfont #1}} \newcommand{\aR}{\proglang{R}\xspace} \newcommand{\MATLAB}{\proglang{MATLAB}\xspace} \newcommand{\eg}{e.g.,\xspace} \newcommand{\SoftwarePackage}{\pkg} \newcommand{\ACOTSP}{\SoftwarePackage{ACOTSP}\xspace} %% How to use this command: % Parameter with one short switch: \defparameter[short]{paramName}{long}{default} % Parameter without short switch: \defparameter{paramName}{long}{default} % Parameter without switch: \defparameter{paramName}{}{default} \newcommand{\defparameter}[4][]{% \item[\code{#2}]\hypertarget{opt:#2}{} ~~ % \ifthenelse{\equal{#3}{}}{}{% \emph{flag:} % \ifthenelse{\equal{#1}{}}{}{% \code{-#1}~~~\emph{or}~~~}% \code{--#3} ~~ }% \emph{default:}~\texttt{#4} \\ } \newcommand{\parameter}[1]{\hyperlink{opt:#1}{\code{#1}}} \newcommand{\iracefun}[1]{\href{https://mlopez-ibanez.github.io/irace/reference/#1.html}{\code{#1()}}} %\usepackage{showlabels} %\showlabels{hypertarget} \newcommand{\irace}{\pkg{irace}\xspace} \newcommand{\Irace}{\pkg{Irace}\xspace} \newcommand{\race}{\pkg{race}\xspace} \newcommand{\FRACE}{\text{F-Race}\xspace} \newcommand{\IFRACE}{\text{I/F-Race}\xspace} \newcommand{\PyImp}{\pkg{PyImp}\xspace} \newcommand{\iraceversion}{\Sexpr{packageVersion("irace")}} \newcommand{\Niter}{\ensuremath{N^\text{iter}}\xspace} \newcommand{\Nparam}{\ensuremath{{N^\text{param}}}\xspace} \newcommand{\iter}{\ensuremath{j}\xspace} \newcommand{\Budget}{\ensuremath{B}\xspace} \newcommand{\Budgetj}{\ensuremath{\Budget_{\iter}}\xspace} \newcommand{\Bused}{\ensuremath{\Budget_\text{used}}\xspace} \newcommand{\Ncand}[1][]{\ensuremath{N_{#1}}\xspace} \newcommand{\Mui}{\ensuremath{\mu_{\iter}}\xspace} \newcommand{\Nmin}{\ensuremath{N^\text{min}}\xspace} \newcommand{\Nsurv}{\ensuremath{N^\text{surv}}\xspace} \newcommand{\Nelite}{\ensuremath{N^\text{elite}}\xspace} \newcommand{\Nnew}{\ensuremath{N^\text{new}}\xspace} \newcommand{\bmax}{\ensuremath{b^\text{max}}\xspace} \newcommand{\bmin}{\ensuremath{b^\text{min}}\xspace} \newcommand{\Celite}{\ensuremath{\Theta^\text{elite}}\xspace} \ifthenelse {\boolean{Release}}{% \newcommand{\MANUEL}[1]{} \newcommand{\LESLIE}[1]{} \newcommand{\THOMAS}[1]{} }{% \newcommand{\MANUEL}[1]{{\footnotesize\noindent\textbf{\color{red}[~MANUEL: #1~]}}} \newcommand{\LESLIE}[1]{\footnote{\noindent\textbf{[ LESLIE: #1 ]}}} \newcommand{\THOMAS}[1]{\footnote{\noindent\textbf{[ THOMAS: #1 ]}}} } \newcommand{\hide}[1]{} \usepackage{tcolorbox} \newcommand{\infoicon}{% \parbox[c]{0.75cm}{\includegraphics[keepaspectratio=true,width=0.75cm]{light-bulb-icon}}% \hspace{1em}} \newcommand{\warningicon}{% \parbox[c]{0.75cm}{\includegraphics[keepaspectratio=true,width=0.75cm]{Warning-icon}}% \hspace{1em}} \definecolor{LightGray}{RGB}{193,193,193} \definecolor{LightYellow}{RGB}{253,247,172} \newlength\macroiconwidth \newenvironment{xwarningbox}{% \setlength{\fboxrule}{3.0\fboxrule}% \setlength{\fboxsep}{0\fboxsep}% \begin{tcolorbox}[colback=LightYellow,colframe=LightGray,boxrule=\fboxrule,boxsep=\fboxsep]% \infoicon% \settowidth{\macroiconwidth}{\infoicon}% \begin{minipage}[c]{\columnwidth - \macroiconwidth - 2.0\fboxrule - 2.0\fboxsep} \raggedright\footnotesize % }{% \end{minipage} \end{tcolorbox} % } % Workaround for broken knitr: https://github.com/yihui/knitr-examples/blob/master/036-latex-if.tex \providecommand{\hldef}[1]{\textcolor[rgb]{0.345,0.345,0.345}{#1}}% \providecommand{\hlsng}[1]{\textcolor[rgb]{0.192,0.494,0.8}{#1}}% <>= library(knitr) knit_hooks$set(inline = function(x) { if (is.numeric(x)) return(knitr:::format_sci(x, 'latex')) highr::hi_latex(x) }) @ \begin{document} <>= library(knitr) @ \author{Manuel L\'opez-Ib\'a\~nez, Leslie P\'erez C\'aceres, J\'er\'emie Dubois-Lacoste,\\ Thomas St\"utzle and Mauro Birattari \\IRIDIA, CoDE, Universit\'e Libre de Bruxelles, Brussels, Belgium} \title{The \irace Package: User Guide} \date{Version \iraceversion, \today} %\keywords{automatic % algorithm configuration, racing, parameter tuning, \aR} \maketitle \tableofcontents %Load files needed for examples <>= library("irace") load("examples.Rdata") # loads "experiment" and "output" iraceResults <- irace::read_logfile(system.file(package="irace", "exdata", "irace-acotsp.Rdata", mustWork=TRUE)) log_ablation_file <- system.file(package="irace", "exdata", "log-ablation.Rdata", mustWork = TRUE) load(log_ablation_file) options(width = 70) @ \newpage %% %% %% %% General info %% %% %% \section{General information} \MANUEL{Some things could be taken from the intro of the irace paper and reformulated.} \MANUEL{It would be good to mention that not only opt algorithms can be configured with irace, we say this in the paper.} \subsection{Background} \MANUEL{I would add a paragraph defining what is irace (a bit longer than the abstract above) and references to the literature so people can find more info. The first reference should be the irace TR.} \LESLIE{Here i guess we should say why tune an algorithm is a good idea, and why using irace is a better one.} The \irace package implements an \emph{iterated racing} procedure, which is an extension of Iterated F-race (\IFRACE)~\cite{BirYuaBal2010:emaoa}. The main use of \irace is the automatic configuration of optimization and decision algorithms, that is, finding the most appropriate settings of an algorithm given a set of instances of a problem. However, it may also be useful for configuring other types of algorithms when performance depends on the used parameter settings. It builds upon the \pkg{race} package by Birattari and it is implemented in \aR. The \irace package is available from CRAN: % \begin{center} \url{https://cran.r-project.org/package=irace} \end{center} % More information about \irace is available at \url{https://mlopez-ibanez.github.io/irace}. \subsection{Version} The current version of the \irace package is \iraceversion. Previous versions of the package can also be found in the \href{https://cran.r-project.org/package=irace}{CRAN website}. The algorithm underlying the current version of \irace and its motivation are described by \citet{LopDubPerStuBir2016irace}. The \textbf{adaptive capping mechanism} available from version $3.0$ is described by \citet{PerLopHooStu2017:lion}. Details of the implementation before version 2.0 can be found in a previous technical report~\cite{LopDubStu2011irace}. % \begin{xwarningbox} Versions of \irace before 2.0 are not compatible with the file formats detailed in this document. \end{xwarningbox} \subsection{License} The \irace package is Copyright \copyright{} \the\year\ and distributed under the GNU General Public License version 3.0 (\url{http://www.gnu.org/licenses/gpl-3.0.en.html}). The \irace package is free software (software libre): You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. The \irace package is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. Please be aware that the fact that this program is released as Free Software does not excuse you from scientific propriety, which obligates you to give appropriate credit! If you write a scientific paper describing research that made substantive use of this program, it is your obligation as a scientist to (a) mention the fashion in which this software was used in the Methods section; (b) mention the algorithm in the References section. The appropriate citation is: \begin{itemize}[leftmargin=3em] \item[] Manuel López-Ibáñez, Jérémie Dubois-Lacoste, Leslie Pérez Cáceres, Thomas Stützle, and Mauro Birattari. The \irace package: Iterated Racing for Automatic Algorithm Configuration. \emph{Operations Research Perspectives}, 3:43--58, 2016. doi:~\href{http://dx.doi.org/10.1016/j.orp.2016.09.002}{10.1016/j.orp.2016.09.002} \end{itemize} \section{Before starting} \MANUEL{I think this could be a bit more detailed by defining what is a parameter, a configuration, an instance, etc. but ok for now.} The \irace package provides an automatic configuration tool for tuning optimization algorithms, that is, automatically finding good configurations for the parameters values of a (target) algorithm saving the effort that normally requires manual tuning. \begin{figure}[t] \centering \includegraphics[width=0.6\textwidth]{irace-scheme} \caption{Scheme of \irace flow of information.} \label{fig:irace-scheme} \end{figure} Figure~\ref{fig:irace-scheme} gives a general scheme of how \irace works. \Irace receives as input a \emph{parameter space definition} corresponding to the parameters of the target algorithm that will be tuned, a set of \emph{instances} for which the parameters must be tuned for and a set of options for \irace that define the \emph{configuration scenario}. Then, \irace searches in the parameter search space for good performing algorithm configurations by executing the target algorithm on different instances and with different parameter configurations. A \parameter{targetRunner} must be provided to execute the target algorithm with a specific parameter configuration ($\theta$) and instance ($i$). The \parameter{targetRunner} function (or program) acts as an interface between the execution of the target algorithm and \irace: It receives the instance and configuration as arguments and must return the evaluation of the execution of the target algorithm. The following user guide contains guidelines for installing \irace, defining configuration scenarios, and using \irace to automatically configure your algorithms. %% %% %% %% Installation %% %% %% \section{Installation} \subsection{System requirements} \begin{itemize} \item \aR ($\text{version} \geq 3.2.0$) is required for running irace, but you don't need to know the \aR language to use it. \aR is freely available and you can download it from the \aR project website (\url{https://www.r-project.org}). See \autoref{sec:installation} for a quick installation guide of \aR. \item For GNU/Linux and OS X, the command-line executable \code{parallel-irace} requires GNU Bash. Individual examples may require additional software. \end{itemize} \subsection{\irace installation} \label{sec:irace install} The \irace package can be installed automatically within \aR or by manual download and installation. We advise to use the automatic installation unless particular circumstances do not allow it. The instructions to install \irace with the two mentioned methods are the following: \subsubsection[Install automatically within R]{Install automatically within \aR{}} Execute the following line in the \aR console to install the package: <>= install.packages("irace") @ Select a mirror close to your location, and test the installation in the \aR console with: <>= library("irace") q() # To exit R @ Alternatively, within the \aR graphical interface, you may use the \code{Packages and data->Package installer} menu on OS X or the \code{Packages} menu on Windows. \subsubsection{Manual download and installation} From the \irace package CRAN website (\url{https://cran.r-project.org/package=irace}), download one of the three versions available depending on your operating system: \begin{itemize} \item \code{irace_\iraceversion.tar.gz} (Unix/BSD/GNU/Linux) \item \code{irace_\iraceversion.tgz} (OS X) \item \code{irace_\iraceversion.zip} (Windows) \end{itemize} To install the package on GNU/Linux and OS X, you must execute the following command at the shell (replace \code{} with the path to the downloaded file, either \code{irace_\iraceversion.tar.gz} or \code{irace_\iraceversion.zip}): % \begin{lstlisting}[style=BashInputStyle] R CMD INSTALL \end{lstlisting} To install the package on Windows, open \aR and execute the following line on the \aR console (replace \code{} with the path to the downloaded file \code{irace_\iraceversion.zip}): %\LESLIE{Check that this actually works on internet says that this: \code{Rscript -e "install.packages('foo.zip', repos = NULL)"} also works} % <>= install.packages("", repos = NULL) @ If the previous installation instructions fail because of insufficient permissions and you do not have sufficient admin rights to install \irace system-wide, then you need to force a local installation. \subsubsection{Local installation} Let's assume you wish to install \irace on a path denoted by \code{}, which is a filesystem path for which you have sufficient rights. This directory \textbf{must} exist before attempting the installation. Moreover, you must provide to \aR the path to this library when loading the package. However, the latter can be avoided by adding the path to the system variable \code{R_LIBS} or to the \aR internal variable \code{.libPaths}, as we will see below.\footnote{% On Windows, see also \url{https://cran.r-project.org/bin/windows/base/rw-FAQ.html\#I-don_0027t-have-permission-to-write-to-the-R_002d3_002e3_002e1_005clibrary-directory}.} On GNU/Linux or OS X, execute the following commands to install the package on a local directory: \begin{lstlisting}[style=BashInputStyle] export R_LIBS_USER="" # Create R_LIBS_USER if it doesn't exist mkdir $R_LIBS_USER # Replace with the path to the downloaded file. R CMD INSTALL --library=$R_LIBS_USER # Tell R where to find R_LIBS_USER export R_LIBS=${R_LIBS_USER}:${R_LIBS} \end{lstlisting} On Windows, you can install the package on a local directory by executing the following lines in the \aR console: <>= # Replace with the path to the downloaded file. # Replace with the path used for installation. install.packages("", repos = NULL, lib = "") # Tell R where to find R_LIBS_USER. # This must be executed for every new session. .libPaths(c("", .libPaths())) @ \subsubsection{Testing the installation and invoking irace} Once \irace has been installed, load the package and test that the installation was successful by opening an \aR console and executing: <>= # Load the package library("irace") # Obtain the installation path system.file(package = "irace") @ The last command must print out the filesystem path where \irace is installed. In the remainder of this guide, the variable \code{\$IRACE_HOME} is used to denote this path. When executing any provided command that includes the \code{\$IRACE_HOME} variable do not forget to replace this variable with the installation path of \irace. On GNU/Linux or OS X, you can let the operating system know where to find \irace by defining the \code{\$IRACE_HOME} variable and adding it to the system \code{PATH}. Append the following commands to \path{~/.bash_profile}, \path{~/.bashrc} or \path{~/.profile}: % %<>= \begin{lstlisting}[style=BashInputStyle] # Replace with the irace installation path export IRACE_HOME= export PATH=${IRACE_HOME}/bin/:$PATH # Tell R where to find R_LIBS_USER # Use the following line only if local installation was forced export R_LIBS=${R_LIBS_USER}:${R_LIBS} \end{lstlisting} %@ Then, open a new terminal and launch \irace as follows: %<>= \begin{lstlisting}[style=BashInputStyle] irace --help \end{lstlisting} %@ On Windows, you need to add both \aR and the installation path of \irace to the environment variable \code{PATH}. To edit the \code{PATH}, search for ``Environment variables'' in the control panel, edit \code{PATH} and add a string similar to \path{C:\R_PATH\bin;C:\IRACE_HOME\bin\x64\} where \code{R_PATH} is the installation path of \aR and \code{IRACE_HOME} is the installation path of \irace. If \irace was installed locally, you also need to edit the environment variable \code{R_LIBS} to add \code{R_LIBS_USER}. Then, open a new terminal (run program \code{cmd.exe}) and launch \irace as: % %<>= \begin{lstlisting}[style=BashInputStyle] irace.exe --help \end{lstlisting} %@ Alternatively, you may directly invoke \irace from within the \aR console by executing: <>= library("irace") irace_cmdline("--help") @ \section{Running irace}\label{sec:execution} Before performing the tuning of your algorithm, it is necessary to define a tuning scenario that will give \irace all the necessary information to optimize the parameters of the algorithm. The tuning scenario is composed of the following elements: \begin{enumerate} \item Target algorithm parameter description (see \autoref{sec:target parameters}). \item Target algorithm runner (see \autoref{sec:runner}). \item Training instances list (see \autoref{sec:training}) \item \irace options (see \autoref{sec:irace options}). \item \textit{Optional:} Initial configurations (see \autoref{sec:initial}). \item \textit{Optional:} Target algorithm evaluator (see \autoref{sec:evaluator}). \end{enumerate} These scenario elements can be provided as plain text files or as \aR objects. This user guide provides examples of both types, but we advise the use of plain text files, which we consider the simpler option. For a step-by-step guide to create the scenario elements for your target algorithm continue to \autoref{sec:step}. For an example execution of \irace using the \ACOTSP scenario go to \autoref{sec:example}. \subsection{Step-by-step setup guide}\label{sec:step} This section provides a guide to setup a basic execution of \irace. The template files provided in the package (\IRACEHOME{/templates}) will be used as basis for creating your new scenario. Please follow carefully the indications provided in each step and in the template files used; if you have doubts check the the sections that describe each option in detail. \begin{enumerate}[leftmargin=*] \item Create a directory (\eg~\path{./tuning/}) for the scenario setup. This directory will contain all the files that describe the scenario. On GNU/Linux or OS X, you can do this as follows: %<>= \begin{lstlisting}[style=BashInputStyle] mkdir ./tuning cd ./tuning \end{lstlisting} %@ \item Initialize the tuning directory with template config files. On GNU/Linux or OS X, you can do this as follows: %<>= \begin{lstlisting}[style=BashInputStyle] irace --init \end{lstlisting} %@ \item Define the target algorithm parameters to be tuned by following the instructions in \code{parameters.txt}. Available parameter types and other guidelines can be found in \autoref{sec:target parameters}. \item \textit{Optional}: Define the initial parameter configuration(s) of your algorithm, which allows you to provide good starting configurations (if you know some) for the tuning. Follow the instructions in \code{configurations.txt} and set \parameter{configurationsFile}\code{="configurations.txt"} in \code{scenario.txt}. More information in \autoref{sec:initial}. If you do not need to define initial configurations remove this file from the directory. \item Place the instances you would like to use for the tuning of your algorithm in the folder \path{./tuning/Instances/}. In addition, you can create a file (\eg~\code{instances-list.txt}) that specifies which instances from that directory should be run and which instance-specific parameters to use. To use such an instance file, set the appropriate option in \code{scenario.txt}, e.g., \parameter{trainInstancesFile} \code{= "instances-list.txt"}. See \autoref{sec:training} for guidelines. \item Uncomment and assign in \code{scenario.txt} only the options for which you need a value different from the default. Some common options that you might want to adjust are: \begin{description} \item[\parameter{execDir}] (\code{--exec-dir}): the directory in which \irace will execute the target algorithm; the default value is the current directory. %\item[\parameter{logFile}] (\code{--log-file}): a file the name of the results \aR data file that produces \irace. \item[\parameter{maxExperiments}] (\code{--max-experiments}): the maximum number of executions of the target algorithm that \irace will perform. \item[\parameter{maxTime}] (\code{--max-time}): maximum total execution time in seconds for the executions of \code{targetRunner}. In this case, \code{targetRunner} must return two values: cost and time. Note that you must provide either \parameter{maxTime} or \parameter{maxExperiments}. \item[\parameter{trainInstancesDir}] (\code{--train-instances-dir}): set to \path{./Instances} if you put the training instances in that folder as instructed above. \end{description} For setting the tuning budget, see \autoref{sec:budget}. For more information on \irace options and their default values, see \autoref{sec:irace options}. \item Modify the \code{target-runner} script to run your algorithm. This script must execute your algorithm with the parameters and instance specified by \irace and return the evaluation of the execution and \textit{optionally} the execution time (\code{cost [time]}). When the \parameter{maxTime} option is used, returning \code{time} is mandatory. The \code{target-runner} template is written in \proglang{GNU Bash} scripting language, which can be executed easily in GNU/Linux and OS X systems. However, you may use any other programming language. We provide examples written in \proglang{Python}, \proglang{MATLAB} and other languages in \IRACEHOME{/examples/}. An example using Julia is available at \url{https://github.com/sbomsdorf/An-example-of-irace-using-Julia-code}. % Follow these instructions to adjust the given \code{target-runner} template to your algorithm: \begin{enumerate} \item Set the \code{EXE} variable with the path to the executable of the target algorithm. \item Set the \code{FIXED_PARAMS} if you need extra arguments in the execution line of your algorithm. An example could be the time that your algorithm is required to run (\code{FIXED_PARAMS}\hspace{0pt}\code{=}\hspace{0pt}\code{"--time 60"}) or the number of evaluations required (\code{FIXED_PARAMS}\hspace{0pt}\code{=}\hspace{0pt}\code{"--evaluations 10000"}). \item The line provided in the template executes the executable described in the \code{EXE} variable. % \begin{center} \code{\$EXE \$\{FIXED_PARAMS\} -i \$\{INSTANCE\} --seed \$\{SEED\} \$\{CONFIG_PARAMS\}} \end{center} % You must change this line according to the way your algorithm is executed. In this example, the algorithm receives the instance to solve with the flag \code{-i} and the seed of the random number generator with the flag \code{--seed}. The variable \code{CONFIG_PARAMS} adds to the command line the parameters that \irace has given for the execution. You must set the command line execution as needed. For example, the instance might not need a flag and might need to be the first argument: \begin{center} \code{\$EXE \$\{INSTANCE\} \$\{FIXED_PARAMS\} --seed \$\{SEED\} \$\{CONFIG_PARAMS\}} \end{center} The output of your algorithm is saved to the file defined in the \code{\$STDOUT} variable, and error output is saved in the file given by \code{\$STDERR}. The line: \begin{center} \code{if [ -s "\${STDOUT}" ]; then} \end{center} checks if the file containing the output of your algorithm is not empty. The example provided in the template assumes that your algorithm prints in the last output line the best result found (only a number). The line: \begin{center} \code{COST=\$(cat \$\{STDOUT\} | grep -e '\^{}[[:space:]]*[+-]\textbackslash{}?[0-9]' | cut -f1)} \end{center} parses the output of your algorithm to obtain the result from the last line. The \code{target-runner} script must print \textbf{only} one number. In the template example, the result is printed with \code{echo "\$COST"} (assuming \parameter{maxExperiments} is used) and the generated files are deleted (you may remove that line if you wish to keep them). \begin{xwarningbox} The \code{target-runner} script must be an executable file, unless you specify \parameter{targetRunnerLauncher}. \end{xwarningbox} You can test the target runner from the \aR console by checking the scenario as explained earlier in \autoref{sec:execution}. If you have problems related to the \code{target-runner} script when executing \irace, see \autoref{sec:check list} for a check list to help diagnose common problems. For more information about the \parameter{targetRunner}, please see \autoref{sec:runner}, \end{enumerate} \item \textit{Optional}: Modify the \code{target-evaluator} file. This is rarely needed and the \code{target-runner} template does not use it. \autoref{sec:evaluator} explains when a \parameter{targetEvaluator} is needed and how to define it. \item The \irace executable provides an option (\parameter{--check}) to check that the scenario is correctly defined. We recommend to perform a check every time you create a new scenario. When performing the check, \irace will verify that the scenario and parameter definitions are correct and will test the execution of the target algorithm. To check your scenario execute the following commands: \begin{itemize} \item From the command-line (on Windows, execute \code{irace.bat}): %<>= \begin{lstlisting}[style=BashInputStyle] # $IRACE_HOME is the installation directory of irace. $IRACE_HOME/bin/irace --scenario scenario.txt --check \end{lstlisting} %@ \item Or from the \aR console: <>= library("irace") scenario <- readScenario(filename = "scenario.txt", scenario = defaultScenario()) checkIraceScenario(scenario = scenario) @ \end{itemize} \item Once all the scenario elements are prepared you can execute \irace, either using the command-line wrappers provided by the package or directly from the \aR console: \begin{itemize} \item \textbf{From the command-line console}, call the command (on Windows, you should execute \code{irace.exe}): \begin{lstlisting}[style=BashInputStyle] cd ./tuning/ # $IRACE_HOME is the installation directory of irace # By default, irace reads scenario.txt, you can specify a different file # with --scenario. $IRACE_HOME/bin/irace \end{lstlisting} For this example, we assume that the needed scenario files have been set properly in the \code{scenario.txt} file using the options described in \autoref{sec:irace options}. Most \irace options can be specified in the command line or directly in the \code{scenario.txt} file. \item \textbf{From the \aR console}, evaluate: <>= library("irace") # Go to the directory containing the scenario files setwd("./tuning") scenario <- readScenario(filename = "scenario.txt", scenario = defaultScenario()) irace_main(scenario = scenario) @ \end{itemize} This will perform one run of \irace. See the output of \code{irace --help} in the command-line or \code{irace_cmdline("--help")} in \aR for quick information on additional \irace options. For more information about \irace options, see \autoref{sec:irace options}. \end{enumerate} \begin{xwarningbox} Command-line options override the same options specified in the \code{scenario.txt} file. \end{xwarningbox} \subsection{Setup example for ACOTSP}\label{sec:example} The \ACOTSP tuning example can be found in the package installation in the folder \IRACEHOME{/examples/acotsp}. % Other example scenarios can be found in the same folder. More examples of tuning scenarios can be found in the Algorithm Configuration Library (AClib, \url{http://www.aclib.net/}). In this section, we describe how to execute the \ACOTSP scenario. If you wish to start setting up your own scenario, continue to the next section. For this example, we assume a GNU/Linux system such as Ubuntu with a working \proglang{C} compiler such as \code{gcc}. To execute this scenario follow these steps: \begin{enumerate} \item Create a directory for the tuning (\eg~\path{./tuning/}) and copy the example scenario files located in the \code{examples} folder to the created directory: %<>= \begin{lstlisting}[style=BashInputStyle] mkdir ./tuning cd ./tuning # $IRACE_HOME is the installation directory of irace. cp $IRACE_HOME/examples/acotsp/* ./ ls ./ # Make sure that target-runner is executable chmod u+x target-runner \end{lstlisting} %@ \item Download the training instances from \url{https://iridia.ulb.ac.be/supp/IridiaSupp2016-003/scenarios/acotsp/instances.tar.gz} to the \path{./tuning/} directory and decompress it, which creates create a folder \path{instances}: %<>= \begin{lstlisting}[style=BashInputStyle] tar -xvf instances.tar.gz ls instances/ \end{lstlisting} %@ If the above gives an error or does not show any files, then the files were not extract correctly. Maybe the \path{instances.tar.gz} file did not download correctly or maybe it is not in the correct place. It should be within the folder \path{tuning}.correctly. \item Download the \ACOTSP software from \url{https://github.com/MLopez-Ibanez/ACOTSPQAP/archive/refs/heads/master.zip} to the \path{./tuning/} directory and compile the \path{acotsp} executable using \code{make}. %<>= \begin{lstlisting}[style=BashInputStyle] unzip master.zip make -C ACOTSPQAP-master acotsp ./ACOTSPQAP-master/acotsp --help \end{lstlisting} %@ If the above gives an error, then the \path{acotsp} executable failed to compile for some reason. Maybe you are missing the C compiler or some files did not extract correctly. \item Create a directory for executing the experiments and execute \irace: %<>= \begin{lstlisting}[style=BashInputStyle] mkdir ./acotsp-arena/ # $IRACE_HOME is the installation directory of irace. $IRACE_HOME/bin/irace \end{lstlisting} %@ Or you can also execute \irace from the \aR console using: <>= library("irace") setwd("./tuning/") irace_cmdline() @ \end{enumerate} The most usual sources of error when running the above commands are: \begin{itemize} \item The \irace package is not correctly installed. Please make sure that installing \irace did not give any errors. \item The location of the files is not correct. Please make sure that you have: \begin{itemize} \item The folder \path{tuning} and that it contains the files \path{scenario.txt}, \path{parameters-acotsp.txt}, \path{target-runner}, and the folders \path{instances}, \path{ACOTSPQAP-master} and \path{acotsp-arena}. \item The folder \path{instances} should contain the TSP instance files (\path{*.tsp}). \item The folder \path{ACOTSPQAP-master} should contain the executable \path{acotsp}. You should be able to invoke \code{./ACOTSPQAP-master/acotsp --help} without an error. \item The folder \path{acotsp-arena} should be empty. \end{itemize} \end{itemize} %% %% %% %% Scenario settings %% %% %% \section{Defining a configuration scenario}\label{sec:scenario} \subsection{Target algorithm parameters} \label{sec:target parameters} The parameters of the target algorithm are defined by a parameter file as described in \autoref{sec:parameters file}. Optionally, when executing \irace from the \aR console, the parameters can be specified directly as an \aR object (see \autoref{sec:parameters object}). For defining your parameters follow the guidelines provided in the following sections. \subsubsection{Parameter types} Each target parameter has an associated type that defines its domain and the way \irace handles them internally. Understanding the nature of the domains of the target parameters is important to select appropriate types. The four basic types supported by \irace are the following: \begin{itemize} \item \textit{Real} parameters are numerical parameters that can take floating-point values within a given range. The range is specified as an interval `\code{(,)}'. This interval is closed, that is, the parameter value may eventually be one of the bounds. The possible values are rounded to a number of \emph{decimal places} specified by the global option \code{digits} (Section~\ref{sec:globaloptions}). For example, given the default number of digits of $4$, the values $0.12345$ and $0.12341$ are both rounded to $0.1234$. Selected real-valued parameters can be optionally sampled on a logarithmic scale (base $e$). \item \textit{Integer} parameters are numerical parameters that can take only integer values within the given range. Their range is specified as the range of real parameters and they can also be optionally sampled on a logarithmic scale (base $e$). \item \textit{Categorical} parameters are defined by a set of possible values specified as `\code{(,} \code{...,} \code{)}'. The values are quoted or unquoted character strings. Empty strings and strings containing commas or spaces must be quoted. \item \emph{Ordinal} parameters are defined by an \emph{ordered} set of possible values in the same format as for categorical parameters. They are handled internally as integer parameters, where the integers correspond to the indexes of the values. \end{itemize} \begin{xwarningbox} Boolean (or logical) parameters are best encoded as categorical ones with just two values rather than integer ones with domain $(0,1)$. Some boolean parameters take an explicit value (0/1 or true/false) such as: \begin{CodeInput} dlb "--dlb " c (0, 1) \end{CodeInput} Others are switches whose presence activates the parameter: \begin{CodeInput} dlb "" c ("", "--dlb") \end{CodeInput} \end{xwarningbox} \subsubsection{Parameter domains} For each target parameter, an interval or a set of values must be defined according to its type, as described above. There is no limit for the size of the set or the length of the interval, but keep in mind that larger domains could increase the difficulty of the tuning task. Choose always values that you consider relevant for the tuning. In case of doubt, we recommend to choose larger intervals, as occasionally best parameter settings may be not intuitive a priori. All intervals are considered as closed intervals. It is possible to define parameters that will have always the same value. Such ``\emph{fixed}'' parameters will not be tuned but their values are used when executing the target algorithm and they are affected by constraints defined on them. All fixed parameters must be defined as categorical parameters and have a domain of one element. \subsubsection{Parameter dependent domains}\label{sec:paramdependant} Domains that are dependent on the values of other parameters can be specified only for numerical parameters (both integer and real). To do so, the dependent domain must be expressed in function of another parameter, which must be a numerical parameter. The expression that defines a dependency must be written between quotes: \code{(value, "expression")} or \code{("expression", value)} or \code{("expression", "expression")}. The expressions can only use the following operators and \aR functions: \code{+}, \code{-}, \code{*}, \code{/}, \code{\%\%}, \code{min}, \code{max}, \code{round}, \code{floor}, \code{ceiling}, \code{trunc}. If you need to use an operator or function not listed here, please contact us. \begin{xwarningbox} The user must ensure that the defined domain is valid at all times since \irace currently is not able to detect possible invalid domains based on the expressions provided. \end{xwarningbox} If you have a parameter \code{p2} that is just a transformation of another \code{p1}, then instead of using a dependent domain (left-hand side of the following example), it will be better to create a dummy parameter that controls the transformation (right-hand side) and do the transformation within \code{target-runner}. For example: % \begin{center} \begin{minipage}{0.4\linewidth} \small\centering% \begin{CodeInput}[frame=single] # With dependent domains p1 "" r (0, 100) p2 "" r ("p1", "p1 + 10") \end{CodeInput} \end{minipage} \hspace{\stretch{1}} should be \hspace{\stretch{1}} \begin{minipage}{0.4\linewidth} \small\centering% \begin{CodeInput}[frame=single] # With a dummy parameter p1 "" r (0, 100) p2dum "" r (0, 10) \end{CodeInput} \end{minipage} \end{center} % and \code{target-runner} will compute $\code{p2} = \code{p2dum} \cdot \code{p1}$. \subsubsection{Conditional parameters}\label{sec:conditional} Conditional parameters are active only when others have certain values. These dependencies define a hierarchical relation between parameters. For example, the target algorithm may have a parameter \code{localsearch} that takes values \code{(sa, ts)} and another parameter \code{ts-length} that only needs to be set if the first parameter takes precisely the value \code{ts}. Thus, parameter \code{ts-length} is conditional on \code{localsearch == "ts"}. \subsubsection{Forbidden parameter configurations}\label{sec:forbidden} A line containing just \texttt{[forbidden]} ends the list of parameters and starts the list of forbidden expressions. Each line is a logical expression (in \aR syntax) containing parameter names as defined by the \parameter{parameterFile} (\autoref{sec:target parameters}), values and logical operators. For a list of \aR logical operators see: \begin{center} \url{https://stat.ethz.ch/R-manual/R-devel/library/base/html/Syntax.html} \end{center} If \irace generates a parameter configuration that makes any of the logical expressions evaluate to \code{TRUE}, then the configuration is considered forbidden and it is never evaluated. This is useful when some combination of parameter values could cause the target algorithm to crash, consume excessive CPU time or memory, or when it is known that they do no produce satisfactory results. \begin{xwarningbox} Initial configuration (\autoref{sec:initial}) that are forbidden will be discarded with a warning. \end{xwarningbox} If the forbidden constraints provided are too strict, \irace may produce the following error: \begin{CodeInput} irace tried 100 times to sample from the model a configuration not forbidden without success, perhaps your constraints are too strict? \end{CodeInput} % In that case, it may be a good idea to reformulate the forbidden constraints as conditional parameters (\autoref{sec:conditional}), parameter-dependent domains (\autoref{sec:paramdependant}), repairing the configurations (\autoref{sec:repairconf}) or post-processing within the target-algorithm (\autoref{sec:complex_domains}). \subsubsection{Global options}\label{sec:globaloptions} A line containing just \texttt{[global]} starts the definition of global options. The only global option currently implemented is \code{digits}, which controls the number of decimal digits for real valued parameters. Its default value is 4. \subsubsection{Parameter file format}\label{sec:parameters file} For simplicity, the description of the parameters space is given as a table. Each line of the table defines a configurable parameter % \begin{center} \code{\