1 /** 2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved. 3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0) 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 */ 6 module distssh.config; 7 8 import core.time : dur, Duration; 9 import logger = std.experimental.logger; 10 import std.algorithm : among, remove, filter, find, map, maxElement; 11 import std.array : array, empty; 12 import std.file : thisExePath, getcwd; 13 import std.format : format; 14 import std.getopt : defaultGetoptPrinter; 15 import std.meta : AliasSeq; 16 import std.path : baseName, buildPath, dirName, absolutePath; 17 import std.range : drop; 18 import std.stdio : writeln, writefln; 19 import std..string : toLower; 20 import std.traits : EnumMembers, hasMember; 21 import std.variant : Algebraic, visit; 22 static import std.getopt; 23 24 import colorlog : VerboseMode; 25 26 import distssh.types; 27 28 version (unittest) { 29 import unit_threaded.assertions; 30 } 31 32 struct Config { 33 struct Global { 34 std.getopt.GetoptResult helpInfo; 35 VerboseMode verbosity; 36 string progName; 37 bool noImportEnv; 38 bool cloneEnv; 39 bool stdinMsgPackEnv; 40 Duration timeout = defaultTimeout_s.dur!"seconds"; 41 42 string selfBinary; 43 string selfDir; 44 45 string importEnv; 46 string workDir; 47 string[] command; 48 49 Path dbPath; 50 51 /// The hosts the cluster consist of. 52 Host[] cluster; 53 } 54 55 struct Help { 56 std.getopt.GetoptResult helpInfo; 57 } 58 59 struct Shell { 60 std.getopt.GetoptResult helpInfo; 61 static string helpDescription = "open an interactive shell on the remote host"; 62 } 63 64 struct Cmd { 65 std.getopt.GetoptResult helpInfo; 66 static string helpDescription = "run a command on a remote host"; 67 } 68 69 struct LocalRun { 70 std.getopt.GetoptResult helpInfo; 71 static string helpDescription = "import env and run the command locally"; 72 bool useFakeTerminal; 73 } 74 75 struct Install { 76 std.getopt.GetoptResult helpInfo; 77 static string helpDescription = "install distssh by setting up the correct symlinks"; 78 } 79 80 struct MeasureHosts { 81 std.getopt.GetoptResult helpInfo; 82 static string helpDescription = "measure the login time and load of all remote hosts"; 83 } 84 85 struct LocalLoad { 86 std.getopt.GetoptResult helpInfo; 87 static string helpDescription = "measure the load on the current host"; 88 } 89 90 struct RunOnAll { 91 std.getopt.GetoptResult helpInfo; 92 static string helpDescription = "run the command on all remote hosts"; 93 } 94 95 struct LocalShell { 96 std.getopt.GetoptResult helpInfo; 97 static string helpDescription = "run the shell locally"; 98 } 99 100 struct Env { 101 std.getopt.GetoptResult helpInfo; 102 static string helpDescription = "manipulate the stored environment"; 103 /// Print the environment. 104 bool print; 105 /// Env variable to set in the config specified in importEnv. 106 string[] envSet; 107 /// Env variables to remove from the onespecified in importEnv. 108 string[] envDel; 109 /// Export the current environment 110 bool exportEnv; 111 } 112 113 struct Daemon { 114 std.getopt.GetoptResult helpInfo; 115 static string helpDescription = "update the cluster statistics in the database"; 116 Duration timeout; 117 /// If the daemon should persist in the background after it has measured the cluster once. 118 bool background; 119 /// Force the server to start even though there may be one running in the background 120 bool forceStart; 121 } 122 123 struct Purge { 124 std.getopt.GetoptResult helpInfo; 125 static string helpDescription = "purge the cluster of rogue processes"; 126 /// Only prints those that would be removed 127 bool print; 128 /// Kill rogue process 129 bool kill; 130 /// regex whitelist. Only processes in this list is not killed. 131 string[] whiteList; 132 /// restrict killing of processes to the current user 133 bool userFilter; 134 } 135 136 struct LocalPurge { 137 std.getopt.GetoptResult helpInfo; 138 static string helpDescription = "purge the current host rogue processes"; 139 /// Only prints those that would be removed 140 bool print; 141 /// Kill rogue process 142 bool kill; 143 /// regex whitelist. Only processes in this list is not killed. 144 string[] whiteList; 145 /// restrict killing of processes to the current user 146 bool userFilter; 147 } 148 149 alias Type = Algebraic!(Help, Shell, Cmd, LocalRun, Install, MeasureHosts, 150 LocalLoad, RunOnAll, LocalShell, Env, Daemon, Purge, LocalPurge); 151 Type data; 152 153 Global global; 154 155 void printHelp() { 156 static void printGroup(T)(std.getopt.GetoptResult global, 157 std.getopt.GetoptResult helpInfo, string progName) { 158 const helpDescription = () { 159 static if (hasMember!(T, "helpDescription")) 160 return T.helpDescription ~ "\n"; 161 else 162 return null; 163 }(); 164 defaultGetoptPrinter(format("usage: %s %s <options>\n%s", progName, 165 T.stringof.toLower, helpDescription), global.options); 166 defaultGetoptPrinter(null, helpInfo.options.filter!(a => a.optShort != "-h").array); 167 } 168 169 static void printHelpGroup(std.getopt.GetoptResult helpInfo, string progName) { 170 defaultGetoptPrinter(format("usage: %s <command>\n", progName), helpInfo.options); 171 writeln("sub-commands"); 172 string[2][] subCommands; 173 static foreach (T; Type.AllowedTypes) { 174 static if (hasMember!(T, "helpDescription")) 175 subCommands ~= [T.stringof.toLower, T.helpDescription]; 176 else 177 subCommands ~= [T.stringof.toLower, null]; 178 } 179 const width = subCommands.map!(a => a[0].length).maxElement + 1; 180 foreach (cmd; subCommands) 181 writefln(" %s%*s %s", cmd[0], width - cmd[0].length, " ", cmd[1]); 182 } 183 184 template printers(T...) { 185 static if (T.length == 1) { 186 static if (is(T[0] == Config.Help)) 187 alias printers = (T[0] a) => printHelpGroup(global.helpInfo, global.progName); 188 else 189 alias printers = (T[0] a) => printGroup!(T[0])(global.helpInfo, 190 a.helpInfo, global.progName); 191 } else { 192 alias printers = AliasSeq!(printers!(T[0]), printers!(T[1 .. $])); 193 } 194 } 195 196 data.visit!(printers!(Type.AllowedTypes)); 197 } 198 } 199 200 /** 201 * #SPC-remote_command_parse 202 * 203 * Params: 204 * args = the command line arguments to parse. 205 */ 206 Config parseUserArgs(string[] args) { 207 import my.xdg : xdgRuntimeDir; 208 209 Config conf; 210 conf.data = Config.Help.init; 211 conf.global.progName = args[0].baseName; 212 conf.global.selfBinary = buildPath(thisExePath.dirName, args[0].baseName); 213 conf.global.selfDir = conf.global.selfBinary.dirName; 214 conf.global.workDir = getcwd; 215 conf.global.dbPath = xdgRuntimeDir ~ Path("distssh.sqlite3"); 216 217 switch (conf.global.selfBinary.baseName) { 218 case distShell: 219 conf.data = Config.Shell.init; 220 conf.global.cluster = hostsFromEnv; 221 return conf; 222 case distCmd: 223 if (args.length > 1 && args[1].among("-h", "--help")) 224 conf.data = Config.Help.init; 225 else { 226 conf.data = Config.Cmd.init; 227 conf.global.command = args.length > 1 ? args[1 .. $] : null; 228 conf.global.cluster = hostsFromEnv; 229 configImportEnvFile(conf); 230 } 231 return conf; 232 default: 233 } 234 235 string group; 236 if (args.length > 1 && args[1][0] != '-') { 237 group = args[1]; 238 args = args.remove(1); 239 } 240 241 try { 242 void globalParse() { 243 string export_env_file; 244 ulong timeout_s = defaultTimeout_s; 245 246 // dfmt off 247 conf.global.helpInfo = std.getopt.getopt(args, std.getopt.config.passThrough, std.getopt.config.keepEndOfOptions, 248 "clone-env", "clone the current environment to the remote host without an intermediate file", &conf.global.cloneEnv, 249 "env-file", "file to load the environment from", &export_env_file, 250 "i|import-env", "import the env from the file (default: " ~ distsshEnvExport ~ ")", &conf.global.importEnv, 251 "no-import-env", "do not automatically import the environment from " ~ distsshEnvExport, &conf.global.noImportEnv, 252 "stdin-msgpack-env", "import env from stdin as a msgpack stream", &conf.global.stdinMsgPackEnv, 253 "timeout", "timeout to use when checking remote hosts", &timeout_s, 254 "v|verbose", format("Set the verbosity (%-(%s, %))", [EnumMembers!(VerboseMode)]), &conf.global.verbosity, 255 "workdir", "working directory to run the command in", &conf.global.workDir, 256 ); 257 // dfmt on 258 if (conf.global.helpInfo.helpWanted) 259 args ~= "-h"; 260 261 // must convert e.g. "." 262 conf.global.workDir = conf.global.workDir.absolutePath; 263 264 conf.global.timeout = timeout_s.dur!"seconds"; 265 266 if (!export_env_file.empty) 267 conf.global.importEnv = export_env_file; 268 } 269 270 void helpParse() { 271 conf.data = Config.Help.init; 272 } 273 274 void envParse() { 275 Config.Env data; 276 scope (success) 277 conf.data = data; 278 279 // dfmt off 280 data.helpInfo = std.getopt.getopt(args, 281 "d|delete", "remove a variable from the exported environment", &data.envDel, 282 "e|export", "export the current environment to a file that is used on the remote host", &data.exportEnv, 283 "p|print", "print the content of an exported environment", &data.print, 284 "s|set", "set a variable in the exported environment. Example: FOO=42", &data.envSet, 285 ); 286 // dfmt on 287 } 288 289 void shellParse() { 290 conf.data = Config.Shell.init; 291 conf.global.cluster = hostsFromEnv; 292 } 293 294 void cmdParse() { 295 conf.data = Config.Cmd.init; 296 conf.global.cluster = hostsFromEnv; 297 } 298 299 void localrunParse() { 300 Config.LocalRun data; 301 scope (success) 302 conf.data = data; 303 304 // dfmt off 305 data.helpInfo = std.getopt.getopt(args, std.getopt.config.passThrough, std.getopt.config.keepEndOfOptions, 306 "pseudo-terminal", "force that a pseudo-terminal is used when running the command", &data.useFakeTerminal, 307 ); 308 // dfmt on 309 } 310 311 void installParse() { 312 conf.data = Config.Install.init; 313 } 314 315 void measurehostsParse() { 316 conf.data = Config.MeasureHosts.init; 317 conf.global.cluster = hostsFromEnv; 318 } 319 320 void localloadParse() { 321 conf.data = Config.LocalLoad.init; 322 } 323 324 void runonallParse() { 325 conf.data = Config.RunOnAll.init; 326 conf.global.cluster = hostsFromEnv; 327 } 328 329 void localshellParse() { 330 conf.data = Config.LocalShell.init; 331 } 332 333 void daemonParse() { 334 conf.global.cluster = hostsFromEnv; 335 Config.Daemon data; 336 scope (success) 337 conf.data = data; 338 339 ulong timeout = 30; 340 // dfmt off 341 data.helpInfo = std.getopt.getopt(args, 342 "b|background", "persist in the background", &data.background, 343 "force-start", "force the server to start", &data.forceStart, 344 "t|timeout", "shutdown background process if unused not used for this time (default: 30 minutes)", &timeout, 345 ); 346 // dfmt on 347 data.timeout = timeout.dur!"minutes"; 348 } 349 350 void purgeParse() { 351 conf.global.cluster = hostsFromEnv; 352 Config.Purge data; 353 scope (success) 354 conf.data = data; 355 356 // dfmt off 357 data.helpInfo = std.getopt.getopt(args, 358 "k|kill", "kill rogue processes", &data.kill, 359 "p|print", "print rogue process", &data.print, 360 "user-filter", "only purge those processes owned by the current user", &data.userFilter, 361 "whitelist", "one or more regex (case insensitive) that whitelist processes as not rogue", &data.whiteList, 362 ); 363 // dfmt on 364 } 365 366 void localpurgeParse() { 367 Config.LocalPurge data; 368 scope (success) 369 conf.data = data; 370 371 // dfmt off 372 data.helpInfo = std.getopt.getopt(args, 373 "k|kill", "kill rogue processes", &data.kill, 374 "p|print", "print rogue process", &data.print, 375 "user-filter", "only purge those processes owned by the current user", &data.userFilter, 376 "whitelist", "one or more regex (case insensitive) that whitelist processes as not rogue", &data.whiteList, 377 ); 378 // dfmt on 379 } 380 381 alias ParseFn = void delegate(); 382 ParseFn[string] parsers; 383 384 static foreach (T; Config.Type.AllowedTypes) { 385 mixin(format(`parsers["%1$s"] = &%1$sParse;`, T.stringof.toLower)); 386 } 387 388 globalParse; 389 390 if (auto p = group in parsers) { 391 (*p)(); 392 } 393 394 if (args.length > 1) { 395 conf.global.command = args.find("--").drop(1).array(); 396 } 397 configImportEnvFile(conf); 398 } catch (std.getopt.GetOptException e) { 399 // unknown option 400 logger.error(e.msg); 401 } catch (Exception e) { 402 logger.error(e.msg); 403 } 404 405 return conf; 406 } 407 408 /** Update a Configs object's file to import the environment from. 409 * 410 * This should only be called after all other command line parsing has been 411 * done. It is because this function take into consideration the priority as 412 * specified in the requirement: 413 * #SPC-configure_env_import_file 414 * 415 * Params: 416 * opts = config to update the file to import the environment from. 417 */ 418 void configImportEnvFile(ref Config opts) nothrow { 419 import std.process : environment; 420 421 if (opts.global.noImportEnv) { 422 opts.global.importEnv = null; 423 } else if (opts.global.importEnv.length != 0) { 424 // do nothing. the user has specified a file 425 } else { 426 try { 427 opts.global.importEnv = environment.get(globalEnvFileKey, distsshEnvExport); 428 } catch (Exception e) { 429 } 430 } 431 } 432 433 @("shall determine the absolute path of self") 434 unittest { 435 import std.path; 436 import std.file; 437 438 auto opts = parseUserArgs(["distssh", "ls"]); 439 assert(opts.global.selfBinary[0] == '/'); 440 assert(opts.global.selfBinary.baseName == "distssh"); 441 442 opts = parseUserArgs(["distshell"]); 443 assert(opts.global.selfBinary[0] == '/'); 444 assert(opts.global.selfBinary.baseName == "distshell"); 445 446 opts = parseUserArgs(["distcmd"]); 447 assert(opts.global.selfBinary[0] == '/'); 448 assert(opts.global.selfBinary.baseName == "distcmd"); 449 450 opts = parseUserArgs(["distcmd_recv", getcwd, distsshEnvExport]); 451 assert(opts.global.selfBinary[0] == '/'); 452 assert(opts.global.selfBinary.baseName == "distcmd_recv"); 453 } 454 455 @("shall either return the default timeout or the user specified timeout") 456 unittest { 457 import core.time : dur; 458 import std.conv; 459 460 auto opts = parseUserArgs(["distssh", "ls"]); 461 assert(opts.global.timeout == defaultTimeout_s.dur!"seconds"); 462 opts = parseUserArgs(["distssh", "--timeout", "10", "ls"]); 463 assert(opts.global.timeout == 10.dur!"seconds"); 464 465 opts = parseUserArgs(["distshell"]); 466 opts.global.timeout.shouldEqual(defaultTimeout_s.dur!"seconds"); 467 opts = parseUserArgs(["distshell", "--timeout", "10"]); 468 assert(opts.global.timeout == defaultTimeout_s.dur!"seconds"); 469 } 470 471 @("shall only be the default timeout because --timeout should be passed on to the command") 472 unittest { 473 import core.time : dur; 474 import std.conv; 475 476 auto opts = parseUserArgs(["distcmd", "ls"]); 477 assert(opts.global.timeout == defaultTimeout_s.dur!"seconds"); 478 479 opts = parseUserArgs(["distcmd", "--timeout", "10"]); 480 assert(opts.global.timeout == defaultTimeout_s.dur!"seconds"); 481 } 482 483 @("shall convert relative workdirs to absolute when parsing user args") 484 unittest { 485 import std.path : isAbsolute; 486 487 auto opts = parseUserArgs(["distssh", "--workdir", "."]); 488 assert(opts.global.workDir.isAbsolute, "expected an absolute path"); 489 } 490 491 Host[] hostsFromEnv() nothrow { 492 import std.algorithm : splitter, map; 493 import std.array : array; 494 import std.exception : collectException; 495 import std.process : environment; 496 import std..string : strip; 497 498 typeof(return) rval; 499 500 try { 501 string hosts_env = environment.get(globalEnvHostKey, "").strip; 502 rval = hosts_env.splitter(";").map!(a => a.strip) 503 .filter!(a => a.length > 0) 504 .map!(a => Host(a)) 505 .array; 506 507 if (rval.length == 0) { 508 logger.errorf("No remote host configured (%s='%s')", globalEnvHostKey, hosts_env); 509 } 510 } catch (Exception e) { 511 logger.error(e.msg).collectException; 512 } 513 514 return rval; 515 }