1 /** 2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved. 3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0) 4 Author: Joakim Brännström (joakim.brannstrom@gmx.com) 5 */ 6 module distssh.config; 7 8 import core.time : dur, Duration; 9 import logger = std.experimental.logger; 10 import std.algorithm : among, remove, filter, find, map, maxElement; 11 import std.array : array, empty; 12 import std.file : thisExePath, getcwd; 13 import std.format : format; 14 import std.getopt : defaultGetoptPrinter; 15 import std.meta : AliasSeq; 16 import std.path : baseName, buildPath, dirName, absolutePath; 17 import std.range : drop; 18 import std.stdio : writeln, writefln; 19 import std..string : toLower; 20 import std.traits : EnumMembers, hasMember; 21 import std.variant : Algebraic, visit; 22 static import std.getopt; 23 24 import colorlog : VerboseMode; 25 import my.named_type; 26 27 import distssh.types; 28 29 version (unittest) { 30 import unit_threaded.assertions; 31 } 32 33 struct Config { 34 struct Global { 35 std.getopt.GetoptResult helpInfo; 36 VerboseMode verbosity; 37 38 NamedType!(bool, Tag!"NoImportEnv") noImportEnv; 39 NamedType!(bool, Tag!"CloneEnv") cloneEnv; 40 NamedType!(bool, Tag!"StdinMsgPackEnv") stdinMsgPackEnv; 41 NamedType!(Duration, Tag!"Timeout", Duration.zero, ImplicitConvertable) timeout = defaultTimeout_s 42 .dur!"seconds"; 43 44 NamedType!(string, Tag!"ProgName") progName; 45 NamedType!(string, Tag!"SelfBinary") selfBinary; 46 NamedType!(string, Tag!"SelfDir") selfDir; 47 48 NamedType!(string, Tag!"ImportEnvFile") importEnv; 49 NamedType!(string, Tag!"Workdir") workDir; 50 NamedType!(string[], Tag!"Command", null, Lengthable) command; 51 52 Path dbPath; 53 54 /// The hosts the cluster consist of. 55 Host[] cluster; 56 } 57 58 struct Help { 59 std.getopt.GetoptResult helpInfo; 60 } 61 62 struct Shell { 63 std.getopt.GetoptResult helpInfo; 64 static string helpDescription = "open an interactive shell on the remote host"; 65 } 66 67 struct Cmd { 68 std.getopt.GetoptResult helpInfo; 69 static string helpDescription = "run a command on a remote host"; 70 } 71 72 struct LocalRun { 73 std.getopt.GetoptResult helpInfo; 74 static string helpDescription = "import env and run the command locally"; 75 bool useFakeTerminal; 76 } 77 78 struct Install { 79 std.getopt.GetoptResult helpInfo; 80 static string helpDescription = "install distssh by setting up the correct symlinks"; 81 } 82 83 struct MeasureHosts { 84 std.getopt.GetoptResult helpInfo; 85 static string helpDescription = "measure the login time and load of all remote hosts"; 86 } 87 88 struct LocalLoad { 89 std.getopt.GetoptResult helpInfo; 90 static string helpDescription = "measure the load on the current host"; 91 } 92 93 struct RunOnAll { 94 std.getopt.GetoptResult helpInfo; 95 static string helpDescription = "run the command on all remote hosts"; 96 } 97 98 struct LocalShell { 99 std.getopt.GetoptResult helpInfo; 100 static string helpDescription = "run the shell locally"; 101 } 102 103 struct Env { 104 std.getopt.GetoptResult helpInfo; 105 static string helpDescription = "manipulate the stored environment"; 106 /// Print the environment. 107 bool print; 108 /// Env variable to set in the config specified in importEnv. 109 string[] envSet; 110 /// Env variables to remove from the onespecified in importEnv. 111 string[] envDel; 112 /// Export the current environment 113 bool exportEnv; 114 } 115 116 struct Daemon { 117 std.getopt.GetoptResult helpInfo; 118 static string helpDescription = "update the cluster statistics in the database"; 119 Duration timeout; 120 /// If the daemon should persist in the background after it has measured the cluster once. 121 bool background; 122 /// Force the server to start even though there may be one running in the background 123 bool forceStart; 124 } 125 126 struct Purge { 127 std.getopt.GetoptResult helpInfo; 128 static string helpDescription = "purge the cluster of rogue processes"; 129 /// Only prints those that would be removed 130 bool print; 131 /// Kill rogue process 132 bool kill; 133 /// regex whitelist. Only processes in this list is not killed. 134 string[] whiteList; 135 /// restrict killing of processes to the current user 136 bool userFilter; 137 } 138 139 struct LocalPurge { 140 std.getopt.GetoptResult helpInfo; 141 static string helpDescription = "purge the current host rogue processes"; 142 /// Only prints those that would be removed 143 bool print; 144 /// Kill rogue process 145 bool kill; 146 /// regex whitelist. Only processes in this list is not killed. 147 string[] whiteList; 148 /// restrict killing of processes to the current user 149 bool userFilter; 150 } 151 152 alias Type = Algebraic!(Help, Shell, Cmd, LocalRun, Install, MeasureHosts, 153 LocalLoad, RunOnAll, LocalShell, Env, Daemon, Purge, LocalPurge); 154 Type data; 155 156 Global global; 157 158 void printHelp() { 159 static void printGroup(T)(std.getopt.GetoptResult global, 160 std.getopt.GetoptResult helpInfo, string progName) { 161 const helpDescription = () { 162 static if (hasMember!(T, "helpDescription")) 163 return T.helpDescription ~ "\n"; 164 else 165 return null; 166 }(); 167 defaultGetoptPrinter(format("usage: %s %s <options>\n%s", progName, 168 T.stringof.toLower, helpDescription), global.options); 169 defaultGetoptPrinter(null, helpInfo.options.filter!(a => a.optShort != "-h").array); 170 } 171 172 static void printHelpGroup(std.getopt.GetoptResult helpInfo, string progName) { 173 defaultGetoptPrinter(format("usage: %s <command>\n", progName), helpInfo.options); 174 writeln("sub-commands"); 175 string[2][] subCommands; 176 static foreach (T; Type.AllowedTypes) { 177 static if (hasMember!(T, "helpDescription")) 178 subCommands ~= [T.stringof.toLower, T.helpDescription]; 179 else 180 subCommands ~= [T.stringof.toLower, null]; 181 } 182 const width = subCommands.map!(a => a[0].length).maxElement + 1; 183 foreach (cmd; subCommands) 184 writefln(" %s%*s %s", cmd[0], width - cmd[0].length, " ", cmd[1]); 185 } 186 187 template printers(T...) { 188 static if (T.length == 1) { 189 static if (is(T[0] == Config.Help)) 190 alias printers = (T[0] a) => printHelpGroup(global.helpInfo, 191 global.progName.get); 192 else 193 alias printers = (T[0] a) => printGroup!(T[0])(global.helpInfo, 194 a.helpInfo, global.progName.get); 195 } else { 196 alias printers = AliasSeq!(printers!(T[0]), printers!(T[1 .. $])); 197 } 198 } 199 200 data.visit!(printers!(Type.AllowedTypes)); 201 } 202 } 203 204 /** 205 * #SPC-remote_command_parse 206 * 207 * Params: 208 * args = the command line arguments to parse. 209 */ 210 Config parseUserArgs(string[] args) { 211 import my.xdg : xdgRuntimeDir; 212 213 Config conf; 214 conf.data = Config.Help.init; 215 conf.global.progName = typeof(conf.global.progName)(args[0].baseName); 216 conf.global.selfBinary = typeof(conf.global.selfBinary)( 217 buildPath(thisExePath.dirName, args[0].baseName)); 218 conf.global.selfDir = typeof(conf.global.selfDir)(conf.global.selfBinary.get.dirName); 219 conf.global.workDir = typeof(conf.global.workDir)(getcwd); 220 conf.global.dbPath = xdgRuntimeDir ~ Path("distssh.sqlite3"); 221 222 switch (conf.global.selfBinary.get.baseName) { 223 case distShell: 224 conf.data = Config.Shell.init; 225 conf.global.cluster = hostsFromEnv; 226 return conf; 227 case distCmd: 228 if (args.length > 1 && args[1].among("-h", "--help")) 229 conf.data = Config.Help.init; 230 else { 231 conf.data = Config.Cmd.init; 232 conf.global.command = typeof(conf.global.command)(args.length > 1 ? args[1 .. $] : null); 233 conf.global.cluster = hostsFromEnv; 234 configImportEnvFile(conf); 235 } 236 return conf; 237 default: 238 } 239 240 string group; 241 if (args.length > 1 && args[1][0] != '-') { 242 group = args[1]; 243 args = args.remove(1); 244 } 245 246 try { 247 void globalParse() { 248 string export_env_file; 249 ulong timeout_s = defaultTimeout_s; 250 251 // dfmt off 252 conf.global.helpInfo = std.getopt.getopt(args, std.getopt.config.passThrough, std.getopt.config.keepEndOfOptions, 253 "clone-env", "clone the current environment to the remote host without an intermediate file", conf.global.cloneEnv.getPtr, 254 "env-file", "file to load the environment from", &export_env_file, 255 "i|import-env", "import the env from the file (default: " ~ distsshEnvExport ~ ")", conf.global.importEnv.getPtr, 256 "no-import-env", "do not automatically import the environment from " ~ distsshEnvExport, conf.global.noImportEnv.getPtr, 257 "stdin-msgpack-env", "import env from stdin as a msgpack stream", conf.global.stdinMsgPackEnv.getPtr, 258 "timeout", "timeout to use when checking remote hosts", &timeout_s, 259 "v|verbose", format("Set the verbosity (%-(%s, %))", [EnumMembers!(VerboseMode)]), &conf.global.verbosity, 260 "workdir", "working directory to run the command in", conf.global.workDir.getPtr, 261 ); 262 // dfmt on 263 if (conf.global.helpInfo.helpWanted) 264 args ~= "-h"; 265 266 // must convert e.g. "." 267 conf.global.workDir = typeof(conf.global.workDir)(conf.global.workDir.get.absolutePath); 268 269 conf.global.timeout.get = typeof(conf.global.timeout)(timeout_s.dur!"seconds"); 270 271 if (!export_env_file.empty) 272 conf.global.importEnv = typeof(conf.global.importEnv)(export_env_file); 273 } 274 275 void helpParse() { 276 conf.data = Config.Help.init; 277 } 278 279 void envParse() { 280 Config.Env data; 281 scope (success) 282 conf.data = data; 283 284 // dfmt off 285 data.helpInfo = std.getopt.getopt(args, 286 "d|delete", "remove a variable from the exported environment", &data.envDel, 287 "e|export", "export the current environment to a file that is used on the remote host", &data.exportEnv, 288 "p|print", "print the content of an exported environment", &data.print, 289 "s|set", "set a variable in the exported environment. Example: FOO=42", &data.envSet, 290 ); 291 // dfmt on 292 } 293 294 void shellParse() { 295 conf.data = Config.Shell.init; 296 conf.global.cluster = hostsFromEnv; 297 } 298 299 void cmdParse() { 300 conf.data = Config.Cmd.init; 301 conf.global.cluster = hostsFromEnv; 302 } 303 304 void localrunParse() { 305 Config.LocalRun data; 306 scope (success) 307 conf.data = data; 308 309 // dfmt off 310 data.helpInfo = std.getopt.getopt(args, std.getopt.config.passThrough, std.getopt.config.keepEndOfOptions, 311 "pseudo-terminal", "force that a pseudo-terminal is used when running the command", &data.useFakeTerminal, 312 ); 313 // dfmt on 314 } 315 316 void installParse() { 317 conf.data = Config.Install.init; 318 } 319 320 void measurehostsParse() { 321 conf.data = Config.MeasureHosts.init; 322 conf.global.cluster = hostsFromEnv; 323 } 324 325 void localloadParse() { 326 conf.data = Config.LocalLoad.init; 327 } 328 329 void runonallParse() { 330 conf.data = Config.RunOnAll.init; 331 conf.global.cluster = hostsFromEnv; 332 } 333 334 void localshellParse() { 335 conf.data = Config.LocalShell.init; 336 } 337 338 void daemonParse() { 339 conf.global.cluster = hostsFromEnv; 340 Config.Daemon data; 341 scope (success) 342 conf.data = data; 343 344 ulong timeout = 30; 345 // dfmt off 346 data.helpInfo = std.getopt.getopt(args, 347 "b|background", "persist in the background", &data.background, 348 "force-start", "force the server to start", &data.forceStart, 349 "t|timeout", "shutdown background process if unused not used for this time (default: 30 minutes)", &timeout, 350 ); 351 // dfmt on 352 data.timeout = timeout.dur!"minutes"; 353 } 354 355 void purgeParse() { 356 conf.global.cluster = hostsFromEnv; 357 Config.Purge data; 358 scope (success) 359 conf.data = data; 360 361 // dfmt off 362 data.helpInfo = std.getopt.getopt(args, 363 "k|kill", "kill rogue processes", &data.kill, 364 "p|print", "print rogue process", &data.print, 365 "user-filter", "only purge those processes owned by the current user", &data.userFilter, 366 "whitelist", "one or more regex (case insensitive) that whitelist processes as not rogue", &data.whiteList, 367 ); 368 // dfmt on 369 } 370 371 void localpurgeParse() { 372 Config.LocalPurge data; 373 scope (success) 374 conf.data = data; 375 376 // dfmt off 377 data.helpInfo = std.getopt.getopt(args, 378 "k|kill", "kill rogue processes", &data.kill, 379 "p|print", "print rogue process", &data.print, 380 "user-filter", "only purge those processes owned by the current user", &data.userFilter, 381 "whitelist", "one or more regex (case insensitive) that whitelist processes as not rogue", &data.whiteList, 382 ); 383 // dfmt on 384 } 385 386 alias ParseFn = void delegate(); 387 ParseFn[string] parsers; 388 389 static foreach (T; Config.Type.AllowedTypes) { 390 mixin(format(`parsers["%1$s"] = &%1$sParse;`, T.stringof.toLower)); 391 } 392 393 globalParse; 394 395 if (auto p = group in parsers) { 396 (*p)(); 397 } 398 399 if (args.length > 1) { 400 conf.global.command = typeof(conf.global.command)(args.find("--").drop(1).array); 401 } 402 configImportEnvFile(conf); 403 } catch (std.getopt.GetOptException e) { 404 // unknown option 405 logger.error(e.msg); 406 } catch (Exception e) { 407 logger.error(e.msg); 408 } 409 410 return conf; 411 } 412 413 /** Update a Configs object's file to import the environment from. 414 * 415 * This should only be called after all other command line parsing has been 416 * done. It is because this function take into consideration the priority as 417 * specified in the requirement: 418 * #SPC-configure_env_import_file 419 * 420 * Params: 421 * opts = config to update the file to import the environment from. 422 */ 423 void configImportEnvFile(ref Config opts) nothrow { 424 import std.process : environment; 425 426 if (opts.global.noImportEnv) { 427 opts.global.importEnv.get = null; 428 } else if (opts.global.importEnv.get.length != 0) { 429 // do nothing. the user has specified a file 430 } else { 431 try { 432 opts.global.importEnv = typeof(opts.global.importEnv)(environment.get(globalEnvFileKey, 433 distsshEnvExport)); 434 } catch (Exception e) { 435 } 436 } 437 } 438 439 @("shall determine the absolute path of self") 440 unittest { 441 import std.path; 442 import std.file; 443 444 auto opts = parseUserArgs(["distssh", "ls"]); 445 assert(opts.global.selfBinary.get[0] == '/'); 446 assert(opts.global.selfBinary.get.baseName == "distssh"); 447 448 opts = parseUserArgs(["distshell"]); 449 assert(opts.global.selfBinary.get[0] == '/'); 450 assert(opts.global.selfBinary.get.baseName == "distshell"); 451 452 opts = parseUserArgs(["distcmd"]); 453 assert(opts.global.selfBinary.get[0] == '/'); 454 assert(opts.global.selfBinary.get.baseName == "distcmd"); 455 456 opts = parseUserArgs(["distcmd_recv", getcwd, distsshEnvExport]); 457 assert(opts.global.selfBinary.get[0] == '/'); 458 assert(opts.global.selfBinary.get.baseName == "distcmd_recv"); 459 } 460 461 @("shall either return the default timeout or the user specified timeout") 462 unittest { 463 import core.time : dur; 464 import std.conv; 465 466 auto opts = parseUserArgs(["distssh", "ls"]); 467 assert(opts.global.timeout == defaultTimeout_s.dur!"seconds"); 468 opts = parseUserArgs(["distssh", "--timeout", "10", "ls"]); 469 assert(opts.global.timeout == 10.dur!"seconds"); 470 471 opts = parseUserArgs(["distshell"]); 472 opts.global.timeout.shouldEqual(defaultTimeout_s.dur!"seconds"); 473 opts = parseUserArgs(["distshell", "--timeout", "10"]); 474 assert(opts.global.timeout == defaultTimeout_s.dur!"seconds"); 475 } 476 477 @("shall only be the default timeout because --timeout should be passed on to the command") 478 unittest { 479 import core.time : dur; 480 import std.conv; 481 482 auto opts = parseUserArgs(["distcmd", "ls"]); 483 assert(opts.global.timeout == defaultTimeout_s.dur!"seconds"); 484 485 opts = parseUserArgs(["distcmd", "--timeout", "10"]); 486 assert(opts.global.timeout == defaultTimeout_s.dur!"seconds"); 487 } 488 489 @("shall convert relative workdirs to absolute when parsing user args") 490 unittest { 491 import std.path : isAbsolute; 492 493 auto opts = parseUserArgs(["distssh", "--workdir", "."]); 494 assert(opts.global.workDir.get.isAbsolute, "expected an absolute path"); 495 } 496 497 Host[] hostsFromEnv() nothrow { 498 import std.algorithm : splitter, map; 499 import std.array : array; 500 import std.exception : collectException; 501 import std.process : environment; 502 import std..string : strip; 503 504 typeof(return) rval; 505 506 try { 507 string hosts_env = environment.get(globalEnvHostKey, "").strip; 508 rval = hosts_env.splitter(";").map!(a => a.strip) 509 .filter!(a => a.length > 0) 510 .map!(a => Host(a)) 511 .array; 512 513 if (rval.length == 0) { 514 logger.errorf("No remote host configured (%s='%s')", globalEnvHostKey, hosts_env); 515 } 516 } catch (Exception e) { 517 logger.error(e.msg).collectException; 518 } 519 520 return rval; 521 }