1 /**
2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved.
3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0)
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 */
6 module distssh.config;
7 
8 import core.time : dur, Duration;
9 import logger = std.experimental.logger;
10 import std.algorithm : among, remove, filter, find, map, maxElement;
11 import std.array : array, empty;
12 import std.file : thisExePath, getcwd;
13 import std.format : format;
14 import std.getopt : defaultGetoptPrinter;
15 import std.meta : AliasSeq;
16 import std.path : baseName, buildPath, dirName, absolutePath;
17 import std.range : drop;
18 import std.stdio : writeln, writefln;
19 import std..string : toLower;
20 import std.traits : EnumMembers, hasMember;
21 import std.variant : Algebraic, visit;
22 static import std.getopt;
23 
24 import colorlog : VerboseMode;
25 import my.named_type;
26 
27 import distssh.types;
28 
29 version (unittest) {
30     import unit_threaded.assertions;
31 }
32 
33 struct Config {
34     struct Global {
35         std.getopt.GetoptResult helpInfo;
36         VerboseMode verbosity;
37 
38         NamedType!(bool, Tag!"NoImportEnv") noImportEnv;
39         NamedType!(bool, Tag!"CloneEnv") cloneEnv;
40         NamedType!(bool, Tag!"StdinMsgPackEnv") stdinMsgPackEnv;
41         NamedType!(Duration, Tag!"Timeout", Duration.zero, ImplicitConvertable) timeout = defaultTimeout_s
42             .dur!"seconds";
43 
44         NamedType!(string, Tag!"ProgName") progName;
45         NamedType!(string, Tag!"SelfBinary") selfBinary;
46         NamedType!(string, Tag!"SelfDir") selfDir;
47 
48         NamedType!(string, Tag!"ImportEnvFile") importEnv;
49         NamedType!(string, Tag!"Workdir") workDir;
50         NamedType!(string[], Tag!"Command", null, Lengthable) command;
51 
52         Path dbPath;
53 
54         /// The hosts the cluster consist of.
55         Host[] cluster;
56     }
57 
58     struct Help {
59         std.getopt.GetoptResult helpInfo;
60     }
61 
62     struct Shell {
63         std.getopt.GetoptResult helpInfo;
64         static string helpDescription = "open an interactive shell on the remote host";
65     }
66 
67     struct Cmd {
68         std.getopt.GetoptResult helpInfo;
69         static string helpDescription = "run a command on a remote host";
70     }
71 
72     struct LocalRun {
73         std.getopt.GetoptResult helpInfo;
74         static string helpDescription = "import env and run the command locally";
75         bool useFakeTerminal;
76     }
77 
78     struct Install {
79         std.getopt.GetoptResult helpInfo;
80         static string helpDescription = "install distssh by setting up the correct symlinks";
81     }
82 
83     struct MeasureHosts {
84         std.getopt.GetoptResult helpInfo;
85         static string helpDescription = "measure the login time and load of all remote hosts";
86     }
87 
88     struct LocalLoad {
89         std.getopt.GetoptResult helpInfo;
90         static string helpDescription = "measure the load on the current host";
91     }
92 
93     struct RunOnAll {
94         std.getopt.GetoptResult helpInfo;
95         static string helpDescription = "run the command on all remote hosts";
96     }
97 
98     struct LocalShell {
99         std.getopt.GetoptResult helpInfo;
100         static string helpDescription = "run the shell locally";
101     }
102 
103     struct Env {
104         std.getopt.GetoptResult helpInfo;
105         static string helpDescription = "manipulate the stored environment";
106         /// Print the environment.
107         bool print;
108         /// Env variable to set in the config specified in importEnv.
109         string[] envSet;
110         /// Env variables to remove from the onespecified in importEnv.
111         string[] envDel;
112         /// Export the current environment
113         bool exportEnv;
114     }
115 
116     struct Daemon {
117         std.getopt.GetoptResult helpInfo;
118         static string helpDescription = "update the cluster statistics in the database";
119         Duration timeout;
120         /// If the daemon should persist in the background after it has measured the cluster once.
121         bool background;
122         /// Force the server to start even though there may be one running in the background
123         bool forceStart;
124     }
125 
126     struct Purge {
127         std.getopt.GetoptResult helpInfo;
128         static string helpDescription = "purge the cluster of rogue processes";
129         /// Only prints those that would be removed
130         bool print;
131         /// Kill rogue process
132         bool kill;
133         /// regex whitelist. Only processes in this list is not killed.
134         string[] whiteList;
135         /// restrict killing of processes to the current user
136         bool userFilter;
137     }
138 
139     struct LocalPurge {
140         std.getopt.GetoptResult helpInfo;
141         static string helpDescription = "purge the current host rogue processes";
142         /// Only prints those that would be removed
143         bool print;
144         /// Kill rogue process
145         bool kill;
146         /// regex whitelist. Only processes in this list is not killed.
147         string[] whiteList;
148         /// restrict killing of processes to the current user
149         bool userFilter;
150     }
151 
152     alias Type = Algebraic!(Help, Shell, Cmd, LocalRun, Install, MeasureHosts,
153             LocalLoad, RunOnAll, LocalShell, Env, Daemon, Purge, LocalPurge);
154     Type data;
155 
156     Global global;
157 
158     void printHelp() {
159         static void printGroup(T)(std.getopt.GetoptResult global,
160                 std.getopt.GetoptResult helpInfo, string progName) {
161             const helpDescription = () {
162                 static if (hasMember!(T, "helpDescription"))
163                     return T.helpDescription ~ "\n";
164                 else
165                     return null;
166             }();
167             defaultGetoptPrinter(format("usage: %s %s <options>\n%s", progName,
168                     T.stringof.toLower, helpDescription), global.options);
169             defaultGetoptPrinter(null, helpInfo.options.filter!(a => a.optShort != "-h").array);
170         }
171 
172         static void printHelpGroup(std.getopt.GetoptResult helpInfo, string progName) {
173             defaultGetoptPrinter(format("usage: %s <command>\n", progName), helpInfo.options);
174             writeln("sub-commands");
175             string[2][] subCommands;
176             static foreach (T; Type.AllowedTypes) {
177                 static if (hasMember!(T, "helpDescription"))
178                     subCommands ~= [T.stringof.toLower, T.helpDescription];
179                 else
180                     subCommands ~= [T.stringof.toLower, null];
181             }
182             const width = subCommands.map!(a => a[0].length).maxElement + 1;
183             foreach (cmd; subCommands)
184                 writefln(" %s%*s %s", cmd[0], width - cmd[0].length, " ", cmd[1]);
185         }
186 
187         template printers(T...) {
188             static if (T.length == 1) {
189                 static if (is(T[0] == Config.Help))
190                     alias printers = (T[0] a) => printHelpGroup(global.helpInfo,
191                             global.progName.get);
192                 else
193                     alias printers = (T[0] a) => printGroup!(T[0])(global.helpInfo,
194                             a.helpInfo, global.progName.get);
195             } else {
196                 alias printers = AliasSeq!(printers!(T[0]), printers!(T[1 .. $]));
197             }
198         }
199 
200         data.visit!(printers!(Type.AllowedTypes));
201     }
202 }
203 
204 /**
205  * #SPC-remote_command_parse
206  *
207  * Params:
208  *  args = the command line arguments to parse.
209  */
210 Config parseUserArgs(string[] args) {
211     import my.xdg : xdgRuntimeDir;
212 
213     Config conf;
214     conf.data = Config.Help.init;
215     conf.global.progName = typeof(conf.global.progName)(args[0].baseName);
216     conf.global.selfBinary = typeof(conf.global.selfBinary)(
217             buildPath(thisExePath.dirName, args[0].baseName));
218     conf.global.selfDir = typeof(conf.global.selfDir)(conf.global.selfBinary.get.dirName);
219     conf.global.workDir = typeof(conf.global.workDir)(getcwd);
220     conf.global.dbPath = xdgRuntimeDir ~ Path("distssh.sqlite3");
221 
222     switch (conf.global.selfBinary.get.baseName) {
223     case distShell:
224         conf.data = Config.Shell.init;
225         conf.global.cluster = hostsFromEnv;
226         return conf;
227     case distCmd:
228         if (args.length > 1 && args[1].among("-h", "--help"))
229             conf.data = Config.Help.init;
230         else {
231             conf.data = Config.Cmd.init;
232             conf.global.command = typeof(conf.global.command)(args.length > 1 ? args[1 .. $] : null);
233             conf.global.cluster = hostsFromEnv;
234             configImportEnvFile(conf);
235         }
236         return conf;
237     default:
238     }
239 
240     string group;
241     if (args.length > 1 && args[1][0] != '-') {
242         group = args[1];
243         args = args.remove(1);
244     }
245 
246     try {
247         void globalParse() {
248             string export_env_file;
249             ulong timeout_s = defaultTimeout_s;
250 
251             // dfmt off
252             conf.global.helpInfo = std.getopt.getopt(args, std.getopt.config.passThrough, std.getopt.config.keepEndOfOptions,
253                 "clone-env", "clone the current environment to the remote host without an intermediate file", conf.global.cloneEnv.getPtr,
254                 "env-file", "file to load the environment from", &export_env_file,
255                 "i|import-env", "import the env from the file (default: " ~ distsshEnvExport ~ ")", conf.global.importEnv.getPtr,
256                 "no-import-env", "do not automatically import the environment from " ~ distsshEnvExport, conf.global.noImportEnv.getPtr,
257                 "stdin-msgpack-env", "import env from stdin as a msgpack stream", conf.global.stdinMsgPackEnv.getPtr,
258                 "timeout", "timeout to use when checking remote hosts", &timeout_s,
259                 "v|verbose", format("Set the verbosity (%-(%s, %))", [EnumMembers!(VerboseMode)]), &conf.global.verbosity,
260                 "workdir", "working directory to run the command in", conf.global.workDir.getPtr,
261                 );
262             // dfmt on
263             if (conf.global.helpInfo.helpWanted)
264                 args ~= "-h";
265 
266             // must convert e.g. "."
267             conf.global.workDir = typeof(conf.global.workDir)(conf.global.workDir.get.absolutePath);
268 
269             conf.global.timeout.get = typeof(conf.global.timeout)(timeout_s.dur!"seconds");
270 
271             if (!export_env_file.empty)
272                 conf.global.importEnv = typeof(conf.global.importEnv)(export_env_file);
273         }
274 
275         void helpParse() {
276             conf.data = Config.Help.init;
277         }
278 
279         void envParse() {
280             Config.Env data;
281             scope (success)
282                 conf.data = data;
283 
284             // dfmt off
285             data.helpInfo = std.getopt.getopt(args,
286                 "d|delete", "remove a variable from the exported environment", &data.envDel,
287                 "e|export", "export the current environment to a file that is used on the remote host", &data.exportEnv,
288                 "p|print", "print the content of an exported environment", &data.print,
289                 "s|set", "set a variable in the exported environment. Example: FOO=42", &data.envSet,
290                 );
291             // dfmt on
292         }
293 
294         void shellParse() {
295             conf.data = Config.Shell.init;
296             conf.global.cluster = hostsFromEnv;
297         }
298 
299         void cmdParse() {
300             conf.data = Config.Cmd.init;
301             conf.global.cluster = hostsFromEnv;
302         }
303 
304         void localrunParse() {
305             Config.LocalRun data;
306             scope (success)
307                 conf.data = data;
308 
309             // dfmt off
310             data.helpInfo = std.getopt.getopt(args, std.getopt.config.passThrough, std.getopt.config.keepEndOfOptions,
311                 "pseudo-terminal", "force that a pseudo-terminal is used when running the command", &data.useFakeTerminal,
312             );
313             // dfmt on
314         }
315 
316         void installParse() {
317             conf.data = Config.Install.init;
318         }
319 
320         void measurehostsParse() {
321             conf.data = Config.MeasureHosts.init;
322             conf.global.cluster = hostsFromEnv;
323         }
324 
325         void localloadParse() {
326             conf.data = Config.LocalLoad.init;
327         }
328 
329         void runonallParse() {
330             conf.data = Config.RunOnAll.init;
331             conf.global.cluster = hostsFromEnv;
332         }
333 
334         void localshellParse() {
335             conf.data = Config.LocalShell.init;
336         }
337 
338         void daemonParse() {
339             conf.global.cluster = hostsFromEnv;
340             Config.Daemon data;
341             scope (success)
342                 conf.data = data;
343 
344             ulong timeout = 30;
345             // dfmt off
346             data.helpInfo = std.getopt.getopt(args,
347                 "b|background", "persist in the background", &data.background,
348                 "force-start", "force the server to start", &data.forceStart,
349                 "t|timeout", "shutdown background process if unused not used for this time (default: 30 minutes)", &timeout,
350             );
351             // dfmt on
352             data.timeout = timeout.dur!"minutes";
353         }
354 
355         void purgeParse() {
356             conf.global.cluster = hostsFromEnv;
357             Config.Purge data;
358             scope (success)
359                 conf.data = data;
360 
361             // dfmt off
362             data.helpInfo = std.getopt.getopt(args,
363                 "k|kill", "kill rogue processes", &data.kill,
364                 "p|print", "print rogue process", &data.print,
365                 "user-filter", "only purge those processes owned by the current user", &data.userFilter,
366                 "whitelist", "one or more regex (case insensitive) that whitelist processes as not rogue", &data.whiteList,
367                 );
368             // dfmt on
369         }
370 
371         void localpurgeParse() {
372             Config.LocalPurge data;
373             scope (success)
374                 conf.data = data;
375 
376             // dfmt off
377             data.helpInfo = std.getopt.getopt(args,
378                 "k|kill", "kill rogue processes", &data.kill,
379                 "p|print", "print rogue process", &data.print,
380                 "user-filter", "only purge those processes owned by the current user", &data.userFilter,
381                 "whitelist", "one or more regex (case insensitive) that whitelist processes as not rogue", &data.whiteList,
382                 );
383             // dfmt on
384         }
385 
386         alias ParseFn = void delegate();
387         ParseFn[string] parsers;
388 
389         static foreach (T; Config.Type.AllowedTypes) {
390             mixin(format(`parsers["%1$s"] = &%1$sParse;`, T.stringof.toLower));
391         }
392 
393         globalParse;
394 
395         if (auto p = group in parsers) {
396             (*p)();
397         }
398 
399         if (args.length > 1) {
400             conf.global.command = typeof(conf.global.command)(args.find("--").drop(1).array);
401         }
402         configImportEnvFile(conf);
403     } catch (std.getopt.GetOptException e) {
404         // unknown option
405         logger.error(e.msg);
406     } catch (Exception e) {
407         logger.error(e.msg);
408     }
409 
410     return conf;
411 }
412 
413 /** Update a Configs object's file to import the environment from.
414  *
415  * This should only be called after all other command line parsing has been
416  * done. It is because this function take into consideration the priority as
417  * specified in the requirement:
418  * #SPC-configure_env_import_file
419  *
420  * Params:
421  *  opts = config to update the file to import the environment from.
422  */
423 void configImportEnvFile(ref Config opts) nothrow {
424     import std.process : environment;
425 
426     if (opts.global.noImportEnv) {
427         opts.global.importEnv.get = null;
428     } else if (opts.global.importEnv.get.length != 0) {
429         // do nothing. the user has specified a file
430     } else {
431         try {
432             opts.global.importEnv = typeof(opts.global.importEnv)(environment.get(globalEnvFileKey,
433                     distsshEnvExport));
434         } catch (Exception e) {
435         }
436     }
437 }
438 
439 @("shall determine the absolute path of self")
440 unittest {
441     import std.path;
442     import std.file;
443 
444     auto opts = parseUserArgs(["distssh", "ls"]);
445     assert(opts.global.selfBinary.get[0] == '/');
446     assert(opts.global.selfBinary.get.baseName == "distssh");
447 
448     opts = parseUserArgs(["distshell"]);
449     assert(opts.global.selfBinary.get[0] == '/');
450     assert(opts.global.selfBinary.get.baseName == "distshell");
451 
452     opts = parseUserArgs(["distcmd"]);
453     assert(opts.global.selfBinary.get[0] == '/');
454     assert(opts.global.selfBinary.get.baseName == "distcmd");
455 
456     opts = parseUserArgs(["distcmd_recv", getcwd, distsshEnvExport]);
457     assert(opts.global.selfBinary.get[0] == '/');
458     assert(opts.global.selfBinary.get.baseName == "distcmd_recv");
459 }
460 
461 @("shall either return the default timeout or the user specified timeout")
462 unittest {
463     import core.time : dur;
464     import std.conv;
465 
466     auto opts = parseUserArgs(["distssh", "ls"]);
467     assert(opts.global.timeout == defaultTimeout_s.dur!"seconds");
468     opts = parseUserArgs(["distssh", "--timeout", "10", "ls"]);
469     assert(opts.global.timeout == 10.dur!"seconds");
470 
471     opts = parseUserArgs(["distshell"]);
472     opts.global.timeout.shouldEqual(defaultTimeout_s.dur!"seconds");
473     opts = parseUserArgs(["distshell", "--timeout", "10"]);
474     assert(opts.global.timeout == defaultTimeout_s.dur!"seconds");
475 }
476 
477 @("shall only be the default timeout because --timeout should be passed on to the command")
478 unittest {
479     import core.time : dur;
480     import std.conv;
481 
482     auto opts = parseUserArgs(["distcmd", "ls"]);
483     assert(opts.global.timeout == defaultTimeout_s.dur!"seconds");
484 
485     opts = parseUserArgs(["distcmd", "--timeout", "10"]);
486     assert(opts.global.timeout == defaultTimeout_s.dur!"seconds");
487 }
488 
489 @("shall convert relative workdirs to absolute when parsing user args")
490 unittest {
491     import std.path : isAbsolute;
492 
493     auto opts = parseUserArgs(["distssh", "--workdir", "."]);
494     assert(opts.global.workDir.get.isAbsolute, "expected an absolute path");
495 }
496 
497 Host[] hostsFromEnv() nothrow {
498     import std.algorithm : splitter, map;
499     import std.array : array;
500     import std.exception : collectException;
501     import std.process : environment;
502     import std..string : strip;
503 
504     typeof(return) rval;
505 
506     try {
507         string hosts_env = environment.get(globalEnvHostKey, "").strip;
508         rval = hosts_env.splitter(";").map!(a => a.strip)
509             .filter!(a => a.length > 0)
510             .map!(a => Host(a))
511             .array;
512 
513         if (rval.length == 0) {
514             logger.errorf("No remote host configured (%s='%s')", globalEnvHostKey, hosts_env);
515         }
516     } catch (Exception e) {
517         logger.error(e.msg).collectException;
518     }
519 
520     return rval;
521 }