1 /**
2 Copyright: Copyright (c) 2019, Joakim Brännström. All rights reserved.
3 License: $(LINK2 http://www.boost.org/LICENSE_1_0.txt, Boost Software License 1.0)
4 Author: Joakim Brännström (joakim.brannstrom@gmx.com)
5 */
6 module distssh.config;
7 
8 import core.time : dur, Duration;
9 import logger = std.experimental.logger;
10 import std.algorithm : among, remove, filter, find, map, maxElement;
11 import std.array : array, empty;
12 import std.file : thisExePath, getcwd;
13 import std.format : format;
14 import std.getopt : defaultGetoptPrinter;
15 import std.meta : AliasSeq;
16 import std.path : baseName, buildPath, dirName, absolutePath;
17 import std.range : drop;
18 import std.stdio : writeln, writefln;
19 import std..string : toLower;
20 import std.traits : EnumMembers, hasMember;
21 import std.variant : Algebraic, visit;
22 static import std.getopt;
23 
24 import colorlog : VerboseMode;
25 
26 import distssh.types;
27 
28 version (unittest) {
29     import unit_threaded.assertions;
30 }
31 
32 struct Config {
33     struct Global {
34         std.getopt.GetoptResult helpInfo;
35         VerboseMode verbosity;
36         string progName;
37         bool noImportEnv;
38         bool cloneEnv;
39         bool stdinMsgPackEnv;
40         Duration timeout = defaultTimeout_s.dur!"seconds";
41 
42         string selfBinary;
43         string selfDir;
44 
45         string importEnv;
46         string workDir;
47         string[] command;
48 
49         Path dbPath;
50 
51         /// The hosts the cluster consist of.
52         Host[] cluster;
53     }
54 
55     struct Help {
56         std.getopt.GetoptResult helpInfo;
57     }
58 
59     struct Shell {
60         std.getopt.GetoptResult helpInfo;
61         static string helpDescription = "open an interactive shell on the remote host";
62     }
63 
64     struct Cmd {
65         std.getopt.GetoptResult helpInfo;
66         static string helpDescription = "run a command on a remote host";
67     }
68 
69     struct LocalRun {
70         std.getopt.GetoptResult helpInfo;
71         static string helpDescription = "import env and run the command locally";
72         bool useFakeTerminal;
73     }
74 
75     struct Install {
76         std.getopt.GetoptResult helpInfo;
77         static string helpDescription = "install distssh by setting up the correct symlinks";
78     }
79 
80     struct MeasureHosts {
81         std.getopt.GetoptResult helpInfo;
82         static string helpDescription = "measure the login time and load of all remote hosts";
83     }
84 
85     struct LocalLoad {
86         std.getopt.GetoptResult helpInfo;
87         static string helpDescription = "measure the load on the current host";
88     }
89 
90     struct RunOnAll {
91         std.getopt.GetoptResult helpInfo;
92         static string helpDescription = "run the command on all remote hosts";
93     }
94 
95     struct LocalShell {
96         std.getopt.GetoptResult helpInfo;
97         static string helpDescription = "run the shell locally";
98     }
99 
100     struct Env {
101         std.getopt.GetoptResult helpInfo;
102         static string helpDescription = "manipulate the stored environment";
103         /// Print the environment.
104         bool print;
105         /// Env variable to set in the config specified in importEnv.
106         string[] envSet;
107         /// Env variables to remove from the onespecified in importEnv.
108         string[] envDel;
109         /// Export the current environment
110         bool exportEnv;
111     }
112 
113     struct Daemon {
114         std.getopt.GetoptResult helpInfo;
115         static string helpDescription = "update the cluster statistics in the database";
116         Duration timeout;
117         /// If the daemon should persist in the background after it has measured the cluster once.
118         bool background;
119         /// Force the server to start even though there may be one running in the background
120         bool forceStart;
121     }
122 
123     struct Purge {
124         std.getopt.GetoptResult helpInfo;
125         static string helpDescription = "purge the cluster of rogue processes";
126         /// Only prints those that would be removed
127         bool print;
128         /// Kill rogue process
129         bool kill;
130         /// regex whitelist. Only processes in this list is not killed.
131         string[] whiteList;
132         /// restrict killing of processes to the current user
133         bool userFilter;
134     }
135 
136     struct LocalPurge {
137         std.getopt.GetoptResult helpInfo;
138         static string helpDescription = "purge the current host rogue processes";
139         /// Only prints those that would be removed
140         bool print;
141         /// Kill rogue process
142         bool kill;
143         /// regex whitelist. Only processes in this list is not killed.
144         string[] whiteList;
145         /// restrict killing of processes to the current user
146         bool userFilter;
147     }
148 
149     alias Type = Algebraic!(Help, Shell, Cmd, LocalRun, Install, MeasureHosts,
150             LocalLoad, RunOnAll, LocalShell, Env, Daemon, Purge, LocalPurge);
151     Type data;
152 
153     Global global;
154 
155     void printHelp() {
156         static void printGroup(T)(std.getopt.GetoptResult global,
157                 std.getopt.GetoptResult helpInfo, string progName) {
158             const helpDescription = () {
159                 static if (hasMember!(T, "helpDescription"))
160                     return T.helpDescription ~ "\n";
161                 else
162                     return null;
163             }();
164             defaultGetoptPrinter(format("usage: %s %s <options>\n%s", progName,
165                     T.stringof.toLower, helpDescription), global.options);
166             defaultGetoptPrinter(null, helpInfo.options.filter!(a => a.optShort != "-h").array);
167         }
168 
169         static void printHelpGroup(std.getopt.GetoptResult helpInfo, string progName) {
170             defaultGetoptPrinter(format("usage: %s <command>\n", progName), helpInfo.options);
171             writeln("sub-commands");
172             string[2][] subCommands;
173             static foreach (T; Type.AllowedTypes) {
174                 static if (hasMember!(T, "helpDescription"))
175                     subCommands ~= [T.stringof.toLower, T.helpDescription];
176                 else
177                     subCommands ~= [T.stringof.toLower, null];
178             }
179             const width = subCommands.map!(a => a[0].length).maxElement + 1;
180             foreach (cmd; subCommands)
181                 writefln(" %s%*s %s", cmd[0], width - cmd[0].length, " ", cmd[1]);
182         }
183 
184         template printers(T...) {
185             static if (T.length == 1) {
186                 static if (is(T[0] == Config.Help))
187                     alias printers = (T[0] a) => printHelpGroup(global.helpInfo, global.progName);
188                 else
189                     alias printers = (T[0] a) => printGroup!(T[0])(global.helpInfo,
190                             a.helpInfo, global.progName);
191             } else {
192                 alias printers = AliasSeq!(printers!(T[0]), printers!(T[1 .. $]));
193             }
194         }
195 
196         data.visit!(printers!(Type.AllowedTypes));
197     }
198 }
199 
200 /**
201  * #SPC-remote_command_parse
202  *
203  * Params:
204  *  args = the command line arguments to parse.
205  */
206 Config parseUserArgs(string[] args) {
207     import my.xdg : xdgRuntimeDir;
208 
209     Config conf;
210     conf.data = Config.Help.init;
211     conf.global.progName = args[0].baseName;
212     conf.global.selfBinary = buildPath(thisExePath.dirName, args[0].baseName);
213     conf.global.selfDir = conf.global.selfBinary.dirName;
214     conf.global.workDir = getcwd;
215     conf.global.dbPath = xdgRuntimeDir ~ Path("distssh.sqlite3");
216 
217     switch (conf.global.selfBinary.baseName) {
218     case distShell:
219         conf.data = Config.Shell.init;
220         conf.global.cluster = hostsFromEnv;
221         return conf;
222     case distCmd:
223         if (args.length > 1 && args[1].among("-h", "--help"))
224             conf.data = Config.Help.init;
225         else {
226             conf.data = Config.Cmd.init;
227             conf.global.command = args.length > 1 ? args[1 .. $] : null;
228             conf.global.cluster = hostsFromEnv;
229             configImportEnvFile(conf);
230         }
231         return conf;
232     default:
233     }
234 
235     string group;
236     if (args.length > 1 && args[1][0] != '-') {
237         group = args[1];
238         args = args.remove(1);
239     }
240 
241     try {
242         void globalParse() {
243             string export_env_file;
244             ulong timeout_s = defaultTimeout_s;
245 
246             // dfmt off
247             conf.global.helpInfo = std.getopt.getopt(args, std.getopt.config.passThrough, std.getopt.config.keepEndOfOptions,
248                 "clone-env", "clone the current environment to the remote host without an intermediate file", &conf.global.cloneEnv,
249                 "env-file", "file to load the environment from", &export_env_file,
250                 "i|import-env", "import the env from the file (default: " ~ distsshEnvExport ~ ")", &conf.global.importEnv,
251                 "no-import-env", "do not automatically import the environment from " ~ distsshEnvExport, &conf.global.noImportEnv,
252                 "stdin-msgpack-env", "import env from stdin as a msgpack stream", &conf.global.stdinMsgPackEnv,
253                 "timeout", "timeout to use when checking remote hosts", &timeout_s,
254                 "v|verbose", format("Set the verbosity (%-(%s, %))", [EnumMembers!(VerboseMode)]), &conf.global.verbosity,
255                 "workdir", "working directory to run the command in", &conf.global.workDir,
256                 );
257             // dfmt on
258             if (conf.global.helpInfo.helpWanted)
259                 args ~= "-h";
260 
261             // must convert e.g. "."
262             conf.global.workDir = conf.global.workDir.absolutePath;
263 
264             conf.global.timeout = timeout_s.dur!"seconds";
265 
266             if (!export_env_file.empty)
267                 conf.global.importEnv = export_env_file;
268         }
269 
270         void helpParse() {
271             conf.data = Config.Help.init;
272         }
273 
274         void envParse() {
275             Config.Env data;
276             scope (success)
277                 conf.data = data;
278 
279             // dfmt off
280             data.helpInfo = std.getopt.getopt(args,
281                 "d|delete", "remove a variable from the exported environment", &data.envDel,
282                 "e|export", "export the current environment to a file that is used on the remote host", &data.exportEnv,
283                 "p|print", "print the content of an exported environment", &data.print,
284                 "s|set", "set a variable in the exported environment. Example: FOO=42", &data.envSet,
285                 );
286             // dfmt on
287         }
288 
289         void shellParse() {
290             conf.data = Config.Shell.init;
291             conf.global.cluster = hostsFromEnv;
292         }
293 
294         void cmdParse() {
295             conf.data = Config.Cmd.init;
296             conf.global.cluster = hostsFromEnv;
297         }
298 
299         void localrunParse() {
300             Config.LocalRun data;
301             scope (success)
302                 conf.data = data;
303 
304             // dfmt off
305             data.helpInfo = std.getopt.getopt(args, std.getopt.config.passThrough, std.getopt.config.keepEndOfOptions,
306                 "pseudo-terminal", "force that a pseudo-terminal is used when running the command", &data.useFakeTerminal,
307             );
308             // dfmt on
309         }
310 
311         void installParse() {
312             conf.data = Config.Install.init;
313         }
314 
315         void measurehostsParse() {
316             conf.data = Config.MeasureHosts.init;
317             conf.global.cluster = hostsFromEnv;
318         }
319 
320         void localloadParse() {
321             conf.data = Config.LocalLoad.init;
322         }
323 
324         void runonallParse() {
325             conf.data = Config.RunOnAll.init;
326             conf.global.cluster = hostsFromEnv;
327         }
328 
329         void localshellParse() {
330             conf.data = Config.LocalShell.init;
331         }
332 
333         void daemonParse() {
334             conf.global.cluster = hostsFromEnv;
335             Config.Daemon data;
336             scope (success)
337                 conf.data = data;
338 
339             ulong timeout = 30;
340             // dfmt off
341             data.helpInfo = std.getopt.getopt(args,
342                 "b|background", "persist in the background", &data.background,
343                 "force-start", "force the server to start", &data.forceStart,
344                 "t|timeout", "shutdown background process if unused not used for this time (default: 30 minutes)", &timeout,
345             );
346             // dfmt on
347             data.timeout = timeout.dur!"minutes";
348         }
349 
350         void purgeParse() {
351             conf.global.cluster = hostsFromEnv;
352             Config.Purge data;
353             scope (success)
354                 conf.data = data;
355 
356             // dfmt off
357             data.helpInfo = std.getopt.getopt(args,
358                 "k|kill", "kill rogue processes", &data.kill,
359                 "p|print", "print rogue process", &data.print,
360                 "user-filter", "only purge those processes owned by the current user", &data.userFilter,
361                 "whitelist", "one or more regex (case insensitive) that whitelist processes as not rogue", &data.whiteList,
362                 );
363             // dfmt on
364         }
365 
366         void localpurgeParse() {
367             Config.LocalPurge data;
368             scope (success)
369                 conf.data = data;
370 
371             // dfmt off
372             data.helpInfo = std.getopt.getopt(args,
373                 "k|kill", "kill rogue processes", &data.kill,
374                 "p|print", "print rogue process", &data.print,
375                 "user-filter", "only purge those processes owned by the current user", &data.userFilter,
376                 "whitelist", "one or more regex (case insensitive) that whitelist processes as not rogue", &data.whiteList,
377                 );
378             // dfmt on
379         }
380 
381         alias ParseFn = void delegate();
382         ParseFn[string] parsers;
383 
384         static foreach (T; Config.Type.AllowedTypes) {
385             mixin(format(`parsers["%1$s"] = &%1$sParse;`, T.stringof.toLower));
386         }
387 
388         globalParse;
389 
390         if (auto p = group in parsers) {
391             (*p)();
392         }
393 
394         if (args.length > 1) {
395             conf.global.command = args.find("--").drop(1).array();
396         }
397         configImportEnvFile(conf);
398     } catch (std.getopt.GetOptException e) {
399         // unknown option
400         logger.error(e.msg);
401     } catch (Exception e) {
402         logger.error(e.msg);
403     }
404 
405     return conf;
406 }
407 
408 /** Update a Configs object's file to import the environment from.
409  *
410  * This should only be called after all other command line parsing has been
411  * done. It is because this function take into consideration the priority as
412  * specified in the requirement:
413  * #SPC-configure_env_import_file
414  *
415  * Params:
416  *  opts = config to update the file to import the environment from.
417  */
418 void configImportEnvFile(ref Config opts) nothrow {
419     import std.process : environment;
420 
421     if (opts.global.noImportEnv) {
422         opts.global.importEnv = null;
423     } else if (opts.global.importEnv.length != 0) {
424         // do nothing. the user has specified a file
425     } else {
426         try {
427             opts.global.importEnv = environment.get(globalEnvFileKey, distsshEnvExport);
428         } catch (Exception e) {
429         }
430     }
431 }
432 
433 @("shall determine the absolute path of self")
434 unittest {
435     import std.path;
436     import std.file;
437 
438     auto opts = parseUserArgs(["distssh", "ls"]);
439     assert(opts.global.selfBinary[0] == '/');
440     assert(opts.global.selfBinary.baseName == "distssh");
441 
442     opts = parseUserArgs(["distshell"]);
443     assert(opts.global.selfBinary[0] == '/');
444     assert(opts.global.selfBinary.baseName == "distshell");
445 
446     opts = parseUserArgs(["distcmd"]);
447     assert(opts.global.selfBinary[0] == '/');
448     assert(opts.global.selfBinary.baseName == "distcmd");
449 
450     opts = parseUserArgs(["distcmd_recv", getcwd, distsshEnvExport]);
451     assert(opts.global.selfBinary[0] == '/');
452     assert(opts.global.selfBinary.baseName == "distcmd_recv");
453 }
454 
455 @("shall either return the default timeout or the user specified timeout")
456 unittest {
457     import core.time : dur;
458     import std.conv;
459 
460     auto opts = parseUserArgs(["distssh", "ls"]);
461     assert(opts.global.timeout == defaultTimeout_s.dur!"seconds");
462     opts = parseUserArgs(["distssh", "--timeout", "10", "ls"]);
463     assert(opts.global.timeout == 10.dur!"seconds");
464 
465     opts = parseUserArgs(["distshell"]);
466     opts.global.timeout.shouldEqual(defaultTimeout_s.dur!"seconds");
467     opts = parseUserArgs(["distshell", "--timeout", "10"]);
468     assert(opts.global.timeout == defaultTimeout_s.dur!"seconds");
469 }
470 
471 @("shall only be the default timeout because --timeout should be passed on to the command")
472 unittest {
473     import core.time : dur;
474     import std.conv;
475 
476     auto opts = parseUserArgs(["distcmd", "ls"]);
477     assert(opts.global.timeout == defaultTimeout_s.dur!"seconds");
478 
479     opts = parseUserArgs(["distcmd", "--timeout", "10"]);
480     assert(opts.global.timeout == defaultTimeout_s.dur!"seconds");
481 }
482 
483 @("shall convert relative workdirs to absolute when parsing user args")
484 unittest {
485     import std.path : isAbsolute;
486 
487     auto opts = parseUserArgs(["distssh", "--workdir", "."]);
488     assert(opts.global.workDir.isAbsolute, "expected an absolute path");
489 }
490 
491 Host[] hostsFromEnv() nothrow {
492     import std.algorithm : splitter, map;
493     import std.array : array;
494     import std.exception : collectException;
495     import std.process : environment;
496     import std..string : strip;
497 
498     typeof(return) rval;
499 
500     try {
501         string hosts_env = environment.get(globalEnvHostKey, "").strip;
502         rval = hosts_env.splitter(";").map!(a => a.strip)
503             .filter!(a => a.length > 0)
504             .map!(a => Host(a))
505             .array;
506 
507         if (rval.length == 0) {
508             logger.errorf("No remote host configured (%s='%s')", globalEnvHostKey, hosts_env);
509         }
510     } catch (Exception e) {
511         logger.error(e.msg).collectException;
512     }
513 
514     return rval;
515 }