1 /**
2 * Simple execution of shell commands,
3 * and wrappers for common utilities.
4 *
5 * License:
6 * This Source Code Form is subject to the terms of
7 * the Mozilla Public License, v. 2.0. If a copy of
8 * the MPL was not distributed with this file, You
9 * can obtain one at http://mozilla.org/MPL/2.0/.
10 *
11 * Authors:
12 * Vladimir Panteleev <ae@cy.md>
13 */
14
15 module ae.sys.cmd;
16
17 import core.thread;
18
19 import std.exception;
20 import std.process;
21 import std.stdio;
22 import std.string;
23 import std.traits;
24
25 import ae.sys.data : TData;
26 import ae.sys.file;
27
28 /// Returns a very unique name for a temporary file.
29 string getTempFileName(string extension)
30 {
31 import std.random : uniform;
32 import std.file : tempDir;
33 import std.path : buildPath;
34
35 static int counter;
36 return buildPath(tempDir(), format("run-%d-%d-%d-%d.%s",
37 getpid(),
38 getCurrentThreadID(),
39 uniform!uint(),
40 counter++,
41 extension
42 ));
43 }
44
45 version (linux)
46 {
47 import core.sys.posix.sys.types : pid_t;
48 private extern(C) pid_t gettid();
49 }
50
51 /// Like `thisProcessID`, but for threads.
52 ulong getCurrentThreadID()
53 {
54 version (Windows)
55 {
56 import core.sys.windows.windows : GetCurrentThreadId;
57 return GetCurrentThreadId();
58 }
59 else
60 version (linux)
61 {
62 return gettid();
63 }
64 else
65 version (Posix)
66 {
67 import core.sys.posix.pthread : pthread_self;
68 return cast(ulong)pthread_self();
69 }
70 }
71
72 // ************************************************************************
73
74 private struct ProcessParams
75 {
76 string shellCommand;
77 string[] processArgs;
78
79 string toShellCommand() { return shellCommand ? shellCommand : escapeShellCommand(processArgs); }
80 // Note: a portable toProcessArgs() cannot exist because CMD.EXE does not use CommandLineToArgvW.
81
82 const(string[string]) environment = null;
83 std.process.Config config = std.process.Config.none;
84 size_t maxOutput = size_t.max;
85 string workDir = null;
86 File[3] files;
87 size_t numFiles;
88
89 this(Params...)(Params params)
90 {
91 files = [stdin, stdout, stderr];
92
93 static foreach (i; 0 .. params.length)
94 {{
95 auto arg = params[i];
96 static if (i == 0)
97 {
98 static if (is(typeof(arg) == string))
99 shellCommand = arg;
100 else
101 static if (is(typeof(arg) == string[]))
102 processArgs = arg;
103 else
104 static assert(false, "Unknown type for process invocation command line: " ~ typeof(arg).stringof);
105 assert(arg, "Null command");
106 }
107 else
108 {
109 static if (is(typeof(arg) == string))
110 workDir = arg;
111 else
112 static if (is(typeof(arg) : const(string[string])))
113 environment = arg;
114 else
115 static if (is(typeof(arg) == size_t))
116 maxOutput = arg;
117 else
118 static if (is(typeof(arg) == std.process.Config))
119 config |= arg;
120 else
121 static if (is(typeof(arg) == File))
122 files[numFiles++] = arg;
123 else
124 static assert(false, "Unknown type for process invocation parameter: " ~ typeof(arg).stringof);
125 }
126 }}
127 }
128 }
129
130 private void invoke(alias runner)(string command)
131 {
132 //debug scope(failure) std.stdio.writeln("[CWD] ", getcwd());
133 debug(CMD) std.stdio.stderr.writeln("invoke: ", command);
134 auto status = runner();
135 enforce(status == 0,
136 "Command `%s` failed with status %d".format(command, status));
137 }
138
139 /// std.process helper.
140 /// Run a command, and throw if it exited with a non-zero status.
141 void run(Params...)(Params params)
142 {
143 auto parsed = ProcessParams(params);
144 invoke!({
145 auto pid = parsed.processArgs
146 ? spawnProcess(
147 parsed.processArgs, parsed.files[0], parsed.files[1], parsed.files[2],
148 parsed.environment, parsed.config, parsed.workDir
149 )
150 : spawnShell(
151 parsed.shellCommand, parsed.files[0], parsed.files[1], parsed.files[2],
152 parsed.environment, parsed.config, parsed.workDir
153 );
154 return pid.wait();
155 })(parsed.toShellCommand());
156 }
157
158 /// std.process helper.
159 /// Run a command and collect its output.
160 /// Throw if it exited with a non-zero status.
161 string query(Params...)(Params params)
162 {
163 auto parsed = ProcessParams(params, Config.stderrPassThrough);
164 assert(parsed.numFiles == 0, "Can't specify files with query");
165 string output;
166 invoke!({
167 auto result = parsed.processArgs
168 ? execute(parsed.processArgs, parsed.environment, parsed.config, parsed.maxOutput, parsed.workDir)
169 : executeShell(parsed.shellCommand, parsed.environment, parsed.config, parsed.maxOutput, parsed.workDir);
170 output = result.output.stripRight();
171 return result.status;
172 })(parsed.toShellCommand());
173 return output;
174 }
175
176 /// std.process helper.
177 /// Run a command, feed it the given input, and collect its output.
178 /// Throw if it exited with non-zero status. Return output.
179 T[] pipe(T, Params...)(in T[] input, Params params)
180 if (!hasIndirections!T)
181 {
182 auto parsed = ProcessParams(params);
183 assert(parsed.numFiles == 0, "Can't specify files with pipe");
184 T[] output;
185 invoke!({
186 auto pipes = parsed.processArgs
187 ? pipeProcess(parsed.processArgs, Redirect.stdin | Redirect.stdout,
188 parsed.environment, parsed.config, parsed.workDir)
189 : pipeShell(parsed.shellCommand, Redirect.stdin | Redirect.stdout,
190 parsed.environment, parsed.config, parsed.workDir);
191 auto f = pipes.stdin;
192 auto writer = writeFileAsync(f, input);
193 scope(exit) writer.join();
194 output = cast(T[])readFile(pipes.stdout);
195 return pipes.pid.wait();
196 })(parsed.toShellCommand());
197 return output;
198 }
199
200 TData!T pipe(T, Params...)(in TData!T input, Params params)
201 if (!hasIndirections!T)
202 {
203 import ae.sys.dataio : readFileData;
204
205 auto parsed = ProcessParams(params);
206 assert(parsed.numFiles == 0, "Can't specify files with pipe");
207 TData!T output;
208 invoke!({
209 auto pipes = parsed.processArgs
210 ? pipeProcess(parsed.processArgs, Redirect.stdin | Redirect.stdout,
211 parsed.environment, parsed.config, parsed.workDir)
212 : pipeShell(parsed.shellCommand, Redirect.stdin | Redirect.stdout,
213 parsed.environment, parsed.config, parsed.workDir);
214 auto f = pipes.stdin;
215 auto writer = writeFileAsync(f, input.unsafeContents);
216 scope(exit) writer.join();
217 output = readFileData(pipes.stdout).asDataOf!T;
218 return pipes.pid.wait();
219 })(parsed.toShellCommand());
220 return output;
221 }
222
223 deprecated T[] pipe(T, Params...)(string[] args, in T[] input, Params params)
224 if (!hasIndirections!T)
225 {
226 return pipe(input, args, params);
227 }
228
229 debug(ae_unittest) unittest
230 {
231 if (false) // Instantiation test
232 {
233 import ae.sys.data : Data;
234 import ae.utils.array : asBytes;
235
236 run("cat");
237 run(["cat"]);
238 query(["cat"]);
239 query("cat | cat");
240 pipe("hello", "rev");
241 pipe(Data("hello".asBytes), "rev");
242 }
243 }
244
245 // ************************************************************************
246
247 /// Wrapper for the `iconv` program.
248 ubyte[] iconv(const(void)[] data, string inputEncoding, string outputEncoding)
249 {
250 auto args = ["timeout", "30", "iconv", "-f", inputEncoding, "-t", outputEncoding];
251 auto result = data.pipe(args);
252 return cast(ubyte[])result;
253 }
254
255 /// ditto
256 string iconv(const(void)[] data, string inputEncoding)
257 {
258 import std.utf : validate;
259 auto result = cast(string)iconv(data, inputEncoding, "UTF-8");
260 validate(result);
261 return result;
262 }
263
264 version (HAVE_UNIX)
265 debug(ae_unittest) unittest
266 {
267 assert(iconv("Hello"w, "UTF-16LE") == "Hello");
268 }
269
270 /// Wrapper for the `sha1sum` program.
271 string sha1sum(const(void)[] data)
272 {
273 auto output = cast(string)data.pipe(["sha1sum", "-b", "-"]);
274 return output[0..40];
275 }
276
277 version (HAVE_UNIX)
278 debug(ae_unittest) unittest
279 {
280 assert(sha1sum("") == "da39a3ee5e6b4b0d3255bfef95601890afd80709");
281 assert(sha1sum("a b\nc\r\nd") == "667c71ffe2ac8a4fe500e3b96435417e4c5ec13b");
282 }
283
284 // ************************************************************************
285
286 import ae.utils.path;
287 deprecated alias NULL_FILE = nullFileName;
288
289 // ************************************************************************
290
291 /// Reverse of std.process.environment.toAA
292 void setEnvironment(string[string] env)
293 {
294 foreach (k, v; env)
295 if (k.length)
296 environment[k] = v;
297 foreach (k, v; environment.toAA())
298 if (k.length && k !in env)
299 environment.remove(k);
300 }
301
302 /// Expand Windows-like variable placeholders (`"%VAR%"`) in the given string.
303 string expandWindowsEnvVars(alias getenv = environment.get)(string s)
304 {
305 import std.array : appender;
306 auto buf = appender!string();
307
308 size_t lastPercent = 0;
309 bool inPercent = false;
310
311 foreach (i, c; s)
312 if (c == '%')
313 {
314 if (inPercent)
315 buf.put(lastPercent == i ? "%" : getenv(s[lastPercent .. i]));
316 else
317 buf.put(s[lastPercent .. i]);
318 inPercent = !inPercent;
319 lastPercent = i + 1;
320 }
321 enforce(!inPercent, "Unterminated environment variable name");
322 buf.put(s[lastPercent .. $]);
323 return buf.data;
324 }
325
326 debug(ae_unittest) unittest
327 {
328 std.process.environment[`FOOTEST`] = `bar`;
329 assert("a%FOOTEST%b".expandWindowsEnvVars() == "abarb");
330 }
331
332 // ************************************************************************
333
334 /// Like `std.process.wait`, but with a timeout.
335 /// If the timeout is exceeded, the program is killed.
336 int waitTimeout(Pid pid, Duration time)
337 {
338 bool ok = false;
339 auto t = new Thread({
340 Thread.sleep(time);
341 if (!ok)
342 try
343 pid.kill();
344 catch (Exception) {} // Ignore race condition
345 }).start();
346 scope(exit) t.join();
347
348 auto result = pid.wait();
349 ok = true;
350 return result;
351 }
352
353 /// Wait for process to exit asynchronously.
354 /// Call callback when it exits.
355 /// WARNING: the callback will be invoked in another thread!
356 Thread waitAsync(Pid pid, void delegate(int) callback = null)
357 {
358 return new Thread({
359 auto result = pid.wait();
360 if (callback)
361 callback(result);
362 }).start();
363 }