Rename the "parallel-sync" option to "output-sync".

This commit is contained in:
Paul Smith 2013-04-14 16:31:18 -04:00
parent cb64352c75
commit 4bf3c33f83
7 changed files with 154 additions and 162 deletions

View file

@ -4057,12 +4057,13 @@ If there is nothing looking like an integer after the @samp{-j} option,
there is no limit on the number of job slots. The default number of job
slots is one, which means serial execution (one thing at a time).
One consequence of running several recipes simultaneously is that by
default, output from each recipe appears as soon as it is generated,
with the result that messages from different recipes may be interspersed.
This may create problems in interpreting output. If the @samp{-P} option
is used, however, recipes will save their output until completion and
then take turns writing it, with a more coherent result.
When running several recipes simultaneously the output from each
recipe appears as soon as it is generated, with the result that
messages from different recipes may be interspersed. To avoid this
you can use the @samp{--output-sync} (@samp{-O}) option; if this
option is provided then the output from each recipe will be saved
until the recipe is complete, then written all at once. This ensures
that output from different recipes is not mixed together.
Another problem is that two processes cannot both take input from the
same device; so to make sure that only one recipe tries to take input
@ -8615,22 +8616,25 @@ The data base output contains file name and line number information for
recipe and variable definitions, so it can be a useful debugging tool
in complex environments.
@item -P
@cindex @code{-P}
@itemx --parallel-sync
@cindex @code{--parallel-sync}
@cindex parallel recipe execution, output
When jobs are running in parallel under @samp{--jobs}, the output of
each job is held until the job is complete thus ensuring that the output
of each recipe is grouped together.
@item -O
@cindex @code{-O}
@itemx --output-sync
@cindex @code{--output-sync}
@cindex output of parallel execution
@cindex parallel execution, output of
Ensure that the complete output from each recipe is printed in one
uninterrupted sequence. This option is only useful when using the
@code{--jobs} option to run multiple recipes simultaneously
(@pxref{Parallel, ,Parallel Execution}). Without this option output
will be displayed as it is generated by the recipes.
With no argument or the argument @samp{1}, messages from each job in
recursive makes are grouped together. With the argument @samp{2}, the
complete output from any recursive make is grouped together. The latter
achieves better grouping of output from related jobs, but causes longer
delay, since messages do not appear until the recursive make has
completed. Therefore @samp{-P} is more useful when watching the output
while make runs, and @samp{-P2} is better suited when running a complex
completed. Therefore @samp{-O} is more useful when watching the output
while make runs, and @samp{-O2} is better suited when running a complex
parallel build in the background and checking its output afterwards.
@item -q

172
job.c
View file

@ -243,15 +243,15 @@ unsigned long job_counter = 0;
unsigned int jobserver_tokens = 0;
#ifdef PARALLEL_SYNC
/* Semaphore for use in -j mode with parallel_sync. */
#ifdef OUTPUT_SYNC
/* Semaphore for use in -j mode with output_sync. */
int sync_handle = -1;
#define STREAM_OK(strm) ((fcntl (fileno ((strm)), F_GETFD) != -1) || (errno != EBADF))
#define STREAM_OK(_s) ((fcntl (fileno (_s), F_GETFD) != -1) || (errno != EBADF))
#define FD_NOT_EMPTY(FD) ((FD) >= 0 && lseek ((FD), 0, SEEK_CUR) > 0)
#endif /* PARALLEL_SYNC */
#define FD_NOT_EMPTY(_f) ((_f) >= 0 && lseek ((_f), 0, SEEK_CUR) > 0)
#endif /* OUTPUT_SYNC */
#ifdef WINDOWS32
/*
@ -549,8 +549,8 @@ child_handler (int sig UNUSED)
*/
}
#ifdef PARALLEL_SYNC
/* Adds file descriptors to the child structure to support parallel_sync;
#ifdef OUTPUT_SYNC
/* Adds file descriptors to the child structure to support output_sync;
one for stdout and one for stderr as long as they are (a) open and
(b) separate. If stdout and stderr share a device they can share a
temp file too. */
@ -559,38 +559,38 @@ assign_child_tempfiles (struct child *c, int combined)
{
FILE *outstrm = NULL, *errstrm = NULL;
int o_ok, e_ok;
const char *suppressed = "parallel-sync suppressed: ";
const char *suppressed = "output-sync suppressed: ";
char *failmode = NULL;
/* Check status of stdout and stderr before hooking up temp files. */
o_ok = STREAM_OK (stdout);
e_ok = STREAM_OK (stderr);
/* The tmpfile() function returns a FILE pointer but those can be in
/* The tmpfile() function returns a FILE pointer but those can be in
limited supply, so we immediately dup its file descriptor and keep
only that, closing the FILE pointer. */
if (combined)
{
if (!(outstrm = tmpfile ()))
failmode = "tmpfile()";
failmode = "tmpfile()";
else
errstrm = outstrm;
errstrm = outstrm;
}
else if (o_ok && e_ok)
{
if (!(outstrm = tmpfile ()) || !(errstrm = tmpfile ()))
failmode = "tmpfile()";
failmode = "tmpfile()";
}
else if (o_ok)
{
if (!(outstrm = tmpfile ()))
failmode = "tmpfile()";
failmode = "tmpfile()";
}
else if (e_ok)
{
if (!(errstrm = tmpfile ()))
failmode = "tmpfile()";
failmode = "tmpfile()";
}
else
failmode = "stdout";
@ -598,22 +598,22 @@ assign_child_tempfiles (struct child *c, int combined)
if (!failmode && outstrm)
{
if ((c->outfd = dup (fileno (outstrm))) == -1)
failmode = "dup2()";
failmode = "dup2()";
else
CLOSE_ON_EXEC (c->outfd);
CLOSE_ON_EXEC (c->outfd);
}
if (!failmode && errstrm)
{
if (combined)
c->errfd = c->outfd;
c->errfd = c->outfd;
else
{
if ((c->errfd = dup (fileno (errstrm))) == -1)
failmode = "dup2()";
else
CLOSE_ON_EXEC (c->errfd);
}
{
if ((c->errfd = dup (fileno (errstrm))) == -1)
failmode = "dup2()";
else
CLOSE_ON_EXEC (c->errfd);
}
}
if (failmode)
@ -643,9 +643,9 @@ pump_from_tmp_fd (int from_fd, int to_fd)
char *write_buf = buffer;
EINTRLOOP (nleft, read (from_fd, buffer, sizeof (buffer)));
if (nleft < 0)
perror ("read()");
perror ("read()");
if (nleft <= 0)
break;
break;
while (nleft > 0)
{
EINTRLOOP (nwrite, write (to_fd, write_buf, nleft));
@ -702,10 +702,8 @@ sync_output (struct child *c)
if ((outfd_not_empty || errfd_not_empty) && (sem = acquire_semaphore ()))
{
/*
* We've entered the "critical section" during which a lock is held.
* We want to keep it as short as possible.
*/
/* We've entered the "critical section" during which a lock is held.
We want to keep it as short as possible. */
if (outfd_not_empty)
{
log_working_directory (1, 1);
@ -715,7 +713,7 @@ sync_output (struct child *c)
if (errfd_not_empty && c->errfd != c->outfd)
pump_from_tmp_fd (c->errfd, fileno (stderr));
/* Exit the critical section */
/* Exit the critical section. */
release_semaphore (sem);
}
@ -725,7 +723,7 @@ sync_output (struct child *c)
close (c->errfd);
c->outfd = c->errfd = -1;
}
#endif /* PARALLEL_SYNC */
#endif /* OUTPUT_SYNC */
extern int shell_function_pid, shell_function_completed;
@ -1021,11 +1019,11 @@ reap_children (int block, int err)
c->sh_batch_file = NULL;
}
#ifdef PARALLEL_SYNC
#ifdef OUTPUT_SYNC
/* Synchronize parallel output if requested */
if (parallel_sync)
sync_output (c);
#endif /* PARALLEL_SYNC */
if (output_sync)
sync_output (c);
#endif /* OUTPUT_SYNC */
/* If this child had the good stdin, say it is now free. */
if (c->good_stdin)
@ -1607,44 +1605,44 @@ start_job_command (struct child *child)
#else /* !__EMX__ */
#ifdef PARALLEL_SYNC
if (parallel_sync)
{
static int combined_output;
/* If parallel_sync is turned on, find a resource to
synchronize on. This block is traversed only once. */
if (sync_handle == -1)
{
struct stat stbuf_o, stbuf_e;
#ifdef OUTPUT_SYNC
if (output_sync)
{
static int combined_output;
/* If output_sync is turned on, find a resource to
synchronize on. This block is traversed only once. */
if (sync_handle == -1)
{
struct stat stbuf_o, stbuf_e;
if (STREAM_OK (stdout))
{
sync_handle = fileno (stdout);
combined_output =
fstat (fileno (stdout), &stbuf_o) == 0 &&
fstat (fileno (stderr), &stbuf_e) == 0 &&
stbuf_o.st_dev == stbuf_e.st_dev &&
stbuf_o.st_ino == stbuf_e.st_ino;
}
else if (STREAM_OK (stderr))
sync_handle = fileno (stderr);
else
{
perror_with_name ("parallel-sync suppressed: ", "stderr");
parallel_sync = 0;
}
}
if (STREAM_OK (stdout))
{
sync_handle = fileno (stdout);
combined_output =
fstat (fileno (stdout), &stbuf_o) == 0 &&
fstat (fileno (stderr), &stbuf_e) == 0 &&
stbuf_o.st_dev == stbuf_e.st_dev &&
stbuf_o.st_ino == stbuf_e.st_ino;
}
else if (STREAM_OK (stderr))
sync_handle = fileno (stderr);
else
{
perror_with_name ("output-sync suppressed: ", "stderr");
output_sync = 0;
}
}
/* If it still looks like we can synchronize, create a temp
file to hold stdout (and one for stderr if separate). */
if (parallel_sync >= PARALLEL_SYNC_COARSE
|| (parallel_sync == PARALLEL_SYNC_FINE && !(flags & COMMANDS_RECURSE)))
{
if (!assign_child_tempfiles (child, combined_output))
parallel_sync = 0;
}
}
#endif /* PARALLEL_SYNC */
/* If it still looks like we can synchronize, create a temp
file to hold stdout (and one for stderr if separate). */
if (output_sync >= OUTPUT_SYNC_COARSE
|| (output_sync == OUTPUT_SYNC_FINE && !(flags & COMMANDS_RECURSE)))
{
if (!assign_child_tempfiles (child, combined_output))
output_sync = 0;
}
}
#endif /* OUTPUT_SYNC */
child->pid = vfork ();
environ = parent_environ; /* Restore value child may have clobbered. */
@ -1669,22 +1667,22 @@ start_job_command (struct child *child)
setrlimit (RLIMIT_STACK, &stack_limit);
#endif
#ifdef PARALLEL_SYNC
/* Divert child output into tempfile(s) if parallel_sync in use. */
if (parallel_sync)
{
int outfd = fileno (stdout);
int errfd = fileno (stderr);
#ifdef OUTPUT_SYNC
/* Divert child output into tempfile(s) if output_sync in use. */
if (output_sync)
{
int outfd = fileno (stdout);
int errfd = fileno (stderr);
if ((child->outfd >= 0 &&
(close (outfd) == -1 || dup2 (child->outfd, outfd) == -1))
|| (child->errfd >= 0 &&
(close (errfd) == -1 || dup2 (child->errfd, errfd) == -1)))
{
perror_with_name ("parallel-sync: ", "dup2()");
}
}
#endif /* PARALLEL_SYNC */
if ((child->outfd >= 0 &&
(close (outfd) == -1 || dup2 (child->outfd, outfd) == -1))
|| (child->errfd >= 0 &&
(close (errfd) == -1 || dup2 (child->errfd, errfd) == -1)))
{
perror_with_name ("output-sync: ", "dup2()");
}
}
#endif /* OUTPUT_SYNC */
child_execute_job (child->good_stdin ? 0 : bad_stdin, 1,
argv, child->environment);
@ -2019,7 +2017,7 @@ new_job (struct file *file)
c->file = file;
c->command_lines = lines;
c->sh_batch_file = NULL;
#ifdef PARALLEL_SYNC
#ifdef OUTPUT_SYNC
c->outfd = c->errfd = -1;
#endif

14
job.h
View file

@ -34,9 +34,9 @@ this program. If not, see <http://www.gnu.org/licenses/>. */
# define CLOSE_ON_EXEC(_d) (void) fcntl ((_d), F_SETFD, FD_CLOEXEC)
#endif
#ifdef POSIX /* PARALLEL_SYNC */
#define PARALLEL_SYNC
#endif /* POSIX */
#ifdef POSIX
# define OUTPUT_SYNC
#endif
/* Structure describing a running or dead child process. */
@ -64,10 +64,10 @@ struct child
unsigned int good_stdin:1; /* Nonzero if this child has a good stdin. */
unsigned int deleted:1; /* Nonzero if targets have been deleted. */
unsigned int dontcare:1; /* Saved dontcare flag. */
#ifdef PARALLEL_SYNC
int outfd; /* Optional file descriptor for saving stdout */
int errfd; /* Optional file descriptor for saving stderr */
#endif /* PARALLEL_SYNC */
#ifdef OUTPUT_SYNC
int outfd; /* File descriptor for saving stdout */
int errfd; /* File descriptor for saving stderr */
#endif
};
extern struct child *children;

34
main.c
View file

@ -228,12 +228,12 @@ static unsigned int master_job_slots = 0;
static unsigned int inf_jobs = 0;
#ifdef PARALLEL_SYNC
#ifdef OUTPUT_SYNC
/* Default value for parallel sync without an argument. */
/* Default value for output-sync without an argument. */
static unsigned int no_parallel_sync = 0;
static unsigned int default_parallel_sync = PARALLEL_SYNC_FINE;
static unsigned int no_output_sync = 0;
static unsigned int default_output_sync = OUTPUT_SYNC_FINE;
#endif
@ -351,12 +351,12 @@ static const char *const usage[] =
N_("\
-o FILE, --old-file=FILE, --assume-old=FILE\n\
Consider FILE to be very old and don't remake it.\n"),
#ifdef OUTPUT_SYNC
N_("\
-O [2], --output-sync[=2] Synchronize output of parallel jobs [coarse].\n"),
#endif
N_("\
-p, --print-data-base Print make's internal database.\n"),
#ifdef PARALLEL_SYNC
N_("\
-P [2], --parallel-sync[=2] Synchronize output of parallel jobs [coarse].\n"),
#endif
N_("\
-q, --question Run no recipe; exit status says if up to date.\n"),
N_("\
@ -420,12 +420,12 @@ static const struct command_switch switches[] =
{ 'm', ignore, 0, 0, 0, 0, 0, 0, 0 },
{ 'n', flag, &just_print_flag, 1, 1, 1, 0, 0, "just-print" },
{ 'o', filename, &old_files, 0, 0, 0, 0, 0, "old-file" },
{ 'p', flag, &print_data_base_flag, 1, 1, 0, 0, 0, "print-data-base" },
#ifdef PARALLEL_SYNC
// { 'P', flag, &parallel_sync, 1, 1, 0, 0, 0, "parallel-sync" }, // two-state
{ 'P', positive_int, &parallel_sync, 1, 1, 0, &default_parallel_sync,
&no_parallel_sync, "parallel-sync" },
#ifdef OUTPUT_SYNC
// { 'O', flag, &output_sync, 1, 1, 0, 0, 0, "output-sync" }, // two-state
{ 'O', positive_int, &output_sync, 1, 1, 0, &default_output_sync,
&no_output_sync, "output-sync" },
#endif
{ 'p', flag, &print_data_base_flag, 1, 1, 0, 0, 0, "print-data-base" },
{ 'q', flag, &question_flag, 1, 1, 1, 0, 0, "question" },
{ 'r', flag, &no_builtin_rules_flag, 1, 1, 0, 0, 0, "no-builtin-rules" },
{ 'R', flag, &no_builtin_variables_flag, 1, 1, 0, 0, 0,
@ -521,13 +521,13 @@ int second_expansion;
int one_shell;
/* Either PARALLEL_SYNC_FINE or PARALLEL_SYNC_COARSE
if the "--parallel-sync" option was given.
/* Either OUTPUT_SYNC_FINE or OUTPUT_SYNC_COARSE
if the "--output-sync" option was given.
This attempts to synchronize the output of parallel
jobs such that the results of each job stay together.
It works best in combination with .ONESHELL. */
int parallel_sync;
int output_sync;
/* Nonzero if we have seen the '.NOTPARALLEL' target.
This turns off parallel builds for this invocation of make. */
@ -1763,7 +1763,7 @@ main (int argc, char **argv, char **envp)
if (! open_jobserver_semaphore(cp))
{
DWORD err = GetLastError();
fatal (NILF, _("internal error: unable to open jobserver semaphore '%s': (Error %ld: %s)"),
fatal (NILF, _("internal error: unable to open jobserver semaphore '%s': (Error %ld: %s)"),
cp, err, map_windows32_error_to_string(err));
}
DB (DB_JOBS, (_("Jobserver client (semaphore %s)\n"), cp));

View file

@ -525,8 +525,8 @@ int strncasecmp (const char *s1, const char *s2, int n);
# endif
#endif
#define PARALLEL_SYNC_FINE 1
#define PARALLEL_SYNC_COARSE 2
#define OUTPUT_SYNC_FINE 1
#define OUTPUT_SYNC_COARSE 2
extern const gmk_floc *reading_file;
extern const gmk_floc **expanding_var;
@ -539,7 +539,7 @@ extern int env_overrides, no_builtin_rules_flag, no_builtin_variables_flag;
extern int print_version_flag, print_directory_flag, check_symlink_flag;
extern int warn_undefined_variables_flag, trace_flag, posix_pedantic;
extern int not_parallel, second_expansion, clock_skew_detected;
extern int rebuilding_makefiles, one_shell, parallel_sync;
extern int rebuilding_makefiles, one_shell, output_sync;
/* can we run commands via 'sh -c xxx' or must we use batch files? */
extern int batch_mode_shell;

12
misc.c
View file

@ -235,7 +235,7 @@ message (prefix, fmt, va_alist)
if (fmt != 0)
{
if (parallel_sync)
if (output_sync)
log_working_directory (1, 1);
if (prefix)
@ -250,7 +250,7 @@ message (prefix, fmt, va_alist)
VA_END (args);
putchar ('\n');
if (parallel_sync)
if (output_sync)
log_working_directory (0, 1);
}
@ -273,7 +273,7 @@ error (flocp, fmt, va_alist)
va_list args;
#endif
if (parallel_sync)
if (output_sync)
log_working_directory (1, 1);
else
log_working_directory (1, 0);
@ -292,7 +292,7 @@ error (flocp, fmt, va_alist)
putc ('\n', stderr);
fflush (stderr);
if (parallel_sync)
if (output_sync)
log_working_directory (0, 1);
}
@ -312,7 +312,7 @@ fatal (flocp, fmt, va_alist)
va_list args;
#endif
if (parallel_sync)
if (output_sync)
log_working_directory (1, 1);
else
log_working_directory (1, 0);
@ -330,7 +330,7 @@ fatal (flocp, fmt, va_alist)
fputs (_(". Stop.\n"), stderr);
if (parallel_sync)
if (output_sync)
log_working_directory (0, 1);
die (2);

View file

@ -1,6 +1,6 @@
# -*-perl-*-
$description = "Test parallel-sync (-P) option.";
$description = "Test --output-sync (-O) option.";
$details = "Test the synchronization of output from parallel jobs.";
@ -24,11 +24,9 @@ open(MAKEFILE,"> foo/Makefile");
print MAKEFILE <<EOF;
all: foo
foo:
\@echo foo: start; $sleep_command 2; echo foo: end
foo: ; \@echo foo: start; $sleep_command 2; echo foo: end
foo-fail:
\@$sleep_command 2; false
foo-fail: ; \@$sleep_command 2; false
EOF
close(MAKEFILE);
@ -36,11 +34,9 @@ open(MAKEFILE,"> bar/Makefile");
print MAKEFILE <<EOF;
all: bar baz
bar:
\@echo bar: start; $sleep_command 1; echo bar: end
bar: ; \@echo bar: start; $sleep_command 1; echo bar: end
baz:
\@echo baz: start; $sleep_command 4; echo baz: end
baz: ; \@echo baz: start; $sleep_command 4; echo baz: end
EOF
close(MAKEFILE);
@ -48,12 +44,10 @@ close(MAKEFILE);
run_make_test('
all: make-foo make-bar
make-foo:
$(MAKE) -C foo
make-foo: ; $(MAKE) -C foo
make-bar:
$(MAKE) -C bar',
'-j -P2',
make-bar: ; $(MAKE) -C bar',
'-j -O2',
"#MAKEPATH# -C foo
#MAKEPATH# -C bar
#MAKE#[1]: Entering directory '#PWD#/foo'
@ -77,12 +71,10 @@ baz: end
run_make_test('
all: make-foo make-bar
make-foo:
$(MAKE) -C foo
make-foo: ; $(MAKE) -C foo
make-bar:
$(MAKE) -C bar',
'-j -P',
make-bar: ; $(MAKE) -C bar',
'-j --output-sync',
"#MAKEPATH# -C foo
#MAKEPATH# -C bar
#MAKE#[1]: Entering directory '#PWD#/foo'
@ -107,12 +99,10 @@ baz: end
run_make_test('
all: make-foo-fail make-bar-bar
make-foo-fail:
$(MAKE) -C foo foo-fail
make-foo-fail: ; $(MAKE) -C foo foo-fail
make-bar-bar:
$(MAKE) -C bar bar',
'-j -P',
make-bar-bar: ; $(MAKE) -C bar bar',
'-j -O',
"#MAKEPATH# -C foo foo-fail
#MAKEPATH# -C bar bar
#MAKE#[1]: Entering directory '#PWD#/foo'
@ -123,13 +113,13 @@ bar: end
#MAKE#[1]: Leaving directory '#PWD#/bar'
#MAKE#[1]: Leaving directory '#PWD#/bar'
#MAKE#[1]: Entering directory '#PWD#/foo'
Makefile:7: recipe for target 'foo-fail' failed
Makefile:5: recipe for target 'foo-fail' failed
#MAKE#[1]: Leaving directory '#PWD#/foo'
#MAKE#[1]: Entering directory '#PWD#/foo'
#MAKE#[1]: *** [foo-fail] Error 1
#MAKE#[1]: Leaving directory '#PWD#/foo'
#MAKE#[1]: Leaving directory '#PWD#/foo'
#MAKEFILE#:5: recipe for target 'make-foo-fail' failed
#MAKEFILE#:4: recipe for target 'make-foo-fail' failed
#MAKE#: *** [make-foo-fail] Error 2\n",
512);