From 370f0305ecdc05540f492b44ef10102acf0ae59a Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 18 Jul 2015 15:44:53 -0500 Subject: [PATCH 1/2] rtapi: add rtapi_barrier() --- docs/man/man3/rtapi_barrier.3rtapi | 108 +++++++++++++++++++++++++++++++++++++ src/rtapi/rtapi_bitops.h | 2 + 2 files changed, 110 insertions(+) create mode 100644 docs/man/man3/rtapi_barrier.3rtapi diff --git a/docs/man/man3/rtapi_barrier.3rtapi b/docs/man/man3/rtapi_barrier.3rtapi new file mode 100644 index 0000000..4b9bb3c --- /dev/null +++ b/docs/man/man3/rtapi_barrier.3rtapi @@ -0,0 +1,108 @@ +.\" Copyright (c) 2015 Jeff Epler +.\" +.\" This is free documentation; you can redistribute it and/or +.\" modify it under the terms of the GNU General Public License as +.\" published by the Free Software Foundation; either version 2 of +.\" the License, or (at your option) any later version. +.\" +.\" The GNU General Public License's references to "object code" +.\" and "executables" are to be interpreted as the output of any +.\" document formatting or typesetting system, including +.\" intermediate and printed output. +.\" +.\" This manual is distributed in the hope that it will be useful, +.\" but WITHOUT ANY WARRANTY; without even the implied warranty of +.\" MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +.\" GNU General Public License for more details. +.\" +.\" You should have received a copy of the GNU General Public +.\" License along with this manual; if not, write to the Free +.\" Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111, +.\" USA. +.TH funct "3rtapi" "2014-06-28" "LinuxCNC Documentation" "RTAPI" +.SH NAME + +rtapi_barrier \- enforce memory ordering between tasks + +.SH SYNTAX +.HP +#include +.HP +#define rtapi_barrier() /* architecture- and compiler-specific */ + +.SH DESCRIPTION +In multithreaded applications, not all CPU architectures make the same +promises about the order in which data written in one thread of +execution is seen in another thread. Generally, common x86 systems +innately provide stronger and more useful guarantees than the ARM +architecture. + +As a concrete example, LinuxCNC originally used a very simple method for +treating a shared memory area as a FIFO. One task, the writer, would +execute + +.RS 2 +.EX +message = 42; +message_available = 1; +.EE +.RE + +and the other task, the reader, would execute + +.RS 2 +.EX +while(!message_available) /* NOTHING */; +printf("%d\\n", message); +.EE +.RE + +(assume that before either task enters the code above, message and +message_available are both 0) However, in the ARM memory model, it is +possible for the message '0' to be printed, because writes from one +thread can become visible in another thread in a different ordering +(this is not the case on x86 or x86_64). + +The simplest way to fix this is the insertion of a barrier instruction +on each side: + +.RS 2 +.EX +message = 42; +rtapi_barrier(); +message_available = 1; +.EE +.RE + +and + +.RS 2 +.EX +while(!message_available) /* NOTHING */; +rtapi_barrier(); +printf("%d\\n", message); +.EE +.RE + +Typically, this macro expands to a "full memory barrier", gcc's +__sync_synchronize. + +.SH REALTIME CONSIDERATIONS +Can be called at any time. Generally a single instruction, albeit a +costly one (possibly in the range of 50 to 100 cycles). + +.SH RETURN VALUE +None + +.SH BUGS +Due to linuxcnc's extensive x86 heritage, may not be called in all the +right places. + +Failure to use a barrier where one is needed is often a difficult bug to +detect, occuring with a probability under 1/100000 per message in the +message passing case which originally brought the problem to light. + +.SH SEE ALSO +.UR https://en.wikipedia.org/wiki/Memory_barrier +Memory Barrier on wikipedia +.UE . diff --git a/src/rtapi/rtapi_bitops.h b/src/rtapi/rtapi_bitops.h index 0b675b1..d0c2871 100644 --- a/src/rtapi/rtapi_bitops.h +++ b/src/rtapi/rtapi_bitops.h @@ -333,4 +333,6 @@ static __inline__ long test_and_clear_bit(int nr, volatile void *addr) { return (oldval & (1lu << boff)) != 0; } #endif + +#define rtapi_barrier() (__sync_synchronize()) #endif -- 1.9.1 From 190004da51721858f0f2190a48e77c8fce0eba8e Mon Sep 17 00:00:00 2001 From: Jeff Epler Date: Sat, 18 Jul 2015 15:45:17 -0500 Subject: [PATCH 2/2] hal_stream: use rtapi_barrier to fix intermittent bugs --- src/hal/hal_lib.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/hal/hal_lib.c b/src/hal/hal_lib.c index ee2a306..e5e917f 100644 --- a/src/hal/hal_lib.c +++ b/src/hal/hal_lib.c @@ -3568,6 +3568,7 @@ int hal_stream_write(hal_stream_t *stream, union hal_stream_data *buf) { union hal_stream_data *dptr = &stream->fifo->data[in * stride]; memcpy(dptr, buf, sizeof(union hal_stream_data) * num_pins); dptr[num_pins].s = ++stream->fifo->this_sample; + rtapi_barrier(); stream->fifo->in = newin; return 0; } @@ -3577,6 +3578,7 @@ int hal_stream_read(hal_stream_t *stream, union hal_stream_data *buf, unsigned * stream->fifo->num_underruns ++; return -ENOSPC; } + rtapi_barrier(); int out = stream->fifo->out, newout = hal_stream_advance(stream, out); int num_pins = stream->fifo->num_pins; int stride = num_pins + 1; -- 1.9.1