Re: [Linuxptp-devel] [PATCH V2] ptp and igb: implement POSIX timer (alarm)

[Richard C. <ric...@gm...>]

On Mon, Oct 03, 2016 at 09:08:55PM +0200, Richard Cochran wrote:
> On Thu, Sep 15, 2016 at 01:23:20PM +0100, Kieran Tyrrell wrote:
> > Signed-off-by: Kieran Tyrrell <ki...@si...>
> 
> Overall, this is looking much better.  I will come back to this with
> some comments, but I would like to run some tests first, hopefully
> this week or next.

Once again, thanks for providing the proof-of-concept.  The patch has
some technical issues, and those would be fixable.  However, a little
while ago I ran some tests to see how well this works, and the result
convinced me that the performance of the PHC timers doesn't justify
the implementation effort.

Here is what I did.

I started with Linux 4.1.33-rt38 and backported your patch.  In
addition, I hacked the SDPs on the i210 to show up as gpios under
/sys.  Then I used a simple program (see below) running at RT priority
to toggle a gpio every 10 milliseconds according to the PTP time.
This gpio output was time stamped on an input to the PTP master.

During the test, I ran hackbench to simulate a load on the host.
I ran ptp4l on the i210 host and synchronized to an external master
connected via a crossover cable.  (Prior to running this test, I
verified the host was synchronized to within 200 nanoseconds using the
PPS output.)  I used phc2sys to synchronize the Linux system time to
the i210 PHC.

I ran the test program in two ways.

1. using nanosleep(CLOCK_REALTIME)

2. using the PHC timer provided by your patch.

The resulting graphic shows the offsets from the 100 PPS on the gpio
over a ten minute test.

   http://linuxptp.sourceforge.net/phc-timer/phc-timer-vs-nanosleep.png

These are the statistics.

  |        |     nanosleep |     PHC Timer |
  |--------+---------------+---------------|
  | min    | +8.933000e+03 | +2.080500e+04 |
  | max    | +2.962100e+04 | +4.319700e+04 |
  | pk-pk  | +2.068800e+04 | +2.239200e+04 |
  | mean   | +1.517014e+04 | +2.845244e+04 |
  | stddev | +2.449972e+03 | +3.137972e+03 |

Both results display jitter that is to be expected using a program
running under RT Linux.  What is really interesting is the average
offset.  There is an addition 13 microsecond delay when using the PHC
timer on the i210, but why?

Remember that reading the PCIe bus costs microseconds for each
operation.  The additional overhead of accessing the card over PCIe
completely dwarfs any additional time accuracy.

So the conclusion is, using PHC + phc2sys + nanosleep provides the
best performance that we can offer under Linux for globally
synchronized distributed applications.

Thanks,
Richard

---
/*
 * This program tests synchronization to a global time source
 * in a distributed system.
 *
 * Portions taken from the sgd test program.
 * Copyright (C) 2015 linutronix GmbH
 *
 * Portions taken from linux/Documentation/ptp/testptp.c.
 * Copyright (C) 2010 OMICRON electronics GmbH
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License along
 * with this program; if not, write to the Free Software Foundation, Inc.,
 * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 */
#define _GNU_SOURCE /*for CPU_SET*/
#include <errno.h>
#include <fcntl.h>
#include <pthread.h>
#include <sched.h>
#include <signal.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>

#define PERIOD_SEC	0
#define PERIOD_NSEC	10000000

static int gpio_fd = -1, running = 1, toggle;

static clockid_t get_clockid(int fd)
{
#define CLOCKFD 3
#define FD_TO_CLOCKID(fd)	((~(clockid_t) (fd) << 3) | CLOCKFD)

	return FD_TO_CLOCKID(fd);
}

static void gpio_pulse(void)
{
	char c;

	c = '1';
	write(gpio_fd, &c, sizeof(c));
	c = '0';
	write(gpio_fd, &c, sizeof(c));
}

static void gpio_toggle(void)
{
	static int even;
	char c;

	if (even) {
		c = '1';
		write(gpio_fd, &c, sizeof(c));
	} else {
		c = '0';
		write(gpio_fd, &c, sizeof(c));
	}
	even = 1 - even;
}

static void handle_alarm(int s)
{
	if (toggle)
		gpio_toggle();
	else
		gpio_pulse();
}

static int install_handler(int signum, void (*handler)(int))
{
	struct sigaction action;
	sigset_t mask;

	/* Unblock the signal. */
	sigemptyset(&mask);
	sigaddset(&mask, signum);
	sigprocmask(SIG_UNBLOCK, &mask, NULL);

	/* Install the signal handler. */
	action.sa_handler = handler;
	action.sa_flags = 0;
	sigemptyset(&action.sa_mask);
	sigaction(signum, &action, NULL);

	return 0;
}

static int run_nanosleep(clockid_t clkid)
{
	struct timespec ts;
	int err;

	clock_gettime(clkid, &ts);
	ts.tv_sec += 2;
	ts.tv_nsec = 0;

	while (running) {
		err = clock_nanosleep(clkid, TIMER_ABSTIME, &ts, NULL);
		switch (err) {
		case 0:
			if (toggle) {
				gpio_toggle();
			} else {
				gpio_pulse();
			}
			ts.tv_sec += PERIOD_SEC;
			ts.tv_nsec += PERIOD_NSEC;
			while (ts.tv_nsec > 999999999) {
				ts.tv_sec += 1;
				ts.tv_nsec -= 1000000000;
			}
			break;
		case EINTR:
			continue;
		default:
			fprintf(stderr, "clock_nanosleep returned %d: %s",
				err, strerror(err));
		}
	}

	return 0;
}

static int run_timer(clockid_t clkid)
{
	static timer_t timerid;
	struct itimerspec timeout;
	struct sigevent sigevent;

	install_handler(SIGALRM, handle_alarm);

	/* Create a timer. */
	sigevent.sigev_notify = SIGEV_SIGNAL;
	sigevent.sigev_signo = SIGALRM;
	if (timer_create(clkid, &sigevent, &timerid)) {
		perror("timer_create");
		return -1;
	}

	/* Start the timer. */
	memset(&timeout, 0, sizeof(timeout));
	timeout.it_interval.tv_sec = PERIOD_SEC;
	timeout.it_interval.tv_nsec = PERIOD_NSEC;
	clock_gettime(clkid, &timeout.it_value);
	timeout.it_value.tv_sec += 2;
	timeout.it_value.tv_nsec = 0;

	if (timer_settime(timerid, TIMER_ABSTIME, &timeout, NULL)) {
		perror("timer_settime");
		return -1;
	}

	while (running) {
		pause();
	}

	timer_delete(timerid);
	return 0;
}

static int set_realtime(pthread_t thread, int priority, int cpu)
{
	cpu_set_t cpuset;
	struct sched_param sp;
	int err, policy;

	int min = sched_get_priority_min(SCHED_FIFO);
	int max = sched_get_priority_max(SCHED_FIFO);

	fprintf(stderr, "min %d max %d\n", min, max);

	if (priority < 0) {
		return 0;
	}

	err = pthread_getschedparam(thread, &policy, &sp);
	if (err) {
		fprintf(stderr, "pthread_getschedparam: %s\n", strerror(err));
		return -1;
	}

	sp.sched_priority = priority;

	err = pthread_setschedparam(thread, SCHED_FIFO, &sp);
	if (err) {
		fprintf(stderr, "pthread_setschedparam: %s\n", strerror(err));
		return -1;
	}

	if (cpu < 0) {
		return 0;
	}
	CPU_ZERO(&cpuset);
	CPU_SET(cpu, &cpuset);
	err = pthread_setaffinity_np(thread, sizeof(cpu_set_t), &cpuset);
	if (err) {
		fprintf(stderr, "pthread_setaffinity_np: %s\n", strerror(err));
		return -1;
	}

	return 0;
}

static void usage(char *progname)
{
	fprintf(stderr,
		"\n"
		"usage: %s [options]\n"
		"\n"
		" -c [num]   run on CPU 'num'\n"
		" -d [file]  open device 'file'\n"
		"            omit to use CLOCK_REALTIME instead\n"
		" -h         prints this message and exits\n"
		" -n         use clock_nanosleep() instead of timer_settime()\n"
		" -o [file]  write to gpio 'file'\n"
		" -p [num]   run with RT priorty 'num'\n"
		" -t         toggle instead of making a pulse\n"
		"\n",
		progname);
}

int main(int argc, char *argv[])
{
	int c, cpu = -1, fd, err, nanosleep = 0, priority = -1;
	char *device = NULL, *gpio = NULL, *progname;
	clockid_t clkid;

	/* Process the command line arguments. */
	progname = strrchr(argv[0], '/');
	progname = progname ? 1 + progname : argv[0];
	while (EOF != (c = getopt(argc, argv, "c:d:hno:p:t"))) {
		switch (c) {
		case 'c':
			cpu = atoi(optarg);
			break;
		case 'd':
			device = optarg;
			break;
		case 'h':
			usage(progname);
			return 0;
		case 'n':
			nanosleep = 1;
			break;
		case 'o':
			gpio = optarg;
			break;
		case 'p':
			priority = atoi(optarg);
			break;
		case 't':
			toggle = 1;
			break;
		case '?':
			usage(progname);
			return -1;
		}
	}
	if (!gpio) {
		usage(progname);
		return -1;
	}
	if (set_realtime(pthread_self(), priority, cpu)) {
		return -1;
	}
	gpio_fd = open(gpio, O_RDWR);
	if (gpio_fd < 0) {
		fprintf(stderr, "cannot open %s: %m\n", gpio);
		return -1;
	}
	if (device) {
		fd = open(device, O_RDWR);
		if (fd < 0) {
			fprintf(stderr, "cannot open %s: %m\n", device);
			return -1;
		}
		clkid = get_clockid(fd);
	} else {
		clkid = CLOCK_REALTIME;
	}

	if (nanosleep) {
		err = run_nanosleep(clkid);
	} else {
		err = run_timer(clkid);
	}

	if (device) {
		close(fd);
	}
	close(gpio_fd);
	return err;
}